feedx 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/feedx.gemspec +1 -1
- data/lib/feedx.rb +1 -0
- data/lib/feedx/pusher.rb +12 -6
- data/lib/feedx/pusher/recurring.rb +39 -0
- data/spec/feedx/pusher/recurring_spec.rb +28 -0
- data/spec/feedx/pusher_spec.rb +4 -4
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 268fb5c866f233c844f43b2ba3a19a607bdf719db6a84e0e843ad98f7f4ec899
|
4
|
+
data.tar.gz: 19e359168ab87c5f1049a2a325480d8a48611c3693b1d33b00bed6150dd3bc20
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2a6a8b0282f4a17556ae261ba6f90399c2e8e2633cb6fee594ca529fe0f592cce25cceafa523ba9fda9ea6fd13a794a843c75125abd7a633abd033b1996ffd12
|
7
|
+
data.tar.gz: fb5577e3dd555ee27b9a399f2eabf3b30957be6c41218b74e9daaf9dfef335b50cd529a66d723903f04d2ba7063c4bd7d8b49ceb425ca3d7899f6ebd6ec41680
|
data/Gemfile.lock
CHANGED
data/feedx.gemspec
CHANGED
data/lib/feedx.rb
CHANGED
data/lib/feedx/pusher.rb
CHANGED
@@ -4,6 +4,8 @@ require 'bfs'
|
|
4
4
|
module Feedx
|
5
5
|
# Pushes a relation as a protobuf encoded stream to an S3 location.
|
6
6
|
class Pusher
|
7
|
+
autoload :Recurring, 'feedx/pusher/recurring'
|
8
|
+
|
7
9
|
# See constructor.
|
8
10
|
def self.perform(url, opts={}, &block)
|
9
11
|
new(url, opts, &block).perform
|
@@ -25,13 +27,18 @@ module Feedx
|
|
25
27
|
@compress = detect_compress(opts[:compress])
|
26
28
|
end
|
27
29
|
|
28
|
-
def perform
|
30
|
+
def perform(enum=build_enum)
|
29
31
|
@blob.create do |io|
|
30
|
-
@compress.wrap(io) {|w|
|
32
|
+
@compress.wrap(io) {|w| write_all(enum, w) }
|
31
33
|
end
|
32
34
|
@blob.info.size
|
33
35
|
end
|
34
36
|
|
37
|
+
# @return [Enumerable,ActiveRecord::Relation] the relation or enumerator.
|
38
|
+
def build_enum
|
39
|
+
@enum.is_a?(Proc) ? @enum.call : @enum
|
40
|
+
end
|
41
|
+
|
35
42
|
private
|
36
43
|
|
37
44
|
def detect_format(val)
|
@@ -62,11 +69,10 @@ module Feedx
|
|
62
69
|
end
|
63
70
|
end
|
64
71
|
|
65
|
-
def
|
72
|
+
def write_all(enum, io)
|
66
73
|
stream = @format.new(io)
|
67
|
-
|
68
|
-
iterator
|
69
|
-
relation.send(iterator) {|rec| stream.write(rec) }
|
74
|
+
iterator = enum.respond_to?(:find_each) ? :find_each : :each
|
75
|
+
enum.send(iterator) {|rec| stream.write(rec) }
|
70
76
|
end
|
71
77
|
end
|
72
78
|
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Feedx
|
2
|
+
class Pusher
|
3
|
+
module Recurring
|
4
|
+
DEFAULT_CHECK = proc do |enum|
|
5
|
+
enum.respond_to?(:maximum) ? enum.maximum(:updated_at).to_f : 0.0
|
6
|
+
end.freeze
|
7
|
+
|
8
|
+
# Registered pushers
|
9
|
+
def self.registry
|
10
|
+
@registry ||= {}
|
11
|
+
end
|
12
|
+
|
13
|
+
# Performs a recurring Pusher task. See see Feedx::Pusher.new for additional
|
14
|
+
# information on params and options.
|
15
|
+
#
|
16
|
+
# @param [String] name a unique task name.
|
17
|
+
# @param [Hash] opts options.
|
18
|
+
# @option [Proc] :check a custom revision check to evalute before perform.
|
19
|
+
# Receives the enum as an argument and returns a Numeric revision.
|
20
|
+
# Default: ->(enum) { enum.maximum(:updated_at).to_f }
|
21
|
+
def self.perform(name, url, opts={}, &block)
|
22
|
+
stored = registry[name]
|
23
|
+
unless stored
|
24
|
+
check = opts.delete(:check) || DEFAULT_CHECK
|
25
|
+
stored = { pusher: Feedx::Pusher.new(url, opts, &block), check: check, revision: 0 }
|
26
|
+
registry[name] = stored
|
27
|
+
end
|
28
|
+
|
29
|
+
enum = stored[:pusher].build_enum
|
30
|
+
latest = stored[:check].call(enum).to_f
|
31
|
+
return -1 unless stored[:revision] < latest
|
32
|
+
|
33
|
+
size = stored[:pusher].perform(enum)
|
34
|
+
stored[:revision] = latest
|
35
|
+
size
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe Feedx::Pusher::Recurring do
|
4
|
+
let :enumerable do
|
5
|
+
enum = %w[x y z].map {|t| { title: t } } * 100
|
6
|
+
enum.instance_eval do
|
7
|
+
def maximum(*)
|
8
|
+
1515151515.0
|
9
|
+
end
|
10
|
+
end
|
11
|
+
enum
|
12
|
+
end
|
13
|
+
|
14
|
+
let(:tempdir) { Dir.mktmpdir }
|
15
|
+
after { FileUtils.rm_rf tempdir }
|
16
|
+
after { described_class.registry.clear }
|
17
|
+
|
18
|
+
it 'should perform conditionally' do
|
19
|
+
size = described_class.perform 'test.task', "file://#{tempdir}/file.json", enum: enumerable
|
20
|
+
expect(size).to eq(4200)
|
21
|
+
expect(File.size("#{tempdir}/file.json")).to eq(size)
|
22
|
+
expect(described_class.registry).to include('test.task')
|
23
|
+
expect(described_class.registry['test.task']).to include(revision: 1515151515.0)
|
24
|
+
|
25
|
+
size = described_class.perform 'test.task', "file://#{tempdir}/file.json", enum: enumerable
|
26
|
+
expect(size).to eq(-1)
|
27
|
+
end
|
28
|
+
end
|
data/spec/feedx/pusher_spec.rb
CHANGED
@@ -34,7 +34,7 @@ RSpec.describe Feedx::Pusher do
|
|
34
34
|
|
35
35
|
it 'should push plain JSON' do
|
36
36
|
size = described_class.perform "file://#{tempdir}/file.json", enum: enumerable
|
37
|
-
expect(size).to
|
37
|
+
expect(size).to eq(15900)
|
38
38
|
expect(File.size("#{tempdir}/file.json")).to eq(size)
|
39
39
|
end
|
40
40
|
|
@@ -46,15 +46,15 @@ RSpec.describe Feedx::Pusher do
|
|
46
46
|
|
47
47
|
it 'should push plain PB' do
|
48
48
|
size = described_class.perform "file://#{tempdir}/file.pb", enum: enumerable
|
49
|
-
expect(size).to
|
49
|
+
expect(size).to eq(1200)
|
50
50
|
expect(File.size("#{tempdir}/file.pb")).to eq(size)
|
51
51
|
end
|
52
52
|
|
53
53
|
it 'should support factories' do
|
54
54
|
size = described_class.perform("file://#{tempdir}/file.json") do
|
55
|
-
|
55
|
+
enumerable
|
56
56
|
end
|
57
|
-
expect(size).to
|
57
|
+
expect(size).to eq(15900)
|
58
58
|
expect(File.size("#{tempdir}/file.json")).to eq(size)
|
59
59
|
end
|
60
60
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: feedx
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Black Square Media Ltd
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-11-
|
11
|
+
date: 2018-11-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bfs
|
@@ -121,12 +121,14 @@ files:
|
|
121
121
|
- lib/feedx/format/json.rb
|
122
122
|
- lib/feedx/format/protobuf.rb
|
123
123
|
- lib/feedx/pusher.rb
|
124
|
+
- lib/feedx/pusher/recurring.rb
|
124
125
|
- spec/feedx/compression/gzip_spec.rb
|
125
126
|
- spec/feedx/compression/none_spec.rb
|
126
127
|
- spec/feedx/compression_spec.rb
|
127
128
|
- spec/feedx/format/json_spec.rb
|
128
129
|
- spec/feedx/format/protobuf_spec.rb
|
129
130
|
- spec/feedx/format_spec.rb
|
131
|
+
- spec/feedx/pusher/recurring_spec.rb
|
130
132
|
- spec/feedx/pusher_spec.rb
|
131
133
|
- spec/spec_helper.rb
|
132
134
|
homepage: https://github.com/bsm/feedx
|
@@ -160,5 +162,6 @@ test_files:
|
|
160
162
|
- spec/feedx/format/json_spec.rb
|
161
163
|
- spec/feedx/format/protobuf_spec.rb
|
162
164
|
- spec/feedx/format_spec.rb
|
165
|
+
- spec/feedx/pusher/recurring_spec.rb
|
163
166
|
- spec/feedx/pusher_spec.rb
|
164
167
|
- spec/spec_helper.rb
|