feedx 0.2.0 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 88ccb5e67e8ad202fab870bea54b8ff04c26b644fa3f8e07f8aa3b0c09a75351
4
- data.tar.gz: b832b9867141506986402d5a091f0c35528ee739ea624e7616e7e018748c0cd1
3
+ metadata.gz: 268fb5c866f233c844f43b2ba3a19a607bdf719db6a84e0e843ad98f7f4ec899
4
+ data.tar.gz: 19e359168ab87c5f1049a2a325480d8a48611c3693b1d33b00bed6150dd3bc20
5
5
  SHA512:
6
- metadata.gz: 4e9df1a95a2821b882c6eed8e0d7b36532bb734b43405eb5e90c24984b8c7a0ef5fa4f7f8b35fd371a7ce724ce00e6d9a6174753ae552340773e9367024b8fce
7
- data.tar.gz: 9e2d6714753f66eb2976a591eaf95a1b7caa4476e404d5622820de824bc7212e182b8d3b6cac0ae50cc5b2a6bcfd1cc4c2ab94e1a36b20b99563bae957969a45
6
+ metadata.gz: 2a6a8b0282f4a17556ae261ba6f90399c2e8e2633cb6fee594ca529fe0f592cce25cceafa523ba9fda9ea6fd13a794a843c75125abd7a633abd033b1996ffd12
7
+ data.tar.gz: fb5577e3dd555ee27b9a399f2eabf3b30957be6c41218b74e9daaf9dfef335b50cd529a66d723903f04d2ba7063c4bd7d8b49ceb425ca3d7899f6ebd6ec41680
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- feedx (0.2.0)
4
+ feedx (0.2.1)
5
5
  bfs
6
6
 
7
7
  GEM
data/feedx.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'feedx'
3
- s.version = '0.2.0'
3
+ s.version = '0.2.1'
4
4
  s.authors = ['Black Square Media Ltd']
5
5
  s.email = ['info@blacksquaremedia.com']
6
6
  s.summary = %(Exchange data between components via feeds)
data/lib/feedx.rb CHANGED
@@ -2,4 +2,5 @@ module Feedx
2
2
  autoload :Compression, 'feedx/compression'
3
3
  autoload :Format, 'feedx/format'
4
4
  autoload :Pusher, 'feedx/pusher'
5
+ autoload :TaskState, 'feedx/task_state'
5
6
  end
data/lib/feedx/pusher.rb CHANGED
@@ -4,6 +4,8 @@ require 'bfs'
4
4
  module Feedx
5
5
  # Pushes a relation as a protobuf encoded stream to an S3 location.
6
6
  class Pusher
7
+ autoload :Recurring, 'feedx/pusher/recurring'
8
+
7
9
  # See constructor.
8
10
  def self.perform(url, opts={}, &block)
9
11
  new(url, opts, &block).perform
@@ -25,13 +27,18 @@ module Feedx
25
27
  @compress = detect_compress(opts[:compress])
26
28
  end
27
29
 
28
- def perform
30
+ def perform(enum=build_enum)
29
31
  @blob.create do |io|
30
- @compress.wrap(io) {|w| write_to(w) }
32
+ @compress.wrap(io) {|w| write_all(enum, w) }
31
33
  end
32
34
  @blob.info.size
33
35
  end
34
36
 
37
+ # @return [Enumerable,ActiveRecord::Relation] the relation or enumerator.
38
+ def build_enum
39
+ @enum.is_a?(Proc) ? @enum.call : @enum
40
+ end
41
+
35
42
  private
36
43
 
37
44
  def detect_format(val)
@@ -62,11 +69,10 @@ module Feedx
62
69
  end
63
70
  end
64
71
 
65
- def write_to(io)
72
+ def write_all(enum, io)
66
73
  stream = @format.new(io)
67
- relation = @enum.is_a?(Proc) ? @enum.call : @enum
68
- iterator = relation.respond_to?(:find_each) ? :find_each : :each
69
- relation.send(iterator) {|rec| stream.write(rec) }
74
+ iterator = enum.respond_to?(:find_each) ? :find_each : :each
75
+ enum.send(iterator) {|rec| stream.write(rec) }
70
76
  end
71
77
  end
72
78
  end
@@ -0,0 +1,39 @@
1
+ module Feedx
2
+ class Pusher
3
+ module Recurring
4
+ DEFAULT_CHECK = proc do |enum|
5
+ enum.respond_to?(:maximum) ? enum.maximum(:updated_at).to_f : 0.0
6
+ end.freeze
7
+
8
+ # Registered pushers
9
+ def self.registry
10
+ @registry ||= {}
11
+ end
12
+
13
+ # Performs a recurring Pusher task. See see Feedx::Pusher.new for additional
14
+ # information on params and options.
15
+ #
16
+ # @param [String] name a unique task name.
17
+ # @param [Hash] opts options.
18
+ # @option [Proc] :check a custom revision check to evalute before perform.
19
+ # Receives the enum as an argument and returns a Numeric revision.
20
+ # Default: ->(enum) { enum.maximum(:updated_at).to_f }
21
+ def self.perform(name, url, opts={}, &block)
22
+ stored = registry[name]
23
+ unless stored
24
+ check = opts.delete(:check) || DEFAULT_CHECK
25
+ stored = { pusher: Feedx::Pusher.new(url, opts, &block), check: check, revision: 0 }
26
+ registry[name] = stored
27
+ end
28
+
29
+ enum = stored[:pusher].build_enum
30
+ latest = stored[:check].call(enum).to_f
31
+ return -1 unless stored[:revision] < latest
32
+
33
+ size = stored[:pusher].perform(enum)
34
+ stored[:revision] = latest
35
+ size
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,28 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe Feedx::Pusher::Recurring do
4
+ let :enumerable do
5
+ enum = %w[x y z].map {|t| { title: t } } * 100
6
+ enum.instance_eval do
7
+ def maximum(*)
8
+ 1515151515.0
9
+ end
10
+ end
11
+ enum
12
+ end
13
+
14
+ let(:tempdir) { Dir.mktmpdir }
15
+ after { FileUtils.rm_rf tempdir }
16
+ after { described_class.registry.clear }
17
+
18
+ it 'should perform conditionally' do
19
+ size = described_class.perform 'test.task', "file://#{tempdir}/file.json", enum: enumerable
20
+ expect(size).to eq(4200)
21
+ expect(File.size("#{tempdir}/file.json")).to eq(size)
22
+ expect(described_class.registry).to include('test.task')
23
+ expect(described_class.registry['test.task']).to include(revision: 1515151515.0)
24
+
25
+ size = described_class.perform 'test.task', "file://#{tempdir}/file.json", enum: enumerable
26
+ expect(size).to eq(-1)
27
+ end
28
+ end
@@ -34,7 +34,7 @@ RSpec.describe Feedx::Pusher do
34
34
 
35
35
  it 'should push plain JSON' do
36
36
  size = described_class.perform "file://#{tempdir}/file.json", enum: enumerable
37
- expect(size).to be_within(0).of(15900)
37
+ expect(size).to eq(15900)
38
38
  expect(File.size("#{tempdir}/file.json")).to eq(size)
39
39
  end
40
40
 
@@ -46,15 +46,15 @@ RSpec.describe Feedx::Pusher do
46
46
 
47
47
  it 'should push plain PB' do
48
48
  size = described_class.perform "file://#{tempdir}/file.pb", enum: enumerable
49
- expect(size).to be_within(0).of(1200)
49
+ expect(size).to eq(1200)
50
50
  expect(File.size("#{tempdir}/file.pb")).to eq(size)
51
51
  end
52
52
 
53
53
  it 'should support factories' do
54
54
  size = described_class.perform("file://#{tempdir}/file.json") do
55
- [model.new('xy')] * 10
55
+ enumerable
56
56
  end
57
- expect(size).to be_within(0).of(540)
57
+ expect(size).to eq(15900)
58
58
  expect(File.size("#{tempdir}/file.json")).to eq(size)
59
59
  end
60
60
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: feedx
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Black Square Media Ltd
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-11-22 00:00:00.000000000 Z
11
+ date: 2018-11-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bfs
@@ -121,12 +121,14 @@ files:
121
121
  - lib/feedx/format/json.rb
122
122
  - lib/feedx/format/protobuf.rb
123
123
  - lib/feedx/pusher.rb
124
+ - lib/feedx/pusher/recurring.rb
124
125
  - spec/feedx/compression/gzip_spec.rb
125
126
  - spec/feedx/compression/none_spec.rb
126
127
  - spec/feedx/compression_spec.rb
127
128
  - spec/feedx/format/json_spec.rb
128
129
  - spec/feedx/format/protobuf_spec.rb
129
130
  - spec/feedx/format_spec.rb
131
+ - spec/feedx/pusher/recurring_spec.rb
130
132
  - spec/feedx/pusher_spec.rb
131
133
  - spec/spec_helper.rb
132
134
  homepage: https://github.com/bsm/feedx
@@ -160,5 +162,6 @@ test_files:
160
162
  - spec/feedx/format/json_spec.rb
161
163
  - spec/feedx/format/protobuf_spec.rb
162
164
  - spec/feedx/format_spec.rb
165
+ - spec/feedx/pusher/recurring_spec.rb
163
166
  - spec/feedx/pusher_spec.rb
164
167
  - spec/spec_helper.rb