feedx 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88ccb5e67e8ad202fab870bea54b8ff04c26b644fa3f8e07f8aa3b0c09a75351
-  data.tar.gz: b832b9867141506986402d5a091f0c35528ee739ea624e7616e7e018748c0cd1
+  metadata.gz: 268fb5c866f233c844f43b2ba3a19a607bdf719db6a84e0e843ad98f7f4ec899
+  data.tar.gz: 19e359168ab87c5f1049a2a325480d8a48611c3693b1d33b00bed6150dd3bc20
 SHA512:
-  metadata.gz: 4e9df1a95a2821b882c6eed8e0d7b36532bb734b43405eb5e90c24984b8c7a0ef5fa4f7f8b35fd371a7ce724ce00e6d9a6174753ae552340773e9367024b8fce
-  data.tar.gz: 9e2d6714753f66eb2976a591eaf95a1b7caa4476e404d5622820de824bc7212e182b8d3b6cac0ae50cc5b2a6bcfd1cc4c2ab94e1a36b20b99563bae957969a45
+  metadata.gz: 2a6a8b0282f4a17556ae261ba6f90399c2e8e2633cb6fee594ca529fe0f592cce25cceafa523ba9fda9ea6fd13a794a843c75125abd7a633abd033b1996ffd12
+  data.tar.gz: fb5577e3dd555ee27b9a399f2eabf3b30957be6c41218b74e9daaf9dfef335b50cd529a66d723903f04d2ba7063c4bd7d8b49ceb425ca3d7899f6ebd6ec41680

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    feedx (0.2.0)
+    feedx (0.2.1)
       bfs
 
 GEM

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.2.0'
+  s.version       = '0.2.1'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)

data/lib/feedx.rb

@@ -2,4 +2,5 @@ module Feedx
   autoload :Compression, 'feedx/compression'
   autoload :Format, 'feedx/format'
   autoload :Pusher, 'feedx/pusher'
+  autoload :TaskState, 'feedx/task_state'
 end

data/lib/feedx/pusher.rb

@@ -4,6 +4,8 @@ require 'bfs'
 module Feedx
   # Pushes a relation as a protobuf encoded stream to an S3 location.
   class Pusher
+    autoload :Recurring, 'feedx/pusher/recurring'
+
     # See constructor.
     def self.perform(url, opts={}, &block)
       new(url, opts, &block).perform
@@ -25,13 +27,18 @@ module Feedx
       @compress = detect_compress(opts[:compress])
     end
 
-    def perform
+    def perform(enum=build_enum)
       @blob.create do |io|
-        @compress.wrap(io) {|w| write_to(w) }
+        @compress.wrap(io) {|w| write_all(enum, w) }
       end
       @blob.info.size
     end
 
+    # @return [Enumerable,ActiveRecord::Relation] the relation or enumerator.
+    def build_enum
+      @enum.is_a?(Proc) ? @enum.call : @enum
+    end
+
     private
 
     def detect_format(val)
@@ -62,11 +69,10 @@ module Feedx
       end
     end
 
-    def write_to(io)
+    def write_all(enum, io)
       stream   = @format.new(io)
-      relation = @enum.is_a?(Proc) ? @enum.call : @enum
-      iterator = relation.respond_to?(:find_each) ? :find_each : :each
-      relation.send(iterator) {|rec| stream.write(rec) }
+      iterator = enum.respond_to?(:find_each) ? :find_each : :each
+      enum.send(iterator) {|rec| stream.write(rec) }
     end
   end
 end

data/lib/feedx/pusher/recurring.rb

@@ -0,0 +1,39 @@
+module Feedx
+  class Pusher
+    module Recurring
+      DEFAULT_CHECK = proc do |enum|
+        enum.respond_to?(:maximum) ? enum.maximum(:updated_at).to_f : 0.0
+      end.freeze
+
+      # Registered pushers
+      def self.registry
+        @registry ||= {}
+      end
+
+      # Performs a recurring Pusher task. See see Feedx::Pusher.new for additional
+      # information on params and options.
+      #
+      # @param [String] name a unique task name.
+      # @param [Hash] opts options.
+      # @option [Proc] :check a custom revision check to evalute before perform.
+      #   Receives the enum as an argument and returns a Numeric revision.
+      #   Default: ->(enum) { enum.maximum(:updated_at).to_f }
+      def self.perform(name, url, opts={}, &block)
+        stored = registry[name]
+        unless stored
+          check  = opts.delete(:check) || DEFAULT_CHECK
+          stored = { pusher: Feedx::Pusher.new(url, opts, &block), check: check, revision: 0 }
+          registry[name] = stored
+        end
+
+        enum   = stored[:pusher].build_enum
+        latest = stored[:check].call(enum).to_f
+        return -1 unless stored[:revision] < latest
+
+        size = stored[:pusher].perform(enum)
+        stored[:revision] = latest
+        size
+      end
+    end
+  end
+end

data/spec/feedx/pusher/recurring_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Pusher::Recurring do
+  let :enumerable do
+    enum = %w[x y z].map {|t| { title: t } } * 100
+    enum.instance_eval do
+      def maximum(*)
+        1515151515.0
+      end
+    end
+    enum
+  end
+
+  let(:tempdir) { Dir.mktmpdir }
+  after { FileUtils.rm_rf tempdir }
+  after { described_class.registry.clear }
+
+  it 'should perform conditionally' do
+    size = described_class.perform 'test.task', "file://#{tempdir}/file.json", enum: enumerable
+    expect(size).to eq(4200)
+    expect(File.size("#{tempdir}/file.json")).to eq(size)
+    expect(described_class.registry).to include('test.task')
+    expect(described_class.registry['test.task']).to include(revision: 1515151515.0)
+
+    size = described_class.perform 'test.task', "file://#{tempdir}/file.json", enum: enumerable
+    expect(size).to eq(-1)
+  end
+end

data/spec/feedx/pusher_spec.rb

@@ -34,7 +34,7 @@ RSpec.describe Feedx::Pusher do
 
   it 'should push plain JSON' do
     size = described_class.perform "file://#{tempdir}/file.json", enum: enumerable
-    expect(size).to be_within(0).of(15900)
+    expect(size).to eq(15900)
     expect(File.size("#{tempdir}/file.json")).to eq(size)
   end
 
@@ -46,15 +46,15 @@ RSpec.describe Feedx::Pusher do
 
   it 'should push plain PB' do
     size = described_class.perform "file://#{tempdir}/file.pb", enum: enumerable
-    expect(size).to be_within(0).of(1200)
+    expect(size).to eq(1200)
     expect(File.size("#{tempdir}/file.pb")).to eq(size)
   end
 
   it 'should support factories' do
     size = described_class.perform("file://#{tempdir}/file.json") do
-      [model.new('xy')] * 10
+      enumerable
     end
-    expect(size).to be_within(0).of(540)
+    expect(size).to eq(15900)
     expect(File.size("#{tempdir}/file.json")).to eq(size)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-22 00:00:00.000000000 Z
+date: 2018-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -121,12 +121,14 @@ files:
 - lib/feedx/format/json.rb
 - lib/feedx/format/protobuf.rb
 - lib/feedx/pusher.rb
+- lib/feedx/pusher/recurring.rb
 - spec/feedx/compression/gzip_spec.rb
 - spec/feedx/compression/none_spec.rb
 - spec/feedx/compression_spec.rb
 - spec/feedx/format/json_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
+- spec/feedx/pusher/recurring_spec.rb
 - spec/feedx/pusher_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/bsm/feedx
@@ -160,5 +162,6 @@ test_files:
 - spec/feedx/format/json_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
+- spec/feedx/pusher/recurring_spec.rb
 - spec/feedx/pusher_spec.rb
 - spec/spec_helper.rb