feedx 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 268fb5c866f233c844f43b2ba3a19a607bdf719db6a84e0e843ad98f7f4ec899
-  data.tar.gz: 19e359168ab87c5f1049a2a325480d8a48611c3693b1d33b00bed6150dd3bc20
+  metadata.gz: f277d04178e84d34dbdaf5a34c0583cd8e058f7afd3780986fd4a5ba18e27e1c
+  data.tar.gz: b806a72c9f263a4819d13068fa347a002d5084c8f95c4c1586e843af95f683fa
 SHA512:
-  metadata.gz: 2a6a8b0282f4a17556ae261ba6f90399c2e8e2633cb6fee594ca529fe0f592cce25cceafa523ba9fda9ea6fd13a794a843c75125abd7a633abd033b1996ffd12
-  data.tar.gz: fb5577e3dd555ee27b9a399f2eabf3b30957be6c41218b74e9daaf9dfef335b50cd529a66d723903f04d2ba7063c4bd7d8b49ceb425ca3d7899f6ebd6ec41680
+  metadata.gz: 381ac4112b4b9da045f52f6663f004f80404e58c533b6be0298cf55212db700ae50d7e5f0f6e4064930c266d4c3677f3f6aded9480c6191c936580810a88ee03
+  data.tar.gz: b90fc37870f6767b4b19b15cca79a242f36e5922c93084f2b235430ac6879e8e4877703dd1335d9b44f879db3dc79f680775e95aa3c7b0c1aad9b276ac24b6aa

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    feedx (0.2.1)
-      bfs
+    feedx (0.3.0)
+      bfs (>= 0.3.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.0)
-    bfs (0.3.2)
+    bfs (0.3.3)
     diff-lcs (1.3)
     google-protobuf (3.6.1)
     jaro_winkler (1.5.1)

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.2.1'
+  s.version       = '0.3.0'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)
@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
   s.required_ruby_version = '>= 2.2'
 
-  s.add_dependency 'bfs'
+  s.add_dependency 'bfs', '>= 0.3.3'
 
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'pbio'

data/lib/feedx/pusher.rb

@@ -6,6 +6,8 @@ module Feedx
   class Pusher
     autoload :Recurring, 'feedx/pusher/recurring'
 
+    META_LAST_MODIFIED = 'X-Feedx-Pusher-Last-Modified'.freeze
+
     # See constructor.
     def self.perform(url, opts={}, &block)
       new(url, opts, &block).perform
@@ -16,6 +18,7 @@ module Feedx
     # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
+    # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
     # @yield A block factory to generate the relation or enumerator.
     # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
     def initialize(url, opts={}, &block)
@@ -25,20 +28,26 @@ module Feedx
       @blob     = BFS::Blob.new(url)
       @format   = detect_format(opts[:format])
       @compress = detect_compress(opts[:compress])
+      @last_mod = opts[:last_modified]
     end
 
-    def perform(enum=build_enum)
-      @blob.create do |io|
+    def perform
+      enum = @enum.is_a?(Proc) ? @enum.call : @enum
+      last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
+      current  = (last_mod.to_f * 1000).floor
+
+      begin
+        previous = @blob.info.metadata[META_LAST_MODIFIED].to_i
+        return -1 unless current > previous
+      rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
+      end if current.positive?
+
+      @blob.create metadata: { META_LAST_MODIFIED => current.to_s } do |io|
         @compress.wrap(io) {|w| write_all(enum, w) }
       end
       @blob.info.size
     end
 
-    # @return [Enumerable,ActiveRecord::Relation] the relation or enumerator.
-    def build_enum
-      @enum.is_a?(Proc) ? @enum.call : @enum
-    end
-
     private
 
     def detect_format(val)

data/spec/feedx/pusher_spec.rb

@@ -17,44 +17,63 @@ RSpec.describe Feedx::Pusher do
     %w[x y z].map {|t| model.new(t) } * 100
   end
 
-  let(:tempdir) { Dir.mktmpdir }
-  after { FileUtils.rm_rf tempdir }
+  let(:bucket) { BFS::Bucket::InMem.new }
+  before { allow(BFS).to receive(:resolve).and_return(bucket) }
 
   it 'should reject invalid inputs' do
     expect do
-      described_class.perform "file://#{tempdir}/file.txt", enum: enumerable
+      described_class.perform 'mock:///dir/file.txt', enum: enumerable
     end.to raise_error(/unable to detect format/)
   end
 
   it 'should push compressed JSON' do
-    size = described_class.perform "file://#{tempdir}/file.jsonz", enum: enumerable
+    size = described_class.perform 'mock:///dir/file.jsonz', enum: enumerable
     expect(size).to be_within(20).of(166)
-    expect(File.size("#{tempdir}/file.jsonz")).to eq(size)
+    expect(bucket.info('dir/file.jsonz').size).to eq(size)
   end
 
   it 'should push plain JSON' do
-    size = described_class.perform "file://#{tempdir}/file.json", enum: enumerable
+    size = described_class.perform 'mock:///dir/file.json', enum: enumerable
     expect(size).to eq(15900)
-    expect(File.size("#{tempdir}/file.json")).to eq(size)
+    expect(bucket.info('dir/file.json').size).to eq(size)
   end
 
   it 'should push compressed PB' do
-    size = described_class.perform "file://#{tempdir}/file.pbz", enum: enumerable
+    size = described_class.perform 'mock:///dir/file.pbz', enum: enumerable
     expect(size).to be_within(20).of(41)
-    expect(File.size("#{tempdir}/file.pbz")).to eq(size)
+    expect(bucket.info('dir/file.pbz').size).to eq(size)
   end
 
   it 'should push plain PB' do
-    size = described_class.perform "file://#{tempdir}/file.pb", enum: enumerable
+    size = described_class.perform 'mock:///dir/file.pb', enum: enumerable
     expect(size).to eq(1200)
-    expect(File.size("#{tempdir}/file.pb")).to eq(size)
+    expect(bucket.info('dir/file.pb').size).to eq(size)
   end
 
   it 'should support factories' do
-    size = described_class.perform("file://#{tempdir}/file.json") do
+    size = described_class.perform('mock:///dir/file.json') do
       enumerable
     end
     expect(size).to eq(15900)
-    expect(File.size("#{tempdir}/file.json")).to eq(size)
+    expect(bucket.info('dir/file.json').size).to eq(size)
+  end
+
+  it 'should support last-modified' do
+    described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
+    expect(bucket.info('dir/file.json').metadata).to eq('X-Feedx-Pusher-Last-Modified' => '1515151515000')
+  end
+
+  it 'should perform conditionally' do
+    size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
+    expect(size).to eq(15900)
+
+    size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
+    expect(size).to eq(-1)
+
+    size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151514), enum: enumerable
+    expect(size).to eq(-1)
+
+    size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151516), enum: enumerable
+    expect(size).to eq(15900)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-23 00:00:00.000000000 Z
+date: 2018-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -121,14 +121,12 @@ files:
 - lib/feedx/format/json.rb
 - lib/feedx/format/protobuf.rb
 - lib/feedx/pusher.rb
-- lib/feedx/pusher/recurring.rb
 - spec/feedx/compression/gzip_spec.rb
 - spec/feedx/compression/none_spec.rb
 - spec/feedx/compression_spec.rb
 - spec/feedx/format/json_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
-- spec/feedx/pusher/recurring_spec.rb
 - spec/feedx/pusher_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/bsm/feedx
@@ -162,6 +160,5 @@ test_files:
 - spec/feedx/format/json_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
-- spec/feedx/pusher/recurring_spec.rb
 - spec/feedx/pusher_spec.rb
 - spec/spec_helper.rb

data/lib/feedx/pusher/recurring.rb

@@ -1,39 +0,0 @@
-module Feedx
-  class Pusher
-    module Recurring
-      DEFAULT_CHECK = proc do |enum|
-        enum.respond_to?(:maximum) ? enum.maximum(:updated_at).to_f : 0.0
-      end.freeze
-
-      # Registered pushers
-      def self.registry
-        @registry ||= {}
-      end
-
-      # Performs a recurring Pusher task. See see Feedx::Pusher.new for additional
-      # information on params and options.
-      #
-      # @param [String] name a unique task name.
-      # @param [Hash] opts options.
-      # @option [Proc] :check a custom revision check to evalute before perform.
-      #   Receives the enum as an argument and returns a Numeric revision.
-      #   Default: ->(enum) { enum.maximum(:updated_at).to_f }
-      def self.perform(name, url, opts={}, &block)
-        stored = registry[name]
-        unless stored
-          check  = opts.delete(:check) || DEFAULT_CHECK
-          stored = { pusher: Feedx::Pusher.new(url, opts, &block), check: check, revision: 0 }
-          registry[name] = stored
-        end
-
-        enum   = stored[:pusher].build_enum
-        latest = stored[:check].call(enum).to_f
-        return -1 unless stored[:revision] < latest
-
-        size = stored[:pusher].perform(enum)
-        stored[:revision] = latest
-        size
-      end
-    end
-  end
-end

data/spec/feedx/pusher/recurring_spec.rb

@@ -1,28 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Pusher::Recurring do
-  let :enumerable do
-    enum = %w[x y z].map {|t| { title: t } } * 100
-    enum.instance_eval do
-      def maximum(*)
-        1515151515.0
-      end
-    end
-    enum
-  end
-
-  let(:tempdir) { Dir.mktmpdir }
-  after { FileUtils.rm_rf tempdir }
-  after { described_class.registry.clear }
-
-  it 'should perform conditionally' do
-    size = described_class.perform 'test.task', "file://#{tempdir}/file.json", enum: enumerable
-    expect(size).to eq(4200)
-    expect(File.size("#{tempdir}/file.json")).to eq(size)
-    expect(described_class.registry).to include('test.task')
-    expect(described_class.registry['test.task']).to include(revision: 1515151515.0)
-
-    size = described_class.perform 'test.task', "file://#{tempdir}/file.json", enum: enumerable
-    expect(size).to eq(-1)
-  end
-end