feedx 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2769dc5b097e9d5c17d467070cd3fa638ab7573f7e3cd310843c5c023a84aa5c
-  data.tar.gz: 6edf76ff62393e0ea29aa7a3b8c74fb66d2a0328ddcf4a256f6fda297806f6f3
+  metadata.gz: c294d0811dec65d1fce8a44fe2e18dbe08ac0bffee8d56c6b1d96bdac63e9366
+  data.tar.gz: e6172cbf1f728569624a928dcf2a22c3a9a87824ed6cfd8fddaf834d1e526d8e
 SHA512:
-  metadata.gz: 27c60344dfd2fab31e8c38f0a2368440e43ed5ab698b3df16dfb7820625f6d6646bdfc81211ce3c9c15ec9ffd44a12cea8b33fbaa357bc5cc664d1bf7d0599ad
-  data.tar.gz: db9073b92564898bb535e725f09c32e76c07f32a050da2ad7bf7d513c4fdf8e6402c8733dc1de53238e528bc2986c18e2635ae0e16e006052b6827de350fcd13
+  metadata.gz: 64f69d35aa2cdca873e223b0f959a404dacebe03b07a8ca77796a10d5d6dfc06b4f7b9e7ee3bbfcfa03c23656399038324088a19d8e643636d0d3f3408af7b4b
+  data.tar.gz: fb2e106e250693c5756edecf6d98df7115f8e8b33f4fca01ade0c3c568eb756215e6efa967d36099fdb055e4f77a26f68aebc207d9f62bb82c4748fb193d8ec7

data/.rubocop.yml

@@ -1,3 +1,6 @@
 require: rubocop-performance
 inherit_from:
   - https://gitlab.com/bsm/misc/raw/master/rubocop/default.yml
+
+AllCops:
+  TargetRubyVersion: "2.4"

data/.travis.yml

@@ -15,6 +15,11 @@ matrix:
         - 2.4
       before_install:
         - gem install bundler
+    - language: go
+      go:
+        - 1.12.x
+      env:
+        - GO111MODULE=on
     - language: go
       go:
         - 1.11.x

data/Gemfile.lock

@@ -1,17 +1,17 @@
 PATH
   remote: .
   specs:
-    feedx (0.7.1)
-      bfs (>= 0.3.4)
+    feedx (0.7.2)
+      bfs (>= 0.4.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.0)
-    bfs (0.3.7)
+    bfs (0.4.0)
     diff-lcs (1.3)
     google-protobuf (3.8.0)
-    jaro_winkler (1.5.2)
+    jaro_winkler (1.5.3)
     parallel (1.17.0)
     parser (2.6.3.0)
       ast (~> 2.4.0)
@@ -23,12 +23,12 @@ GEM
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
+    rspec-core (3.8.1)
       rspec-support (~> 3.8.0)
     rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+    rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.2)
@@ -39,8 +39,8 @@ GEM
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-performance (1.3.0)
-      rubocop (>= 0.68.0)
+    rubocop-performance (1.4.0)
+      rubocop (>= 0.71.0)
     ruby-progressbar (1.10.1)
     unicode-display_width (1.6.0)
 

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.7.1'
+  s.version       = '0.7.2'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)
@@ -11,9 +11,9 @@ Gem::Specification.new do |s|
   s.files         = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^spec/}) }
   s.test_files    = `git ls-files -z -- spec/*`.split("\x0")
   s.require_paths = ['lib']
-  s.required_ruby_version = '>= 2.3'
+  s.required_ruby_version = '>= 2.4'
 
-  s.add_dependency 'bfs', '>= 0.3.4'
+  s.add_dependency 'bfs', '>= 0.4.0'
 
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'pbio'

data/lib/feedx.rb

@@ -1,7 +1,9 @@
 module Feedx
   META_LAST_MODIFIED = 'x-feedx-last-modified'.freeze
 
+  autoload :Cache, 'feedx/cache'
   autoload :Compression, 'feedx/compression'
+  autoload :Consumer, 'feedx/consumer'
   autoload :Format, 'feedx/format'
   autoload :Stream, 'feedx/stream'
   autoload :Producer, 'feedx/producer'

data/lib/feedx/cache.rb

@@ -0,0 +1,7 @@
+module Feedx
+  module Cache
+    autoload :Abstract, 'feedx/cache/abstract'
+    autoload :Memory, 'feedx/cache/memory'
+    autoload :Value, 'feedx/cache/value'
+  end
+end

data/lib/feedx/cache/abstract.rb

@@ -0,0 +1,35 @@
+class Feedx::Cache::Abstract
+  # Clears cache.
+  def clear
+    raise 'Not implemented'
+  end
+
+  # Read reads a key.
+  def read(_key, **_opts)
+    raise 'Not implemented'
+  end
+
+  # Write writes a key/value pair.
+  def write(_key, _value, **_opts)
+    raise 'Not implemented'
+  end
+
+  # Fetches data from the cache, using the given key.
+  # The optional block will be evaluated and the result stored in the cache
+  # in the event of a cache miss.
+  def fetch(key, **opts)
+    value = read(key, **opts)
+
+    if block_given?
+      value ||= yield
+      write(key, value, **opts) if value
+    end
+
+    value
+  end
+
+  # @return [Feedx::Abstract::Value] returns a wrapper around a single value.
+  def value(key)
+    Feedx::Cache::Value.new(self, key)
+  end
+end

data/lib/feedx/cache/memory.rb

@@ -0,0 +1,30 @@
+require 'monitor'
+
+# Thread-safe in-memory cache. Use for testing only.
+class Feedx::Cache::Memory < Feedx::Cache::Abstract
+  def initialize
+    @monitor = Monitor.new
+    @entries = {}
+  end
+
+  # Clear empties cache.
+  def clear
+    @monitor.synchronize do
+      @entries.clear
+    end
+  end
+
+  # Read reads a key.
+  def read(key, **)
+    @monitor.synchronize do
+      @entries[key]
+    end
+  end
+
+  # Write writes a key.
+  def write(key, value, **)
+    @monitor.synchronize do
+      @entries[key] = value.to_s
+    end
+  end
+end

data/lib/feedx/cache/value.rb

@@ -0,0 +1,25 @@
+# A single value inside a cache.
+class Feedx::Cache::Value
+  attr_reader :key
+
+  def initialize(cache, key)
+    @cache = cache
+    @key = key
+  end
+
+  # Read the key.
+  def read(**opts)
+    @cache.read(@key, **opts)
+  end
+
+  # Write a value.
+  def write(value, **opts)
+    @cache.write(@key, value, **opts)
+  end
+
+  # Fetches data. The optional block will be evaluated and the
+  # result stored in the cache under the key in the event of a cache miss.
+  def fetch(**opts, &block)
+    @cache.fetch(@key, **opts, &block)
+  end
+end

data/lib/feedx/consumer.rb

@@ -0,0 +1,47 @@
+require 'uri'
+require 'bfs'
+require 'feedx'
+
+module Feedx
+  # Consumes an enumerates over a feed.
+  class Consumer
+    include Enumerable
+
+    # See constructor.
+    def self.each(url, klass, opts={}, &block)
+      new(url, klass, opts).each(&block)
+    end
+
+    # @param [String] url the destination URL.
+    # @param [Class] klass the record class.
+    # @param [Hash] opts options
+    # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
+    # @option opts [Hash] :format_options format decode options. Default: {}.
+    # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
+    # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
+    def initialize(url, klass, opts={})
+      @klass    = klass
+      @stream   = Feedx::Stream.new(url, opts)
+      @fmt_opts = opts[:format_options] || {}
+      @cache    = opts[:cache]
+    end
+
+    # @return [Boolean] returns true if performed.
+    def each(&block)
+      remote_rev = nil
+
+      if @cache
+        local_rev  = @cache.read.to_i
+        remote_rev = @stream.blob.info.metadata[META_LAST_MODIFIED].to_i
+        return false if remote_rev.positive? && remote_rev <= local_rev
+      end
+
+      @stream.open do |fmt|
+        fmt.decode_each(@klass, **@fmt_opts, &block)
+      end
+      @cache.write(remote_rev) if @cache && remote_rev
+
+      true
+    end
+  end
+end

data/lib/feedx/format/abstract.rb

@@ -9,9 +9,9 @@ class Feedx::Format::Abstract
 
   def decode_each(klass, **opts)
     if block_given?
-      yield decode(klass, opts) until eof?
+      yield decode(klass, **opts) until eof?
     else
-      Enumerator.new {|y| y << decode(klass, opts) until eof? }
+      Enumerator.new {|y| y << decode(klass, **opts) until eof? }
     end
   end
 

data/lib/feedx/format/json.rb

@@ -10,7 +10,7 @@ class Feedx::Format::JSON < Feedx::Format::Abstract
     obj
   end
 
-  def encode(msg, **)
-    @io.write msg.to_json << "\n"
+  def encode(msg, **opts)
+    @io.write msg.to_json(**opts) << "\n"
   end
 end

data/lib/feedx/producer.rb

@@ -3,7 +3,7 @@ require 'bfs'
 require 'feedx'
 
 module Feedx
-  # Produces a relation as am encoded stream to a remote location.
+  # Produces a relation as an encoded feed to a remote location.
   class Producer
     # See constructor.
     def self.perform(url, opts={}, &block)
@@ -14,6 +14,7 @@ module Feedx
     # @param [Hash] opts options
     # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
+    # @option opts [Hash] :format_options format encode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
     # @yield A block factory to generate the relation or enumerator.
@@ -22,24 +23,25 @@ module Feedx
       @enum = opts[:enum] || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
-      @stream = Feedx::Stream.new(url, opts)
+      @stream   = Feedx::Stream.new(url, opts)
       @last_mod = opts[:last_modified]
+      @fmt_opts = opts[:format_options] || {}
     end
 
     def perform
-      enum = @enum.is_a?(Proc) ? @enum.call : @enum
-      last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
-      current  = (last_mod.to_f * 1000).floor
+      enum      = @enum.is_a?(Proc) ? @enum.call : @enum
+      last_mod  = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
+      local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
 
       begin
-        previous = @stream.blob.info.metadata[META_LAST_MODIFIED].to_i
-        return -1 unless current > previous
+        remote_rev = @stream.blob.info.metadata[META_LAST_MODIFIED].to_i
+        return -1 unless local_rev > remote_rev
       rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
-      end if current.positive?
+      end if local_rev.positive?
 
-      @stream.create metadata: { META_LAST_MODIFIED => current.to_s } do |fmt|
+      @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
         iter = enum.respond_to?(:find_each) ? :find_each : :each
-        enum.send(iter) {|rec| fmt.encode(rec) }
+        enum.send(iter) {|rec| fmt.encode(rec, **@fmt_opts) }
       end
       @stream.blob.info.size
     end

data/spec/feedx/cache/memory_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Cache::Memory do
+  it 'should read/write' do
+    expect(subject.fetch('key')).to be_nil
+    expect(subject.fetch('key') { 'value' }).to eq('value')
+    expect(subject.fetch('key')).to eq('value')
+    expect(subject.fetch('key') { 'other' }).to eq('value')
+    expect(subject.fetch('key')).to eq('value')
+
+    subject.write('key', 'new-value')
+    expect(subject.read('key')).to eq('new-value')
+    expect(subject.fetch('key')).to eq('new-value')
+
+    subject.clear
+    expect(subject.fetch('key')).to be_nil
+  end
+
+  it 'should write strings' do
+    subject.write('key', 5)
+    expect(subject.read('key')).to eq('5')
+  end
+end

data/spec/feedx/cache/value_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Cache::Value do
+  subject do
+    described_class.new(Feedx::Cache::Memory.new, 'key')
+  end
+
+  it 'should read/write' do
+    expect(subject.fetch).to be_nil
+    expect(subject.fetch { 'value' }).to eq('value')
+    expect(subject.fetch).to eq('value')
+    expect(subject.fetch { 'other' }).to eq('value')
+    expect(subject.fetch).to eq('value')
+
+    subject.write('new-value')
+    expect(subject.read).to eq('new-value')
+    expect(subject.fetch).to eq('new-value')
+  end
+end

data/spec/feedx/consumer_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Consumer do
+  let(:bucket) { BFS::Bucket::InMem.new }
+  let(:klass)  { Feedx::TestCase::Model }
+  let(:cache)  { Feedx::Cache::Memory.new.value('my-consumer') }
+  before { allow(BFS).to receive(:resolve).and_return(bucket) }
+
+  it 'should reject invalid inputs' do
+    expect do
+      described_class.each('mock:///dir/file.txt', klass) {}
+    end.to raise_error(/unable to detect format/)
+  end
+
+  it 'should consume feeds' do
+    url = mock_produce!
+    csm = described_class.new(url, klass)
+    expect(csm).to be_a(Enumerable)
+
+    cnt = csm.count do |rec|
+      expect(rec).to be_instance_of(klass)
+      true
+    end
+    expect(cnt).to eq(300)
+  end
+
+  it 'should perform conditionally' do
+    url = mock_produce! last_modified: Time.at(1515151515)
+    expect(described_class.new(url, klass, cache: cache).count).to eq(300)
+    expect(described_class.new(url, klass, cache: cache).count).to eq(0)
+
+    url = mock_produce!
+    expect(described_class.new(url, klass, cache: cache).count).to eq(300)
+    expect(described_class.new(url, klass, cache: cache).count).to eq(300)
+  end
+
+  private
+
+  def mock_produce!(opts={})
+    url = 'mock:///dir/file.json'
+    opts[:enum] ||= %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
+    Feedx::Producer.perform url, opts
+    url
+  end
+end

data/spec/spec_helper.rb

@@ -19,7 +19,7 @@ module Feedx
         @title = title
       end
 
-      def to_pb
+      def to_pb(*)
         Feedx::TestCase::Message.new title: @title
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-12 00:00:00.000000000 Z
+date: 2019-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.4
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.4
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -137,10 +137,15 @@ files:
 - go.mod
 - go.sum
 - lib/feedx.rb
+- lib/feedx/cache.rb
+- lib/feedx/cache/abstract.rb
+- lib/feedx/cache/memory.rb
+- lib/feedx/cache/value.rb
 - lib/feedx/compression.rb
 - lib/feedx/compression/abstract.rb
 - lib/feedx/compression/gzip.rb
 - lib/feedx/compression/none.rb
+- lib/feedx/consumer.rb
 - lib/feedx/format.rb
 - lib/feedx/format/abstract.rb
 - lib/feedx/format/json.rb
@@ -152,9 +157,12 @@ files:
 - producer_test.go
 - reader.go
 - reader_test.go
+- spec/feedx/cache/memory_spec.rb
+- spec/feedx/cache/value_spec.rb
 - spec/feedx/compression/gzip_spec.rb
 - spec/feedx/compression/none_spec.rb
 - spec/feedx/compression_spec.rb
+- spec/feedx/consumer_spec.rb
 - spec/feedx/format/abstract_spec.rb
 - spec/feedx/format/json_spec.rb
 - spec/feedx/format/protobuf_spec.rb
@@ -176,7 +184,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -188,9 +196,12 @@ signing_key:
 specification_version: 4
 summary: Exchange data between components via feeds
 test_files:
+- spec/feedx/cache/memory_spec.rb
+- spec/feedx/cache/value_spec.rb
 - spec/feedx/compression/gzip_spec.rb
 - spec/feedx/compression/none_spec.rb
 - spec/feedx/compression_spec.rb
+- spec/feedx/consumer_spec.rb
 - spec/feedx/format/abstract_spec.rb
 - spec/feedx/format/json_spec.rb
 - spec/feedx/format/protobuf_spec.rb