feedx 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5662b9773c03d14b3d7d68b8a1ad208986b95c675cbf61d445f74ebdf3b86b09
-  data.tar.gz: 074c6e5b86975aed1b31caa5250d7132a2d6bb68af7c221f89043f5f574d0268
+  metadata.gz: 78c9ab1f475804897ee6610107515f598c14590878a3134f14abecad23e151ab
+  data.tar.gz: 89d4d70a336b47ccba2db349fdcc4283e45ddb877599ac6618ba3da173ba8547
 SHA512:
-  metadata.gz: 3c39e851c7e9266c9b6248f6c6010d351ae9d57809c5f6d153bd6defdf4933fa6f138637e8ad6b0ee5f8cab07dec002b2995ecc0a7ff02222974a49ab1d646e3
-  data.tar.gz: 1846a1ba29397928281d1ba4b5c683c6c4e097e2879acedc52de21d722033290e5aac72a8f001d2012643f67661aa9fd76bac27c925782b6760b2d96295e1e80
+  metadata.gz: 12a539f376dd128eb4eb9d26d7a31812f656c0601375eec8721c814f404326d52feaafa80c6856fc9cf7fe66f1e1e956b9c74621d3ee88f0f8a47307f2ef04c1
+  data.tar.gz: c1de172489a3404f3c811aa3c41bf6d00708f36e756f4db0d4e58db0adfe8aa3908189cc07a25703682c48293c747351781650582ca4f1f1e5d3ab5681237f7c

data/.rubocop.yml

@@ -4,3 +4,5 @@ inherit_from:
 
 AllCops:
   TargetRubyVersion: "2.4"
+Metrics/ParameterLists:
+  Max: 10

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    feedx (0.11.0)
+    feedx (0.12.0)
       bfs (>= 0.5.0)
 
 GEM
@@ -9,7 +9,7 @@ GEM
   specs:
     ast (2.4.1)
     bfs (0.7.0)
-    diff-lcs (1.3)
+    diff-lcs (1.4)
     extpp (0.0.8)
     gio2 (3.4.3)
       gobject-introspection (= 3.4.3)
@@ -20,9 +20,9 @@ GEM
       glib2 (= 3.4.3)
     google-protobuf (3.12.2)
     native-package-installer (1.0.9)
-    parallel (1.19.1)
-    parser (2.7.1.3)
-      ast (~> 2.4.0)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
     pbio (0.2.1)
       google-protobuf
     pkg-config (1.4.1)
@@ -50,13 +50,13 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
-    rubocop (0.85.1)
+    rubocop (0.86.0)
       parallel (~> 1.10)
       parser (>= 2.7.0.1)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.7)
       rexml
-      rubocop-ast (>= 0.0.3)
+      rubocop-ast (>= 0.0.3, < 1.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
     rubocop-ast (0.0.3)

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.11.0'
+  s.version       = '0.12.0'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)

data/lib/feedx/cache/abstract.rb

@@ -5,12 +5,12 @@ class Feedx::Cache::Abstract
   end
 
   # Read reads a key.
-  def read(_key, **_opts)
+  def read(_key, **)
     raise 'Not implemented'
   end
 
   # Write writes a key/value pair.
-  def write(_key, _value, **_opts)
+  def write(_key, _value, **)
     raise 'Not implemented'
   end
 

data/lib/feedx/compression/abstract.rb

@@ -1,9 +1,9 @@
 class Feedx::Compression::Abstract
-  def reader(_io, &_block)
+  def reader(_io, **, &_block)
     raise 'Not implemented'
   end
 
-  def writer(_io, &_block)
+  def writer(_io, **, &_block)
     raise 'Not implemented'
   end
 end

data/lib/feedx/compression/gzip.rb

@@ -1,12 +1,12 @@
 require 'zlib'
 
 class Feedx::Compression::Gzip < Feedx::Compression::Abstract
-  def reader(io, &block)
+  def reader(io, **, &block)
     force_binmode(io)
     Zlib::GzipReader.wrap(io, &block)
   end
 
-  def writer(io, &block)
+  def writer(io, **, &block)
     force_binmode(io)
     Zlib::GzipWriter.wrap(io, &block)
   end

data/lib/feedx/compression/none.rb

@@ -1,9 +1,9 @@
 class Feedx::Compression::None < Feedx::Compression::Abstract
-  def reader(io)
+  def reader(io, **)
     yield(io)
   end
 
-  def writer(io)
+  def writer(io, **)
     yield(io)
   end
 end

data/lib/feedx/consumer.rb

@@ -16,14 +16,17 @@ module Feedx
     # @param [Class] klass the record class.
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
-    # @option opts [Hash] :format_options format decode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
-    def initialize(url, klass, **opts)
-      @klass    = klass
-      @stream   = Feedx::Stream.new(url, **opts)
-      @fmt_opts = opts[:format_options] || {}
-      @cache    = opts[:cache]
+    def initialize(url, klass, format_options: {}, cache: nil, **opts)
+      @klass  = klass
+      @stream = Feedx::Stream.new(url, **opts)
+      @cache  = cache
+      @opts   = opts.merge(format_options)
+
+      return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
+
+      warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
     end
 
     # @return [Boolean] returns true if performed.
@@ -38,7 +41,7 @@ module Feedx
       end
 
       @stream.open do |fmt|
-        fmt.decode_each(@klass, **@fmt_opts, &block)
+        fmt.decode_each(@klass, **@opts, &block)
       end
       @cache.write(remote_rev) if @cache && remote_rev
 

data/lib/feedx/format.rb

@@ -39,6 +39,9 @@ module Feedx
       def registry
         @registry ||= {
           'json'     => :JSON,
+          'jsonl'    => :JSON,
+          'ndjson'   => :JSON,
+          'parquet'  => :Parquet,
           'pb'       => :Protobuf,
           'proto'    => :Protobuf,
           'protobuf' => :Protobuf,

data/lib/feedx/producer.rb

@@ -14,18 +14,21 @@ module Feedx
     # @param [Hash] opts options
     # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
-    # @option opts [Hash] :format_options format encode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
     # @yield A block factory to generate the relation or enumerator.
     # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
-    def initialize(url, **opts, &block)
-      @enum = opts[:enum] || block
+    def initialize(url, last_modified: nil, format_options: {}, enum: nil, **opts, &block)
+      @enum = enum || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
       @stream   = Feedx::Stream.new(url, **opts)
-      @last_mod = opts[:last_modified]
-      @fmt_opts = opts[:format_options] || {}
+      @last_mod = last_modified
+      @opts     = opts.merge(format_options)
+
+      return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
+
+      warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
     end
 
     def perform
@@ -43,7 +46,7 @@ module Feedx
 
       @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
         iter = enum.respond_to?(:find_each) ? :find_each : :each
-        enum.send(iter) {|rec| fmt.encode(rec, **@fmt_opts) }
+        enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
       end
       @stream.blob.info.size
     end

data/lib/feedx/stream.rb

@@ -10,10 +10,11 @@ module Feedx
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
-    def initialize(url, **opts)
+    def initialize(url, format: nil, compress: nil, **opts)
       @blob     = BFS::Blob.new(url)
-      @format   = detect_format(opts[:format])
-      @compress = detect_compress(opts[:compress])
+      @format   = detect_format(format)
+      @compress = detect_compress(compress)
+      @opts     = opts
     end
 
     # Opens the remote for reading.
@@ -22,8 +23,8 @@ module Feedx
     # @yieldparam [Feedx::Format::Abstract] formatted input stream.
     def open(**opts)
       @blob.open(**opts) do |io|
-        @compress.reader(io) do |cio|
-          @format.decoder(cio) do |fmt|
+        @compress.reader(io, **@opts) do |cio|
+          @format.decoder(cio, **@opts) do |fmt|
             yield fmt
           end
         end
@@ -36,8 +37,8 @@ module Feedx
     # @yieldparam [Feedx::Format::Abstract] formatted output stream.
     def create(**opts)
       @blob.create(**opts) do |io|
-        @compress.writer(io) do |cio|
-          @format.encoder(cio) do |fmt|
+        @compress.writer(io, **@opts) do |cio|
+          @format.encoder(cio, **@opts) do |fmt|
             yield fmt
           end
         end

data/spec/feedx/producer_spec.rb

@@ -64,4 +64,10 @@ RSpec.describe Feedx::Producer do
     size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151516), enum: enumerable
     expect(size).to eq(15900)
   end
+
+  it 'should accept downstream options' do
+    expect do
+      described_class.perform 'mock:///dir/file.jsonz', enum: enumerable, x: 1, y: 'v', z: true
+    end.not_to raise_error
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-16 00:00:00.000000000 Z
+date: 2020-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs