feedx 0.11.0 → 0.12.5

data/lib/feedx/cache/abstract.rb

@@ -5,12 +5,12 @@ class Feedx::Cache::Abstract
   end
 
   # Read reads a key.
-  def read(_key, **_opts)
+  def read(_key, **)
     raise 'Not implemented'
   end
 
   # Write writes a key/value pair.
-  def write(_key, _value, **_opts)
+  def write(_key, _value, **)
     raise 'Not implemented'
   end
 

data/lib/feedx/cache/memory.rb

@@ -3,6 +3,7 @@ require 'monitor'
 # Thread-safe in-memory cache. Use for testing only.
 class Feedx::Cache::Memory < Feedx::Cache::Abstract
   def initialize
+    super
     @monitor = Monitor.new
     @entries = {}
   end

data/lib/feedx/compression/abstract.rb

@@ -1,9 +1,9 @@
 class Feedx::Compression::Abstract
-  def reader(_io, &_block)
+  def reader(_io, **, &_block)
     raise 'Not implemented'
   end
 
-  def writer(_io, &_block)
+  def writer(_io, **, &_block)
     raise 'Not implemented'
   end
 end

data/lib/feedx/compression/gzip.rb

@@ -1,12 +1,12 @@
 require 'zlib'
 
 class Feedx::Compression::Gzip < Feedx::Compression::Abstract
-  def reader(io, &block)
+  def reader(io, **, &block)
     force_binmode(io)
     Zlib::GzipReader.wrap(io, &block)
   end
 
-  def writer(io, &block)
+  def writer(io, **, &block)
     force_binmode(io)
     Zlib::GzipWriter.wrap(io, &block)
   end

data/lib/feedx/compression/none.rb

@@ -1,9 +1,9 @@
 class Feedx::Compression::None < Feedx::Compression::Abstract
-  def reader(io)
+  def reader(io, **)
     yield(io)
   end
 
-  def writer(io)
+  def writer(io, **)
     yield(io)
   end
 end

data/lib/feedx/consumer.rb

@@ -16,33 +16,39 @@ module Feedx
     # @param [Class] klass the record class.
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
-    # @option opts [Hash] :format_options format decode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
-    def initialize(url, klass, **opts)
-      @klass    = klass
-      @stream   = Feedx::Stream.new(url, **opts)
-      @fmt_opts = opts[:format_options] || {}
-      @cache    = opts[:cache]
+    def initialize(url, klass, format_options: {}, cache: nil, **opts)
+      @klass = klass
+      @url = url
+      @opts = opts.merge(format_options)
+      @cache = cache
+
+      return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
+
+      warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
     end
 
     # @return [Boolean] returns true if performed.
     def each(&block)
+      stream = Feedx::Stream.new(@url, **@opts)
       remote_rev = nil
 
       if @cache
-        metadata   = @stream.blob.info.metadata
+        metadata   = stream.blob.info.metadata
         local_rev  = @cache.read.to_i
         remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
         return false if remote_rev.positive? && remote_rev <= local_rev
       end
 
-      @stream.open do |fmt|
-        fmt.decode_each(@klass, **@fmt_opts, &block)
+      stream.open do |fmt|
+        fmt.decode_each(@klass, **@opts, &block)
       end
       @cache.write(remote_rev) if @cache && remote_rev
 
       true
+    ensure
+      stream&.close
     end
   end
 end

data/lib/feedx/format.rb

@@ -27,7 +27,7 @@ module Feedx
           ext = File.extname(base)
           raise ArgumentError, 'unable to detect format' if ext.empty?
 
-          kind = _resolve(ext[1..-1]) || _resolve(ext[1..-2])
+          kind = _resolve(ext[1..]) || _resolve(ext[1..-2])
           return kind if kind
 
           base = base[0..-ext.size - 1]
@@ -39,6 +39,9 @@ module Feedx
       def registry
         @registry ||= {
           'json'     => :JSON,
+          'jsonl'    => :JSON,
+          'ndjson'   => :JSON,
+          'parquet'  => :Parquet,
           'pb'       => :Protobuf,
           'proto'    => :Protobuf,
           'protobuf' => :Protobuf,

data/lib/feedx/producer.rb

@@ -14,38 +14,43 @@ module Feedx
     # @param [Hash] opts options
     # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
-    # @option opts [Hash] :format_options format encode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
     # @yield A block factory to generate the relation or enumerator.
     # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
-    def initialize(url, **opts, &block)
-      @enum = opts[:enum] || block
+    def initialize(url, last_modified: nil, format_options: {}, enum: nil, **opts, &block)
+      @enum = enum || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
-      @stream   = Feedx::Stream.new(url, **opts)
-      @last_mod = opts[:last_modified]
-      @fmt_opts = opts[:format_options] || {}
+      @url  = url
+      @opts = opts.merge(format_options)
+      @last_mod = last_modified
+
+      return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
+
+      warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
     end
 
     def perform
-      enum      = @enum.is_a?(Proc) ? @enum.call : @enum
-      last_mod  = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
-      local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
-
-      begin
-        metadata   = @stream.blob.info.metadata
-        remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
-        return -1 unless local_rev > remote_rev
-      rescue BFS::FileNotFound
-        nil
-      end if local_rev.positive?
-
-      @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
-        iter = enum.respond_to?(:find_each) ? :find_each : :each
-        enum.send(iter) {|rec| fmt.encode(rec, **@fmt_opts) }
+      Feedx::Stream.open(@url, **@opts) do |stream|
+        enum = @enum.is_a?(Proc) ? @enum.call : @enum
+        last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
+        local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
+
+        begin
+          metadata   = stream.blob.info.metadata
+          remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
+          return -1 unless local_rev > remote_rev
+        rescue BFS::FileNotFound
+          nil
+        end if local_rev.positive?
+
+        stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
+          iter = enum.respond_to?(:find_each) ? :find_each : :each
+          enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
+        end
+        stream.blob.info.size
       end
-      @stream.blob.info.size
     end
   end
 end

data/lib/feedx/stream.rb

@@ -6,26 +6,40 @@ module Feedx
   class Stream
     attr_reader :blob
 
+    # Behaves like new, but accepts an optional block.
+    # If a block is given, streams are automatically closed after the block is yielded.
+    def self.open(url, **opts)
+      stream = new(url, **opts)
+      return stream unless block_given?
+
+      begin
+        yield stream
+      ensure
+        stream.close
+      end
+    end
+
     # @param [String] url the blob URL.
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
-    def initialize(url, **opts)
+    def initialize(url, format: nil, compress: nil, **opts)
       @blob     = BFS::Blob.new(url)
-      @format   = detect_format(opts[:format])
-      @compress = detect_compress(opts[:compress])
+      @format   = detect_format(format)
+      @compress = detect_compress(compress)
+      @opts     = opts
+
+      BFS.defer(self, :close)
     end
 
     # Opens the remote for reading.
     # @param [Hash] opts BFS::Blob#open options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted input stream.
-    def open(**opts)
+    def open(**opts, &block)
       @blob.open(**opts) do |io|
-        @compress.reader(io) do |cio|
-          @format.decoder(cio) do |fmt|
-            yield fmt
-          end
+        @compress.reader(io, **@opts) do |cio|
+          @format.decoder(cio, **@opts, &block)
         end
       end
     end
@@ -34,16 +48,19 @@ module Feedx
     # @param [Hash] opts BFS::Blob#create options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted output stream.
-    def create(**opts)
+    def create(**opts, &block)
       @blob.create(**opts) do |io|
-        @compress.writer(io) do |cio|
-          @format.encoder(cio) do |fmt|
-            yield fmt
-          end
+        @compress.writer(io, **@opts) do |cio|
+          @format.encoder(cio, **@opts, &block)
         end
       end
     end
 
+    # Closes the underlying connection.
+    def close
+      @blob.close
+    end
+
     private
 
     def detect_format(val)

data/producer_test.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Producer", func() {

data/reader_test.go

@@ -8,8 +8,8 @@ import (
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
 	"github.com/bsm/feedx/internal/testdata"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Reader", func() {

data/spec/feedx/cache/memory_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Cache::Memory do
-  it 'should read/write' do
+  it 'read/writes' do
     expect(subject.fetch('key')).to be_nil
     expect(subject.fetch('key') { 'value' }).to eq('value')
     expect(subject.fetch('key')).to eq('value')
@@ -16,7 +16,7 @@ RSpec.describe Feedx::Cache::Memory do
     expect(subject.fetch('key')).to be_nil
   end
 
-  it 'should write strings' do
+  it 'writes strings' do
     subject.write('key', 5)
     expect(subject.read('key')).to eq('5')
   end

data/spec/feedx/cache/value_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe Feedx::Cache::Value do
     described_class.new(Feedx::Cache::Memory.new, 'key')
   end
 
-  it 'should read/write' do
+  it 'read/writes' do
     expect(subject.fetch).to be_nil
     expect(subject.fetch { 'value' }).to eq('value')
     expect(subject.fetch).to eq('value')

data/spec/feedx/compression/gzip_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression::Gzip do
-  it 'should wrap readers/writers' do
+  it 'wraps readers/writers' do
     wio = StringIO.new
     subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to be_within(20).of(40)

data/spec/feedx/compression/none_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression::None do
-  it 'should wrap readers/writers' do
+  it 'wraps readers/writers' do
     wio = StringIO.new
     subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to eq(3000)

data/spec/feedx/compression_spec.rb

@@ -1,14 +1,14 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression do
-  it 'should resolve' do
+  it 'resolves' do
     expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
     expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
     expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
   end
 
-  it 'should detect' do
+  it 'detects' do
     expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
     expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
     expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)

data/spec/feedx/consumer_spec.rb

@@ -4,15 +4,16 @@ RSpec.describe Feedx::Consumer do
   let(:bucket) { BFS::Bucket::InMem.new }
   let(:klass)  { Feedx::TestCase::Model }
   let(:cache)  { Feedx::Cache::Memory.new.value('my-consumer') }
+
   before { allow(BFS).to receive(:resolve).and_return(bucket) }
 
-  it 'should reject invalid inputs' do
+  it 'rejects invalid inputs' do
     expect do
-      described_class.each('mock:///dir/file.txt', klass) {}
+      described_class.each('mock:///dir/file.txt', klass)
     end.to raise_error(/unable to detect format/)
   end
 
-  it 'should consume feeds' do
+  it 'consumes feeds' do
     url = mock_produce!
     csm = described_class.new(url, klass)
     expect(csm).to be_a(Enumerable)
@@ -24,7 +25,7 @@ RSpec.describe Feedx::Consumer do
     expect(cnt).to eq(300)
   end
 
-  it 'should perform conditionally' do
+  it 'performs conditionally' do
     url = mock_produce! last_modified: Time.at(1515151515)
     expect(described_class.new(url, klass, cache: cache).count).to eq(300)
     expect(described_class.new(url, klass, cache: cache).count).to eq(0)

data/spec/feedx/format/abstract_spec.rb

@@ -2,10 +2,11 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Format::Abstract do
   subject   { Feedx::Format::JSON.new }
+
   let(:wio) { StringIO.new }
   let(:rio) { StringIO.open(wio.string) }
 
-  it 'should decode each' do
+  it 'decodes each' do
     subject.encoder wio do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))

data/spec/feedx/format/json_spec.rb

@@ -4,17 +4,17 @@ RSpec.describe Feedx::Format::JSON do
   let(:wio) { StringIO.new }
   let(:rio) { StringIO.open(wio.string) }
 
-  it 'should encode/decode' do
+  it 'encode/decodes' do
     subject.encoder wio do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))
       enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
     end
-    expect(wio.string.lines).to eq [
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-      %({"title":"Z"}\n),
-    ]
+    expect(wio.string).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Z"}
+    JSON
 
     subject.decoder rio do |dec|
       expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))

data/spec/feedx/format/parquet_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe Feedx::Format::Parquet do
     ])
   end
 
-  it 'should encode/decode' do
+  it 'encode/decodes' do
     subject.encoder wio, schema: schema, batch_size: 2 do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))