feedx 0.10.2 → 0.12.3

data/lib/feedx/producer.rb

@@ -14,38 +14,43 @@ module Feedx
     # @param [Hash] opts options
     # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
-    # @option opts [Hash] :format_options format encode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
     # @yield A block factory to generate the relation or enumerator.
     # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
-    def initialize(url, **opts, &block)
-      @enum = opts[:enum] || block
+    def initialize(url, last_modified: nil, format_options: {}, enum: nil, **opts, &block)
+      @enum = enum || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
-      @stream   = Feedx::Stream.new(url, **opts)
-      @last_mod = opts[:last_modified]
-      @fmt_opts = opts[:format_options] || {}
+      @url  = url
+      @opts = opts.merge(format_options)
+      @last_mod = last_modified
+
+      return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
+
+      warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
     end
 
     def perform
-      enum      = @enum.is_a?(Proc) ? @enum.call : @enum
-      last_mod  = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
-      local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
-
-      begin
-        metadata   = @stream.blob.info.metadata
-        remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
-        return -1 unless local_rev > remote_rev
-      rescue BFS::FileNotFound
-        nil
-      end if local_rev.positive?
-
-      @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
-        iter = enum.respond_to?(:find_each) ? :find_each : :each
-        enum.send(iter) {|rec| fmt.encode(rec, **@fmt_opts) }
+      Feedx::Stream.open(@url, **@opts) do |stream|
+        enum = @enum.is_a?(Proc) ? @enum.call : @enum
+        last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
+        local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
+
+        begin
+          metadata   = stream.blob.info.metadata
+          remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
+          return -1 unless local_rev > remote_rev
+        rescue BFS::FileNotFound
+          nil
+        end if local_rev.positive?
+
+        stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
+          iter = enum.respond_to?(:find_each) ? :find_each : :each
+          enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
+        end
+        stream.blob.info.size
       end
-      @stream.blob.info.size
     end
   end
 end

data/lib/feedx/stream.rb

@@ -6,25 +6,40 @@ module Feedx
   class Stream
     attr_reader :blob
 
+    # Behaves like new, but accepts an optional block.
+    # If a block is given, streams are automatically closed after the block is yielded.
+    def self.open(url, **opts)
+      stream = new(url, **opts)
+      return stream unless block_given?
+
+      begin
+        yield stream
+      ensure
+        stream.close
+      end
+    end
+
     # @param [String] url the blob URL.
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
-    def initialize(url, **opts)
+    def initialize(url, format: nil, compress: nil, **opts)
       @blob     = BFS::Blob.new(url)
-      @format   = detect_format(opts[:format])
-      @compress = detect_compress(opts[:compress])
+      @format   = detect_format(format)
+      @compress = detect_compress(compress)
+      @opts     = opts
+
+      BFS.defer(self, :close)
     end
 
     # Opens the remote for reading.
     # @param [Hash] opts BFS::Blob#open options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted input stream.
-    def open(**opts)
+    def open(**opts, &block)
       @blob.open(**opts) do |io|
-        @compress.reader(io) do |cio|
-          fmt = @format.new(cio)
-          yield fmt
+        @compress.reader(io, **@opts) do |cio|
+          @format.decoder(cio, **@opts, &block)
         end
       end
     end
@@ -33,28 +48,29 @@ module Feedx
     # @param [Hash] opts BFS::Blob#create options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted output stream.
-    def create(**opts)
+    def create(**opts, &block)
       @blob.create(**opts) do |io|
-        @compress.writer(io) do |cio|
-          fmt = @format.new(cio)
-          yield fmt
+        @compress.writer(io, **@opts) do |cio|
+          @format.encoder(cio, **@opts, &block)
         end
       end
     end
 
+    # Closes the underlying connection.
+    def close
+      @blob.close
+    end
+
     private
 
     def detect_format(val)
       case val
       when nil
         Feedx::Format.detect(@blob.path)
-      when Class
-        parent = Feedx::Format::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
+      when String, Symbol
         Feedx::Format.resolve(val)
+      else
+        Feedx::Format.validate!(val)
       end
     end
 
@@ -62,13 +78,10 @@ module Feedx
       case val
       when nil
         Feedx::Compression.detect(@blob.path)
-      when Class
-        parent = Feedx::Compression::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
+      when String, Symbol
         Feedx::Compression.resolve(val)
+      else
+        Feedx::Compression.validate!(val)
       end
     end
   end

data/producer_test.go

@@ -23,8 +23,7 @@ var _ = Describe("Producer", func() {
 			atomic.AddUint32(&numRuns, 1)
 
 			for i := 0; i < 10; i++ {
-				fix := fixture
-				if err := w.Encode(&fix); err != nil {
+				if err := w.Encode(seed()); err != nil {
 					return err
 				}
 			}

data/reader_test.go

@@ -5,9 +5,9 @@ import (
 	"io"
 	"io/ioutil"
 
-	"github.com/bsm/feedx"
-
 	"github.com/bsm/bfs"
+	"github.com/bsm/feedx"
+	"github.com/bsm/feedx/internal/testdata"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -38,18 +38,18 @@ var _ = Describe("Reader", func() {
 	})
 
 	It("should decode", func() {
-		var msgs []MockMessage
+		var msgs []*testdata.MockMessage
 		for {
-			var msg MockMessage
+			var msg testdata.MockMessage
 			err := subject.Decode(&msg)
 			if err == io.EOF {
 				break
 			}
 			Expect(err).NotTo(HaveOccurred())
-			msgs = append(msgs, msg)
+			msgs = append(msgs, &msg)
 		}
 
-		Expect(msgs).To(Equal([]MockMessage{fixture, fixture, fixture}))
+		Expect(msgs).To(ConsistOf(seed(), seed(), seed()))
 		Expect(subject.NumRead()).To(Equal(3))
 	})
 })

data/spec/feedx/compression/gzip_spec.rb

@@ -3,13 +3,13 @@ require 'spec_helper'
 RSpec.describe Feedx::Compression::Gzip do
   it 'should wrap readers/writers' do
     wio = StringIO.new
-    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to be_within(20).of(40)
     expect(wio.string.encoding).to eq(Encoding::BINARY)
 
     data = ''
     StringIO.open(wio.string) do |rio|
-      described_class.reader(rio) {|z| data = z.read }
+      subject.reader(rio) {|z| data = z.read }
     end
     expect(data.size).to eq(3000)
     expect(data.encoding).to eq(Encoding.default_external)

data/spec/feedx/compression/none_spec.rb

@@ -3,12 +3,12 @@ require 'spec_helper'
 RSpec.describe Feedx::Compression::None do
   it 'should wrap readers/writers' do
     wio = StringIO.new
-    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to eq(3000)
 
     data = ''
     StringIO.open(wio.string) do |rio|
-      described_class.reader(rio) {|z| data = z.read }
+      subject.reader(rio) {|z| data = z.read }
     end
     expect(data.size).to eq(3000)
   end

data/spec/feedx/compression_spec.rb

@@ -2,18 +2,18 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Compression do
   it 'should resolve' do
-    expect(described_class.resolve(:gzip)).to eq(described_class::Gzip)
-    expect(described_class.resolve(:gz)).to eq(described_class::Gzip)
-    expect(described_class.resolve(nil)).to eq(described_class::None)
+    expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
+    expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
+    expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
   end
 
   it 'should detect' do
-    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json')).to eq(described_class::None)
-    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb.gz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb')).to eq(described_class::None)
+    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)
+    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb.gz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::None)
   end
 end

data/spec/feedx/consumer_spec.rb

@@ -8,7 +8,7 @@ RSpec.describe Feedx::Consumer do
 
   it 'should reject invalid inputs' do
     expect do
-      described_class.each('mock:///dir/file.txt', klass) {}
+      described_class.each('mock:///dir/file.txt', klass)
     end.to raise_error(/unable to detect format/)
   end
 

data/spec/feedx/format/abstract_spec.rb

@@ -1,17 +1,20 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Abstract do
-  subject   { Feedx::Format::JSON.new(wio) }
+  subject   { Feedx::Format::JSON.new }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should decode each' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
-    StringIO.open(wio.string) do |rio|
-      fmt = subject.class.new(rio)
-      dec = fmt.decode_each(Feedx::TestCase::Model).to_a
-      expect(dec.map(&:title)).to eq(%w[X Y Z])
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
+
+    subject.decoder rio do |dec|
+      acc = dec.decode_each(Feedx::TestCase::Model).to_a
+      expect(acc.map(&:title)).to eq(%w[X Y Z])
     end
   end
 end

data/spec/feedx/format/json_spec.rb

@@ -1,26 +1,27 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::JSON do
-  subject   { described_class.new(wio) }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should encode/decode' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
-    expect(wio.string.lines).to eq [
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-      %({"title":"Z"}\n),
-    ]
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
+    expect(wio.string).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Z"}
+    JSON
 
-    StringIO.open(wio.string) do |rio|
-      fmt = described_class.new(rio)
-      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
-      expect(fmt.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
-      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
-      expect(fmt.decode(Feedx::TestCase::Model)).to be_nil
-      expect(fmt).to be_eof
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(dec.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
+      expect(dec).to be_eof
     end
   end
 end

data/spec/feedx/format/parquet_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Format::Parquet do
+  let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
+
+  let(:schema) do
+    Arrow::Schema.new([
+      Arrow::Field.new('title', :string),
+      Arrow::Field.new('updated_at', type: :timestamp, unit: :second),
+    ])
+  end
+
+  it 'should encode/decode' do
+    subject.encoder wio, schema: schema, batch_size: 2 do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Model.new('Z'))
+    end
+    expect(wio.string.bytesize).to be_within(100).of(1100)
+
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
+      expect(dec).to be_eof
+    end
+  end
+end

data/spec/feedx/format/protobuf_spec.rb

@@ -1,22 +1,23 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Protobuf do
-  subject   { described_class.new(wio) }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should encode/decode' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
     expect(wio.string.bytes).to eq([3, 10, 1, 88] + [3, 10, 1, 89] + [3, 10, 1, 90])
 
-    StringIO.open(wio.string) do |rio|
-      fmt = described_class.new(rio)
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to be_nil
-      expect(fmt).to be_eof
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
+      expect(dec.decode(Feedx::TestCase::Message)).to be_nil
+      expect(dec).to be_eof
     end
   end
 end

data/spec/feedx/format_spec.rb

@@ -2,18 +2,18 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Format do
   it 'should resolve' do
-    expect(described_class.resolve(:json)).to eq(described_class::JSON)
-    expect(described_class.resolve(:pb)).to eq(described_class::Protobuf)
+    expect(described_class.resolve(:json)).to be_instance_of(described_class::JSON)
+    expect(described_class.resolve(:pb)).to be_instance_of(described_class::Protobuf)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid format txt/)
   end
 
   it 'should detect' do
-    expect(described_class.detect('path/to/file.json')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.pb')).to eq(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pb.z')).to eq(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pb.z')).to be_instance_of(described_class::Protobuf)
     expect do
       described_class.detect('path/to/file.txt')
     end.to raise_error(/unable to detect format/)