feedx 0.9.2 → 0.12.0

data/lib/feedx/format/protobuf.rb

@@ -1,16 +1,24 @@
 require 'pbio'
 
 class Feedx::Format::Protobuf < Feedx::Format::Abstract
-  def initialize(io)
-    super PBIO::Delimited.new(io)
-  end
+  class Decoder < Feedx::Format::Abstract::Decoder
+    def initialize(io, **opts)
+      super PBIO::Delimited.new(io), **opts
+    end
 
-  def decode(klass, **)
-    @io.read(klass)
+    def decode(target, **)
+      @io.read(target)
+    end
   end
 
-  def encode(msg, **opts)
-    msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
-    @io.write msg
+  class Encoder < Feedx::Format::Abstract::Encoder
+    def initialize(io, **opts)
+      super PBIO::Delimited.new(io), **opts
+    end
+
+    def encode(msg, **opts)
+      msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
+      @io.write msg
+    end
   end
 end

data/lib/feedx/producer.rb

@@ -6,26 +6,29 @@ module Feedx
   # Produces a relation as an encoded feed to a remote location.
   class Producer
     # See constructor.
-    def self.perform(url, opts = {}, &block)
-      new(url, opts, &block).perform
+    def self.perform(url, **opts, &block)
+      new(url, **opts, &block).perform
     end
 
     # @param [String] url the destination URL.
     # @param [Hash] opts options
     # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
-    # @option opts [Hash] :format_options format encode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
     # @yield A block factory to generate the relation or enumerator.
     # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
-    def initialize(url, opts = {}, &block)
-      @enum = opts[:enum] || block
+    def initialize(url, last_modified: nil, format_options: {}, enum: nil, **opts, &block)
+      @enum = enum || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
-      @stream   = Feedx::Stream.new(url, opts)
-      @last_mod = opts[:last_modified]
-      @fmt_opts = opts[:format_options] || {}
+      @stream   = Feedx::Stream.new(url, **opts)
+      @last_mod = last_modified
+      @opts     = opts.merge(format_options)
+
+      return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
+
+      warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
     end
 
     def perform
@@ -43,7 +46,7 @@ module Feedx
 
       @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
         iter = enum.respond_to?(:find_each) ? :find_each : :each
-        enum.send(iter) {|rec| fmt.encode(rec, **@fmt_opts) }
+        enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
       end
       @stream.blob.info.size
     end

data/lib/feedx/stream.rb

@@ -10,21 +10,23 @@ module Feedx
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
-    def initialize(url, opts = {})
+    def initialize(url, format: nil, compress: nil, **opts)
       @blob     = BFS::Blob.new(url)
-      @format   = detect_format(opts[:format])
-      @compress = detect_compress(opts[:compress])
+      @format   = detect_format(format)
+      @compress = detect_compress(compress)
+      @opts     = opts
     end
 
     # Opens the remote for reading.
     # @param [Hash] opts BFS::Blob#open options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted input stream.
-    def open(opts = {})
-      @blob.open(opts) do |io|
-        @compress.reader(io) do |cio|
-          fmt = @format.new(cio)
-          yield fmt
+    def open(**opts)
+      @blob.open(**opts) do |io|
+        @compress.reader(io, **@opts) do |cio|
+          @format.decoder(cio, **@opts) do |fmt|
+            yield fmt
+          end
         end
       end
     end
@@ -33,11 +35,12 @@ module Feedx
     # @param [Hash] opts BFS::Blob#create options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted output stream.
-    def create(opts = {})
-      @blob.create(opts) do |io|
-        @compress.writer(io) do |cio|
-          fmt = @format.new(cio)
-          yield fmt
+    def create(**opts)
+      @blob.create(**opts) do |io|
+        @compress.writer(io, **@opts) do |cio|
+          @format.encoder(cio, **@opts) do |fmt|
+            yield fmt
+          end
         end
       end
     end
@@ -48,13 +51,10 @@ module Feedx
       case val
       when nil
         Feedx::Format.detect(@blob.path)
-      when Class
-        parent = Feedx::Format::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
+      when String, Symbol
         Feedx::Format.resolve(val)
+      else
+        Feedx::Format.validate!(val)
       end
     end
 
@@ -62,13 +62,10 @@ module Feedx
       case val
       when nil
         Feedx::Compression.detect(@blob.path)
-      when Class
-        parent = Feedx::Compression::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
+      when String, Symbol
         Feedx::Compression.resolve(val)
+      else
+        Feedx::Compression.validate!(val)
       end
     end
   end

data/producer.go

@@ -147,10 +147,7 @@ func (p *Producer) push() (*ProducerPush, error) {
 		return &ProducerPush{Producer: p}, nil
 	}
 
-	writer, err := NewWriter(p.ctx, p.remote, &wopt)
-	if err != nil {
-		return nil, err
-	}
+	writer := NewWriter(p.ctx, p.remote, &wopt)
 	defer writer.Discard()
 
 	if err := p.pfn(writer); err != nil {

data/producer_test.go

@@ -23,8 +23,7 @@ var _ = Describe("Producer", func() {
 			atomic.AddUint32(&numRuns, 1)
 
 			for i := 0; i < 10; i++ {
-				fix := fixture
-				if err := w.Encode(&fix); err != nil {
+				if err := w.Encode(seed()); err != nil {
 					return err
 				}
 			}

data/reader_test.go

@@ -5,10 +5,9 @@ import (
 	"io"
 	"io/ioutil"
 
-	"github.com/bsm/feedx"
-
 	"github.com/bsm/bfs"
-	tbp "github.com/golang/protobuf/proto/proto3_proto"
+	"github.com/bsm/feedx"
+	"github.com/bsm/feedx/internal/testdata"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -34,23 +33,23 @@ var _ = Describe("Reader", func() {
 	It("should read", func() {
 		data, err := ioutil.ReadAll(subject)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(len(data)).To(BeNumerically("~", 140, 20))
+		Expect(len(data)).To(BeNumerically("~", 110, 20))
 		Expect(subject.NumRead()).To(Equal(0))
 	})
 
 	It("should decode", func() {
-		var msgs []tbp.Message
+		var msgs []*testdata.MockMessage
 		for {
-			var msg tbp.Message
+			var msg testdata.MockMessage
 			err := subject.Decode(&msg)
 			if err == io.EOF {
 				break
 			}
 			Expect(err).NotTo(HaveOccurred())
-			msgs = append(msgs, msg)
+			msgs = append(msgs, &msg)
 		}
 
-		Expect(msgs).To(Equal([]tbp.Message{fixture, fixture, fixture}))
+		Expect(msgs).To(ConsistOf(seed(), seed(), seed()))
 		Expect(subject.NumRead()).To(Equal(3))
 	})
 })

data/spec/feedx/compression/gzip_spec.rb

@@ -3,13 +3,15 @@ require 'spec_helper'
 RSpec.describe Feedx::Compression::Gzip do
   it 'should wrap readers/writers' do
     wio = StringIO.new
-    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to be_within(20).of(40)
+    expect(wio.string.encoding).to eq(Encoding::BINARY)
 
     data = ''
     StringIO.open(wio.string) do |rio|
-      described_class.reader(rio) {|z| data = z.read }
+      subject.reader(rio) {|z| data = z.read }
     end
     expect(data.size).to eq(3000)
+    expect(data.encoding).to eq(Encoding.default_external)
   end
 end

data/spec/feedx/compression/none_spec.rb

@@ -3,12 +3,12 @@ require 'spec_helper'
 RSpec.describe Feedx::Compression::None do
   it 'should wrap readers/writers' do
     wio = StringIO.new
-    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to eq(3000)
 
     data = ''
     StringIO.open(wio.string) do |rio|
-      described_class.reader(rio) {|z| data = z.read }
+      subject.reader(rio) {|z| data = z.read }
     end
     expect(data.size).to eq(3000)
   end

data/spec/feedx/compression_spec.rb

@@ -2,18 +2,18 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Compression do
   it 'should resolve' do
-    expect(described_class.resolve(:gzip)).to eq(described_class::Gzip)
-    expect(described_class.resolve(:gz)).to eq(described_class::Gzip)
-    expect(described_class.resolve(nil)).to eq(described_class::None)
+    expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
+    expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
+    expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
   end
 
   it 'should detect' do
-    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json')).to eq(described_class::None)
-    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb.gz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb')).to eq(described_class::None)
+    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)
+    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb.gz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::None)
   end
 end

data/spec/feedx/consumer_spec.rb

@@ -36,10 +36,13 @@ RSpec.describe Feedx::Consumer do
 
   private
 
-  def mock_produce!(opts = {})
+  def mock_produce!(enum: mock_enum, **opts)
     url = 'mock:///dir/file.json'
-    opts[:enum] ||= %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
-    Feedx::Producer.perform url, opts
+    Feedx::Producer.perform url, enum: enum, **opts
     url
   end
+
+  def mock_enum
+    %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
+  end
 end

data/spec/feedx/format/abstract_spec.rb

@@ -1,17 +1,20 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Abstract do
-  subject   { Feedx::Format::JSON.new(wio) }
+  subject   { Feedx::Format::JSON.new }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should decode each' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
-    StringIO.open(wio.string) do |rio|
-      fmt = subject.class.new(rio)
-      dec = fmt.decode_each(Feedx::TestCase::Model).to_a
-      expect(dec.map(&:title)).to eq(%w[X Y Z])
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
+
+    subject.decoder rio do |dec|
+      acc = dec.decode_each(Feedx::TestCase::Model).to_a
+      expect(acc.map(&:title)).to eq(%w[X Y Z])
     end
   end
 end

data/spec/feedx/format/json_spec.rb

@@ -1,26 +1,27 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::JSON do
-  subject   { described_class.new(wio) }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should encode/decode' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
     expect(wio.string.lines).to eq [
       %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
       %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
       %({"title":"Z"}\n),
     ]
 
-    StringIO.open(wio.string) do |rio|
-      fmt = described_class.new(rio)
-      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
-      expect(fmt.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
-      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
-      expect(fmt.decode(Feedx::TestCase::Model)).to be_nil
-      expect(fmt).to be_eof
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(dec.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
+      expect(dec).to be_eof
     end
   end
 end

data/spec/feedx/format/parquet_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Format::Parquet do
+  let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
+
+  let(:schema) do
+    Arrow::Schema.new([
+      Arrow::Field.new('title', :string),
+      Arrow::Field.new('updated_at', type: :timestamp, unit: :second),
+    ])
+  end
+
+  it 'should encode/decode' do
+    subject.encoder wio, schema: schema, batch_size: 2 do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Model.new('Z'))
+    end
+    expect(wio.string.bytesize).to be_within(100).of(1100)
+
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
+      expect(dec).to be_eof
+    end
+  end
+end

data/spec/feedx/format/protobuf_spec.rb

@@ -1,22 +1,23 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Protobuf do
-  subject   { described_class.new(wio) }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should encode/decode' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
     expect(wio.string.bytes).to eq([3, 10, 1, 88] + [3, 10, 1, 89] + [3, 10, 1, 90])
 
-    StringIO.open(wio.string) do |rio|
-      fmt = described_class.new(rio)
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to be_nil
-      expect(fmt).to be_eof
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
+      expect(dec.decode(Feedx::TestCase::Message)).to be_nil
+      expect(dec).to be_eof
     end
   end
 end

data/spec/feedx/format_spec.rb

@@ -2,18 +2,18 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Format do
   it 'should resolve' do
-    expect(described_class.resolve(:json)).to eq(described_class::JSON)
-    expect(described_class.resolve(:pb)).to eq(described_class::Protobuf)
+    expect(described_class.resolve(:json)).to be_instance_of(described_class::JSON)
+    expect(described_class.resolve(:pb)).to be_instance_of(described_class::Protobuf)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid format txt/)
   end
 
   it 'should detect' do
-    expect(described_class.detect('path/to/file.json')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.pb')).to eq(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pb.z')).to eq(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pb.z')).to be_instance_of(described_class::Protobuf)
     expect do
       described_class.detect('path/to/file.txt')
     end.to raise_error(/unable to detect format/)