feedx 0.10.2 → 0.11.0

data/lib/feedx/format/protobuf.rb

@@ -1,16 +1,24 @@
 require 'pbio'
 
 class Feedx::Format::Protobuf < Feedx::Format::Abstract
-  def initialize(io)
-    super PBIO::Delimited.new(io)
-  end
+  class Decoder < Feedx::Format::Abstract::Decoder
+    def initialize(io, **opts)
+      super PBIO::Delimited.new(io), **opts
+    end
 
-  def decode(klass, **)
-    @io.read(klass)
+    def decode(target, **)
+      @io.read(target)
+    end
   end
 
-  def encode(msg, **opts)
-    msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
-    @io.write msg
+  class Encoder < Feedx::Format::Abstract::Encoder
+    def initialize(io, **opts)
+      super PBIO::Delimited.new(io), **opts
+    end
+
+    def encode(msg, **opts)
+      msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
+      @io.write msg
+    end
   end
 end

data/lib/feedx/stream.rb

@@ -23,8 +23,9 @@ module Feedx
     def open(**opts)
       @blob.open(**opts) do |io|
         @compress.reader(io) do |cio|
-          fmt = @format.new(cio)
-          yield fmt
+          @format.decoder(cio) do |fmt|
+            yield fmt
+          end
         end
       end
     end
@@ -36,8 +37,9 @@ module Feedx
     def create(**opts)
       @blob.create(**opts) do |io|
         @compress.writer(io) do |cio|
-          fmt = @format.new(cio)
-          yield fmt
+          @format.encoder(cio) do |fmt|
+            yield fmt
+          end
         end
       end
     end
@@ -48,13 +50,10 @@ module Feedx
       case val
       when nil
         Feedx::Format.detect(@blob.path)
-      when Class
-        parent = Feedx::Format::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
+      when String, Symbol
         Feedx::Format.resolve(val)
+      else
+        Feedx::Format.validate!(val)
       end
     end
 
@@ -62,13 +61,10 @@ module Feedx
       case val
       when nil
         Feedx::Compression.detect(@blob.path)
-      when Class
-        parent = Feedx::Compression::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
+      when String, Symbol
         Feedx::Compression.resolve(val)
+      else
+        Feedx::Compression.validate!(val)
       end
     end
   end

data/producer_test.go

@@ -23,8 +23,7 @@ var _ = Describe("Producer", func() {
 			atomic.AddUint32(&numRuns, 1)
 
 			for i := 0; i < 10; i++ {
-				fix := fixture
-				if err := w.Encode(&fix); err != nil {
+				if err := w.Encode(seed()); err != nil {
 					return err
 				}
 			}

data/reader_test.go

@@ -5,9 +5,9 @@ import (
 	"io"
 	"io/ioutil"
 
-	"github.com/bsm/feedx"
-
 	"github.com/bsm/bfs"
+	"github.com/bsm/feedx"
+	"github.com/bsm/feedx/internal/testdata"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -38,18 +38,18 @@ var _ = Describe("Reader", func() {
 	})
 
 	It("should decode", func() {
-		var msgs []MockMessage
+		var msgs []*testdata.MockMessage
 		for {
-			var msg MockMessage
+			var msg testdata.MockMessage
 			err := subject.Decode(&msg)
 			if err == io.EOF {
 				break
 			}
 			Expect(err).NotTo(HaveOccurred())
-			msgs = append(msgs, msg)
+			msgs = append(msgs, &msg)
 		}
 
-		Expect(msgs).To(Equal([]MockMessage{fixture, fixture, fixture}))
+		Expect(msgs).To(ConsistOf(seed(), seed(), seed()))
 		Expect(subject.NumRead()).To(Equal(3))
 	})
 })

data/spec/feedx/compression/gzip_spec.rb

@@ -3,13 +3,13 @@ require 'spec_helper'
 RSpec.describe Feedx::Compression::Gzip do
   it 'should wrap readers/writers' do
     wio = StringIO.new
-    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to be_within(20).of(40)
     expect(wio.string.encoding).to eq(Encoding::BINARY)
 
     data = ''
     StringIO.open(wio.string) do |rio|
-      described_class.reader(rio) {|z| data = z.read }
+      subject.reader(rio) {|z| data = z.read }
     end
     expect(data.size).to eq(3000)
     expect(data.encoding).to eq(Encoding.default_external)

data/spec/feedx/compression/none_spec.rb

@@ -3,12 +3,12 @@ require 'spec_helper'
 RSpec.describe Feedx::Compression::None do
   it 'should wrap readers/writers' do
     wio = StringIO.new
-    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to eq(3000)
 
     data = ''
     StringIO.open(wio.string) do |rio|
-      described_class.reader(rio) {|z| data = z.read }
+      subject.reader(rio) {|z| data = z.read }
     end
     expect(data.size).to eq(3000)
   end

data/spec/feedx/compression_spec.rb

@@ -2,18 +2,18 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Compression do
   it 'should resolve' do
-    expect(described_class.resolve(:gzip)).to eq(described_class::Gzip)
-    expect(described_class.resolve(:gz)).to eq(described_class::Gzip)
-    expect(described_class.resolve(nil)).to eq(described_class::None)
+    expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
+    expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
+    expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
   end
 
   it 'should detect' do
-    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json')).to eq(described_class::None)
-    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb.gz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb')).to eq(described_class::None)
+    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)
+    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb.gz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::None)
   end
 end

data/spec/feedx/format/abstract_spec.rb

@@ -1,17 +1,20 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Abstract do
-  subject   { Feedx::Format::JSON.new(wio) }
+  subject   { Feedx::Format::JSON.new }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should decode each' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
-    StringIO.open(wio.string) do |rio|
-      fmt = subject.class.new(rio)
-      dec = fmt.decode_each(Feedx::TestCase::Model).to_a
-      expect(dec.map(&:title)).to eq(%w[X Y Z])
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
+
+    subject.decoder rio do |dec|
+      acc = dec.decode_each(Feedx::TestCase::Model).to_a
+      expect(acc.map(&:title)).to eq(%w[X Y Z])
     end
   end
 end

data/spec/feedx/format/json_spec.rb

@@ -1,26 +1,27 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::JSON do
-  subject   { described_class.new(wio) }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should encode/decode' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
     expect(wio.string.lines).to eq [
       %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
       %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
       %({"title":"Z"}\n),
     ]
 
-    StringIO.open(wio.string) do |rio|
-      fmt = described_class.new(rio)
-      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
-      expect(fmt.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
-      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
-      expect(fmt.decode(Feedx::TestCase::Model)).to be_nil
-      expect(fmt).to be_eof
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(dec.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
+      expect(dec).to be_eof
     end
   end
 end

data/spec/feedx/format/parquet_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Format::Parquet do
+  let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
+
+  let(:schema) do
+    Arrow::Schema.new([
+      Arrow::Field.new('title', :string),
+      Arrow::Field.new('updated_at', type: :timestamp, unit: :second),
+    ])
+  end
+
+  it 'should encode/decode' do
+    subject.encoder wio, schema: schema, batch_size: 2 do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Model.new('Z'))
+    end
+    expect(wio.string.bytesize).to be_within(100).of(1100)
+
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
+      expect(dec).to be_eof
+    end
+  end
+end

data/spec/feedx/format/protobuf_spec.rb

@@ -1,22 +1,23 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Protobuf do
-  subject   { described_class.new(wio) }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should encode/decode' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
     expect(wio.string.bytes).to eq([3, 10, 1, 88] + [3, 10, 1, 89] + [3, 10, 1, 90])
 
-    StringIO.open(wio.string) do |rio|
-      fmt = described_class.new(rio)
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to be_nil
-      expect(fmt).to be_eof
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
+      expect(dec.decode(Feedx::TestCase::Message)).to be_nil
+      expect(dec).to be_eof
     end
   end
 end

data/spec/feedx/format_spec.rb

@@ -2,18 +2,18 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Format do
   it 'should resolve' do
-    expect(described_class.resolve(:json)).to eq(described_class::JSON)
-    expect(described_class.resolve(:pb)).to eq(described_class::Protobuf)
+    expect(described_class.resolve(:json)).to be_instance_of(described_class::JSON)
+    expect(described_class.resolve(:pb)).to be_instance_of(described_class::Protobuf)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid format txt/)
   end
 
   it 'should detect' do
-    expect(described_class.detect('path/to/file.json')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.pb')).to eq(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pb.z')).to eq(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pb.z')).to be_instance_of(described_class::Protobuf)
     expect do
       described_class.detect('path/to/file.txt')
     end.to raise_error(/unable to detect format/)

data/spec/feedx/stream_spec.rb

@@ -13,13 +13,32 @@ RSpec.describe Feedx::Stream do
     end.to raise_error(/unable to detect format/)
   end
 
+  it 'should accept custom formats' do
+    format = Class.new do
+      def encoder(io, &block)
+        Feedx::Format::JSON::Encoder.open(io, &block)
+      end
+
+      def decoder(io, &block)
+        Feedx::Format::JSON::Decoder.open(io, &block)
+      end
+    end
+
+    stream = described_class.new('mock:///dir/file.txt', format: format.new)
+    stream.create {|s| s.encode Feedx::TestCase::Model.new('X') }
+
+    expect(bucket.read('dir/file.txt')).to eq(
+      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
+    )
+  end
+
   it 'should encode' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))
     end
 
-    expect(bucket.open('dir/file.json').read).to eq(
+    expect(bucket.read('dir/file.json')).to eq(
       %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n) +
       %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
     )

data/spec/spec_helper.rb

@@ -28,13 +28,29 @@ module Feedx
       end
       alias eql? ==
 
+      def updated_at
+        Time.at(1515151515).utc
+      end
+
       def from_json(data, *)
         hash = ::JSON.parse(data)
         @title = hash['title'] if hash.is_a?(Hash)
       end
 
       def to_json(*)
-        ::JSON.dump(title: @title, updated_at: Time.at(1515151515).utc)
+        ::JSON.dump(title: @title, updated_at: updated_at)
+      end
+
+      def from_parquet(rec)
+        rec.each_pair do |name, value|
+          @title = value if name == 'title'
+        end
+      end
+
+      def to_parquet(schema, *)
+        schema.fields.map do |field|
+          send(field.name)
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.11.0
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-28 00:00:00.000000000 Z
+date: 2020-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: red-parquet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +150,8 @@ files:
 - format_test.go
 - go.mod
 - go.sum
+- internal/testdata/testdata.pb.go
+- internal/testdata/testdata.proto
 - lib/feedx.rb
 - lib/feedx/cache.rb
 - lib/feedx/cache/abstract.rb
@@ -149,6 +165,7 @@ files:
 - lib/feedx/format.rb
 - lib/feedx/format/abstract.rb
 - lib/feedx/format/json.rb
+- lib/feedx/format/parquet.rb
 - lib/feedx/format/protobuf.rb
 - lib/feedx/producer.rb
 - lib/feedx/pusher.rb
@@ -165,6 +182,7 @@ files:
 - spec/feedx/consumer_spec.rb
 - spec/feedx/format/abstract_spec.rb
 - spec/feedx/format/json_spec.rb
+- spec/feedx/format/parquet_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
 - spec/feedx/producer_spec.rb
@@ -191,7 +209,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Exchange data between components via feeds
@@ -204,6 +222,7 @@ test_files:
 - spec/feedx/consumer_spec.rb
 - spec/feedx/format/abstract_spec.rb
 - spec/feedx/format/json_spec.rb
+- spec/feedx/format/parquet_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
 - spec/feedx/producer_spec.rb