feedx 0.10.1 → 0.12.2

data/spec/feedx/format/json_spec.rb

@@ -1,26 +1,27 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::JSON do
-  subject   { described_class.new(wio) }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should encode/decode' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
     expect(wio.string.lines).to eq [
       %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
       %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
       %({"title":"Z"}\n),
     ]
 
-    StringIO.open(wio.string) do |rio|
-      fmt = described_class.new(rio)
-      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
-      expect(fmt.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
-      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
-      expect(fmt.decode(Feedx::TestCase::Model)).to be_nil
-      expect(fmt).to be_eof
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(dec.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
+      expect(dec).to be_eof
     end
   end
 end

data/spec/feedx/format/parquet_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Format::Parquet do
+  let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
+
+  let(:schema) do
+    Arrow::Schema.new([
+      Arrow::Field.new('title', :string),
+      Arrow::Field.new('updated_at', type: :timestamp, unit: :second),
+    ])
+  end
+
+  it 'should encode/decode' do
+    subject.encoder wio, schema: schema, batch_size: 2 do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Model.new('Z'))
+    end
+    expect(wio.string.bytesize).to be_within(100).of(1100)
+
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
+      expect(dec).to be_eof
+    end
+  end
+end

data/spec/feedx/format/protobuf_spec.rb

@@ -1,22 +1,23 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Protobuf do
-  subject   { described_class.new(wio) }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should encode/decode' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
     expect(wio.string.bytes).to eq([3, 10, 1, 88] + [3, 10, 1, 89] + [3, 10, 1, 90])
 
-    StringIO.open(wio.string) do |rio|
-      fmt = described_class.new(rio)
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to be_nil
-      expect(fmt).to be_eof
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
+      expect(dec.decode(Feedx::TestCase::Message)).to be_nil
+      expect(dec).to be_eof
     end
   end
 end

data/spec/feedx/format_spec.rb

@@ -2,18 +2,18 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Format do
   it 'should resolve' do
-    expect(described_class.resolve(:json)).to eq(described_class::JSON)
-    expect(described_class.resolve(:pb)).to eq(described_class::Protobuf)
+    expect(described_class.resolve(:json)).to be_instance_of(described_class::JSON)
+    expect(described_class.resolve(:pb)).to be_instance_of(described_class::Protobuf)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid format txt/)
   end
 
   it 'should detect' do
-    expect(described_class.detect('path/to/file.json')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.pb')).to eq(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pb.z')).to eq(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pb.z')).to be_instance_of(described_class::Protobuf)
     expect do
       described_class.detect('path/to/file.txt')
     end.to raise_error(/unable to detect format/)

data/spec/feedx/producer_spec.rb

@@ -64,4 +64,10 @@ RSpec.describe Feedx::Producer do
     size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151516), enum: enumerable
     expect(size).to eq(15900)
   end
+
+  it 'should accept downstream options' do
+    expect do
+      described_class.perform 'mock:///dir/file.jsonz', enum: enumerable, x: 1, y: 'v', z: true
+    end.not_to raise_error
+  end
 end

data/spec/feedx/stream_spec.rb

@@ -2,10 +2,13 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Stream do
   let(:bucket) { BFS::Bucket::InMem.new }
-  before  { allow(BFS).to receive(:resolve).and_return(bucket) }
+  let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
 
   subject { described_class.new('mock:///dir/file.json') }
-  let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
+
+  before  { allow(BFS).to receive(:resolve).and_return(bucket) }
+  after { subject.close }
+  after { compressed.close }
 
   it 'should reject invalid inputs' do
     expect do
@@ -13,13 +16,35 @@ RSpec.describe Feedx::Stream do
     end.to raise_error(/unable to detect format/)
   end
 
+  it 'should accept custom formats' do
+    format = Class.new do
+      def encoder(io, &block)
+        Feedx::Format::JSON::Encoder.open(io, &block)
+      end
+
+      def decoder(io, &block)
+        Feedx::Format::JSON::Decoder.open(io, &block)
+      end
+    end
+
+    result = described_class.open('mock:///dir/file.txt', format: format.new) do |stream|
+      stream.create {|s| s.encode Feedx::TestCase::Model.new('X') }
+      21
+    end
+    expect(result).to eq(21)
+
+    expect(bucket.read('dir/file.txt')).to eq(
+      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
+    )
+  end
+
   it 'should encode' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))
     end
 
-    expect(bucket.open('dir/file.json').read).to eq(
+    expect(bucket.read('dir/file.json')).to eq(
       %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n) +
       %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
     )

data/spec/spec_helper.rb

@@ -28,13 +28,29 @@ module Feedx
       end
       alias eql? ==
 
+      def updated_at
+        Time.at(1515151515).utc
+      end
+
       def from_json(data, *)
         hash = ::JSON.parse(data)
         @title = hash['title'] if hash.is_a?(Hash)
       end
 
       def to_json(*)
-        ::JSON.dump(title: @title, updated_at: Time.at(1515151515).utc)
+        ::JSON.dump(title: @title, updated_at: updated_at)
+      end
+
+      def from_parquet(rec)
+        rec.each_pair do |name, value|
+          @title = value if name == 'title'
+        end
+      end
+
+      def to_parquet(schema, *)
+        schema.fields.map do |field|
+          send(field.name)
+        end
       end
     end
   end

data/writer_test.go

@@ -56,7 +56,7 @@ var _ = Describe("Writer", func() {
 
 		info, err := plain.Head(ctx)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(info.Size).To(BeNumerically("~", 470, 10))
+		Expect(info.Size).To(BeNumerically("~", 370, 10))
 		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
 
 		info, err = compressed.Head(ctx)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.12.2
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-22 00:00:00.000000000 Z
+date: 2020-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: red-parquet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +150,8 @@ files:
 - format_test.go
 - go.mod
 - go.sum
+- internal/testdata/testdata.pb.go
+- internal/testdata/testdata.proto
 - lib/feedx.rb
 - lib/feedx/cache.rb
 - lib/feedx/cache/abstract.rb
@@ -149,6 +165,7 @@ files:
 - lib/feedx/format.rb
 - lib/feedx/format/abstract.rb
 - lib/feedx/format/json.rb
+- lib/feedx/format/parquet.rb
 - lib/feedx/format/protobuf.rb
 - lib/feedx/producer.rb
 - lib/feedx/pusher.rb
@@ -165,6 +182,7 @@ files:
 - spec/feedx/consumer_spec.rb
 - spec/feedx/format/abstract_spec.rb
 - spec/feedx/format/json_spec.rb
+- spec/feedx/format/parquet_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
 - spec/feedx/producer_spec.rb
@@ -191,7 +209,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Exchange data between components via feeds
@@ -204,6 +222,7 @@ test_files:
 - spec/feedx/consumer_spec.rb
 - spec/feedx/format/abstract_spec.rb
 - spec/feedx/format/json_spec.rb
+- spec/feedx/format/parquet_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
 - spec/feedx/producer_spec.rb