feedx 0.10.2 → 0.12.3

data/spec/feedx/producer_spec.rb

@@ -64,4 +64,10 @@ RSpec.describe Feedx::Producer do
     size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151516), enum: enumerable
     expect(size).to eq(15900)
   end
+
+  it 'should accept downstream options' do
+    expect do
+      described_class.perform 'mock:///dir/file.jsonz', enum: enumerable, x: 1, y: 'v', z: true
+    end.not_to raise_error
+  end
 end

data/spec/feedx/stream_spec.rb

@@ -2,10 +2,13 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Stream do
   let(:bucket) { BFS::Bucket::InMem.new }
-  before  { allow(BFS).to receive(:resolve).and_return(bucket) }
+  let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
 
   subject { described_class.new('mock:///dir/file.json') }
-  let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
+
+  before  { allow(BFS).to receive(:resolve).and_return(bucket) }
+  after { subject.close }
+  after { compressed.close }
 
   it 'should reject invalid inputs' do
     expect do
@@ -13,16 +16,38 @@ RSpec.describe Feedx::Stream do
     end.to raise_error(/unable to detect format/)
   end
 
+  it 'should accept custom formats' do
+    format = Class.new do
+      def encoder(io, &block)
+        Feedx::Format::JSON::Encoder.open(io, &block)
+      end
+
+      def decoder(io, &block)
+        Feedx::Format::JSON::Decoder.open(io, &block)
+      end
+    end
+
+    result = described_class.open('mock:///dir/file.txt', format: format.new) do |stream|
+      stream.create {|s| s.encode Feedx::TestCase::Model.new('X') }
+      21
+    end
+    expect(result).to eq(21)
+
+    expect(bucket.read('dir/file.txt')).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+    JSON
+  end
+
   it 'should encode' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))
     end
 
-    expect(bucket.open('dir/file.json').read).to eq(
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n) +
-      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-    )
+    expect(bucket.read('dir/file.json')).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
+    JSON
   end
 
   it 'should encode compressed' do
@@ -42,6 +67,18 @@ RSpec.describe Feedx::Stream do
     expect(bucket.info('dir/file.json').metadata).to eq('X' => '5')
   end
 
+  it 'should abort encode on errors (if compressed)' do
+    stop = RuntimeError.new('STOP')
+    expect do
+      compressed.create do |s|
+        s.encode(Feedx::TestCase::Model.new('X'))
+        raise stop
+      end
+    end.to raise_error(stop)
+
+    expect(bucket.ls('**').to_a).to be_empty
+  end
+
   it 'should decode' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))

data/spec/spec_helper.rb

@@ -28,13 +28,29 @@ module Feedx
       end
       alias eql? ==
 
+      def updated_at
+        Time.at(1515151515).utc
+      end
+
       def from_json(data, *)
         hash = ::JSON.parse(data)
         @title = hash['title'] if hash.is_a?(Hash)
       end
 
       def to_json(*)
-        ::JSON.dump(title: @title, updated_at: Time.at(1515151515).utc)
+        ::JSON.dump(title: @title, updated_at: updated_at)
+      end
+
+      def from_parquet(rec)
+        rec.each_pair do |name, value|
+          @title = value if name == 'title'
+        end
+      end
+
+      def to_parquet(schema, *)
+        schema.fields.map do |field|
+          send(field.name)
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.12.3
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-28 00:00:00.000000000 Z
+date: 2020-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: red-parquet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -129,6 +143,15 @@ files:
 - compression_test.go
 - consumer.go
 - consumer_test.go
+- ext/parquet/decoder.go
+- ext/parquet/decoder_test.go
+- ext/parquet/go.mod
+- ext/parquet/go.sum
+- ext/parquet/parquet.go
+- ext/parquet/parquet_test.go
+- ext/parquet/reader.go
+- ext/parquet/testdata/alltypes_plain.parquet
+- ext/parquet/types.go
 - feedx.gemspec
 - feedx.go
 - feedx_test.go
@@ -136,6 +159,8 @@ files:
 - format_test.go
 - go.mod
 - go.sum
+- internal/testdata/testdata.pb.go
+- internal/testdata/testdata.proto
 - lib/feedx.rb
 - lib/feedx/cache.rb
 - lib/feedx/cache/abstract.rb
@@ -149,6 +174,7 @@ files:
 - lib/feedx/format.rb
 - lib/feedx/format/abstract.rb
 - lib/feedx/format/json.rb
+- lib/feedx/format/parquet.rb
 - lib/feedx/format/protobuf.rb
 - lib/feedx/producer.rb
 - lib/feedx/pusher.rb
@@ -165,6 +191,7 @@ files:
 - spec/feedx/consumer_spec.rb
 - spec/feedx/format/abstract_spec.rb
 - spec/feedx/format/json_spec.rb
+- spec/feedx/format/parquet_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
 - spec/feedx/producer_spec.rb
@@ -191,7 +218,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Exchange data between components via feeds
@@ -204,6 +231,7 @@ test_files:
 - spec/feedx/consumer_spec.rb
 - spec/feedx/format/abstract_spec.rb
 - spec/feedx/format/json_spec.rb
+- spec/feedx/format/parquet_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
 - spec/feedx/producer_spec.rb