feedx 0.11.0 → 0.12.5

data/spec/feedx/format/protobuf_spec.rb

@@ -4,7 +4,7 @@ RSpec.describe Feedx::Format::Protobuf do
   let(:wio) { StringIO.new }
   let(:rio) { StringIO.open(wio.string) }
 
-  it 'should encode/decode' do
+  it 'encode/decodes' do
     subject.encoder wio do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))

data/spec/feedx/format_spec.rb

@@ -1,13 +1,13 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format do
-  it 'should resolve' do
+  it 'resolves' do
     expect(described_class.resolve(:json)).to be_instance_of(described_class::JSON)
     expect(described_class.resolve(:pb)).to be_instance_of(described_class::Protobuf)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid format txt/)
   end
 
-  it 'should detect' do
+  it 'detects' do
     expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::JSON)
     expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::JSON)
     expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::JSON)

data/spec/feedx/producer_spec.rb

@@ -6,39 +6,40 @@ RSpec.describe Feedx::Producer do
   end
 
   let(:bucket) { BFS::Bucket::InMem.new }
+
   before { allow(BFS).to receive(:resolve).and_return(bucket) }
 
-  it 'should reject invalid inputs' do
+  it 'rejects invalid inputs' do
     expect do
       described_class.perform 'mock:///dir/file.txt', enum: enumerable
     end.to raise_error(/unable to detect format/)
   end
 
-  it 'should push compressed JSON' do
+  it 'pushes compressed JSON' do
     size = described_class.perform 'mock:///dir/file.jsonz', enum: enumerable
     expect(size).to be_within(20).of(166)
     expect(bucket.info('dir/file.jsonz').size).to eq(size)
   end
 
-  it 'should push plain JSON' do
+  it 'pushes plain JSON' do
     size = described_class.perform 'mock:///dir/file.json', enum: enumerable
     expect(size).to eq(15900)
     expect(bucket.info('dir/file.json').size).to eq(size)
   end
 
-  it 'should push compressed PB' do
+  it 'pushes compressed PB' do
     size = described_class.perform 'mock:///dir/file.pbz', enum: enumerable
     expect(size).to be_within(20).of(41)
     expect(bucket.info('dir/file.pbz').size).to eq(size)
   end
 
-  it 'should push plain PB' do
+  it 'pushes plain PB' do
     size = described_class.perform 'mock:///dir/file.pb', enum: enumerable
     expect(size).to eq(1200)
     expect(bucket.info('dir/file.pb').size).to eq(size)
   end
 
-  it 'should support factories' do
+  it 'supports factories' do
     size = described_class.perform('mock:///dir/file.json') do
       enumerable
     end
@@ -46,12 +47,12 @@ RSpec.describe Feedx::Producer do
     expect(bucket.info('dir/file.json').size).to eq(size)
   end
 
-  it 'should support last-modified' do
+  it 'supports last-modified' do
     described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
     expect(bucket.info('dir/file.json').metadata).to eq('X-Feedx-Last-Modified' => '1515151515000')
   end
 
-  it 'should perform conditionally' do
+  it 'performs conditionally' do
     size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
     expect(size).to eq(15900)
 
@@ -64,4 +65,10 @@ RSpec.describe Feedx::Producer do
     size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151516), enum: enumerable
     expect(size).to eq(15900)
   end
+
+  it 'accepts downstream options' do
+    expect do
+      described_class.perform 'mock:///dir/file.jsonz', enum: enumerable, x: 1, y: 'v', z: true
+    end.not_to raise_error
+  end
 end

data/spec/feedx/stream_spec.rb

@@ -1,19 +1,22 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Stream do
+  subject { described_class.new('mock:///dir/file.json') }
+
   let(:bucket) { BFS::Bucket::InMem.new }
+  let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
+
   before  { allow(BFS).to receive(:resolve).and_return(bucket) }
 
-  subject { described_class.new('mock:///dir/file.json') }
-  let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
+  after { subject.close; compressed.close }
 
-  it 'should reject invalid inputs' do
+  it 'rejects invalid inputs' do
     expect do
       described_class.new('mock:///dir/file.txt')
     end.to raise_error(/unable to detect format/)
   end
 
-  it 'should accept custom formats' do
+  it 'accepts custom formats' do
     format = Class.new do
       def encoder(io, &block)
         Feedx::Format::JSON::Encoder.open(io, &block)
@@ -24,27 +27,30 @@ RSpec.describe Feedx::Stream do
       end
     end
 
-    stream = described_class.new('mock:///dir/file.txt', format: format.new)
-    stream.create {|s| s.encode Feedx::TestCase::Model.new('X') }
+    result = described_class.open('mock:///dir/file.txt', format: format.new) do |stream|
+      stream.create {|s| s.encode Feedx::TestCase::Model.new('X') }
+      21
+    end
+    expect(result).to eq(21)
 
-    expect(bucket.read('dir/file.txt')).to eq(
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-    )
+    expect(bucket.read('dir/file.txt')).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+    JSON
   end
 
-  it 'should encode' do
+  it 'encodes' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))
     end
 
-    expect(bucket.read('dir/file.json')).to eq(
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n) +
-      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-    )
+    expect(bucket.read('dir/file.json')).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
+    JSON
   end
 
-  it 'should encode compressed' do
+  it 'encodes compressed' do
     compressed.create do |s|
       100.times do
         s.encode(Feedx::TestCase::Model.new('X'))
@@ -54,14 +60,26 @@ RSpec.describe Feedx::Stream do
     expect(bucket.info('dir/file.json.gz').size).to be_within(10).of(108)
   end
 
-  it 'should encode with create options' do
+  it 'encodes with create options' do
     subject.create metadata: { 'x' => '5' } do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
     end
     expect(bucket.info('dir/file.json').metadata).to eq('X' => '5')
   end
 
-  it 'should decode' do
+  it 'aborts encode on errors (if compressed)' do
+    stop = RuntimeError.new('STOP')
+    expect do
+      compressed.create do |s|
+        s.encode(Feedx::TestCase::Model.new('X'))
+        raise stop
+      end
+    end.to raise_error(stop)
+
+    expect(bucket.ls('**').to_a).to be_empty
+  end
+
+  it 'decodes' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))
@@ -75,7 +93,7 @@ RSpec.describe Feedx::Stream do
     end
   end
 
-  it 'should decode compressed' do
+  it 'decodes compressed' do
     compressed.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))

data/writer_test.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Writer", func() {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.5
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-16 00:00:00.000000000 Z
+date: 2021-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -72,30 +72,22 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+        version: '3.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
-  name: rubocop
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,7 +101,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rubocop-performance
+  name: rubocop-bsm
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -130,9 +122,9 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".editorconfig"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -143,6 +135,15 @@ files:
 - compression_test.go
 - consumer.go
 - consumer_test.go
+- ext/parquet/decoder.go
+- ext/parquet/decoder_test.go
+- ext/parquet/go.mod
+- ext/parquet/go.sum
+- ext/parquet/parquet.go
+- ext/parquet/parquet_test.go
+- ext/parquet/reader.go
+- ext/parquet/testdata/alltypes_plain.parquet
+- ext/parquet/types.go
 - feedx.gemspec
 - feedx.go
 - feedx_test.go
@@ -202,7 +203,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/.travis.yml

@@ -1,24 +0,0 @@
-matrix:
-  include:
-    - language: ruby
-      rvm:
-        - 2.7
-      before_install:
-        - wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
-        - sudo apt install -y ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
-        - sudo apt update
-        - sudo apt install -y libarrow-dev libarrow-glib-dev libarrow-dataset-dev libplasma-dev libplasma-glib-dev libgandiva-dev libgandiva-glib-dev libparquet-dev libparquet-glib-dev
-    - language: ruby
-      rvm:
-        - 2.6
-      before_install:
-        - wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
-        - sudo apt install -y ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
-        - sudo apt update
-        - sudo apt install -y libarrow-dev libarrow-glib-dev libarrow-dataset-dev libplasma-dev libplasma-glib-dev libgandiva-dev libgandiva-glib-dev libparquet-dev libparquet-glib-dev
-    - language: go
-      go:
-        - 1.14.x
-    - language: go
-      go:
-        - 1.13.x