feedx 0.12.2 → 0.12.3

data/lib/feedx/producer.rb

@@ -32,26 +32,25 @@ module Feedx
     end
 
     def perform
-      stream = Feedx::Stream.new(@url, **@opts)
-      enum = @enum.is_a?(Proc) ? @enum.call : @enum
-      last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
-      local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
-
-      begin
-        metadata   = stream.blob.info.metadata
-        remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
-        return -1 unless local_rev > remote_rev
-      rescue BFS::FileNotFound
-        nil
-      end if local_rev.positive?
-
-      stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
-        iter = enum.respond_to?(:find_each) ? :find_each : :each
-        enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
+      Feedx::Stream.open(@url, **@opts) do |stream|
+        enum = @enum.is_a?(Proc) ? @enum.call : @enum
+        last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
+        local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
+
+        begin
+          metadata   = stream.blob.info.metadata
+          remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
+          return -1 unless local_rev > remote_rev
+        rescue BFS::FileNotFound
+          nil
+        end if local_rev.positive?
+
+        stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
+          iter = enum.respond_to?(:find_each) ? :find_each : :each
+          enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
+        end
+        stream.blob.info.size
       end
-      stream.blob.info.size
-    ensure
-      stream&.close
     end
   end
 end

data/lib/feedx/stream.rb

@@ -36,12 +36,10 @@ module Feedx
     # @param [Hash] opts BFS::Blob#open options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted input stream.
-    def open(**opts)
+    def open(**opts, &block)
       @blob.open(**opts) do |io|
         @compress.reader(io, **@opts) do |cio|
-          @format.decoder(cio, **@opts) do |fmt|
-            yield fmt
-          end
+          @format.decoder(cio, **@opts, &block)
         end
       end
     end
@@ -50,12 +48,10 @@ module Feedx
     # @param [Hash] opts BFS::Blob#create options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted output stream.
-    def create(**opts)
+    def create(**opts, &block)
       @blob.create(**opts) do |io|
         @compress.writer(io, **@opts) do |cio|
-          @format.encoder(cio, **@opts) do |fmt|
-            yield fmt
-          end
+          @format.encoder(cio, **@opts, &block)
         end
       end
     end

data/spec/feedx/consumer_spec.rb

@@ -8,7 +8,7 @@ RSpec.describe Feedx::Consumer do
 
   it 'should reject invalid inputs' do
     expect do
-      described_class.each('mock:///dir/file.txt', klass) {}
+      described_class.each('mock:///dir/file.txt', klass)
     end.to raise_error(/unable to detect format/)
   end
 

data/spec/feedx/format/json_spec.rb

@@ -10,11 +10,11 @@ RSpec.describe Feedx::Format::JSON do
       enc.encode(Feedx::TestCase::Model.new('Y'))
       enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
     end
-    expect(wio.string.lines).to eq [
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-      %({"title":"Z"}\n),
-    ]
+    expect(wio.string).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Z"}
+    JSON
 
     subject.decoder rio do |dec|
       expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))

data/spec/feedx/stream_spec.rb

@@ -33,9 +33,9 @@ RSpec.describe Feedx::Stream do
     end
     expect(result).to eq(21)
 
-    expect(bucket.read('dir/file.txt')).to eq(
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-    )
+    expect(bucket.read('dir/file.txt')).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+    JSON
   end
 
   it 'should encode' do
@@ -44,10 +44,10 @@ RSpec.describe Feedx::Stream do
       s.encode(Feedx::TestCase::Model.new('Y'))
     end
 
-    expect(bucket.read('dir/file.json')).to eq(
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n) +
-      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-    )
+    expect(bucket.read('dir/file.json')).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
+    JSON
   end
 
   it 'should encode compressed' do
@@ -67,6 +67,18 @@ RSpec.describe Feedx::Stream do
     expect(bucket.info('dir/file.json').metadata).to eq('X' => '5')
   end
 
+  it 'should abort encode on errors (if compressed)' do
+    stop = RuntimeError.new('STOP')
+    expect do
+      compressed.create do |s|
+        s.encode(Feedx::TestCase::Model.new('X'))
+        raise stop
+      end
+    end.to raise_error(stop)
+
+    expect(bucket.ls('**').to_a).to be_empty
+  end
+
   it 'should decode' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.12.2
+  version: 0.12.3
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-02 00:00:00.000000000 Z
+date: 2020-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -143,6 +143,15 @@ files:
 - compression_test.go
 - consumer.go
 - consumer_test.go
+- ext/parquet/decoder.go
+- ext/parquet/decoder_test.go
+- ext/parquet/go.mod
+- ext/parquet/go.sum
+- ext/parquet/parquet.go
+- ext/parquet/parquet_test.go
+- ext/parquet/reader.go
+- ext/parquet/testdata/alltypes_plain.parquet
+- ext/parquet/types.go
 - feedx.gemspec
 - feedx.go
 - feedx_test.go