feedx 0.12.2 → 0.12.7

data/lib/feedx/cache/memory.rb

@@ -3,6 +3,7 @@ require 'monitor'
 # Thread-safe in-memory cache. Use for testing only.
 class Feedx::Cache::Memory < Feedx::Cache::Abstract
   def initialize
+    super
     @monitor = Monitor.new
     @entries = {}
   end

data/lib/feedx/format.rb

@@ -27,7 +27,7 @@ module Feedx
           ext = File.extname(base)
           raise ArgumentError, 'unable to detect format' if ext.empty?
 
-          kind = _resolve(ext[1..-1]) || _resolve(ext[1..-2])
+          kind = _resolve(ext[1..]) || _resolve(ext[1..-2])
           return kind if kind
 
           base = base[0..-ext.size - 1]

data/lib/feedx/producer.rb

@@ -32,26 +32,25 @@ module Feedx
     end
 
     def perform
-      stream = Feedx::Stream.new(@url, **@opts)
-      enum = @enum.is_a?(Proc) ? @enum.call : @enum
-      last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
-      local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
-
-      begin
-        metadata   = stream.blob.info.metadata
-        remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
-        return -1 unless local_rev > remote_rev
-      rescue BFS::FileNotFound
-        nil
-      end if local_rev.positive?
-
-      stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
-        iter = enum.respond_to?(:find_each) ? :find_each : :each
-        enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
+      Feedx::Stream.open(@url, **@opts) do |stream|
+        enum = @enum.is_a?(Proc) ? @enum.call : @enum
+        last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
+        local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
+
+        begin
+          metadata   = stream.blob.info.metadata
+          remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
+          return -1 unless local_rev > remote_rev
+        rescue BFS::FileNotFound
+          nil
+        end if local_rev.positive?
+
+        stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
+          iter = enum.respond_to?(:find_each) ? :find_each : :each
+          enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
+        end
+        stream.blob.info.size
       end
-      stream.blob.info.size
-    ensure
-      stream&.close
     end
   end
 end

data/lib/feedx/stream.rb

@@ -36,12 +36,10 @@ module Feedx
     # @param [Hash] opts BFS::Blob#open options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted input stream.
-    def open(**opts)
+    def open(**opts, &block)
       @blob.open(**opts) do |io|
         @compress.reader(io, **@opts) do |cio|
-          @format.decoder(cio, **@opts) do |fmt|
-            yield fmt
-          end
+          @format.decoder(cio, **@opts, &block)
         end
       end
     end
@@ -50,12 +48,10 @@ module Feedx
     # @param [Hash] opts BFS::Blob#create options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted output stream.
-    def create(**opts)
+    def create(**opts, &block)
       @blob.create(**opts) do |io|
         @compress.writer(io, **@opts) do |cio|
-          @format.encoder(cio, **@opts) do |fmt|
-            yield fmt
-          end
+          @format.encoder(cio, **@opts, &block)
         end
       end
     end

data/producer_test.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Producer", func() {
@@ -43,7 +43,7 @@ var _ = Describe("Producer", func() {
 		}
 	})
 
-	It("should produce", func() {
+	It("produces", func() {
 		setup(nil)
 
 		Expect(subject.LastPush()).To(BeTemporally("~", time.Now(), time.Second))
@@ -56,7 +56,7 @@ var _ = Describe("Producer", func() {
 		Expect(info.Size).To(BeNumerically("~", 75, 10))
 	})
 
-	It("should produce with custom last-mod check", func() {
+	It("produces with custom last-mod check", func() {
 		setup(&feedx.ProducerOptions{
 			Interval:     50 * time.Millisecond,
 			LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 987654321), nil },

data/reader_test.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
+	"time"
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
 	"github.com/bsm/feedx/internal/testdata"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Reader", func() {
@@ -19,7 +20,7 @@ var _ = Describe("Reader", func() {
 
 	BeforeEach(func() {
 		obj = bfs.NewInMemObject("path/to/file.json")
-		Expect(writeMulti(obj, 3)).To(Succeed())
+		Expect(writeMulti(obj, 3, time.Time{})).To(Succeed())
 
 		var err error
 		subject, err = feedx.NewReader(ctx, obj, nil)
@@ -30,14 +31,14 @@ var _ = Describe("Reader", func() {
 		Expect(subject.Close()).To(Succeed())
 	})
 
-	It("should read", func() {
+	It("reads", func() {
 		data, err := ioutil.ReadAll(subject)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(data)).To(BeNumerically("~", 110, 20))
 		Expect(subject.NumRead()).To(Equal(0))
 	})
 
-	It("should decode", func() {
+	It("decodes", func() {
 		var msgs []*testdata.MockMessage
 		for {
 			var msg testdata.MockMessage

data/spec/feedx/cache/memory_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Cache::Memory do
-  it 'should read/write' do
+  it 'read/writes' do
     expect(subject.fetch('key')).to be_nil
     expect(subject.fetch('key') { 'value' }).to eq('value')
     expect(subject.fetch('key')).to eq('value')
@@ -16,7 +16,7 @@ RSpec.describe Feedx::Cache::Memory do
     expect(subject.fetch('key')).to be_nil
   end
 
-  it 'should write strings' do
+  it 'writes strings' do
     subject.write('key', 5)
     expect(subject.read('key')).to eq('5')
   end

data/spec/feedx/cache/value_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe Feedx::Cache::Value do
     described_class.new(Feedx::Cache::Memory.new, 'key')
   end
 
-  it 'should read/write' do
+  it 'read/writes' do
     expect(subject.fetch).to be_nil
     expect(subject.fetch { 'value' }).to eq('value')
     expect(subject.fetch).to eq('value')

data/spec/feedx/compression/gzip_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression::Gzip do
-  it 'should wrap readers/writers' do
+  it 'wraps readers/writers' do
     wio = StringIO.new
     subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to be_within(20).of(40)

data/spec/feedx/compression/none_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression::None do
-  it 'should wrap readers/writers' do
+  it 'wraps readers/writers' do
     wio = StringIO.new
     subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to eq(3000)

data/spec/feedx/compression_spec.rb

@@ -1,14 +1,14 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression do
-  it 'should resolve' do
+  it 'resolves' do
     expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
     expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
     expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
   end
 
-  it 'should detect' do
+  it 'detects' do
     expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
     expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
     expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)

data/spec/feedx/consumer_spec.rb

@@ -4,15 +4,16 @@ RSpec.describe Feedx::Consumer do
   let(:bucket) { BFS::Bucket::InMem.new }
   let(:klass)  { Feedx::TestCase::Model }
   let(:cache)  { Feedx::Cache::Memory.new.value('my-consumer') }
+
   before { allow(BFS).to receive(:resolve).and_return(bucket) }
 
-  it 'should reject invalid inputs' do
+  it 'rejects invalid inputs' do
     expect do
-      described_class.each('mock:///dir/file.txt', klass) {}
+      described_class.each('mock:///dir/file.txt', klass)
     end.to raise_error(/unable to detect format/)
   end
 
-  it 'should consume feeds' do
+  it 'consumes feeds' do
     url = mock_produce!
     csm = described_class.new(url, klass)
     expect(csm).to be_a(Enumerable)
@@ -24,7 +25,7 @@ RSpec.describe Feedx::Consumer do
     expect(cnt).to eq(300)
   end
 
-  it 'should perform conditionally' do
+  it 'performs conditionally' do
     url = mock_produce! last_modified: Time.at(1515151515)
     expect(described_class.new(url, klass, cache: cache).count).to eq(300)
     expect(described_class.new(url, klass, cache: cache).count).to eq(0)

data/spec/feedx/format/abstract_spec.rb

@@ -2,10 +2,11 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Format::Abstract do
   subject   { Feedx::Format::JSON.new }
+
   let(:wio) { StringIO.new }
   let(:rio) { StringIO.open(wio.string) }
 
-  it 'should decode each' do
+  it 'decodes each' do
     subject.encoder wio do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))

data/spec/feedx/format/json_spec.rb

@@ -4,17 +4,17 @@ RSpec.describe Feedx::Format::JSON do
   let(:wio) { StringIO.new }
   let(:rio) { StringIO.open(wio.string) }
 
-  it 'should encode/decode' do
+  it 'encode/decodes' do
     subject.encoder wio do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))
       enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
     end
-    expect(wio.string.lines).to eq [
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-      %({"title":"Z"}\n),
-    ]
+    expect(wio.string).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Z"}
+    JSON
 
     subject.decoder rio do |dec|
       expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))

data/spec/feedx/format/parquet_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe Feedx::Format::Parquet do
     ])
   end
 
-  it 'should encode/decode' do
+  it 'encode/decodes' do
     subject.encoder wio, schema: schema, batch_size: 2 do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))

data/spec/feedx/format/protobuf_spec.rb

@@ -4,7 +4,7 @@ RSpec.describe Feedx::Format::Protobuf do
   let(:wio) { StringIO.new }
   let(:rio) { StringIO.open(wio.string) }
 
-  it 'should encode/decode' do
+  it 'encode/decodes' do
     subject.encoder wio do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))

data/spec/feedx/format_spec.rb

@@ -1,13 +1,13 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format do
-  it 'should resolve' do
+  it 'resolves' do
     expect(described_class.resolve(:json)).to be_instance_of(described_class::JSON)
     expect(described_class.resolve(:pb)).to be_instance_of(described_class::Protobuf)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid format txt/)
   end
 
-  it 'should detect' do
+  it 'detects' do
     expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::JSON)
     expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::JSON)
     expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::JSON)

data/spec/feedx/producer_spec.rb

@@ -6,39 +6,40 @@ RSpec.describe Feedx::Producer do
   end
 
   let(:bucket) { BFS::Bucket::InMem.new }
+
   before { allow(BFS).to receive(:resolve).and_return(bucket) }
 
-  it 'should reject invalid inputs' do
+  it 'rejects invalid inputs' do
     expect do
       described_class.perform 'mock:///dir/file.txt', enum: enumerable
     end.to raise_error(/unable to detect format/)
   end
 
-  it 'should push compressed JSON' do
+  it 'pushes compressed JSON' do
     size = described_class.perform 'mock:///dir/file.jsonz', enum: enumerable
     expect(size).to be_within(20).of(166)
     expect(bucket.info('dir/file.jsonz').size).to eq(size)
   end
 
-  it 'should push plain JSON' do
+  it 'pushes plain JSON' do
     size = described_class.perform 'mock:///dir/file.json', enum: enumerable
     expect(size).to eq(15900)
     expect(bucket.info('dir/file.json').size).to eq(size)
   end
 
-  it 'should push compressed PB' do
+  it 'pushes compressed PB' do
     size = described_class.perform 'mock:///dir/file.pbz', enum: enumerable
     expect(size).to be_within(20).of(41)
     expect(bucket.info('dir/file.pbz').size).to eq(size)
   end
 
-  it 'should push plain PB' do
+  it 'pushes plain PB' do
     size = described_class.perform 'mock:///dir/file.pb', enum: enumerable
     expect(size).to eq(1200)
     expect(bucket.info('dir/file.pb').size).to eq(size)
   end
 
-  it 'should support factories' do
+  it 'supports factories' do
     size = described_class.perform('mock:///dir/file.json') do
       enumerable
     end
@@ -46,12 +47,12 @@ RSpec.describe Feedx::Producer do
     expect(bucket.info('dir/file.json').size).to eq(size)
   end
 
-  it 'should support last-modified' do
+  it 'supports last-modified' do
     described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
     expect(bucket.info('dir/file.json').metadata).to eq('X-Feedx-Last-Modified' => '1515151515000')
   end
 
-  it 'should perform conditionally' do
+  it 'performs conditionally' do
     size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
     expect(size).to eq(15900)
 
@@ -65,7 +66,7 @@ RSpec.describe Feedx::Producer do
     expect(size).to eq(15900)
   end
 
-  it 'should accept downstream options' do
+  it 'accepts downstream options' do
     expect do
       described_class.perform 'mock:///dir/file.jsonz', enum: enumerable, x: 1, y: 'v', z: true
     end.not_to raise_error

data/spec/feedx/stream_spec.rb

@@ -1,22 +1,22 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Stream do
+  subject { described_class.new('mock:///dir/file.json') }
+
   let(:bucket) { BFS::Bucket::InMem.new }
   let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
 
-  subject { described_class.new('mock:///dir/file.json') }
-
   before  { allow(BFS).to receive(:resolve).and_return(bucket) }
-  after { subject.close }
-  after { compressed.close }
 
-  it 'should reject invalid inputs' do
+  after { subject.close; compressed.close }
+
+  it 'rejects invalid inputs' do
     expect do
       described_class.new('mock:///dir/file.txt')
     end.to raise_error(/unable to detect format/)
   end
 
-  it 'should accept custom formats' do
+  it 'accepts custom formats' do
     format = Class.new do
       def encoder(io, &block)
         Feedx::Format::JSON::Encoder.open(io, &block)
@@ -33,24 +33,24 @@ RSpec.describe Feedx::Stream do
     end
     expect(result).to eq(21)
 
-    expect(bucket.read('dir/file.txt')).to eq(
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-    )
+    expect(bucket.read('dir/file.txt')).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+    JSON
   end
 
-  it 'should encode' do
+  it 'encodes' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))
     end
 
-    expect(bucket.read('dir/file.json')).to eq(
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n) +
-      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-    )
+    expect(bucket.read('dir/file.json')).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
+    JSON
   end
 
-  it 'should encode compressed' do
+  it 'encodes compressed' do
     compressed.create do |s|
       100.times do
         s.encode(Feedx::TestCase::Model.new('X'))
@@ -60,14 +60,26 @@ RSpec.describe Feedx::Stream do
     expect(bucket.info('dir/file.json.gz').size).to be_within(10).of(108)
   end
 
-  it 'should encode with create options' do
+  it 'encodes with create options' do
     subject.create metadata: { 'x' => '5' } do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
     end
     expect(bucket.info('dir/file.json').metadata).to eq('X' => '5')
   end
 
-  it 'should decode' do
+  it 'aborts encode on errors (if compressed)' do
+    stop = RuntimeError.new('STOP')
+    expect do
+      compressed.create do |s|
+        s.encode(Feedx::TestCase::Model.new('X'))
+        raise stop
+      end
+    end.to raise_error(stop)
+
+    expect(bucket.ls('**').to_a).to be_empty
+  end
+
+  it 'decodes' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))
@@ -81,7 +93,7 @@ RSpec.describe Feedx::Stream do
     end
   end
 
-  it 'should decode compressed' do
+  it 'decodes compressed' do
     compressed.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))

data/writer.go

@@ -20,7 +20,7 @@ type WriterOptions struct {
 	Compression Compression
 
 	// Provides an optional last modified timestamp which is stored with the remote metadata.
-	// Default: time.Now().
+	// Default: time.Time{}.
 	LastMod time.Time
 }
 
@@ -31,9 +31,6 @@ func (o *WriterOptions) norm(name string) {
 	if o.Compression == nil {
 		o.Compression = DetectCompression(name)
 	}
-	if o.LastMod.IsZero() {
-		o.LastMod = time.Now()
-	}
 }
 
 // Writer encodes feeds to remote locations.

data/writer_test.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Writer", func() {
@@ -20,7 +20,7 @@ var _ = Describe("Writer", func() {
 		compressed = bfs.NewInMemObject("path/to/file.jsonz")
 	})
 
-	It("should write plain", func() {
+	It("writes plain", func() {
 		w := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
 			LastMod: time.Unix(1515151515, 123456789),
 		})
@@ -35,7 +35,7 @@ var _ = Describe("Writer", func() {
 		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
 	})
 
-	It("should write compressed", func() {
+	It("writes compressed", func() {
 		w := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
 			LastMod: time.Unix(1515151515, 123456789),
 		})
@@ -50,14 +50,14 @@ var _ = Describe("Writer", func() {
 		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
 	})
 
-	It("should encode", func() {
-		Expect(writeMulti(plain, 10)).To(Succeed())
-		Expect(writeMulti(compressed, 10)).To(Succeed())
+	It("encodes", func() {
+		Expect(writeMulti(plain, 10, time.Time{})).To(Succeed())
+		Expect(writeMulti(compressed, 10, mockTime)).To(Succeed())
 
 		info, err := plain.Head(ctx)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(info.Size).To(BeNumerically("~", 370, 10))
-		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
+		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "0"}))
 
 		info, err = compressed.Head(ctx)
 		Expect(err).NotTo(HaveOccurred())