feedx 0.12.0 → 0.12.6

data/producer_test.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Producer", func() {
@@ -43,7 +43,7 @@ var _ = Describe("Producer", func() {
 		}
 	})
 
-	It("should produce", func() {
+	It("produces", func() {
 		setup(nil)
 
 		Expect(subject.LastPush()).To(BeTemporally("~", time.Now(), time.Second))
@@ -56,7 +56,7 @@ var _ = Describe("Producer", func() {
 		Expect(info.Size).To(BeNumerically("~", 75, 10))
 	})
 
-	It("should produce with custom last-mod check", func() {
+	It("produces with custom last-mod check", func() {
 		setup(&feedx.ProducerOptions{
 			Interval:     50 * time.Millisecond,
 			LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 987654321), nil },

data/reader_test.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
+	"time"
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
 	"github.com/bsm/feedx/internal/testdata"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Reader", func() {
@@ -19,7 +20,7 @@ var _ = Describe("Reader", func() {
 
 	BeforeEach(func() {
 		obj = bfs.NewInMemObject("path/to/file.json")
-		Expect(writeMulti(obj, 3)).To(Succeed())
+		Expect(writeMulti(obj, 3, time.Time{})).To(Succeed())
 
 		var err error
 		subject, err = feedx.NewReader(ctx, obj, nil)
@@ -30,14 +31,14 @@ var _ = Describe("Reader", func() {
 		Expect(subject.Close()).To(Succeed())
 	})
 
-	It("should read", func() {
+	It("reads", func() {
 		data, err := ioutil.ReadAll(subject)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(data)).To(BeNumerically("~", 110, 20))
 		Expect(subject.NumRead()).To(Equal(0))
 	})
 
-	It("should decode", func() {
+	It("decodes", func() {
 		var msgs []*testdata.MockMessage
 		for {
 			var msg testdata.MockMessage

data/spec/feedx/cache/memory_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Cache::Memory do
-  it 'should read/write' do
+  it 'read/writes' do
     expect(subject.fetch('key')).to be_nil
     expect(subject.fetch('key') { 'value' }).to eq('value')
     expect(subject.fetch('key')).to eq('value')
@@ -16,7 +16,7 @@ RSpec.describe Feedx::Cache::Memory do
     expect(subject.fetch('key')).to be_nil
   end
 
-  it 'should write strings' do
+  it 'writes strings' do
     subject.write('key', 5)
     expect(subject.read('key')).to eq('5')
   end

data/spec/feedx/cache/value_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe Feedx::Cache::Value do
     described_class.new(Feedx::Cache::Memory.new, 'key')
   end
 
-  it 'should read/write' do
+  it 'read/writes' do
     expect(subject.fetch).to be_nil
     expect(subject.fetch { 'value' }).to eq('value')
     expect(subject.fetch).to eq('value')

data/spec/feedx/compression/gzip_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression::Gzip do
-  it 'should wrap readers/writers' do
+  it 'wraps readers/writers' do
     wio = StringIO.new
     subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to be_within(20).of(40)

data/spec/feedx/compression/none_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression::None do
-  it 'should wrap readers/writers' do
+  it 'wraps readers/writers' do
     wio = StringIO.new
     subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to eq(3000)

data/spec/feedx/compression_spec.rb

@@ -1,14 +1,14 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression do
-  it 'should resolve' do
+  it 'resolves' do
     expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
     expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
     expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
   end
 
-  it 'should detect' do
+  it 'detects' do
     expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
     expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
     expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)

data/spec/feedx/consumer_spec.rb

@@ -4,15 +4,16 @@ RSpec.describe Feedx::Consumer do
   let(:bucket) { BFS::Bucket::InMem.new }
   let(:klass)  { Feedx::TestCase::Model }
   let(:cache)  { Feedx::Cache::Memory.new.value('my-consumer') }
+
   before { allow(BFS).to receive(:resolve).and_return(bucket) }
 
-  it 'should reject invalid inputs' do
+  it 'rejects invalid inputs' do
     expect do
-      described_class.each('mock:///dir/file.txt', klass) {}
+      described_class.each('mock:///dir/file.txt', klass)
     end.to raise_error(/unable to detect format/)
   end
 
-  it 'should consume feeds' do
+  it 'consumes feeds' do
     url = mock_produce!
     csm = described_class.new(url, klass)
     expect(csm).to be_a(Enumerable)
@@ -24,7 +25,7 @@ RSpec.describe Feedx::Consumer do
     expect(cnt).to eq(300)
   end
 
-  it 'should perform conditionally' do
+  it 'performs conditionally' do
     url = mock_produce! last_modified: Time.at(1515151515)
     expect(described_class.new(url, klass, cache: cache).count).to eq(300)
     expect(described_class.new(url, klass, cache: cache).count).to eq(0)

data/spec/feedx/format/abstract_spec.rb

@@ -2,10 +2,11 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Format::Abstract do
   subject   { Feedx::Format::JSON.new }
+
   let(:wio) { StringIO.new }
   let(:rio) { StringIO.open(wio.string) }
 
-  it 'should decode each' do
+  it 'decodes each' do
     subject.encoder wio do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))

data/spec/feedx/format/json_spec.rb

@@ -4,17 +4,17 @@ RSpec.describe Feedx::Format::JSON do
   let(:wio) { StringIO.new }
   let(:rio) { StringIO.open(wio.string) }
 
-  it 'should encode/decode' do
+  it 'encode/decodes' do
     subject.encoder wio do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))
       enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
     end
-    expect(wio.string.lines).to eq [
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-      %({"title":"Z"}\n),
-    ]
+    expect(wio.string).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Z"}
+    JSON
 
     subject.decoder rio do |dec|
       expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))

data/spec/feedx/format/parquet_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe Feedx::Format::Parquet do
     ])
   end
 
-  it 'should encode/decode' do
+  it 'encode/decodes' do
     subject.encoder wio, schema: schema, batch_size: 2 do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))

data/spec/feedx/format/protobuf_spec.rb

@@ -4,7 +4,7 @@ RSpec.describe Feedx::Format::Protobuf do
   let(:wio) { StringIO.new }
   let(:rio) { StringIO.open(wio.string) }
 
-  it 'should encode/decode' do
+  it 'encode/decodes' do
     subject.encoder wio do |enc|
       enc.encode(Feedx::TestCase::Model.new('X'))
       enc.encode(Feedx::TestCase::Model.new('Y'))

data/spec/feedx/format_spec.rb

@@ -1,13 +1,13 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format do
-  it 'should resolve' do
+  it 'resolves' do
     expect(described_class.resolve(:json)).to be_instance_of(described_class::JSON)
     expect(described_class.resolve(:pb)).to be_instance_of(described_class::Protobuf)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid format txt/)
   end
 
-  it 'should detect' do
+  it 'detects' do
     expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::JSON)
     expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::JSON)
     expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::JSON)

data/spec/feedx/producer_spec.rb

@@ -6,39 +6,40 @@ RSpec.describe Feedx::Producer do
   end
 
   let(:bucket) { BFS::Bucket::InMem.new }
+
   before { allow(BFS).to receive(:resolve).and_return(bucket) }
 
-  it 'should reject invalid inputs' do
+  it 'rejects invalid inputs' do
     expect do
       described_class.perform 'mock:///dir/file.txt', enum: enumerable
     end.to raise_error(/unable to detect format/)
   end
 
-  it 'should push compressed JSON' do
+  it 'pushes compressed JSON' do
     size = described_class.perform 'mock:///dir/file.jsonz', enum: enumerable
     expect(size).to be_within(20).of(166)
     expect(bucket.info('dir/file.jsonz').size).to eq(size)
   end
 
-  it 'should push plain JSON' do
+  it 'pushes plain JSON' do
     size = described_class.perform 'mock:///dir/file.json', enum: enumerable
     expect(size).to eq(15900)
     expect(bucket.info('dir/file.json').size).to eq(size)
   end
 
-  it 'should push compressed PB' do
+  it 'pushes compressed PB' do
     size = described_class.perform 'mock:///dir/file.pbz', enum: enumerable
     expect(size).to be_within(20).of(41)
     expect(bucket.info('dir/file.pbz').size).to eq(size)
   end
 
-  it 'should push plain PB' do
+  it 'pushes plain PB' do
     size = described_class.perform 'mock:///dir/file.pb', enum: enumerable
     expect(size).to eq(1200)
     expect(bucket.info('dir/file.pb').size).to eq(size)
   end
 
-  it 'should support factories' do
+  it 'supports factories' do
     size = described_class.perform('mock:///dir/file.json') do
       enumerable
     end
@@ -46,12 +47,12 @@ RSpec.describe Feedx::Producer do
     expect(bucket.info('dir/file.json').size).to eq(size)
   end
 
-  it 'should support last-modified' do
+  it 'supports last-modified' do
     described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
     expect(bucket.info('dir/file.json').metadata).to eq('X-Feedx-Last-Modified' => '1515151515000')
   end
 
-  it 'should perform conditionally' do
+  it 'performs conditionally' do
     size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
     expect(size).to eq(15900)
 
@@ -65,7 +66,7 @@ RSpec.describe Feedx::Producer do
     expect(size).to eq(15900)
   end
 
-  it 'should accept downstream options' do
+  it 'accepts downstream options' do
     expect do
       described_class.perform 'mock:///dir/file.jsonz', enum: enumerable, x: 1, y: 'v', z: true
     end.not_to raise_error

data/spec/feedx/stream_spec.rb

@@ -1,19 +1,22 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Stream do
+  subject { described_class.new('mock:///dir/file.json') }
+
   let(:bucket) { BFS::Bucket::InMem.new }
+  let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
+
   before  { allow(BFS).to receive(:resolve).and_return(bucket) }
 
-  subject { described_class.new('mock:///dir/file.json') }
-  let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
+  after { subject.close; compressed.close }
 
-  it 'should reject invalid inputs' do
+  it 'rejects invalid inputs' do
     expect do
       described_class.new('mock:///dir/file.txt')
     end.to raise_error(/unable to detect format/)
   end
 
-  it 'should accept custom formats' do
+  it 'accepts custom formats' do
     format = Class.new do
       def encoder(io, &block)
         Feedx::Format::JSON::Encoder.open(io, &block)
@@ -24,27 +27,30 @@ RSpec.describe Feedx::Stream do
       end
     end
 
-    stream = described_class.new('mock:///dir/file.txt', format: format.new)
-    stream.create {|s| s.encode Feedx::TestCase::Model.new('X') }
+    result = described_class.open('mock:///dir/file.txt', format: format.new) do |stream|
+      stream.create {|s| s.encode Feedx::TestCase::Model.new('X') }
+      21
+    end
+    expect(result).to eq(21)
 
-    expect(bucket.read('dir/file.txt')).to eq(
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-    )
+    expect(bucket.read('dir/file.txt')).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+    JSON
   end
 
-  it 'should encode' do
+  it 'encodes' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))
     end
 
-    expect(bucket.read('dir/file.json')).to eq(
-      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n) +
-      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
-    )
+    expect(bucket.read('dir/file.json')).to eq(<<~JSON)
+      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
+      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
+    JSON
   end
 
-  it 'should encode compressed' do
+  it 'encodes compressed' do
     compressed.create do |s|
       100.times do
         s.encode(Feedx::TestCase::Model.new('X'))
@@ -54,14 +60,26 @@ RSpec.describe Feedx::Stream do
     expect(bucket.info('dir/file.json.gz').size).to be_within(10).of(108)
   end
 
-  it 'should encode with create options' do
+  it 'encodes with create options' do
     subject.create metadata: { 'x' => '5' } do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
     end
     expect(bucket.info('dir/file.json').metadata).to eq('X' => '5')
   end
 
-  it 'should decode' do
+  it 'aborts encode on errors (if compressed)' do
+    stop = RuntimeError.new('STOP')
+    expect do
+      compressed.create do |s|
+        s.encode(Feedx::TestCase::Model.new('X'))
+        raise stop
+      end
+    end.to raise_error(stop)
+
+    expect(bucket.ls('**').to_a).to be_empty
+  end
+
+  it 'decodes' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))
@@ -75,7 +93,7 @@ RSpec.describe Feedx::Stream do
     end
   end
 
-  it 'should decode compressed' do
+  it 'decodes compressed' do
     compressed.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))

data/writer.go

@@ -20,7 +20,7 @@ type WriterOptions struct {
 	Compression Compression
 
 	// Provides an optional last modified timestamp which is stored with the remote metadata.
-	// Default: time.Now().
+	// Default: time.Time{}.
 	LastMod time.Time
 }
 
@@ -31,9 +31,6 @@ func (o *WriterOptions) norm(name string) {
 	if o.Compression == nil {
 		o.Compression = DetectCompression(name)
 	}
-	if o.LastMod.IsZero() {
-		o.LastMod = time.Now()
-	}
 }
 
 // Writer encodes feeds to remote locations.

data/writer_test.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Writer", func() {
@@ -20,7 +20,7 @@ var _ = Describe("Writer", func() {
 		compressed = bfs.NewInMemObject("path/to/file.jsonz")
 	})
 
-	It("should write plain", func() {
+	It("writes plain", func() {
 		w := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
 			LastMod: time.Unix(1515151515, 123456789),
 		})
@@ -35,7 +35,7 @@ var _ = Describe("Writer", func() {
 		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
 	})
 
-	It("should write compressed", func() {
+	It("writes compressed", func() {
 		w := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
 			LastMod: time.Unix(1515151515, 123456789),
 		})
@@ -50,14 +50,14 @@ var _ = Describe("Writer", func() {
 		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
 	})
 
-	It("should encode", func() {
-		Expect(writeMulti(plain, 10)).To(Succeed())
-		Expect(writeMulti(compressed, 10)).To(Succeed())
+	It("encodes", func() {
+		Expect(writeMulti(plain, 10, time.Time{})).To(Succeed())
+		Expect(writeMulti(compressed, 10, mockTime)).To(Succeed())
 
 		info, err := plain.Head(ctx)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(info.Size).To(BeNumerically("~", 370, 10))
-		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
+		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "0"}))
 
 		info, err = compressed.Head(ctx)
 		Expect(err).NotTo(HaveOccurred())