feedx 0.9.0 → 0.11.0

data/lib/feedx/format/protobuf.rb

@@ -1,16 +1,24 @@
 require 'pbio'
 
 class Feedx::Format::Protobuf < Feedx::Format::Abstract
-  def initialize(io)
-    super PBIO::Delimited.new(io)
-  end
+  class Decoder < Feedx::Format::Abstract::Decoder
+    def initialize(io, **opts)
+      super PBIO::Delimited.new(io), **opts
+    end
 
-  def decode(klass, **)
-    @io.read(klass)
+    def decode(target, **)
+      @io.read(target)
+    end
   end
 
-  def encode(msg, **opts)
-    msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
-    @io.write msg
+  class Encoder < Feedx::Format::Abstract::Encoder
+    def initialize(io, **opts)
+      super PBIO::Delimited.new(io), **opts
+    end
+
+    def encode(msg, **opts)
+      msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
+      @io.write msg
+    end
   end
 end

data/lib/feedx/producer.rb

@@ -6,8 +6,8 @@ module Feedx
   # Produces a relation as an encoded feed to a remote location.
   class Producer
     # See constructor.
-    def self.perform(url, opts={}, &block)
-      new(url, opts, &block).perform
+    def self.perform(url, **opts, &block)
+      new(url, **opts, &block).perform
     end
 
     # @param [String] url the destination URL.
@@ -19,11 +19,11 @@ module Feedx
     # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
     # @yield A block factory to generate the relation or enumerator.
     # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
-    def initialize(url, opts={}, &block)
+    def initialize(url, **opts, &block)
       @enum = opts[:enum] || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
-      @stream   = Feedx::Stream.new(url, opts)
+      @stream   = Feedx::Stream.new(url, **opts)
       @last_mod = opts[:last_modified]
       @fmt_opts = opts[:format_options] || {}
     end
@@ -37,7 +37,8 @@ module Feedx
         metadata   = @stream.blob.info.metadata
         remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
         return -1 unless local_rev > remote_rev
-      rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
+      rescue BFS::FileNotFound
+        nil
       end if local_rev.positive?
 
       @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|

data/lib/feedx/stream.rb

@@ -10,7 +10,7 @@ module Feedx
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
-    def initialize(url, opts={})
+    def initialize(url, **opts)
       @blob     = BFS::Blob.new(url)
       @format   = detect_format(opts[:format])
       @compress = detect_compress(opts[:compress])
@@ -20,11 +20,12 @@ module Feedx
     # @param [Hash] opts BFS::Blob#open options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted input stream.
-    def open(opts={})
-      @blob.open(opts) do |io|
+    def open(**opts)
+      @blob.open(**opts) do |io|
         @compress.reader(io) do |cio|
-          fmt = @format.new(cio)
-          yield fmt
+          @format.decoder(cio) do |fmt|
+            yield fmt
+          end
         end
       end
     end
@@ -33,11 +34,12 @@ module Feedx
     # @param [Hash] opts BFS::Blob#create options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted output stream.
-    def create(opts={})
-      @blob.create(opts) do |io|
+    def create(**opts)
+      @blob.create(**opts) do |io|
         @compress.writer(io) do |cio|
-          fmt = @format.new(cio)
-          yield fmt
+          @format.encoder(cio) do |fmt|
+            yield fmt
+          end
         end
       end
     end
@@ -48,13 +50,10 @@ module Feedx
       case val
       when nil
         Feedx::Format.detect(@blob.path)
-      when Class
-        parent = Feedx::Format::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
+      when String, Symbol
         Feedx::Format.resolve(val)
+      else
+        Feedx::Format.validate!(val)
       end
     end
 
@@ -62,13 +61,10 @@ module Feedx
       case val
       when nil
         Feedx::Compression.detect(@blob.path)
-      when Class
-        parent = Feedx::Compression::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
+      when String, Symbol
         Feedx::Compression.resolve(val)
+      else
+        Feedx::Compression.validate!(val)
       end
     end
   end

data/producer.go

@@ -140,21 +140,14 @@ func (p *Producer) push() (*ProducerPush, error) {
 		wopt.LastMod = modTime
 	}
 
-	// retrieve original last modified time
-	lastMod, err := remoteLastModified(p.ctx, p.remote)
-	if err != nil {
+	// retrieve original last modified time, skip if not modified
+	if rts, err := remoteLastModified(p.ctx, p.remote); err != nil {
 		return nil, err
-	}
-
-	// skip push if not modified
-	if lastMod.Time().Equal(wopt.LastMod) {
+	} else if rts == timestampFromTime(wopt.LastMod) {
 		return &ProducerPush{Producer: p}, nil
 	}
 
-	writer, err := NewWriter(p.ctx, p.remote, &wopt)
-	if err != nil {
-		return nil, err
-	}
+	writer := NewWriter(p.ctx, p.remote, &wopt)
 	defer writer.Discard()
 
 	if err := p.pfn(writer); err != nil {

data/producer_test.go

@@ -2,6 +2,7 @@ package feedx_test
 
 import (
 	"context"
+	"sync/atomic"
 	"time"
 
 	"github.com/bsm/bfs"
@@ -13,14 +14,16 @@ import (
 var _ = Describe("Producer", func() {
 	var subject *feedx.Producer
 	var obj *bfs.Object
+	var numRuns uint32
 	var ctx = context.Background()
 
 	setup := func(o *feedx.ProducerOptions) {
 		var err error
 		subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(w *feedx.Writer) error {
+			atomic.AddUint32(&numRuns, 1)
+
 			for i := 0; i < 10; i++ {
-				fix := fixture
-				if err := w.Encode(&fix); err != nil {
+				if err := w.Encode(seed()); err != nil {
 					return err
 				}
 			}
@@ -30,6 +33,7 @@ var _ = Describe("Producer", func() {
 	}
 
 	BeforeEach(func() {
+		atomic.StoreUint32(&numRuns, 0)
 		obj = bfs.NewInMemObject("path/to/file.jsonz")
 	})
 
@@ -54,16 +58,22 @@ var _ = Describe("Producer", func() {
 
 	It("should produce with custom last-mod check", func() {
 		setup(&feedx.ProducerOptions{
-			LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 0), nil },
+			Interval:     50 * time.Millisecond,
+			LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 987654321), nil },
 		})
 
-		Expect(subject.LastPush()).To(BeTemporally("~", time.Now(), time.Second))
-		Expect(subject.LastModified()).To(Equal(time.Unix(1515151515, 0)))
+		firstPush := subject.LastPush()
+		Expect(firstPush).To(BeTemporally("~", time.Now(), time.Second))
+		Expect(subject.LastModified()).To(Equal(time.Unix(1515151515, 987000000)))
 		Expect(subject.NumWritten()).To(Equal(10))
+		Expect(atomic.LoadUint32(&numRuns)).To(Equal(uint32(1)))
 
 		info, err := obj.Head(ctx)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(info.Size).To(BeNumerically("~", 75, 10))
-		Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515000"))
+		Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515987"))
+
+		Eventually(func() bool { return subject.LastPush().After(firstPush) }).Should(BeTrue())
+		Expect(atomic.LoadUint32(&numRuns)).To(Equal(uint32(1)))
 	})
 })

data/reader_test.go

@@ -5,10 +5,9 @@ import (
 	"io"
 	"io/ioutil"
 
-	"github.com/bsm/feedx"
-
 	"github.com/bsm/bfs"
-	tbp "github.com/golang/protobuf/proto/proto3_proto"
+	"github.com/bsm/feedx"
+	"github.com/bsm/feedx/internal/testdata"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -34,23 +33,23 @@ var _ = Describe("Reader", func() {
 	It("should read", func() {
 		data, err := ioutil.ReadAll(subject)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(len(data)).To(BeNumerically("~", 140, 20))
+		Expect(len(data)).To(BeNumerically("~", 110, 20))
 		Expect(subject.NumRead()).To(Equal(0))
 	})
 
 	It("should decode", func() {
-		var msgs []tbp.Message
+		var msgs []*testdata.MockMessage
 		for {
-			var msg tbp.Message
+			var msg testdata.MockMessage
 			err := subject.Decode(&msg)
 			if err == io.EOF {
 				break
 			}
 			Expect(err).NotTo(HaveOccurred())
-			msgs = append(msgs, msg)
+			msgs = append(msgs, &msg)
 		}
 
-		Expect(msgs).To(Equal([]tbp.Message{fixture, fixture, fixture}))
+		Expect(msgs).To(ConsistOf(seed(), seed(), seed()))
 		Expect(subject.NumRead()).To(Equal(3))
 	})
 })

data/spec/feedx/compression/gzip_spec.rb

@@ -3,13 +3,15 @@ require 'spec_helper'
 RSpec.describe Feedx::Compression::Gzip do
   it 'should wrap readers/writers' do
     wio = StringIO.new
-    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to be_within(20).of(40)
+    expect(wio.string.encoding).to eq(Encoding::BINARY)
 
     data = ''
     StringIO.open(wio.string) do |rio|
-      described_class.reader(rio) {|z| data = z.read }
+      subject.reader(rio) {|z| data = z.read }
     end
     expect(data.size).to eq(3000)
+    expect(data.encoding).to eq(Encoding.default_external)
   end
 end

data/spec/feedx/compression/none_spec.rb

@@ -3,12 +3,12 @@ require 'spec_helper'
 RSpec.describe Feedx::Compression::None do
   it 'should wrap readers/writers' do
     wio = StringIO.new
-    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to eq(3000)
 
     data = ''
     StringIO.open(wio.string) do |rio|
-      described_class.reader(rio) {|z| data = z.read }
+      subject.reader(rio) {|z| data = z.read }
     end
     expect(data.size).to eq(3000)
   end

data/spec/feedx/compression_spec.rb

@@ -2,18 +2,18 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Compression do
   it 'should resolve' do
-    expect(described_class.resolve(:gzip)).to eq(described_class::Gzip)
-    expect(described_class.resolve(:gz)).to eq(described_class::Gzip)
-    expect(described_class.resolve(nil)).to eq(described_class::None)
+    expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
+    expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
+    expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
   end
 
   it 'should detect' do
-    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json')).to eq(described_class::None)
-    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb.gz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb')).to eq(described_class::None)
+    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)
+    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb.gz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::None)
   end
 end

data/spec/feedx/consumer_spec.rb

@@ -36,10 +36,13 @@ RSpec.describe Feedx::Consumer do
 
   private
 
-  def mock_produce!(opts={})
+  def mock_produce!(enum: mock_enum, **opts)
     url = 'mock:///dir/file.json'
-    opts[:enum] ||= %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
-    Feedx::Producer.perform url, opts
+    Feedx::Producer.perform url, enum: enum, **opts
     url
   end
+
+  def mock_enum
+    %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
+  end
 end

data/spec/feedx/format/abstract_spec.rb

@@ -1,17 +1,20 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Abstract do
-  subject   { Feedx::Format::JSON.new(wio) }
+  subject   { Feedx::Format::JSON.new }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should decode each' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
-    StringIO.open(wio.string) do |rio|
-      fmt = subject.class.new(rio)
-      dec = fmt.decode_each(Feedx::TestCase::Model).to_a
-      expect(dec.map(&:title)).to eq(%w[X Y Z])
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
+
+    subject.decoder rio do |dec|
+      acc = dec.decode_each(Feedx::TestCase::Model).to_a
+      expect(acc.map(&:title)).to eq(%w[X Y Z])
     end
   end
 end

data/spec/feedx/format/json_spec.rb

@@ -1,26 +1,27 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::JSON do
-  subject   { described_class.new(wio) }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should encode/decode' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
     expect(wio.string.lines).to eq [
       %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
       %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
       %({"title":"Z"}\n),
     ]
 
-    StringIO.open(wio.string) do |rio|
-      fmt = described_class.new(rio)
-      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
-      expect(fmt.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
-      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
-      expect(fmt.decode(Feedx::TestCase::Model)).to be_nil
-      expect(fmt).to be_eof
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(dec.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
+      expect(dec).to be_eof
     end
   end
 end

data/spec/feedx/format/parquet_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Format::Parquet do
+  let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
+
+  let(:schema) do
+    Arrow::Schema.new([
+      Arrow::Field.new('title', :string),
+      Arrow::Field.new('updated_at', type: :timestamp, unit: :second),
+    ])
+  end
+
+  it 'should encode/decode' do
+    subject.encoder wio, schema: schema, batch_size: 2 do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Model.new('Z'))
+    end
+    expect(wio.string.bytesize).to be_within(100).of(1100)
+
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
+      expect(dec).to be_eof
+    end
+  end
+end