feedx 0.10.1 → 0.12.2

data/lib/feedx/format/parquet.rb

@@ -0,0 +1,102 @@
+require 'parquet'
+require 'tmpdir'
+
+class Feedx::Format::Parquet < Feedx::Format::Abstract
+  class Record < Arrow::Record
+    def each_pair
+      container.columns.each do |col|
+        yield col.name, col[index]
+      end
+    end
+  end
+
+  class Decoder < Feedx::Format::Abstract::Decoder
+    def initialize(io, **)
+      super(io)
+
+      @table  = read_table
+      @cursor = 0
+    end
+
+    def eof?
+      @cursor >= @table.n_rows
+    end
+
+    def decode(target, **)
+      return if eof?
+
+      rec = Record.new(@table, @cursor)
+      @cursor += 1
+
+      target = target.allocate if target.is_a?(Class)
+      target.from_parquet(rec)
+      target
+    end
+
+    private
+
+    def read_table
+      tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
+      IO.copy_stream(@io, tmpname)
+
+      @table = Arrow::Table.load(tmpname, format: 'parquet')
+    ensure
+      unlink!(tmpname) if tmpname
+    end
+
+    def unlink!(tmpname)
+      File.unlink(tmpname)
+    rescue Errno::ENOENT
+      nil
+    end
+  end
+
+  class Encoder < Feedx::Format::Abstract::Encoder
+    attr_reader :schema
+
+    def initialize(io, schema:, buffer_size: 1 << 20, batch_size: 10_000)
+      super(io)
+
+      @schema = schema
+      @batch_size = batch_size.to_i
+      @buffer_size = buffer_size.to_i
+
+      @tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
+      @output  = Arrow::FileOutputStream.new(@tmpname, append: false)
+      @writer  = Parquet::ArrowFileWriter.new(@schema, @output)
+      @batch   = []
+    end
+
+    def encode(msg, **opts)
+      msg = msg.to_parquet(@schema, **opts) if msg.respond_to?(:to_parquet)
+
+      res = @batch.push(msg)
+      flush_table if @batch.size >= @batch_size
+      res
+    end
+
+    def close
+      flush_table unless @batch.empty?
+
+      @writer.close
+      @output.close
+      IO.copy_stream(@tmpname, @io)
+    ensure
+      unlink!
+    end
+
+    private
+
+    def flush_table
+      table = Arrow::RecordBatch.new(@schema, @batch).to_table
+      @writer.write_table table, @buffer_size
+      @batch.clear
+    end
+
+    def unlink!
+      File.unlink(@tmpname)
+    rescue Errno::ENOENT
+      nil
+    end
+  end
+end

data/lib/feedx/format/protobuf.rb

@@ -1,16 +1,24 @@
 require 'pbio'
 
 class Feedx::Format::Protobuf < Feedx::Format::Abstract
-  def initialize(io)
-    super PBIO::Delimited.new(io)
-  end
+  class Decoder < Feedx::Format::Abstract::Decoder
+    def initialize(io, **opts)
+      super PBIO::Delimited.new(io), **opts
+    end
 
-  def decode(klass, **)
-    @io.read(klass)
+    def decode(target, **)
+      @io.read(target)
+    end
   end
 
-  def encode(msg, **opts)
-    msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
-    @io.write msg
+  class Encoder < Feedx::Format::Abstract::Encoder
+    def initialize(io, **opts)
+      super PBIO::Delimited.new(io), **opts
+    end
+
+    def encode(msg, **opts)
+      msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
+      @io.write msg
+    end
   end
 end

data/lib/feedx/producer.rb

@@ -6,46 +6,52 @@ module Feedx
   # Produces a relation as an encoded feed to a remote location.
   class Producer
     # See constructor.
-    def self.perform(url, opts = {}, &block)
-      new(url, opts, &block).perform
+    def self.perform(url, **opts, &block)
+      new(url, **opts, &block).perform
     end
 
     # @param [String] url the destination URL.
     # @param [Hash] opts options
     # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
-    # @option opts [Hash] :format_options format encode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
     # @yield A block factory to generate the relation or enumerator.
     # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
-    def initialize(url, opts = {}, &block)
-      @enum = opts[:enum] || block
+    def initialize(url, last_modified: nil, format_options: {}, enum: nil, **opts, &block)
+      @enum = enum || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
-      @stream   = Feedx::Stream.new(url, opts)
-      @last_mod = opts[:last_modified]
-      @fmt_opts = opts[:format_options] || {}
+      @url  = url
+      @opts = opts.merge(format_options)
+      @last_mod = last_modified
+
+      return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
+
+      warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
     end
 
     def perform
-      enum      = @enum.is_a?(Proc) ? @enum.call : @enum
-      last_mod  = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
+      stream = Feedx::Stream.new(@url, **@opts)
+      enum = @enum.is_a?(Proc) ? @enum.call : @enum
+      last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
       local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
 
       begin
-        metadata   = @stream.blob.info.metadata
+        metadata   = stream.blob.info.metadata
         remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
         return -1 unless local_rev > remote_rev
       rescue BFS::FileNotFound
         nil
       end if local_rev.positive?
 
-      @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
+      stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
         iter = enum.respond_to?(:find_each) ? :find_each : :each
-        enum.send(iter) {|rec| fmt.encode(rec, **@fmt_opts) }
+        enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
       end
-      @stream.blob.info.size
+      stream.blob.info.size
+    ensure
+      stream&.close
     end
   end
 end

data/lib/feedx/stream.rb

@@ -6,25 +6,42 @@ module Feedx
   class Stream
     attr_reader :blob
 
+    # Behaves like new, but accepts an optional block.
+    # If a block is given, streams are automatically closed after the block is yielded.
+    def self.open(url, **opts)
+      stream = new(url, **opts)
+      return stream unless block_given?
+
+      begin
+        yield stream
+      ensure
+        stream.close
+      end
+    end
+
     # @param [String] url the blob URL.
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
-    def initialize(url, opts = {})
+    def initialize(url, format: nil, compress: nil, **opts)
       @blob     = BFS::Blob.new(url)
-      @format   = detect_format(opts[:format])
-      @compress = detect_compress(opts[:compress])
+      @format   = detect_format(format)
+      @compress = detect_compress(compress)
+      @opts     = opts
+
+      BFS.defer(self, :close)
     end
 
     # Opens the remote for reading.
     # @param [Hash] opts BFS::Blob#open options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted input stream.
-    def open(opts = {})
-      @blob.open(opts) do |io|
-        @compress.reader(io) do |cio|
-          fmt = @format.new(cio)
-          yield fmt
+    def open(**opts)
+      @blob.open(**opts) do |io|
+        @compress.reader(io, **@opts) do |cio|
+          @format.decoder(cio, **@opts) do |fmt|
+            yield fmt
+          end
         end
       end
     end
@@ -33,28 +50,31 @@ module Feedx
     # @param [Hash] opts BFS::Blob#create options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted output stream.
-    def create(opts = {})
-      @blob.create(opts) do |io|
-        @compress.writer(io) do |cio|
-          fmt = @format.new(cio)
-          yield fmt
+    def create(**opts)
+      @blob.create(**opts) do |io|
+        @compress.writer(io, **@opts) do |cio|
+          @format.encoder(cio, **@opts) do |fmt|
+            yield fmt
+          end
         end
       end
     end
 
+    # Closes the underlying connection.
+    def close
+      @blob.close
+    end
+
     private
 
     def detect_format(val)
       case val
       when nil
         Feedx::Format.detect(@blob.path)
-      when Class
-        parent = Feedx::Format::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
+      when String, Symbol
         Feedx::Format.resolve(val)
+      else
+        Feedx::Format.validate!(val)
       end
     end
 
@@ -62,13 +82,10 @@ module Feedx
       case val
       when nil
         Feedx::Compression.detect(@blob.path)
-      when Class
-        parent = Feedx::Compression::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
+      when String, Symbol
         Feedx::Compression.resolve(val)
+      else
+        Feedx::Compression.validate!(val)
       end
     end
   end

data/producer_test.go

@@ -23,8 +23,7 @@ var _ = Describe("Producer", func() {
 			atomic.AddUint32(&numRuns, 1)
 
 			for i := 0; i < 10; i++ {
-				fix := fixture
-				if err := w.Encode(&fix); err != nil {
+				if err := w.Encode(seed()); err != nil {
 					return err
 				}
 			}

data/reader_test.go

@@ -5,10 +5,9 @@ import (
 	"io"
 	"io/ioutil"
 
-	"github.com/bsm/feedx"
-
 	"github.com/bsm/bfs"
-	tbp "github.com/golang/protobuf/proto/proto3_proto"
+	"github.com/bsm/feedx"
+	"github.com/bsm/feedx/internal/testdata"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -34,23 +33,23 @@ var _ = Describe("Reader", func() {
 	It("should read", func() {
 		data, err := ioutil.ReadAll(subject)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(len(data)).To(BeNumerically("~", 140, 20))
+		Expect(len(data)).To(BeNumerically("~", 110, 20))
 		Expect(subject.NumRead()).To(Equal(0))
 	})
 
 	It("should decode", func() {
-		var msgs []tbp.Message
+		var msgs []*testdata.MockMessage
 		for {
-			var msg tbp.Message
+			var msg testdata.MockMessage
 			err := subject.Decode(&msg)
 			if err == io.EOF {
 				break
 			}
 			Expect(err).NotTo(HaveOccurred())
-			msgs = append(msgs, msg)
+			msgs = append(msgs, &msg)
 		}
 
-		Expect(msgs).To(Equal([]tbp.Message{fixture, fixture, fixture}))
+		Expect(msgs).To(ConsistOf(seed(), seed(), seed()))
 		Expect(subject.NumRead()).To(Equal(3))
 	})
 })

data/spec/feedx/compression/gzip_spec.rb

@@ -3,13 +3,13 @@ require 'spec_helper'
 RSpec.describe Feedx::Compression::Gzip do
   it 'should wrap readers/writers' do
     wio = StringIO.new
-    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to be_within(20).of(40)
     expect(wio.string.encoding).to eq(Encoding::BINARY)
 
     data = ''
     StringIO.open(wio.string) do |rio|
-      described_class.reader(rio) {|z| data = z.read }
+      subject.reader(rio) {|z| data = z.read }
     end
     expect(data.size).to eq(3000)
     expect(data.encoding).to eq(Encoding.default_external)

data/spec/feedx/compression/none_spec.rb

@@ -3,12 +3,12 @@ require 'spec_helper'
 RSpec.describe Feedx::Compression::None do
   it 'should wrap readers/writers' do
     wio = StringIO.new
-    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
     expect(wio.size).to eq(3000)
 
     data = ''
     StringIO.open(wio.string) do |rio|
-      described_class.reader(rio) {|z| data = z.read }
+      subject.reader(rio) {|z| data = z.read }
     end
     expect(data.size).to eq(3000)
   end

data/spec/feedx/compression_spec.rb

@@ -2,18 +2,18 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Compression do
   it 'should resolve' do
-    expect(described_class.resolve(:gzip)).to eq(described_class::Gzip)
-    expect(described_class.resolve(:gz)).to eq(described_class::Gzip)
-    expect(described_class.resolve(nil)).to eq(described_class::None)
+    expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
+    expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
+    expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
   end
 
   it 'should detect' do
-    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json')).to eq(described_class::None)
-    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb.gz')).to eq(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb')).to eq(described_class::None)
+    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)
+    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb.gz')).to be_instance_of(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::None)
   end
 end

data/spec/feedx/consumer_spec.rb

@@ -36,10 +36,13 @@ RSpec.describe Feedx::Consumer do
 
   private
 
-  def mock_produce!(opts = {})
+  def mock_produce!(enum: mock_enum, **opts)
     url = 'mock:///dir/file.json'
-    opts[:enum] ||= %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
-    Feedx::Producer.perform url, opts
+    Feedx::Producer.perform url, enum: enum, **opts
     url
   end
+
+  def mock_enum
+    %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
+  end
 end

data/spec/feedx/format/abstract_spec.rb

@@ -1,17 +1,20 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Abstract do
-  subject   { Feedx::Format::JSON.new(wio) }
+  subject   { Feedx::Format::JSON.new }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should decode each' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
-    StringIO.open(wio.string) do |rio|
-      fmt = subject.class.new(rio)
-      dec = fmt.decode_each(Feedx::TestCase::Model).to_a
-      expect(dec.map(&:title)).to eq(%w[X Y Z])
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
+
+    subject.decoder rio do |dec|
+      acc = dec.decode_each(Feedx::TestCase::Model).to_a
+      expect(acc.map(&:title)).to eq(%w[X Y Z])
     end
   end
 end