feedx 0.10.2 → 0.12.3

data/internal/testdata/testdata.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+package feedx.internal.testdata;
+option go_package = "github.com/feedx/internal/testdata";
+
+enum MockEnum {
+  UNKNOWN = 0;
+  FIRST = 3;
+}
+
+message MockMessage {
+  string name = 1;
+  MockEnum enum = 2;
+  uint32 height = 3;
+}

data/lib/feedx/cache/abstract.rb

@@ -5,12 +5,12 @@ class Feedx::Cache::Abstract
   end
 
   # Read reads a key.
-  def read(_key, **_opts)
+  def read(_key, **)
     raise 'Not implemented'
   end
 
   # Write writes a key/value pair.
-  def write(_key, _value, **_opts)
+  def write(_key, _value, **)
     raise 'Not implemented'
   end
 

data/lib/feedx/cache/memory.rb

@@ -3,6 +3,7 @@ require 'monitor'
 # Thread-safe in-memory cache. Use for testing only.
 class Feedx::Cache::Memory < Feedx::Cache::Abstract
   def initialize
+    super
     @monitor = Monitor.new
     @entries = {}
   end

data/lib/feedx/compression.rb

@@ -5,12 +5,19 @@ module Feedx
     autoload :Gzip, 'feedx/compression/gzip'
 
     class << self
+      def validate!(kind)
+        raise ArgumentError, "#{kind} does not implement #reader(io, &block)" unless kind.respond_to?(:reader)
+        raise ArgumentError, "#{kind} does not implement #writer(io, &block)" unless kind.respond_to?(:writer)
+
+        kind
+      end
+
       def resolve(name)
         case name.to_s
         when 'gz', 'gzip'
-          Gzip
+          Gzip.new
         when ''
-          None
+          None.new
         else
           raise ArgumentError, "invalid compression #{name}"
         end
@@ -18,9 +25,9 @@ module Feedx
 
       def detect(path)
         if File.extname(path)[-1] == 'z'
-          Gzip
+          Gzip.new
         else
-          None
+          None.new
         end
       end
     end

data/lib/feedx/compression/abstract.rb

@@ -1,9 +1,9 @@
 class Feedx::Compression::Abstract
-  def self.reader(_io, &_block)
+  def reader(_io, **, &_block)
     raise 'Not implemented'
   end
 
-  def self.writer(_io, &_block)
+  def writer(_io, **, &_block)
     raise 'Not implemented'
   end
 end

data/lib/feedx/compression/gzip.rb

@@ -1,25 +1,23 @@
 require 'zlib'
 
 class Feedx::Compression::Gzip < Feedx::Compression::Abstract
-  class << self
-    def reader(io, &block)
-      force_binmode(io)
-      Zlib::GzipReader.wrap(io, &block)
-    end
+  def reader(io, **, &block)
+    force_binmode(io)
+    Zlib::GzipReader.wrap(io, &block)
+  end
 
-    def writer(io, &block)
-      force_binmode(io)
-      Zlib::GzipWriter.wrap(io, &block)
-    end
+  def writer(io, **, &block)
+    force_binmode(io)
+    Zlib::GzipWriter.wrap(io, &block)
+  end
 
-    private
+  private
 
-    def force_binmode(io)
-      if io.respond_to?(:binmode)
-        io.binmode
-      elsif io.respond_to?(:set_encoding)
-        io.set_encoding(Encoding::BINARY)
-      end
+  def force_binmode(io)
+    if io.respond_to?(:binmode)
+      io.binmode
+    elsif io.respond_to?(:set_encoding)
+      io.set_encoding(Encoding::BINARY)
     end
   end
 end

data/lib/feedx/compression/none.rb

@@ -1,9 +1,9 @@
 class Feedx::Compression::None < Feedx::Compression::Abstract
-  def self.reader(io, &block)
-    block.call(io)
+  def reader(io, **)
+    yield(io)
   end
 
-  def self.writer(io, &block)
-    block.call(io)
+  def writer(io, **)
+    yield(io)
   end
 end

data/lib/feedx/consumer.rb

@@ -16,33 +16,39 @@ module Feedx
     # @param [Class] klass the record class.
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
-    # @option opts [Hash] :format_options format decode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
-    def initialize(url, klass, **opts)
-      @klass    = klass
-      @stream   = Feedx::Stream.new(url, **opts)
-      @fmt_opts = opts[:format_options] || {}
-      @cache    = opts[:cache]
+    def initialize(url, klass, format_options: {}, cache: nil, **opts)
+      @klass = klass
+      @url = url
+      @opts = opts.merge(format_options)
+      @cache = cache
+
+      return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
+
+      warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
     end
 
     # @return [Boolean] returns true if performed.
     def each(&block)
+      stream = Feedx::Stream.new(@url, **@opts)
       remote_rev = nil
 
       if @cache
-        metadata   = @stream.blob.info.metadata
+        metadata   = stream.blob.info.metadata
         local_rev  = @cache.read.to_i
         remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
         return false if remote_rev.positive? && remote_rev <= local_rev
       end
 
-      @stream.open do |fmt|
-        fmt.decode_each(@klass, **@fmt_opts, &block)
+      stream.open do |fmt|
+        fmt.decode_each(@klass, **@opts, &block)
       end
       @cache.write(remote_rev) if @cache && remote_rev
 
       true
+    ensure
+      stream&.close
     end
   end
 end

data/lib/feedx/format.rb

@@ -2,13 +2,19 @@ module Feedx
   module Format
     autoload :Abstract, 'feedx/format/abstract'
     autoload :JSON, 'feedx/format/json'
+    autoload :Parquet, 'feedx/format/parquet'
     autoload :Protobuf, 'feedx/format/protobuf'
 
     class << self
-      def register(ext, kind)
-        raise ArgumentError, "#{kind} is not a subclass of Feedx::Format::Abstract" unless kind.is_a?(Class) && kind < Abstract
+      def validate!(kind)
+        raise ArgumentError, "#{kind} does not implement #encoder(io, &block)" unless kind.respond_to?(:encoder)
+        raise ArgumentError, "#{kind} does not implement #decoder(io, &block)" unless kind.respond_to?(:decoder)
+
+        kind
+      end
 
-        registry[ext.to_s] = kind
+      def register(ext, kind)
+        registry[ext.to_s] = validate!(kind)
       end
 
       def resolve(name)
@@ -33,6 +39,9 @@ module Feedx
       def registry
         @registry ||= {
           'json'     => :JSON,
+          'jsonl'    => :JSON,
+          'ndjson'   => :JSON,
+          'parquet'  => :Parquet,
           'pb'       => :Protobuf,
           'proto'    => :Protobuf,
           'protobuf' => :Protobuf,
@@ -40,13 +49,13 @@ module Feedx
       end
 
       def _resolve(name)
-        name  = name.to_s
-        klass = registry[name]
-        if klass.is_a?(Symbol)
-          klass = const_get(klass)
-          registry[name.to_s] = klass
+        name = name.to_s
+        kind = registry[name]
+        if kind.is_a?(Symbol)
+          kind = const_get(kind).new
+          registry[name.to_s] = kind
         end
-        klass
+        kind
       end
     end
   end

data/lib/feedx/format/abstract.rb

@@ -1,25 +1,54 @@
 class Feedx::Format::Abstract
-  def initialize(io)
-    @io = io
+  def decoder(io, **opts, &block)
+    self.class::Decoder.open(io, **opts, &block)
   end
 
-  def eof?
-    @io.eof?
+  def encoder(io, **opts, &block)
+    self.class::Encoder.open(io, **opts, &block)
   end
 
-  def decode_each(klass, **opts)
-    if block_given?
-      yield decode(klass, **opts) until eof?
-    else
-      Enumerator.new {|y| y << decode(klass, **opts) until eof? }
+  class Wrapper
+    def self.open(io, **opts)
+      inst = new(io, **opts)
+      yield inst
+    ensure
+      inst&.close
+    end
+
+    def initialize(io, **)
+      @io = io
     end
   end
 
-  def decode(_klass, **)
-    raise 'Not implemented'
+  class Decoder < Wrapper
+    def eof?
+      @io.eof?
+    end
+
+    def decode_each(target, **opts)
+      if block_given?
+        yield decode(target, **opts) until eof?
+      else
+        Enumerator.new do |acc|
+          acc << decode(target, **opts) until eof?
+        end
+      end
+    end
+
+    def decode(_target, **)
+      raise 'Not implemented'
+    end
+
+    def close; end
   end
 
-  def encode(_msg, **)
-    raise 'Not implemented'
+  class Encoder < Wrapper
+    def encode(_msg, **)
+      raise 'Not implemented'
+    end
+
+    def close
+      @io.flush if @io.respond_to?(:flush)
+    end
   end
 end

data/lib/feedx/format/json.rb

@@ -1,16 +1,20 @@
 require 'json'
 
 class Feedx::Format::JSON < Feedx::Format::Abstract
-  def decode(obj, **)
-    line = @io.gets
-    return unless line
+  class Decoder < Feedx::Format::Abstract::Decoder
+    def decode(target, **)
+      line = @io.gets
+      return unless line
 
-    obj = obj.allocate if obj.is_a?(Class)
-    obj.from_json(line)
-    obj
+      target = target.allocate if target.is_a?(Class)
+      target.from_json(line)
+      target
+    end
   end
 
-  def encode(msg, **opts)
-    @io.write msg.to_json(**opts) << "\n"
+  class Encoder < Feedx::Format::Abstract::Encoder
+    def encode(msg, **opts)
+      @io.write msg.to_json(**opts) << "\n"
+    end
   end
 end

data/lib/feedx/format/parquet.rb

@@ -0,0 +1,102 @@
+require 'parquet'
+require 'tmpdir'
+
+class Feedx::Format::Parquet < Feedx::Format::Abstract
+  class Record < Arrow::Record
+    def each_pair
+      container.columns.each do |col|
+        yield col.name, col[index]
+      end
+    end
+  end
+
+  class Decoder < Feedx::Format::Abstract::Decoder
+    def initialize(io, **)
+      super(io)
+
+      @table  = read_table
+      @cursor = 0
+    end
+
+    def eof?
+      @cursor >= @table.n_rows
+    end
+
+    def decode(target, **)
+      return if eof?
+
+      rec = Record.new(@table, @cursor)
+      @cursor += 1
+
+      target = target.allocate if target.is_a?(Class)
+      target.from_parquet(rec)
+      target
+    end
+
+    private
+
+    def read_table
+      tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
+      IO.copy_stream(@io, tmpname)
+
+      @table = Arrow::Table.load(tmpname, format: 'parquet')
+    ensure
+      unlink!(tmpname) if tmpname
+    end
+
+    def unlink!(tmpname)
+      File.unlink(tmpname)
+    rescue Errno::ENOENT
+      nil
+    end
+  end
+
+  class Encoder < Feedx::Format::Abstract::Encoder
+    attr_reader :schema
+
+    def initialize(io, schema:, buffer_size: 1 << 20, batch_size: 10_000)
+      super(io)
+
+      @schema = schema
+      @batch_size = batch_size.to_i
+      @buffer_size = buffer_size.to_i
+
+      @tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
+      @output  = Arrow::FileOutputStream.new(@tmpname, append: false)
+      @writer  = Parquet::ArrowFileWriter.new(@schema, @output)
+      @batch   = []
+    end
+
+    def encode(msg, **opts)
+      msg = msg.to_parquet(@schema, **opts) if msg.respond_to?(:to_parquet)
+
+      res = @batch.push(msg)
+      flush_table if @batch.size >= @batch_size
+      res
+    end
+
+    def close
+      flush_table unless @batch.empty?
+
+      @writer.close
+      @output.close
+      IO.copy_stream(@tmpname, @io)
+    ensure
+      unlink!
+    end
+
+    private
+
+    def flush_table
+      table = Arrow::RecordBatch.new(@schema, @batch).to_table
+      @writer.write_table table, @buffer_size
+      @batch.clear
+    end
+
+    def unlink!
+      File.unlink(@tmpname)
+    rescue Errno::ENOENT
+      nil
+    end
+  end
+end

data/lib/feedx/format/protobuf.rb

@@ -1,16 +1,24 @@
 require 'pbio'
 
 class Feedx::Format::Protobuf < Feedx::Format::Abstract
-  def initialize(io)
-    super PBIO::Delimited.new(io)
-  end
+  class Decoder < Feedx::Format::Abstract::Decoder
+    def initialize(io, **opts)
+      super PBIO::Delimited.new(io), **opts
+    end
 
-  def decode(klass, **)
-    @io.read(klass)
+    def decode(target, **)
+      @io.read(target)
+    end
   end
 
-  def encode(msg, **opts)
-    msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
-    @io.write msg
+  class Encoder < Feedx::Format::Abstract::Encoder
+    def initialize(io, **opts)
+      super PBIO::Delimited.new(io), **opts
+    end
+
+    def encode(msg, **opts)
+      msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
+      @io.write msg
+    end
   end
 end