feedx 0.9.2 → 0.12.0

data/internal/testdata/testdata.pb.go

@@ -0,0 +1,124 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: internal/testdata/testdata.proto
+
+package testdata
+
+import (
+	fmt "fmt"
+	proto "github.com/gogo/protobuf/proto"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type MockEnum int32
+
+const (
+	MockEnum_UNKNOWN MockEnum = 0
+	MockEnum_FIRST   MockEnum = 3
+)
+
+var MockEnum_name = map[int32]string{
+	0: "UNKNOWN",
+	3: "FIRST",
+}
+
+var MockEnum_value = map[string]int32{
+	"UNKNOWN": 0,
+	"FIRST":   3,
+}
+
+func (x MockEnum) String() string {
+	return proto.EnumName(MockEnum_name, int32(x))
+}
+
+func (MockEnum) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor_076a9f61cb4a1904, []int{0}
+}
+
+type MockMessage struct {
+	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Enum                 MockEnum `protobuf:"varint,2,opt,name=enum,proto3,enum=feedx.internal.testdata.MockEnum" json:"enum,omitempty"`
+	Height               uint32   `protobuf:"varint,3,opt,name=height,proto3" json:"height,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *MockMessage) Reset()         { *m = MockMessage{} }
+func (m *MockMessage) String() string { return proto.CompactTextString(m) }
+func (*MockMessage) ProtoMessage()    {}
+func (*MockMessage) Descriptor() ([]byte, []int) {
+	return fileDescriptor_076a9f61cb4a1904, []int{0}
+}
+func (m *MockMessage) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_MockMessage.Unmarshal(m, b)
+}
+func (m *MockMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_MockMessage.Marshal(b, m, deterministic)
+}
+func (m *MockMessage) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_MockMessage.Merge(m, src)
+}
+func (m *MockMessage) XXX_Size() int {
+	return xxx_messageInfo_MockMessage.Size(m)
+}
+func (m *MockMessage) XXX_DiscardUnknown() {
+	xxx_messageInfo_MockMessage.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MockMessage proto.InternalMessageInfo
+
+func (m *MockMessage) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+func (m *MockMessage) GetEnum() MockEnum {
+	if m != nil {
+		return m.Enum
+	}
+	return MockEnum_UNKNOWN
+}
+
+func (m *MockMessage) GetHeight() uint32 {
+	if m != nil {
+		return m.Height
+	}
+	return 0
+}
+
+func init() {
+	proto.RegisterEnum("feedx.internal.testdata.MockEnum", MockEnum_name, MockEnum_value)
+	proto.RegisterType((*MockMessage)(nil), "feedx.internal.testdata.MockMessage")
+}
+
+func init() { proto.RegisterFile("internal/testdata/testdata.proto", fileDescriptor_076a9f61cb4a1904) }
+
+var fileDescriptor_076a9f61cb4a1904 = []byte{
+	// 199 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0xc8, 0xcc, 0x2b, 0x49,
+	0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x2f, 0x49, 0x2d, 0x2e, 0x49, 0x49, 0x2c, 0x49, 0x84, 0x33, 0xf4,
+	0x0a, 0x8a, 0xf2, 0x4b, 0xf2, 0x85, 0xc4, 0xd3, 0x52, 0x53, 0x53, 0x2a, 0xf4, 0x60, 0xea, 0xf4,
+	0x60, 0xd2, 0x4a, 0x05, 0x5c, 0xdc, 0xbe, 0xf9, 0xc9, 0xd9, 0xbe, 0xa9, 0xc5, 0xc5, 0x89, 0xe9,
+	0xa9, 0x42, 0x42, 0x5c, 0x2c, 0x79, 0x89, 0xb9, 0xa9, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41,
+	0x60, 0xb6, 0x90, 0x29, 0x17, 0x4b, 0x6a, 0x5e, 0x69, 0xae, 0x04, 0x93, 0x02, 0xa3, 0x06, 0x9f,
+	0x91, 0xa2, 0x1e, 0x0e, 0xa3, 0xf4, 0x40, 0xe6, 0xb8, 0xe6, 0x95, 0xe6, 0x06, 0x81, 0x95, 0x0b,
+	0x89, 0x71, 0xb1, 0x65, 0xa4, 0x66, 0xa6, 0x67, 0x94, 0x48, 0x30, 0x2b, 0x30, 0x6a, 0xf0, 0x06,
+	0x41, 0x79, 0x5a, 0x4a, 0x5c, 0x1c, 0x30, 0x95, 0x42, 0xdc, 0x5c, 0xec, 0xa1, 0x7e, 0xde, 0x7e,
+	0xfe, 0xe1, 0x7e, 0x02, 0x0c, 0x42, 0x9c, 0x5c, 0xac, 0x6e, 0x9e, 0x41, 0xc1, 0x21, 0x02, 0xcc,
+	0x4e, 0x2a, 0x51, 0x4a, 0xe9, 0x99, 0x25, 0x19, 0xa5, 0x49, 0x7a, 0xc9, 0xf9, 0xb9, 0xfa, 0x60,
+	0x0b, 0xf5, 0x31, 0xfc, 0x98, 0xc4, 0x06, 0xf6, 0x9b, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x42,
+	0xf6, 0x49, 0xb3, 0xff, 0x00, 0x00, 0x00,
+}

data/internal/testdata/testdata.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+package feedx.internal.testdata;
+option go_package = "github.com/feedx/internal/testdata";
+
+enum MockEnum {
+  UNKNOWN = 0;
+  FIRST = 3;
+}
+
+message MockMessage {
+  string name = 1;
+  MockEnum enum = 2;
+  uint32 height = 3;
+}

data/lib/feedx/cache/abstract.rb

@@ -5,12 +5,12 @@ class Feedx::Cache::Abstract
   end
 
   # Read reads a key.
-  def read(_key, **_opts)
+  def read(_key, **)
     raise 'Not implemented'
   end
 
   # Write writes a key/value pair.
-  def write(_key, _value, **_opts)
+  def write(_key, _value, **)
     raise 'Not implemented'
   end
 

data/lib/feedx/compression.rb

@@ -5,12 +5,19 @@ module Feedx
     autoload :Gzip, 'feedx/compression/gzip'
 
     class << self
+      def validate!(kind)
+        raise ArgumentError, "#{kind} does not implement #reader(io, &block)" unless kind.respond_to?(:reader)
+        raise ArgumentError, "#{kind} does not implement #writer(io, &block)" unless kind.respond_to?(:writer)
+
+        kind
+      end
+
       def resolve(name)
         case name.to_s
         when 'gz', 'gzip'
-          Gzip
+          Gzip.new
         when ''
-          None
+          None.new
         else
           raise ArgumentError, "invalid compression #{name}"
         end
@@ -18,9 +25,9 @@ module Feedx
 
       def detect(path)
         if File.extname(path)[-1] == 'z'
-          Gzip
+          Gzip.new
         else
-          None
+          None.new
         end
       end
     end

data/lib/feedx/compression/abstract.rb

@@ -1,9 +1,9 @@
 class Feedx::Compression::Abstract
-  def self.reader(_io, &_block)
+  def reader(_io, **, &_block)
     raise 'Not implemented'
   end
 
-  def self.writer(_io, &_block)
+  def writer(_io, **, &_block)
     raise 'Not implemented'
   end
 end

data/lib/feedx/compression/gzip.rb

@@ -1,11 +1,23 @@
 require 'zlib'
 
 class Feedx::Compression::Gzip < Feedx::Compression::Abstract
-  def self.reader(io, &block)
+  def reader(io, **, &block)
+    force_binmode(io)
     Zlib::GzipReader.wrap(io, &block)
   end
 
-  def self.writer(io, &block)
+  def writer(io, **, &block)
+    force_binmode(io)
     Zlib::GzipWriter.wrap(io, &block)
   end
+
+  private
+
+  def force_binmode(io)
+    if io.respond_to?(:binmode)
+      io.binmode
+    elsif io.respond_to?(:set_encoding)
+      io.set_encoding(Encoding::BINARY)
+    end
+  end
 end

data/lib/feedx/compression/none.rb

@@ -1,9 +1,9 @@
 class Feedx::Compression::None < Feedx::Compression::Abstract
-  def self.reader(io, &block)
-    block.call(io)
+  def reader(io, **)
+    yield(io)
   end
 
-  def self.writer(io, &block)
-    block.call(io)
+  def writer(io, **)
+    yield(io)
   end
 end

data/lib/feedx/consumer.rb

@@ -8,22 +8,25 @@ module Feedx
     include Enumerable
 
     # See constructor.
-    def self.each(url, klass, opts = {}, &block)
-      new(url, klass, opts).each(&block)
+    def self.each(url, klass, **opts, &block)
+      new(url, klass, **opts).each(&block)
     end
 
     # @param [String] url the destination URL.
     # @param [Class] klass the record class.
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
-    # @option opts [Hash] :format_options format decode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
-    def initialize(url, klass, opts = {})
-      @klass    = klass
-      @stream   = Feedx::Stream.new(url, opts)
-      @fmt_opts = opts[:format_options] || {}
-      @cache    = opts[:cache]
+    def initialize(url, klass, format_options: {}, cache: nil, **opts)
+      @klass  = klass
+      @stream = Feedx::Stream.new(url, **opts)
+      @cache  = cache
+      @opts   = opts.merge(format_options)
+
+      return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
+
+      warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
     end
 
     # @return [Boolean] returns true if performed.
@@ -38,7 +41,7 @@ module Feedx
       end
 
       @stream.open do |fmt|
-        fmt.decode_each(@klass, **@fmt_opts, &block)
+        fmt.decode_each(@klass, **@opts, &block)
       end
       @cache.write(remote_rev) if @cache && remote_rev
 

data/lib/feedx/format.rb

@@ -2,13 +2,19 @@ module Feedx
   module Format
     autoload :Abstract, 'feedx/format/abstract'
     autoload :JSON, 'feedx/format/json'
+    autoload :Parquet, 'feedx/format/parquet'
     autoload :Protobuf, 'feedx/format/protobuf'
 
     class << self
-      def register(ext, kind)
-        raise ArgumentError, "#{kind} is not a subclass of Feedx::Format::Abstract" unless kind.is_a?(Class) && kind < Abstract
+      def validate!(kind)
+        raise ArgumentError, "#{kind} does not implement #encoder(io, &block)" unless kind.respond_to?(:encoder)
+        raise ArgumentError, "#{kind} does not implement #decoder(io, &block)" unless kind.respond_to?(:decoder)
+
+        kind
+      end
 
-        registry[ext.to_s] = kind
+      def register(ext, kind)
+        registry[ext.to_s] = validate!(kind)
       end
 
       def resolve(name)
@@ -33,6 +39,9 @@ module Feedx
       def registry
         @registry ||= {
           'json'     => :JSON,
+          'jsonl'    => :JSON,
+          'ndjson'   => :JSON,
+          'parquet'  => :Parquet,
           'pb'       => :Protobuf,
           'proto'    => :Protobuf,
           'protobuf' => :Protobuf,
@@ -40,13 +49,13 @@ module Feedx
       end
 
       def _resolve(name)
-        name  = name.to_s
-        klass = registry[name]
-        if klass.is_a?(Symbol)
-          klass = const_get(klass)
-          registry[name.to_s] = klass
+        name = name.to_s
+        kind = registry[name]
+        if kind.is_a?(Symbol)
+          kind = const_get(kind).new
+          registry[name.to_s] = kind
         end
-        klass
+        kind
       end
     end
   end

data/lib/feedx/format/abstract.rb

@@ -1,25 +1,54 @@
 class Feedx::Format::Abstract
-  def initialize(io)
-    @io = io
+  def decoder(io, **opts, &block)
+    self.class::Decoder.open(io, **opts, &block)
   end
 
-  def eof?
-    @io.eof?
+  def encoder(io, **opts, &block)
+    self.class::Encoder.open(io, **opts, &block)
   end
 
-  def decode_each(klass, **opts)
-    if block_given?
-      yield decode(klass, **opts) until eof?
-    else
-      Enumerator.new {|y| y << decode(klass, **opts) until eof? }
+  class Wrapper
+    def self.open(io, **opts)
+      inst = new(io, **opts)
+      yield inst
+    ensure
+      inst&.close
+    end
+
+    def initialize(io, **)
+      @io = io
     end
   end
 
-  def decode(_klass, **)
-    raise 'Not implemented'
+  class Decoder < Wrapper
+    def eof?
+      @io.eof?
+    end
+
+    def decode_each(target, **opts)
+      if block_given?
+        yield decode(target, **opts) until eof?
+      else
+        Enumerator.new do |acc|
+          acc << decode(target, **opts) until eof?
+        end
+      end
+    end
+
+    def decode(_target, **)
+      raise 'Not implemented'
+    end
+
+    def close; end
   end
 
-  def encode(_msg, **)
-    raise 'Not implemented'
+  class Encoder < Wrapper
+    def encode(_msg, **)
+      raise 'Not implemented'
+    end
+
+    def close
+      @io.flush if @io.respond_to?(:flush)
+    end
   end
 end

data/lib/feedx/format/json.rb

@@ -1,16 +1,20 @@
 require 'json'
 
 class Feedx::Format::JSON < Feedx::Format::Abstract
-  def decode(obj, **)
-    line = @io.gets
-    return unless line
+  class Decoder < Feedx::Format::Abstract::Decoder
+    def decode(target, **)
+      line = @io.gets
+      return unless line
 
-    obj = obj.allocate if obj.is_a?(Class)
-    obj.from_json(line)
-    obj
+      target = target.allocate if target.is_a?(Class)
+      target.from_json(line)
+      target
+    end
   end
 
-  def encode(msg, **opts)
-    @io.write msg.to_json(**opts) << "\n"
+  class Encoder < Feedx::Format::Abstract::Encoder
+    def encode(msg, **opts)
+      @io.write msg.to_json(**opts) << "\n"
+    end
   end
 end

data/lib/feedx/format/parquet.rb

@@ -0,0 +1,102 @@
+require 'parquet'
+require 'tmpdir'
+
+class Feedx::Format::Parquet < Feedx::Format::Abstract
+  class Record < Arrow::Record
+    def each_pair
+      container.columns.each do |col|
+        yield col.name, col[index]
+      end
+    end
+  end
+
+  class Decoder < Feedx::Format::Abstract::Decoder
+    def initialize(io, **)
+      super(io)
+
+      @table  = read_table
+      @cursor = 0
+    end
+
+    def eof?
+      @cursor >= @table.n_rows
+    end
+
+    def decode(target, **)
+      return if eof?
+
+      rec = Record.new(@table, @cursor)
+      @cursor += 1
+
+      target = target.allocate if target.is_a?(Class)
+      target.from_parquet(rec)
+      target
+    end
+
+    private
+
+    def read_table
+      tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
+      IO.copy_stream(@io, tmpname)
+
+      @table = Arrow::Table.load(tmpname, format: 'parquet')
+    ensure
+      unlink!(tmpname) if tmpname
+    end
+
+    def unlink!(tmpname)
+      File.unlink(tmpname)
+    rescue Errno::ENOENT
+      nil
+    end
+  end
+
+  class Encoder < Feedx::Format::Abstract::Encoder
+    attr_reader :schema
+
+    def initialize(io, schema:, buffer_size: 1 << 20, batch_size: 10_000)
+      super(io)
+
+      @schema = schema
+      @batch_size = batch_size.to_i
+      @buffer_size = buffer_size.to_i
+
+      @tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
+      @output  = Arrow::FileOutputStream.new(@tmpname, append: false)
+      @writer  = Parquet::ArrowFileWriter.new(@schema, @output)
+      @batch   = []
+    end
+
+    def encode(msg, **opts)
+      msg = msg.to_parquet(@schema, **opts) if msg.respond_to?(:to_parquet)
+
+      res = @batch.push(msg)
+      flush_table if @batch.size >= @batch_size
+      res
+    end
+
+    def close
+      flush_table unless @batch.empty?
+
+      @writer.close
+      @output.close
+      IO.copy_stream(@tmpname, @io)
+    ensure
+      unlink!
+    end
+
+    private
+
+    def flush_table
+      table = Arrow::RecordBatch.new(@schema, @batch).to_table
+      @writer.write_table table, @buffer_size
+      @batch.clear
+    end
+
+    def unlink!
+      File.unlink(@tmpname)
+    rescue Errno::ENOENT
+      nil
+    end
+  end
+end