feedx 0.12.6 → 0.14.0

data/ext/parquet/parquet.go

@@ -1,78 +0,0 @@
-package parquet
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-
-	"github.com/bsm/feedx"
-)
-
-// Format is a parquet format.
-type Format struct {
-	TempDir   string
-	Columns   []string // column names to include
-	BatchSize int      // batch size, default: 1,000
-}
-
-// NewDecoder implements Format.
-func (f *Format) NewDecoder(r io.Reader) (feedx.FormatDecoder, error) {
-	if rs, ok := r.(io.ReadSeeker); ok {
-		return newDecoder(rs, f.Columns, f.BatchSize)
-	}
-
-	tmp, err := copyToTempFile(f.TempDir, r)
-	if err != nil {
-		return nil, err
-	}
-
-	dec, err := newDecoder(tmp, f.Columns, f.BatchSize)
-	if err != nil {
-		_ = tmp.Close()
-		return nil, err
-	}
-	dec.closers = append(dec.closers, tmp)
-	return dec, nil
-}
-
-// NewEncoder implements Format.
-func (*Format) NewEncoder(w io.Writer) (feedx.FormatEncoder, error) {
-	return nil, fmt.Errorf("not implemented")
-}
-
-// --------------------------------------------------------------------
-
-type tempFile struct{ *os.File }
-
-func copyToTempFile(dir string, r io.Reader) (*tempFile, error) {
-	w, err := ioutil.TempFile(dir, "feedx-ext-parquet")
-	if err != nil {
-		return nil, err
-	}
-	if _, err := io.Copy(w, r); err != nil {
-		_ = w.Close()
-		_ = os.Remove(w.Name())
-		return nil, err
-	}
-	if err := w.Close(); err != nil {
-		_ = os.Remove(w.Name())
-		return nil, err
-	}
-
-	f, err := os.Open(w.Name())
-	if err != nil {
-		_ = os.Remove(w.Name())
-		return nil, err
-	}
-
-	return &tempFile{File: f}, nil
-}
-
-func (f tempFile) Close() error {
-	err := f.File.Close()
-	if e := os.Remove(f.Name()); e != nil {
-		err = e
-	}
-	return err
-}

data/ext/parquet/parquet_test.go

@@ -1,28 +0,0 @@
-package parquet_test
-
-import (
-	"testing"
-	"time"
-
-	. "github.com/bsm/ginkgo"
-	. "github.com/bsm/gomega"
-)
-
-type mockStruct struct {
-	ID         int       `parquet:"id"`
-	Bool       bool      `parquet:"bool_col"`
-	TinyInt    int8      `parquet:"tinyint_col"`
-	SmallUint  uint16    `parquet:"smallint_col"`
-	StdInt     int       `parquet:"int_col"`
-	BigInt     int64     `parquet:"bigint_col"`
-	Float      *float32  `parquet:"float_col"`
-	Double     float64   `parquet:"double_col"`
-	DateString string    `parquet:"date_string_col"`
-	ByteString []byte    `parquet:"string_col"`
-	Timestamp  time.Time `parquet:"timestamp_col"`
-}
-
-func TestSuite(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "feedx/ext/parquet")
-}

data/ext/parquet/reader.go

@@ -1,89 +0,0 @@
-package parquet
-
-import (
-	"io"
-
-	kpq "github.com/bsm/parquet-go/parquet"
-)
-
-type columnReader struct {
-	file     *kpq.File
-	col      kpq.Column
-	rowGroup int
-
-	chunk   *kpq.ColumnChunkReader
-	values  []interface{}
-	dLevels []uint16
-	rLevels []uint16
-
-	n, i, vi int // chunk stats
-}
-
-func newColumnReader(file *kpq.File, col kpq.Column, batchSize int) *columnReader {
-	return &columnReader{
-		file:    file,
-		col:     col,
-		values:  make([]interface{}, batchSize),
-		dLevels: make([]uint16, batchSize),
-		rLevels: make([]uint16, batchSize),
-	}
-}
-
-func (c *columnReader) Name() string {
-	return c.col.String()
-}
-
-func (c *columnReader) Next() (interface{}, error) {
-	if err := c.ensureChunk(); err != nil {
-		return nil, err
-	}
-
-	if err := c.ensureValues(); err == kpq.EndOfChunk {
-		c.chunk = nil
-		return c.Next()
-	} else if err != nil {
-		return nil, err
-	}
-
-	dLevel := c.dLevels[c.i]
-	c.i++
-
-	if notNull := dLevel == c.col.MaxD(); notNull {
-		val := c.values[c.vi]
-		c.vi++
-		return val, nil
-	}
-
-	return nil, nil
-}
-
-func (c *columnReader) ensureChunk() error {
-	if c.chunk != nil {
-		return nil
-	}
-	if c.rowGroup >= len(c.file.MetaData.RowGroups) {
-		return io.EOF
-	}
-
-	rd, err := c.file.NewReader(c.col, c.rowGroup)
-	if err != nil {
-		return err
-	}
-	c.chunk = rd
-	c.rowGroup++
-	return nil
-}
-
-func (c *columnReader) ensureValues() error {
-	if c.n != 0 && c.i < c.n {
-		return nil
-	}
-
-	n, err := c.chunk.Read(c.values, c.dLevels, c.rLevels)
-	if err != nil {
-		return err
-	}
-
-	c.n, c.i, c.vi = n, 0, 0
-	return nil
-}

data/ext/parquet/types.go

@@ -1,51 +0,0 @@
-package parquet
-
-import (
-	"reflect"
-	"strings"
-	"sync"
-)
-
-type structFields map[string]int
-
-var fieldCache sync.Map // map[reflect.Type]structFields
-
-// cachedTypeFields is like typeFields but uses a cache to avoid repeated work.
-//
-// "Inspired" by https://golang.org/src/encoding/json/encode.go
-// Copyright 2010 The Go Authors. All rights reserved.
-func cachedTypeFields(t reflect.Type) structFields {
-	if f, ok := fieldCache.Load(t); ok {
-		return f.(structFields)
-	}
-
-	f, _ := fieldCache.LoadOrStore(t, typeFields(t))
-	return f.(structFields)
-}
-
-func tagName(tag string) string {
-	if pos := strings.Index(tag, ","); pos != -1 {
-		return tag[:pos]
-	}
-	return tag
-}
-
-// "Inspired" by https://golang.org/src/encoding/json/encode.go
-// Copyright 2010 The Go Authors. All rights reserved.
-func typeFields(t reflect.Type) structFields {
-	index := make(map[string]int, t.NumField())
-	for i := 0; i < t.NumField(); i++ {
-		field := t.Field(i)
-		tag := field.Tag.Get("parquet")
-		if tag == "-" {
-			continue
-		}
-
-		name := field.Name
-		if s := tagName(tag); s != "" {
-			name = s
-		}
-		index[name] = i
-	}
-	return index
-}

data/lib/feedx/format/parquet.rb

@@ -1,102 +0,0 @@
-require 'parquet'
-require 'tmpdir'
-
-class Feedx::Format::Parquet < Feedx::Format::Abstract
-  class Record < Arrow::Record
-    def each_pair
-      container.columns.each do |col|
-        yield col.name, col[index]
-      end
-    end
-  end
-
-  class Decoder < Feedx::Format::Abstract::Decoder
-    def initialize(io, **)
-      super(io)
-
-      @table  = read_table
-      @cursor = 0
-    end
-
-    def eof?
-      @cursor >= @table.n_rows
-    end
-
-    def decode(target, **)
-      return if eof?
-
-      rec = Record.new(@table, @cursor)
-      @cursor += 1
-
-      target = target.allocate if target.is_a?(Class)
-      target.from_parquet(rec)
-      target
-    end
-
-    private
-
-    def read_table
-      tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
-      IO.copy_stream(@io, tmpname)
-
-      @table = Arrow::Table.load(tmpname, format: 'parquet')
-    ensure
-      unlink!(tmpname) if tmpname
-    end
-
-    def unlink!(tmpname)
-      File.unlink(tmpname)
-    rescue Errno::ENOENT
-      nil
-    end
-  end
-
-  class Encoder < Feedx::Format::Abstract::Encoder
-    attr_reader :schema
-
-    def initialize(io, schema:, buffer_size: 1 << 20, batch_size: 10_000)
-      super(io)
-
-      @schema = schema
-      @batch_size = batch_size.to_i
-      @buffer_size = buffer_size.to_i
-
-      @tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
-      @output  = Arrow::FileOutputStream.new(@tmpname, append: false)
-      @writer  = Parquet::ArrowFileWriter.new(@schema, @output)
-      @batch   = []
-    end
-
-    def encode(msg, **opts)
-      msg = msg.to_parquet(@schema, **opts) if msg.respond_to?(:to_parquet)
-
-      res = @batch.push(msg)
-      flush_table if @batch.size >= @batch_size
-      res
-    end
-
-    def close
-      flush_table unless @batch.empty?
-
-      @writer.close
-      @output.close
-      IO.copy_stream(@tmpname, @io)
-    ensure
-      unlink!
-    end
-
-    private
-
-    def flush_table
-      table = Arrow::RecordBatch.new(@schema, @batch).to_table
-      @writer.write_table table, @buffer_size
-      @batch.clear
-    end
-
-    def unlink!
-      File.unlink(@tmpname)
-    rescue Errno::ENOENT
-      nil
-    end
-  end
-end

data/spec/feedx/cache/memory_spec.rb

@@ -1,23 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Cache::Memory do
-  it 'read/writes' do
-    expect(subject.fetch('key')).to be_nil
-    expect(subject.fetch('key') { 'value' }).to eq('value')
-    expect(subject.fetch('key')).to eq('value')
-    expect(subject.fetch('key') { 'other' }).to eq('value')
-    expect(subject.fetch('key')).to eq('value')
-
-    subject.write('key', 'new-value')
-    expect(subject.read('key')).to eq('new-value')
-    expect(subject.fetch('key')).to eq('new-value')
-
-    subject.clear
-    expect(subject.fetch('key')).to be_nil
-  end
-
-  it 'writes strings' do
-    subject.write('key', 5)
-    expect(subject.read('key')).to eq('5')
-  end
-end

data/spec/feedx/cache/value_spec.rb

@@ -1,19 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Cache::Value do
-  subject do
-    described_class.new(Feedx::Cache::Memory.new, 'key')
-  end
-
-  it 'read/writes' do
-    expect(subject.fetch).to be_nil
-    expect(subject.fetch { 'value' }).to eq('value')
-    expect(subject.fetch).to eq('value')
-    expect(subject.fetch { 'other' }).to eq('value')
-    expect(subject.fetch).to eq('value')
-
-    subject.write('new-value')
-    expect(subject.read).to eq('new-value')
-    expect(subject.fetch).to eq('new-value')
-  end
-end

data/spec/feedx/compression/gzip_spec.rb

@@ -1,17 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Compression::Gzip do
-  it 'wraps readers/writers' do
-    wio = StringIO.new
-    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
-    expect(wio.size).to be_within(20).of(40)
-    expect(wio.string.encoding).to eq(Encoding::BINARY)
-
-    data = ''
-    StringIO.open(wio.string) do |rio|
-      subject.reader(rio) {|z| data = z.read }
-    end
-    expect(data.size).to eq(3000)
-    expect(data.encoding).to eq(Encoding.default_external)
-  end
-end

data/spec/feedx/compression/none_spec.rb

@@ -1,15 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Compression::None do
-  it 'wraps readers/writers' do
-    wio = StringIO.new
-    subject.writer(wio) {|w| w.write 'xyz' * 1000 }
-    expect(wio.size).to eq(3000)
-
-    data = ''
-    StringIO.open(wio.string) do |rio|
-      subject.reader(rio) {|z| data = z.read }
-    end
-    expect(data.size).to eq(3000)
-  end
-end

data/spec/feedx/compression_spec.rb

@@ -1,19 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Compression do
-  it 'resolves' do
-    expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
-    expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
-    expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
-    expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
-  end
-
-  it 'detects' do
-    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
-    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)
-    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb.gz')).to be_instance_of(described_class::Gzip)
-    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::None)
-  end
-end

data/spec/feedx/consumer_spec.rb

@@ -1,49 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Consumer do
-  let(:bucket) { BFS::Bucket::InMem.new }
-  let(:klass)  { Feedx::TestCase::Model }
-  let(:cache)  { Feedx::Cache::Memory.new.value('my-consumer') }
-
-  before { allow(BFS).to receive(:resolve).and_return(bucket) }
-
-  it 'rejects invalid inputs' do
-    expect do
-      described_class.each('mock:///dir/file.txt', klass)
-    end.to raise_error(/unable to detect format/)
-  end
-
-  it 'consumes feeds' do
-    url = mock_produce!
-    csm = described_class.new(url, klass)
-    expect(csm).to be_a(Enumerable)
-
-    cnt = csm.count do |rec|
-      expect(rec).to be_instance_of(klass)
-      true
-    end
-    expect(cnt).to eq(300)
-  end
-
-  it 'performs conditionally' do
-    url = mock_produce! last_modified: Time.at(1515151515)
-    expect(described_class.new(url, klass, cache: cache).count).to eq(300)
-    expect(described_class.new(url, klass, cache: cache).count).to eq(0)
-
-    url = mock_produce!
-    expect(described_class.new(url, klass, cache: cache).count).to eq(300)
-    expect(described_class.new(url, klass, cache: cache).count).to eq(300)
-  end
-
-  private
-
-  def mock_produce!(enum: mock_enum, **opts)
-    url = 'mock:///dir/file.json'
-    Feedx::Producer.perform url, enum: enum, **opts
-    url
-  end
-
-  def mock_enum
-    %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
-  end
-end

data/spec/feedx/format/abstract_spec.rb

@@ -1,21 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Format::Abstract do
-  subject   { Feedx::Format::JSON.new }
-
-  let(:wio) { StringIO.new }
-  let(:rio) { StringIO.open(wio.string) }
-
-  it 'decodes each' do
-    subject.encoder wio do |enc|
-      enc.encode(Feedx::TestCase::Model.new('X'))
-      enc.encode(Feedx::TestCase::Model.new('Y'))
-      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
-    end
-
-    subject.decoder rio do |dec|
-      acc = dec.decode_each(Feedx::TestCase::Model).to_a
-      expect(acc.map(&:title)).to eq(%w[X Y Z])
-    end
-  end
-end

data/spec/feedx/format/json_spec.rb

@@ -1,27 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Format::JSON do
-  let(:wio) { StringIO.new }
-  let(:rio) { StringIO.open(wio.string) }
-
-  it 'encode/decodes' do
-    subject.encoder wio do |enc|
-      enc.encode(Feedx::TestCase::Model.new('X'))
-      enc.encode(Feedx::TestCase::Model.new('Y'))
-      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
-    end
-    expect(wio.string).to eq(<<~JSON)
-      {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
-      {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
-      {"title":"Z"}
-    JSON
-
-    subject.decoder rio do |dec|
-      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
-      expect(dec.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
-      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
-      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
-      expect(dec).to be_eof
-    end
-  end
-end

data/spec/feedx/format/parquet_spec.rb

@@ -1,30 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Format::Parquet do
-  let(:wio) { StringIO.new }
-  let(:rio) { StringIO.open(wio.string) }
-
-  let(:schema) do
-    Arrow::Schema.new([
-      Arrow::Field.new('title', :string),
-      Arrow::Field.new('updated_at', type: :timestamp, unit: :second),
-    ])
-  end
-
-  it 'encode/decodes' do
-    subject.encoder wio, schema: schema, batch_size: 2 do |enc|
-      enc.encode(Feedx::TestCase::Model.new('X'))
-      enc.encode(Feedx::TestCase::Model.new('Y'))
-      enc.encode(Feedx::TestCase::Model.new('Z'))
-    end
-    expect(wio.string.bytesize).to be_within(100).of(1100)
-
-    subject.decoder rio do |dec|
-      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
-      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Y'))
-      expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
-      expect(dec.decode(Feedx::TestCase::Model)).to be_nil
-      expect(dec).to be_eof
-    end
-  end
-end

data/spec/feedx/format/protobuf_spec.rb

@@ -1,23 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Format::Protobuf do
-  let(:wio) { StringIO.new }
-  let(:rio) { StringIO.open(wio.string) }
-
-  it 'encode/decodes' do
-    subject.encoder wio do |enc|
-      enc.encode(Feedx::TestCase::Model.new('X'))
-      enc.encode(Feedx::TestCase::Model.new('Y'))
-      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
-    end
-    expect(wio.string.bytes).to eq([3, 10, 1, 88] + [3, 10, 1, 89] + [3, 10, 1, 90])
-
-    subject.decoder rio do |dec|
-      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
-      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
-      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
-      expect(dec.decode(Feedx::TestCase::Message)).to be_nil
-      expect(dec).to be_eof
-    end
-  end
-end

data/spec/feedx/format_spec.rb

@@ -1,21 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Feedx::Format do
-  it 'resolves' do
-    expect(described_class.resolve(:json)).to be_instance_of(described_class::JSON)
-    expect(described_class.resolve(:pb)).to be_instance_of(described_class::Protobuf)
-    expect { described_class.resolve(:txt) }.to raise_error(/invalid format txt/)
-  end
-
-  it 'detects' do
-    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::JSON)
-    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::JSON)
-    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::JSON)
-    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pb.z')).to be_instance_of(described_class::Protobuf)
-    expect do
-      described_class.detect('path/to/file.txt')
-    end.to raise_error(/unable to detect format/)
-  end
-end