feedx 0.9.0 → 0.11.0

data/spec/feedx/format/protobuf_spec.rb

@@ -1,22 +1,23 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Protobuf do
-  subject   { described_class.new(wio) }
   let(:wio) { StringIO.new }
+  let(:rio) { StringIO.open(wio.string) }
 
   it 'should encode/decode' do
-    subject.encode(Feedx::TestCase::Model.new('X'))
-    subject.encode(Feedx::TestCase::Model.new('Y'))
-    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    subject.encoder wio do |enc|
+      enc.encode(Feedx::TestCase::Model.new('X'))
+      enc.encode(Feedx::TestCase::Model.new('Y'))
+      enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    end
     expect(wio.string.bytes).to eq([3, 10, 1, 88] + [3, 10, 1, 89] + [3, 10, 1, 90])
 
-    StringIO.open(wio.string) do |rio|
-      fmt = described_class.new(rio)
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
-      expect(fmt.decode(Feedx::TestCase::Message)).to be_nil
-      expect(fmt).to be_eof
+    subject.decoder rio do |dec|
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
+      expect(dec.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
+      expect(dec.decode(Feedx::TestCase::Message)).to be_nil
+      expect(dec).to be_eof
     end
   end
 end

data/spec/feedx/format_spec.rb

@@ -2,18 +2,18 @@ require 'spec_helper'
 
 RSpec.describe Feedx::Format do
   it 'should resolve' do
-    expect(described_class.resolve(:json)).to eq(described_class::JSON)
-    expect(described_class.resolve(:pb)).to eq(described_class::Protobuf)
+    expect(described_class.resolve(:json)).to be_instance_of(described_class::JSON)
+    expect(described_class.resolve(:pb)).to be_instance_of(described_class::Protobuf)
     expect { described_class.resolve(:txt) }.to raise_error(/invalid format txt/)
   end
 
   it 'should detect' do
-    expect(described_class.detect('path/to/file.json')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::JSON)
-    expect(described_class.detect('path/to/file.pb')).to eq(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Protobuf)
-    expect(described_class.detect('path/to/file.pb.z')).to eq(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::JSON)
+    expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pb.z')).to be_instance_of(described_class::Protobuf)
     expect do
       described_class.detect('path/to/file.txt')
     end.to raise_error(/unable to detect format/)

data/spec/feedx/stream_spec.rb

@@ -13,13 +13,32 @@ RSpec.describe Feedx::Stream do
     end.to raise_error(/unable to detect format/)
   end
 
+  it 'should accept custom formats' do
+    format = Class.new do
+      def encoder(io, &block)
+        Feedx::Format::JSON::Encoder.open(io, &block)
+      end
+
+      def decoder(io, &block)
+        Feedx::Format::JSON::Decoder.open(io, &block)
+      end
+    end
+
+    stream = described_class.new('mock:///dir/file.txt', format: format.new)
+    stream.create {|s| s.encode Feedx::TestCase::Model.new('X') }
+
+    expect(bucket.read('dir/file.txt')).to eq(
+      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
+    )
+  end
+
   it 'should encode' do
     subject.create do |s|
       s.encode(Feedx::TestCase::Model.new('X'))
       s.encode(Feedx::TestCase::Model.new('Y'))
     end
 
-    expect(bucket.open('dir/file.json').read).to eq(
+    expect(bucket.read('dir/file.json')).to eq(
       %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n) +
       %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
     )

data/spec/spec_helper.rb

@@ -28,13 +28,29 @@ module Feedx
       end
       alias eql? ==
 
+      def updated_at
+        Time.at(1515151515).utc
+      end
+
       def from_json(data, *)
         hash = ::JSON.parse(data)
         @title = hash['title'] if hash.is_a?(Hash)
       end
 
       def to_json(*)
-        ::JSON.dump(title: @title, updated_at: Time.at(1515151515).utc)
+        ::JSON.dump(title: @title, updated_at: updated_at)
+      end
+
+      def from_parquet(rec)
+        rec.each_pair do |name, value|
+          @title = value if name == 'title'
+        end
+      end
+
+      def to_parquet(schema, *)
+        schema.fields.map do |field|
+          send(field.name)
+        end
       end
     end
   end

data/writer.go

@@ -39,33 +39,29 @@ func (o *WriterOptions) norm(name string) {
 // Writer encodes feeds to remote locations.
 type Writer struct {
 	ctx    context.Context
-	cancel context.CancelFunc
-
 	remote *bfs.Object
 	opt    WriterOptions
 	num    int
 
-	bw io.WriteCloser // bfs writer
+	bw bfs.Writer
 	cw io.WriteCloser // compression writer
 	ww *bufio.Writer
 	fe FormatEncoder
 }
 
 // NewWriter inits a new feed writer.
-func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) (*Writer, error) {
+func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) *Writer {
 	var o WriterOptions
 	if opt != nil {
 		o = *opt
 	}
 	o.norm(remote.Name())
 
-	ctx, cancel := context.WithCancel(ctx)
 	return &Writer{
 		ctx:    ctx,
-		cancel: cancel,
 		remote: remote,
 		opt:    o,
-	}, nil
+	}
 }
 
 // Write write raw bytes to the feed.
@@ -113,13 +109,27 @@ func (w *Writer) NumWritten() int {
 
 // Discard closes the writer and discards the contents.
 func (w *Writer) Discard() error {
-	w.cancel()
-	return w.Commit()
+	err := w.close()
+	if w.bw != nil {
+		if e := w.bw.Discard(); e != nil {
+			err = e
+		}
+	}
+	return err
 }
 
 // Commit closes the writer and persists the contents.
 func (w *Writer) Commit() error {
-	var err error
+	err := w.close()
+	if w.bw != nil {
+		if e := w.bw.Commit(); e != nil {
+			err = e
+		}
+	}
+	return err
+}
+
+func (w *Writer) close() (err error) {
 	if w.fe != nil {
 		if e := w.fe.Close(); e != nil {
 			err = e
@@ -135,11 +145,6 @@ func (w *Writer) Commit() error {
 			err = e
 		}
 	}
-	if w.bw != nil {
-		if e := w.bw.Close(); e != nil {
-			err = e
-		}
-	}
 	return err
 }
 

data/writer_test.go

@@ -21,10 +21,9 @@ var _ = Describe("Writer", func() {
 	})
 
 	It("should write plain", func() {
-		w, err := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
+		w := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
 			LastMod: time.Unix(1515151515, 123456789),
 		})
-		Expect(err).NotTo(HaveOccurred())
 		defer w.Discard()
 
 		Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
@@ -37,10 +36,9 @@ var _ = Describe("Writer", func() {
 	})
 
 	It("should write compressed", func() {
-		w, err := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
+		w := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
 			LastMod: time.Unix(1515151515, 123456789),
 		})
-		Expect(err).NotTo(HaveOccurred())
 		defer w.Discard()
 
 		Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
@@ -58,7 +56,7 @@ var _ = Describe("Writer", func() {
 
 		info, err := plain.Head(ctx)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(info.Size).To(BeNumerically("~", 470, 10))
+		Expect(info.Size).To(BeNumerically("~", 370, 10))
 		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
 
 		info, err = compressed.Head(ctx)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-10 00:00:00.000000000 Z
+date: 2020-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: red-parquet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +150,8 @@ files:
 - format_test.go
 - go.mod
 - go.sum
+- internal/testdata/testdata.pb.go
+- internal/testdata/testdata.proto
 - lib/feedx.rb
 - lib/feedx/cache.rb
 - lib/feedx/cache/abstract.rb
@@ -149,6 +165,7 @@ files:
 - lib/feedx/format.rb
 - lib/feedx/format/abstract.rb
 - lib/feedx/format/json.rb
+- lib/feedx/format/parquet.rb
 - lib/feedx/format/protobuf.rb
 - lib/feedx/producer.rb
 - lib/feedx/pusher.rb
@@ -165,6 +182,7 @@ files:
 - spec/feedx/consumer_spec.rb
 - spec/feedx/format/abstract_spec.rb
 - spec/feedx/format/json_spec.rb
+- spec/feedx/format/parquet_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
 - spec/feedx/producer_spec.rb
@@ -191,7 +209,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Exchange data between components via feeds
@@ -204,6 +222,7 @@ test_files:
 - spec/feedx/consumer_spec.rb
 - spec/feedx/format/abstract_spec.rb
 - spec/feedx/format/json_spec.rb
+- spec/feedx/format/parquet_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
 - spec/feedx/producer_spec.rb