feedx 0.6.3 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f4710e181ac3c987f7b02ebc976cf70bbc4c9826139823bca58620d94aa994a
-  data.tar.gz: 273951be43e9e8ac3473bfccf428cf2860c34dd4c0869b86f726446c477c2e7f
+  metadata.gz: b306ae446c9cae375b6ad538cc41d88a621bbb5cb59d9adf5aa114326567b2ca
+  data.tar.gz: 218b0aabdf6d1dda76e520b226f7d211a27ba14421db4ed8e34a9f59451e1e20
 SHA512:
-  metadata.gz: 862eeb21fda2add182866fdb686adab45091d585a9880565b7e12c2ef3240ecc3982ec09e0370f1e4734d57ac44aa97cd5fcd702215c84d9534e5eab5ba9b648
-  data.tar.gz: fc10bdbfea9922ca29d3d0967ab8a1aa14330b914b1e07972ca61e040620d2ccdcd710e0eb63860001feb3c366097652b825fd4660e847449edfeecdbd3232e0
+  metadata.gz: 246d39971aa5d6fe96267b95fba832f90d24ae8c8a16c4ea949fb3bca3f8777fd036e311f8d5f97b7cfed301b3782c2035e4e4ba1b62b7373e1f1e7d090ab00d
+  data.tar.gz: a151e2c9cde8820505912ceca55afa58f3a44cb25581eda9b66f830e057fe1254093c93ad5be77848fa8e3b0033d14885cb4a412bb4308bf0109b2bb57a47771

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    feedx (0.6.3)
+    feedx (0.7.0)
       bfs (>= 0.3.4)
 
 GEM
@@ -32,7 +32,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    rubocop (0.69.0)
+    rubocop (0.70.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.6)

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.6.3'
+  s.version       = '0.7.0'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)

data/lib/feedx.rb

@@ -3,6 +3,7 @@ module Feedx
 
   autoload :Compression, 'feedx/compression'
   autoload :Format, 'feedx/format'
+  autoload :Stream, 'feedx/stream'
   autoload :Producer, 'feedx/producer'
   autoload :Pusher, 'feedx/pusher'
   autoload :TaskState, 'feedx/task_state'

data/lib/feedx/compression/abstract.rb

@@ -1,5 +1,9 @@
 class Feedx::Compression::Abstract
-  def self.wrap(_io, &_block)
+  def self.reader(_io, &_block)
+    raise 'Not implemented'
+  end
+
+  def self.writer(_io, &_block)
     raise 'Not implemented'
   end
 end

data/lib/feedx/compression/gzip.rb

@@ -1,7 +1,11 @@
 require 'zlib'
 
 class Feedx::Compression::Gzip < Feedx::Compression::Abstract
-  def self.wrap(io, &block)
+  def self.reader(io, &block)
+    Zlib::GzipReader.wrap(io, &block)
+  end
+
+  def self.writer(io, &block)
     Zlib::GzipWriter.wrap(io, &block)
   end
 end

data/lib/feedx/compression/none.rb

@@ -1,5 +1,9 @@
 class Feedx::Compression::None < Feedx::Compression::Abstract
-  def self.wrap(io, &block)
+  def self.reader(io, &block)
+    block.call(io)
+  end
+
+  def self.writer(io, &block)
     block.call(io)
   end
 end

data/lib/feedx/format/abstract.rb

@@ -3,7 +3,15 @@ class Feedx::Format::Abstract
     @io = io
   end
 
-  def write(_msg)
+  def eof?
+    @io.eof?
+  end
+
+  def decode(_klass)
+    raise 'Not implemented'
+  end
+
+  def encode(_msg)
     raise 'Not implemented'
   end
 end

data/lib/feedx/format/json.rb

@@ -1,7 +1,16 @@
 require 'json'
 
 class Feedx::Format::JSON < Feedx::Format::Abstract
-  def write(msg)
+  def decode(obj)
+    line = @io.gets
+    return unless line
+
+    obj = obj.allocate if obj.is_a?(Class)
+    obj.from_json(line)
+    obj
+  end
+
+  def encode(msg)
     @io.write msg.to_json << "\n"
   end
 end

data/lib/feedx/format/protobuf.rb

@@ -5,7 +5,12 @@ class Feedx::Format::Protobuf < Feedx::Format::Abstract
     super PBIO::Delimited.new(io)
   end
 
-  def write(msg)
-    @io.write msg.to_pb
+  def decode(klass)
+    @io.read(klass)
+  end
+
+  def encode(msg)
+    msg = msg.to_pb if msg.respond_to?(:to_pb)
+    @io.write msg
   end
 end

data/lib/feedx/producer.rb

@@ -22,9 +22,7 @@ module Feedx
       @enum = opts[:enum] || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
-      @blob     = BFS::Blob.new(url)
-      @format   = detect_format(opts[:format])
-      @compress = detect_compress(opts[:compress])
+      @stream = Feedx::Stream.new(url, opts)
       @last_mod = opts[:last_modified]
     end
 
@@ -34,51 +32,16 @@ module Feedx
       current  = (last_mod.to_f * 1000).floor
 
       begin
-        previous = @blob.info.metadata[META_LAST_MODIFIED].to_i
+        previous = @stream.blob.info.metadata[META_LAST_MODIFIED].to_i
         return -1 unless current > previous
       rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
       end if current.positive?
 
-      @blob.create metadata: { META_LAST_MODIFIED => current.to_s } do |io|
-        @compress.wrap(io) {|w| write_all(enum, w) }
+      @stream.create metadata: { META_LAST_MODIFIED => current.to_s } do |fmt|
+        iter = enum.respond_to?(:find_each) ? :find_each : :each
+        enum.send(iter) {|rec| fmt.encode(rec) }
       end
-      @blob.info.size
-    end
-
-    private
-
-    def detect_format(val)
-      case val
-      when nil
-        Feedx::Format.detect(@blob.path)
-      when Class
-        parent = Feedx::Format::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
-        Feedx::Format.resolve(val)
-      end
-    end
-
-    def detect_compress(val)
-      case val
-      when nil
-        Feedx::Compression.detect(@blob.path)
-      when Class
-        parent = Feedx::Compression::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
-        Feedx::Compression.resolve(val)
-      end
-    end
-
-    def write_all(enum, io)
-      stream   = @format.new(io)
-      iterator = enum.respond_to?(:find_each) ? :find_each : :each
-      enum.send(iterator) {|rec| stream.write(rec) }
+      @stream.blob.info.size
     end
   end
 end

data/lib/feedx/stream.rb

@@ -0,0 +1,75 @@
+require 'bfs'
+require 'feedx'
+
+module Feedx
+  # Abstract stream handler around a remote blob.
+  class Stream
+    attr_reader :blob
+
+    # @param [String] url the blob URL.
+    # @param [Hash] opts options
+    # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
+    # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
+    def initialize(url, opts={})
+      @blob     = BFS::Blob.new(url)
+      @format   = detect_format(opts[:format])
+      @compress = detect_compress(opts[:compress])
+    end
+
+    # Opens the remote for reading.
+    # @param [Hash] opts BFS::Blob#open options
+    # @yield A block over a formatted stream.
+    # @yieldparam [Feedx::Format::Abstract] formatted input stream.
+    def open(opts={})
+      @blob.open(opts) do |io|
+        @compress.reader(io) do |cio|
+          fmt = @format.new(cio)
+          yield fmt
+        end
+      end
+    end
+
+    # Opens the remote for writing.
+    # @param [Hash] opts BFS::Blob#create options
+    # @yield A block over a formatted stream.
+    # @yieldparam [Feedx::Format::Abstract] formatted output stream.
+    def create(opts={})
+      @blob.create(opts) do |io|
+        @compress.writer(io) do |cio|
+          fmt = @format.new(cio)
+          yield fmt
+        end
+      end
+    end
+
+    private
+
+    def detect_format(val)
+      case val
+      when nil
+        Feedx::Format.detect(@blob.path)
+      when Class
+        parent = Feedx::Format::Abstract
+        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
+
+        val
+      else
+        Feedx::Format.resolve(val)
+      end
+    end
+
+    def detect_compress(val)
+      case val
+      when nil
+        Feedx::Compression.detect(@blob.path)
+      when Class
+        parent = Feedx::Compression::Abstract
+        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
+
+        val
+      else
+        Feedx::Compression.resolve(val)
+      end
+    end
+  end
+end

data/spec/feedx/compression/gzip_spec.rb

@@ -1,9 +1,15 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression::Gzip do
-  it 'should wrap' do
-    io = StringIO.new
-    described_class.wrap(io) {|w| w.write 'xyz' * 1000 }
-    expect(io.size).to be_within(20).of(40)
+  it 'should wrap readers/writers' do
+    wio = StringIO.new
+    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    expect(wio.size).to be_within(20).of(40)
+
+    data = ''
+    StringIO.open(wio.string) do |rio|
+      described_class.reader(rio) {|z| data = z.read }
+    end
+    expect(data.size).to eq(3000)
   end
 end

data/spec/feedx/compression/none_spec.rb

@@ -1,9 +1,15 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Compression::None do
-  it 'should wrap' do
-    io = StringIO.new
-    described_class.wrap(io) {|w| w.write 'xyz' * 1000 }
-    expect(io.size).to eq(3000)
+  it 'should wrap readers/writers' do
+    wio = StringIO.new
+    described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
+    expect(wio.size).to eq(3000)
+
+    data = ''
+    StringIO.open(wio.string) do |rio|
+      described_class.reader(rio) {|z| data = z.read }
+    end
+    expect(data.size).to eq(3000)
   end
 end

data/spec/feedx/format/json_spec.rb

@@ -1,12 +1,26 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::JSON do
-  subject  { described_class.new(io) }
-  let(:io) { StringIO.new }
+  subject   { described_class.new(wio) }
+  let(:wio) { StringIO.new }
 
-  it 'should write' do
-    subject.write(a: 1, b: 2)
-    subject.write(c: ['x'], d: true)
-    expect(io.string).to eq %({"a":1,"b":2}\n{"c":["x"],"d":true}\n)
+  it 'should encode/decode' do
+    subject.encode(Feedx::TestCase::Model.new('X'))
+    subject.encode(Feedx::TestCase::Model.new('Y'))
+    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    expect(wio.string.lines).to eq [
+      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
+      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
+      %({"title":"Z"}\n),
+    ]
+
+    StringIO.open(wio.string) do |rio|
+      fmt = described_class.new(rio)
+      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(fmt.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(fmt.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
+      expect(fmt.decode(Feedx::TestCase::Model)).to be_nil
+      expect(fmt).to be_eof
+    end
   end
 end

data/spec/feedx/format/protobuf_spec.rb

@@ -1,20 +1,22 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Format::Protobuf do
-  subject  { described_class.new(io) }
-  let(:io) { StringIO.new }
+  subject   { described_class.new(wio) }
+  let(:wio) { StringIO.new }
 
-  let(:model) do
-    Class.new Struct.new(:title) do
-      def to_pb
-        Feedx::TestCase::Message.new title: title
-      end
-    end
-  end
+  it 'should encode/decode' do
+    subject.encode(Feedx::TestCase::Model.new('X'))
+    subject.encode(Feedx::TestCase::Model.new('Y'))
+    subject.encode(Feedx::TestCase::Message.new(title: 'Z'))
+    expect(wio.string.bytes).to eq([3, 10, 1, 88] + [3, 10, 1, 89] + [3, 10, 1, 90])
 
-  it 'should write' do
-    subject.write(model.new('X'))
-    subject.write(model.new('Y'))
-    expect(io.string.bytes).to eq([3, 10, 1, 88] + [3, 10, 1, 89])
+    StringIO.open(wio.string) do |rio|
+      fmt = described_class.new(rio)
+      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'X'))
+      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Y'))
+      expect(fmt.decode(Feedx::TestCase::Message)).to eq(Feedx::TestCase::Message.new(title: 'Z'))
+      expect(fmt.decode(Feedx::TestCase::Message)).to be_nil
+      expect(fmt).to be_eof
+    end
   end
 end

data/spec/feedx/producer_spec.rb

@@ -1,20 +1,8 @@
 require 'spec_helper'
 
 RSpec.describe Feedx::Producer do
-  let :model do
-    Class.new Struct.new(:title) do
-      def to_pb
-        Feedx::TestCase::Message.new title: title
-      end
-
-      def to_json(*)
-        ::JSON.dump(title: title, updated_at: Time.at(1515151515).utc)
-      end
-    end
-  end
-
   let :enumerable do
-    %w[x y z].map {|t| model.new(t) } * 100
+    %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
   end
 
   let(:bucket) { BFS::Bucket::InMem.new }

data/spec/feedx/stream_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Stream do
+  let(:bucket) { BFS::Bucket::InMem.new }
+  before  { allow(BFS).to receive(:resolve).and_return(bucket) }
+
+  subject { described_class.new('mock:///dir/file.json') }
+  let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
+
+  it 'should reject invalid inputs' do
+    expect do
+      described_class.new('mock:///dir/file.txt')
+    end.to raise_error(/unable to detect format/)
+  end
+
+  it 'should encode' do
+    subject.create do |s|
+      s.encode(Feedx::TestCase::Model.new('X'))
+      s.encode(Feedx::TestCase::Model.new('Y'))
+    end
+
+    expect(bucket.open('dir/file.json').read).to eq(
+      %({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n) +
+      %({"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}\n),
+    )
+  end
+
+  it 'should encode compressed' do
+    compressed.create do |s|
+      100.times do
+        s.encode(Feedx::TestCase::Model.new('X'))
+      end
+    end
+
+    expect(bucket.info('dir/file.json.gz').size).to be_within(10).of(108)
+  end
+
+  it 'should encode with create options' do
+    subject.create metadata: { 'x' => '5' } do |s|
+      s.encode(Feedx::TestCase::Model.new('X'))
+    end
+    expect(bucket.info('dir/file.json').metadata).to eq('x' => '5')
+  end
+
+  it 'should decode' do
+    subject.create do |s|
+      s.encode(Feedx::TestCase::Model.new('X'))
+      s.encode(Feedx::TestCase::Model.new('Y'))
+    end
+
+    subject.open do |s|
+      expect(s.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(s.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(s.decode(Feedx::TestCase::Model)).to be_nil
+      expect(s).to be_eof
+    end
+  end
+
+  it 'should decode compressed' do
+    compressed.create do |s|
+      s.encode(Feedx::TestCase::Model.new('X'))
+      s.encode(Feedx::TestCase::Model.new('Y'))
+    end
+
+    compressed.open do |s|
+      expect(s.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
+      expect(s.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Y'))
+      expect(s.decode(Feedx::TestCase::Model)).to be_nil
+      expect(s).to be_eof
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -11,5 +11,31 @@ end
 module Feedx
   module TestCase
     Message = Google::Protobuf::DescriptorPool.generated_pool.lookup('com.blacksquaremedia.feedx.testcase.Message').msgclass
+
+    class Model
+      attr_reader :title
+
+      def initialize(title)
+        @title = title
+      end
+
+      def to_pb
+        Feedx::TestCase::Message.new title: @title
+      end
+
+      def ==(other)
+        title == other.title
+      end
+      alias eql? ==
+
+      def from_json(data, *)
+        hash = ::JSON.parse(data)
+        @title = hash['title'] if hash.is_a?(Hash)
+      end
+
+      def to_json(*)
+        ::JSON.dump(title: @title, updated_at: Time.at(1515151515).utc)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.7.0
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-20 00:00:00.000000000 Z
+date: 2019-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -147,6 +147,7 @@ files:
 - lib/feedx/format/protobuf.rb
 - lib/feedx/producer.rb
 - lib/feedx/pusher.rb
+- lib/feedx/stream.rb
 - producer.go
 - producer_test.go
 - reader.go
@@ -158,6 +159,7 @@ files:
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
 - spec/feedx/producer_spec.rb
+- spec/feedx/stream_spec.rb
 - spec/spec_helper.rb
 - writer.go
 - writer_test.go
@@ -192,4 +194,5 @@ test_files:
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
 - spec/feedx/producer_spec.rb
+- spec/feedx/stream_spec.rb
 - spec/spec_helper.rb