feedx 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 93c6d8da8f0485d4cde2b935d55d176bd85265ed2983d5fda8b86e229635c666
+  data.tar.gz: 0d32c1595c97c42937bbdb8765da4347d8041fc6e4db8c335a88a26c715eed02
+SHA512:
+  metadata.gz: 689dd5925778d4895cd1dbfd35bf717070dbfdbe4fe3c831d01108ccef637fb1eaeb1166945b152393cff653f71426dbdd326de78633b6ed4aab36640e11693e
+  data.tar.gz: ea603ed38643a9484875d626ff9be7b42ae0a56f20e7a85ccdcbb0ca8101846084bfc4cdb4373793df3d4be36ca31eb92e0351186c85712937d88f1b46677e07

data/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true

data/.gitignore

@@ -0,0 +1,2 @@
+.rubocop-*
+pkg/

data/.rubocop.yml

@@ -0,0 +1,5 @@
+inherit_from:
+  - https://storage.googleapis.com/bsm-misc/rubocop.yml?1541767352
+
+AllCops:
+  TargetRubyVersion: "2.2"

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.5
+  - 2.4

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,59 @@
+PATH
+  remote: .
+  specs:
+    feedx (0.1.0)
+      bfs
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    bfs (0.2.2)
+    diff-lcs (1.3)
+    google-protobuf (3.6.1)
+    jaro_winkler (1.5.1)
+    parallel (1.12.1)
+    parser (2.5.3.0)
+      ast (~> 2.4.0)
+    pbio (0.1.0)
+      google-protobuf
+    powerpack (0.1.2)
+    rainbow (3.0.0)
+    rake (12.3.1)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubocop (0.60.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.4.0)
+    ruby-progressbar (1.10.0)
+    unicode-display_width (1.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  feedx!
+  pbio
+  rake
+  rspec
+  rubocop
+
+BUNDLED WITH
+   1.16.4

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2018 Black Square Media Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,23 @@
+# Feedx
+
+[![Build Status](https://travis-ci.org/bsm/feedx.png?branch=master)](https://travis-ci.org/bsm/feedx)
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+
+Feed-based data exchange between services.
+
+## Usage
+
+```ruby
+require 'bfs/s3'
+require 'feedx'
+
+# Init a new pusher with an S3 destination
+relation = Post.includes(:author)
+pusher = Feedx::Pusher.new relation, 's3://my-bucket/feeds/users.json.gz'
+
+# Push a new feed every hour
+loop do
+  pusher.perform
+  sleep(3600)
+end
+```

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/setup'
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new(:rubocop)
+
+task default: %i[spec rubocop]

data/feedx.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |s|
+  s.name          = 'feedx'
+  s.version       = '0.1.0'
+  s.authors       = ['Black Square Media Ltd']
+  s.email         = ['info@blacksquaremedia.com']
+  s.summary       = %(Exchange data between components via feeds)
+  s.description   = %(Use feeds to exchange data between (micro-)services.)
+  s.homepage      = 'https://github.com/bsm/feedx'
+  s.license       = 'Apache-2.0'
+
+  s.files         = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^spec/}) }
+  s.test_files    = `git ls-files -z -- spec/*`.split("\x0")
+  s.require_paths = ['lib']
+  s.required_ruby_version = '>= 2.2'
+
+  s.add_dependency 'bfs'
+
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'pbio'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'rubocop'
+end

data/lib/feedx.rb

@@ -0,0 +1,5 @@
+module Feedx
+  autoload :Compression, 'feedx/compression'
+  autoload :Format, 'feedx/format'
+  autoload :Pusher, 'feedx/pusher'
+end

data/lib/feedx/compression.rb

@@ -0,0 +1,28 @@
+module Feedx
+  module Compression
+    autoload :Abstract, 'feedx/compression/abstract'
+    autoload :None, 'feedx/compression/none'
+    autoload :Gzip, 'feedx/compression/gzip'
+
+    class << self
+      def resolve(name)
+        case name.to_s
+        when 'gz', 'gzip'
+          Gzip
+        when ''
+          None
+        else
+          raise ArgumentError, "invalid compression #{name}"
+        end
+      end
+
+      def detect(path)
+        if File.extname(path)[-1] == 'z'
+          Gzip
+        else
+          None
+        end
+      end
+    end
+  end
+end

data/lib/feedx/compression/abstract.rb

@@ -0,0 +1,5 @@
+class Feedx::Compression::Abstract
+  def self.wrap(_io, &_block)
+    raise 'Not implemented'
+  end
+end

data/lib/feedx/compression/gzip.rb

@@ -0,0 +1,7 @@
+require 'zlib'
+
+class Feedx::Compression::Gzip < Feedx::Compression::Abstract
+  def self.wrap(io, &block)
+    Zlib::GzipWriter.wrap(io, &block)
+  end
+end

data/lib/feedx/compression/none.rb

@@ -0,0 +1,5 @@
+class Feedx::Compression::None < Feedx::Compression::Abstract
+  def self.wrap(io, &block)
+    block.call(io)
+  end
+end

data/lib/feedx/format.rb

@@ -0,0 +1,37 @@
+module Feedx
+  module Format
+    autoload :Abstract, 'feedx/format/abstract'
+    autoload :JSON, 'feedx/format/json'
+    autoload :Protobuf, 'feedx/format/protobuf'
+
+    class << self
+      def resolve(name)
+        _resolve(name) || raise(ArgumentError, "invalid format #{name}")
+      end
+
+      def detect(path)
+        base = File.basename(path)
+        loop do
+          ext = File.extname(base)
+          raise ArgumentError, 'unable to detect format' if ext.empty?
+
+          kind = _resolve(ext[1..-1]) || _resolve(ext[1..-2])
+          return kind if kind
+
+          base = base[0..-ext.size - 1]
+        end
+      end
+
+      private
+
+      def _resolve(name)
+        case name.to_s
+        when 'pb', 'proto', 'protobuf'
+          Protobuf
+        when 'json'
+          JSON
+        end
+      end
+    end
+  end
+end

data/lib/feedx/format/abstract.rb

@@ -0,0 +1,9 @@
+class Feedx::Format::Abstract
+  def initialize(io)
+    @io = io
+  end
+
+  def write(_msg)
+    raise 'Not implemented'
+  end
+end

data/lib/feedx/format/json.rb

@@ -0,0 +1,7 @@
+require 'json'
+
+class Feedx::Format::JSON < Feedx::Format::Abstract
+  def write(msg)
+    @io.write msg.to_json << "\n"
+  end
+end

data/lib/feedx/format/protobuf.rb

@@ -0,0 +1,11 @@
+require 'pbio'
+
+class Feedx::Format::Protobuf < Feedx::Format::Abstract
+  def initialize(io)
+    super PBIO::Delimited.new(io)
+  end
+
+  def write(msg)
+    @io.write msg.to_pb
+  end
+end

data/lib/feedx/pusher.rb

@@ -0,0 +1,62 @@
+require 'uri'
+require 'bfs'
+
+module Feedx
+  # Pushes a relation as a protobuf encoded stream to an S3 location.
+  class Pusher
+    # @param [Enumerable,ActiveRecord::Relation] relation to stream.
+    # @param [String] url the destination URL.
+    # @param [Hash] opts options
+    # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
+    # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
+    def initialize(relation, url, opts={})
+      @relation = relation
+      @blob     = BFS::Blob.new(url)
+      @format   = detect_format(opts[:format])
+      @compress = detect_compress(opts[:compress])
+    end
+
+    def perform
+      @blob.create do |io|
+        @compress.wrap(io) {|w| write_to(w) }
+      end
+      @blob.info.size
+    end
+
+    private
+
+    def detect_format(val)
+      case val
+      when nil
+        Feedx::Format.detect(@blob.path)
+      when Class
+        parent = Feedx::Format::Abstract
+        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
+
+        val
+      else
+        Feedx::Format.resolve(val)
+      end
+    end
+
+    def detect_compress(val)
+      case val
+      when nil
+        Feedx::Compression.detect(@blob.path)
+      when Class
+        parent = Feedx::Compression::Abstract
+        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
+
+        val
+      else
+        Feedx::Compression.resolve(val)
+      end
+    end
+
+    def write_to(io)
+      stream = @format.new(io)
+      enum = @relation.respond_to?(:find_each) ? :find_each : :each
+      @relation.send(enum) {|rec| stream.write(rec) }
+    end
+  end
+end

data/spec/feedx/compression/gzip_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Compression::Gzip do
+  it 'should wrap' do
+    io = StringIO.new
+    described_class.wrap(io) {|w| w.write 'xyz' * 1000 }
+    expect(io.size).to be_within(20).of(40)
+  end
+end

data/spec/feedx/compression/none_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Compression::None do
+  it 'should wrap' do
+    io = StringIO.new
+    described_class.wrap(io) {|w| w.write 'xyz' * 1000 }
+    expect(io.size).to eq(3000)
+  end
+end

data/spec/feedx/compression_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Compression do
+  it 'should resolve' do
+    expect(described_class.resolve(:gzip)).to eq(described_class::Gzip)
+    expect(described_class.resolve(:gz)).to eq(described_class::Gzip)
+    expect(described_class.resolve(nil)).to eq(described_class::None)
+    expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
+  end
+
+  it 'should detect' do
+    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::Gzip)
+    expect(described_class.detect('path/to/file.json')).to eq(described_class::None)
+    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb.gz')).to eq(described_class::Gzip)
+    expect(described_class.detect('path/to/file.pb')).to eq(described_class::None)
+  end
+end

data/spec/feedx/format/json_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Format::JSON do
+  subject  { described_class.new(io) }
+  let(:io) { StringIO.new }
+
+  it 'should write' do
+    subject.write(a: 1, b: 2)
+    subject.write(c: ['x'], d: true)
+    expect(io.string).to eq %({"a":1,"b":2}\n{"c":["x"],"d":true}\n)
+  end
+end

data/spec/feedx/format/protobuf_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Format::Protobuf do
+  subject  { described_class.new(io) }
+  let(:io) { StringIO.new }
+
+  let(:model) do
+    Class.new Struct.new(:title) do
+      def to_pb
+        Feedx::TestCase::Message.new title: title
+      end
+    end
+  end
+
+  it 'should write' do
+    subject.write(model.new('X'))
+    subject.write(model.new('Y'))
+    expect(io.string.bytes).to eq([3, 10, 1, 88] + [3, 10, 1, 89])
+  end
+end

data/spec/feedx/format_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Format do
+  it 'should resolve' do
+    expect(described_class.resolve(:json)).to eq(described_class::JSON)
+    expect(described_class.resolve(:pb)).to eq(described_class::Protobuf)
+    expect { described_class.resolve(:txt) }.to raise_error(/invalid format txt/)
+  end
+
+  it 'should detect' do
+    expect(described_class.detect('path/to/file.json')).to eq(described_class::JSON)
+    expect(described_class.detect('path/to/file.jsonz')).to eq(described_class::JSON)
+    expect(described_class.detect('path/to/file.json.gz')).to eq(described_class::JSON)
+    expect(described_class.detect('path/to/file.pb')).to eq(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pbz')).to eq(described_class::Protobuf)
+    expect(described_class.detect('path/to/file.pb.z')).to eq(described_class::Protobuf)
+    expect do
+      described_class.detect('path/to/file.txt')
+    end.to raise_error(/unable to detect format/)
+  end
+end

data/spec/feedx/pusher_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+RSpec.describe Feedx::Pusher do
+  let :model do
+    Class.new Struct.new(:title) do
+      def to_pb
+        Feedx::TestCase::Message.new title: title
+      end
+
+      def to_json
+        ::JSON.dump(title: title, updated_at: Time.at(1515151515).utc)
+      end
+    end
+  end
+
+  let :relation do
+    %w[x y z].map {|_t| model.new('t') } * 100
+  end
+
+  let(:tempdir) { Dir.mktmpdir }
+  after { FileUtils.rm_rf tempdir }
+
+  it 'should reject invalid inputs' do
+    expect do
+      described_class.new relation, "file://#{tempdir}/file.txt"
+    end.to raise_error(/unable to detect format/)
+  end
+
+  it 'should push compressed JSON' do
+    pusher = described_class.new relation, "file://#{tempdir}/file.jsonz"
+    size   = pusher.perform
+    expect(size).to be_within(20).of(140)
+    expect(File.size("#{tempdir}/file.jsonz")).to eq(size)
+  end
+
+  it 'should push plain JSON' do
+    pusher = described_class.new relation, "file://#{tempdir}/file.json"
+    size = pusher.perform
+    expect(size).to be_within(0).of(15900)
+    expect(File.size("#{tempdir}/file.json")).to eq(size)
+  end
+
+  it 'should push compressed PB' do
+    pusher = described_class.new relation, "file://#{tempdir}/file.pbz"
+    size = pusher.perform
+    expect(size).to be_within(20).of(41)
+    expect(File.size("#{tempdir}/file.pbz")).to eq(size)
+  end
+
+  it 'should push plain PB' do
+    pusher = described_class.new relation, "file://#{tempdir}/file.pb"
+    size = pusher.perform
+    expect(size).to be_within(0).of(1200)
+    expect(File.size("#{tempdir}/file.pb")).to eq(size)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require 'rspec'
+require 'feedx'
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message 'com.blacksquaremedia.feedx.testcase.Message' do
+    optional :title, :string, 1
+  end
+end
+
+module Feedx
+  module TestCase
+    Message = Google::Protobuf::DescriptorPool.generated_pool.lookup('com.blacksquaremedia.feedx.testcase.Message').msgclass
+  end
+end

metadata

@@ -0,0 +1,164 @@
+--- !ruby/object:Gem::Specification
+name: feedx
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Black Square Media Ltd
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-11-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bfs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pbio
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Use feeds to exchange data between (micro-)services.
+email:
+- info@blacksquaremedia.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".editorconfig"
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- feedx.gemspec
+- lib/feedx.rb
+- lib/feedx/compression.rb
+- lib/feedx/compression/abstract.rb
+- lib/feedx/compression/gzip.rb
+- lib/feedx/compression/none.rb
+- lib/feedx/format.rb
+- lib/feedx/format/abstract.rb
+- lib/feedx/format/json.rb
+- lib/feedx/format/protobuf.rb
+- lib/feedx/pusher.rb
+- spec/feedx/compression/gzip_spec.rb
+- spec/feedx/compression/none_spec.rb
+- spec/feedx/compression_spec.rb
+- spec/feedx/format/json_spec.rb
+- spec/feedx/format/protobuf_spec.rb
+- spec/feedx/format_spec.rb
+- spec/feedx/pusher_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/bsm/feedx
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Exchange data between components via feeds
+test_files:
+- spec/feedx/compression/gzip_spec.rb
+- spec/feedx/compression/none_spec.rb
+- spec/feedx/compression_spec.rb
+- spec/feedx/format/json_spec.rb
+- spec/feedx/format/protobuf_spec.rb
+- spec/feedx/format_spec.rb
+- spec/feedx/pusher_spec.rb
+- spec/spec_helper.rb