rodimus 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7fc221f69a67693a1f94810a09b297e345a9248c
+  data.tar.gz: b2fc619efd740b162e0968d92c30ce4f8eb9562d
+SHA512:
+  metadata.gz: 9ce961dff9778ea69aaf5593084840f7239228bb9e720a31325e961db15ee8fde1aa1d57ab5b7e53bfc390fdeb2cfc1bd019f9d68930855306ccc1d11b37dc06
+  data.tar.gz: 2b4e42e96f08dbd9c42b8a486cbcebd2a20ed7fe9c61cc3beeb16d8fcb772caeacf621e605f0ff8f6d426099b05205e1d06d371816ef61dd68d4d746b22e0633

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.*.swp

data/.ruby-gemset

@@ -0,0 +1 @@
+rodimus

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.1.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rb_etl.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Brandon Rice
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# Rodimus
+
+ETL stands for Extract-Transform-Load. Sometimes, you have data in Source A
+that needs to be moved to Destination B.  Along the way, it needs to be
+manipulated in some way.  This is a common scenario when working with a data
+warehouse.  There are lots of ETL solutions in the wild, but very few of them
+are open source.  None of them (that I know of) are Ruby.  So, I started
+hacking on one for my own use.
+
+__Why the name?__ Rodimus Prime is one of the leaders of the Autobots, and he
+has a cool name.  Naming a data transformation library after a Transformer
+increases the coolness factor.  It's science.
+
+__NOTE:__ This library is still in the earliest phases of development.  Things
+are prone to change suddenly and rapidly.  Use at your own risk.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rodimus'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rodimus
+
+## Usage
+
+See the examples directory for the quickest path to success.
+
+## Contributing
+
+1. Fork it ( http://github.com/nevern02/rodimus/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,14 @@
+$: << File.expand_path('../lib', __FILE__)
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'rodimus'
+
+task :default => :test
+
+Rake::TestTask.new do |t|
+  t.libs.push 'lib'
+  t.test_files = FileList['test/*_test.rb']
+  t.verbose = true
+end
+

data/examples/mongo_input.rb

@@ -0,0 +1,40 @@
+require 'rodimus'
+require 'mongo'
+require 'json'
+
+class MongoInput
+  attr_reader :client, :db, :collection
+
+  include Rodimus::Step
+
+  def initialize
+    @client = Mongo::MongoClient.new('localhost', 27017)
+    @db = client['inventory_events']
+    @collection = db['model_events']
+    @incoming = collection.find.limit(4)
+  end
+
+  def process_row(row)
+    row.to_json
+  end
+end
+
+class TempfileOut
+  include Rodimus::Step
+
+  def initialize
+    @outgoing = File.new('output.txt', 'w')
+  end
+
+  def process_row(row)
+    JSON.parse(row).keys.join(',')
+  end
+end
+
+t = Rodimus::Transformation.new
+s1 = MongoInput.new
+s2 = TempfileOut.new
+t.steps << s1
+t.steps << s2
+t.run
+puts "Transformation to #{s2.outgoing.path} complete!"

data/lib/rodimus/step.rb

@@ -0,0 +1,30 @@
+module Rodimus
+
+  module Step
+    attr_accessor :incoming, :outgoing
+
+    def run
+      incoming.each do |row|
+        transformed_row = process_row(row)
+        handle_output(transformed_row)
+      end
+      finalize
+    end
+
+    private
+
+    # Override this for custom functionality
+    def finalize; end
+
+    # Override this for custom functionality
+    def handle_output(transformed_row)
+      outgoing.puts(transformed_row)
+    end
+
+    # Override this for custom functionality
+    def process_row(row)
+      row.to_s
+    end
+  end
+
+end

data/lib/rodimus/transformation.rb

@@ -0,0 +1,37 @@
+module Rodimus
+
+  class Transformation
+    attr_reader :steps
+
+    def initialize
+      @steps = []
+    end
+
+    def run
+      prepare
+
+      steps.each do |step|
+        fork do
+          step.run
+        end
+        step.incoming && step.incoming.close
+        step.outgoing && step.outgoing.close
+      end
+
+      Process.waitall
+    end
+
+    private
+
+    def prepare
+      # [1, 2, 3, 4] => [1, 2], [2, 3], [3, 4]
+      steps.inject do |first, second|
+        read, write = IO.pipe
+        first.outgoing = write
+        second.incoming = read
+        second
+      end
+    end
+  end
+
+end

data/lib/rodimus/version.rb

@@ -0,0 +1,3 @@
+module Rodimus
+  VERSION = "0.0.1"
+end

data/lib/rodimus.rb

@@ -0,0 +1,6 @@
+require 'rodimus/version'
+require 'rodimus/step'
+require 'rodimus/transformation'
+
+module Rodimus
+end

data/rodimus.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rodimus/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rodimus"
+  spec.version       = Rodimus::VERSION
+  spec.authors       = ["Brandon Rice"]
+  spec.email         = ["brice84@gmail.com"]
+  spec.summary       = "An ETL (Extract-Transform-Load) library that uses a forking process model for concurrency."
+  spec.description   = "ETL is hard.  There are lots of solutions, but few are open-source and none (that I know of) are Ruby."
+  spec.homepage      = "https://github.com/nevern02/rodimus"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "mongo"
+end

data/test/step_test.rb

@@ -0,0 +1,40 @@
+require 'minitest/autorun'
+require 'rodimus'
+
+module Rodimus
+  
+  class TestStep < MiniTest::Unit::TestCase
+    def test_streaming_rows
+      test_string = "row 1\nrow 2"
+      step = Object.new
+      step.extend(Rodimus::Step)
+      step.define_singleton_method(:transform) { |i| i }
+      incoming = StringIO.new(test_string)
+      outgoing = StringIO.new
+      step.incoming = incoming
+      step.outgoing = outgoing
+      step.run
+      outgoing.rewind
+      assert_equal test_string, outgoing.read.chomp
+    end
+
+    def test_transformation_called
+      test_string = "row 1\nrow 2"
+      step = Class.new do
+        include Rodimus::Step
+
+        def process_row(row)
+          row.upcase
+        end
+      end.new
+      incoming = StringIO.new(test_string)
+      outgoing = StringIO.new
+      step.incoming = incoming
+      step.outgoing = outgoing
+      step.run
+      outgoing.rewind
+      assert_equal test_string.upcase, outgoing.read.chomp
+    end
+  end
+
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: rodimus
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Brandon Rice
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ETL is hard.  There are lots of solutions, but few are open-source and
+  none (that I know of) are Ruby.
+email:
+- brice84@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/mongo_input.rb
+- lib/rodimus.rb
+- lib/rodimus/step.rb
+- lib/rodimus/transformation.rb
+- lib/rodimus/version.rb
+- rodimus.gemspec
+- test/step_test.rb
+homepage: https://github.com/nevern02/rodimus
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: An ETL (Extract-Transform-Load) library that uses a forking process model
+  for concurrency.
+test_files:
+- test/step_test.rb