rodimus 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5492a6a648768bd138246fd1137174a1d424da95
-  data.tar.gz: 6f8a62c081ba530005953def9e3be5e95b56a8e5
+  metadata.gz: 86ccd8d549d00dbd8a6c1ba56ce9eb1fe41aa496
+  data.tar.gz: efdf0cac67bec3a2c2cd469164c9f7bdc812ebd2
 SHA512:
-  metadata.gz: a2d871ad7e844d562a60b0d39d08712b3a53ad56cc866326780c60cf86615aaecdd57da6fa05f84cb02644848ab4876ab47e2b1b45431424a839c4ffdf459c13
-  data.tar.gz: 4ce9fa3f5d99e63aef2643f25b56f50ad14081cd1cae52f8b2223de5e8184e06effc91d7da19d9e85ab98e5f329ddb831ba442bbf30cb888f0fd71e566ebe977
+  metadata.gz: fbfe473e374f6aa661d3cffca3c313b5811f3f332bdfd2860919022806c679383d7b20053ef5e56c4ee2a2fb8f13814c678b14463d0ba17a27b7dc9b76714344
+  data.tar.gz: 1d45b630f57c78f956f7b170ca5d2e8533bda42e88ad3240df112c197579bb95a6032e08deadc1ffe653a1a393eb61553f475815d4d178a1b5c82251afd478b7

data/.travis.yml

@@ -3,4 +3,4 @@ rvm:
   - ruby-head
   - 2.0.0
   - 1.9.3
-  - jruby-head
+  - 1.9.2

data/README.md

@@ -31,7 +31,33 @@ Or install it yourself as:
 
 ## Usage
 
-See the examples directory for the quickest path to success.
+tl;dr: See the examples directory for the quickest path to success.
+
+A transformation is an operation that consists of many steps.  Each step may
+manipulate the data in some way.  Typically, the first step is reserved for
+reading from your data source, and the last step is used to write to the new
+destination.  
+
+In Rodimus, you create a transformation object, and then you add
+one or more steps to its array of steps.  You typically create steps by writing 
+your own classes that include the Rodimus::Step mixin.  When the transformation is
+subsequently run, a new process is forked for each step.  All processes are
+connected together using pipes except for the first and last steps (those being the
+source and destination steps).  Each step then consumes rows of data from its
+incoming pipe and performs some operation on it before writing it to the
+outgoing pipe.  
+
+There are several methods on the Rodimus::Step mixin that are able to be
+overridden for custom processing behavior before, during, or after the each
+row is handled.  If those aren't enough, you're also free to manipulate the
+input/output objects (i.e. to redirect to standard out).
+
+The Rodimus approach is to provide a minimal, flexible framework upon which
+custom ETL solutions can be built.  ETL is complex, and there tend to be many
+subtle differences between projects which can make things like establishing
+conventions and encouraging code reuse difficult.  Rodimus is an attempt to
+codify those things which are probably useful to a majority of ETL projects
+with as little overhead as possible.
 
 ## Contributing
 

data/examples/csv_input_stdout.rb

@@ -0,0 +1,37 @@
+require 'rodimus'
+require 'csv'
+require 'json'
+
+class CsvInput
+  include Rodimus::Step
+
+  def initialize
+    @incoming = CSV.open('worldbank-sample.csv')
+    @incoming.readline # skip the headers
+  end
+
+  def process_row(row)
+    row.to_json
+  end
+end
+
+class FormattedText
+  include Rodimus::Step
+
+  def initialize
+    @outgoing = STDOUT.dup
+  end
+
+  def process_row(row)
+    data = JSON.parse(row)
+    "In #{data.first} during #{data[1]}, CO2 emissions were #{data[2]} metric tons  per capita." 
+  end
+end
+
+t = Rodimus::Transformation.new
+s1 = CsvInput.new
+s2 = FormattedText.new
+t.steps << s1
+t.steps << s2
+t.run
+puts "Transformation complete!"

data/examples/worldbank-sample.csv

@@ -0,0 +1,49 @@
+Country,Year,CO2 emissions (metric tons per capita),Electric power consumption (kWh per capita),Energy use (kg of oil equivalent per capita),"Fertility rate, total (births per woman)","GNI per capita, Atlas method (current US$)","Internet users (per 1,000 people)","Life expectancy at birth, total (years)",Military expenditure (% of GDP),"Population, total","Prevalence of HIV, total (% of population ages 15-49)"
+Belarus,2000,5.91,2988.71,2459.67,1.29,1.38E+03,18.69,68.01,1.26,1.00E+07,
+Belarus,2001,5.87,2996.81,2476.16,,1300,43.15,,1.44,9970260,
+Belarus,2002,6.03,2982.77,2539.95,1.25,1370,89.8,68.21,1.4,9925000,
+Belarus,2003,6.33,3039.1,2628.83,1.25,1610,162.76,,1.3,9873968,0.31
+Belarus,2004,,3143.58,2725.44,1.24,2150,250.51,68.39,1.36,9824469,
+Belarus,2005,,,,1.24,2760,347.23,68.48,1.24,9775591,0.34
+Philippines,2000,1.03,514.02,559.83,,1.06E+03,20.33,69.53,1.08,7.58E+07,
+Philippines,2001,0.99,535.18,539.4,,1.05E+03,25.89,,0.99,7.72E+07,
+Philippines,2002,0.99,539.74,537.47,3.5,1.03E+03,44.47,70.19,0.98,7.87E+07,
+Philippines,2003,0.96,574.47,524.9,,1.08E+03,49.9,,1.03,8.02E+07,0.1
+Philippines,2004,,597.06,542.39,,1.20E+03,53.91,,0.9,8.16E+07,
+Philippines,2005,,,,3.2,1.29E+03,,71.04,0.82,8.31E+07,0.1
+Morocco,2000,1.2,489.04,350.32,2.62,1.19E+03,7.03,68.81,4.13,2.85E+07,
+Morocco,2001,1.32,508.1,370.93,2.5,1.20E+03,13.87,,4.08,2.88E+07,
+Morocco,2002,1.32,526.4,373.38,2.5,1.19E+03,23.99,69.48,4.27,2.92E+07,
+Morocco,2003,1.28,562.93,370.01,2.5,1.34E+03,33.87,,4.22,2.95E+07,0.09
+Morocco,2004,,594.3,383.8,2.43,1.57E+03,117.3,,4.54,2.98E+07,
+Morocco,2005,,,,2.4,1.75E+03,152.61,70.38,4.28,3.01E+07,0.1
+Afghanistan,2000,0.04,,,,,,,,,
+Afghanistan,2001,0.03,,,,,,,,,
+Afghanistan,2002,0.03,,,,,0.04,,,,
+Afghanistan,2003,0.03,,,,,0.73,,,,0.1
+Afghanistan,2004,,,,,,0.87,,,,
+Afghanistan,2005,,,,,,1,,,,0.1
+Nicaragua,2000,0.77,349.37,558.5,3.54,760,10.16,68.87,0.78,4920286,
+Nicaragua,2001,0.8,349.6,559.75,,760,15.03,,0.71,4991475,
+Nicaragua,2002,0.78,373.83,552.24,3.3,760,17.82,69.48,0.87,5050368,
+Nicaragua,2003,0.77,373.59,613.75,,790,19.62,,0.87,5096507,0.21
+Nicaragua,2004,,416.57,643.39,,870,24.4,,0.7,5122841,
+Nicaragua,2005,,,,3.08,950,27.19,70.39,0.69,5149311,0.24
+"Korea, Dem. Rep.",2000,3.52,747.14,903.3,2.02E+00,,,63.1,,2.19E+07,
+"Korea, Dem. Rep.",2001,3.63,772.66,928.63,,,0,,,2.20E+07,
+"Korea, Dem. Rep.",2002,3.43,751.85,882.16,2.00E+00,,,63.04,,2.21E+07,
+"Korea, Dem. Rep.",2003,3.48,793.87,894.09,,,,,,2.23E+07,0.2
+"Korea, Dem. Rep.",2004,,826.54,910.17,,,,,,2.24E+07,
+"Korea, Dem. Rep.",2005,,,,1.96E+00,,,63.92,,2.25E+07,0.2
+Kyrgyz Republic,2000,0.94,1687.38,497.43,2.4,280,10.5,68.56,2.85,4915300,
+Kyrgyz Republic,2001,0.78,1443.05,451.08,2.4,280,30.39,68.71,2.35,4954800,
+Kyrgyz Republic,2002,0.99,1365.26,507.89,2.5,290,30.44,68.11,2.73,4993200,
+Kyrgyz Republic,2003,1.06,1646.69,531.5,2.5,340,39.69,68.26,2.87,5038600,0.1
+Kyrgyz Republic,2004,,1421.22,545.87,2.45,400,51.64,68.15,2.85,5092802,
+Kyrgyz Republic,2005,,,,2.41,450,54.44,68.34,,5143500,0.14
+Middle East & North Africa,2000,3.21,1075.27,1076.66,3.27,1.66E+03,9.02,68.14,4.32,2.79E+08,
+Middle East & North Africa,2001,3.21,1122.92,1104.58,,1.71E+03,11.96,,4.59,2.84E+08,
+Middle East & North Africa,2002,3.5,1176.9,1145.34,3.11,1.70E+03,28.94,68.79,4.18,2.90E+08,
+Middle East & North Africa,2003,3.43,1221.77,1157.82,,1.81E+03,41.89,,4.29,2.95E+08,0.12
+Middle East & North Africa,2004,,1290.24,1190.15,,1.98E+03,60.28,,4.28,3.00E+08,
+Middle East & North Africa,2005,,,,2.97,2.22E+03,88.9,69.67,3.73,3.06E+08,0.15

data/lib/rodimus/step.rb

@@ -32,9 +32,12 @@ module Rodimus
 
     def run
       Rodimus.logger.info "Running #{self}"
+      @row_count = 1
       incoming.each do |row|
         transformed_row = process_row(row)
         handle_output(transformed_row)
+        Rodimus.logger.info(self) { "#{@row_count} rows processed" } if @row_count % 50000 == 0
+        @row_count += 1
       end
       finalize
       Rodimus.logger.info "Finished #{self}"
@@ -43,7 +46,7 @@ module Rodimus
     end
 
     def to_s
-      "#{self.class} connected to input: #{incoming} and output: #{outgoing}"
+      "#{self.class} connected to input: #{incoming || 'nil'} and output: #{outgoing || 'nil'}"
     end
   end
 

data/lib/rodimus/transformation.rb

@@ -3,22 +3,24 @@ require 'drb'
 module Rodimus
 
   class Transformation
-    attr_reader :drb_server, :steps
+    attr_reader :drb_server, :pids, :steps
 
     # User-data accessible across all running steps.
     attr_reader :shared_data
 
     def initialize
       @steps = []
+      @pids = []
       @shared_data = {} # TODO: This needs to be thread safe
     end
 
     def run
       @drb_server = DRb.start_service(nil, shared_data)
+      pids.clear
       prepare
 
       steps.each do |step|
-        fork do
+        pids << fork do
           DRb.start_service # the parent DRb thread dies across the fork
           step.shared_data = DRbObject.new_with_uri(drb_server.uri)
           step.run

data/lib/rodimus/version.rb

@@ -1,3 +1,3 @@
 module Rodimus
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/rodimus.gemspec

@@ -4,14 +4,15 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'rodimus/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "rodimus"
-  spec.version       = Rodimus::VERSION
-  spec.authors       = ["Brandon Rice"]
-  spec.email         = ["brice84@gmail.com"]
-  spec.summary       = "An ETL (Extract-Transform-Load) library that uses a forking process model for concurrency."
-  spec.description   = "An ETL (Extract-Transform-Load) library that uses a forking process model for concurrency."
-  spec.homepage      = "https://github.com/nevern02/rodimus"
-  spec.license       = "MIT"
+  spec.name                  = "rodimus"
+  spec.version               = Rodimus::VERSION
+  spec.authors               = ["Brandon Rice"]
+  spec.email                 = ["brice84@gmail.com"]
+  spec.summary               = "An ETL (Extract-Transform-Load) library that uses a forking process model for concurrency."
+  spec.description           = "An ETL (Extract-Transform-Load) library that uses a forking process model for concurrency."
+  spec.homepage              = "https://github.com/nevern02/rodimus"
+  spec.license               = "MIT"
+  spec.required_ruby_version = ">= 1.9.2"
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
@@ -20,5 +21,4 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "mongo"
 end

data/test/transformation_test.rb

@@ -0,0 +1,27 @@
+require 'minitest/autorun'
+require 'rodimus'
+
+module Rodimus
+
+  class TestTransformation < MiniTest::Unit::TestCase
+    Rodimus.configure do |config|
+      config.logger = Logger.new(nil)
+    end
+
+    def test_forking_processes
+      incoming = StringIO.new
+      transformation = Transformation.new
+      steps = []
+      number_of_steps = 2 + rand(5)
+      number_of_steps.times { steps << Object.new }
+      steps.each do |step| 
+        step.extend(Rodimus::Step) 
+        transformation.steps << step
+      end
+      steps.first.incoming = incoming
+      transformation.run
+      assert_equal(steps.count, transformation.pids.count)
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rodimus
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Brandon Rice
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-20 00:00:00.000000000 Z
+date: 2014-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: mongo
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: An ETL (Extract-Transform-Load) library that uses a forking process model
   for concurrency.
 email:
@@ -68,7 +54,8 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- examples/mongo_input.rb
+- examples/csv_input_stdout.rb
+- examples/worldbank-sample.csv
 - lib/rodimus.rb
 - lib/rodimus/configuration.rb
 - lib/rodimus/step.rb
@@ -76,6 +63,7 @@ files:
 - lib/rodimus/version.rb
 - rodimus.gemspec
 - test/step_test.rb
+- test/transformation_test.rb
 homepage: https://github.com/nevern02/rodimus
 licenses:
 - MIT
@@ -88,7 +76,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -103,3 +91,4 @@ summary: An ETL (Extract-Transform-Load) library that uses a forking process mod
   for concurrency.
 test_files:
 - test/step_test.rb
+- test/transformation_test.rb

data/examples/mongo_input.rb

@@ -1,40 +0,0 @@
-require 'rodimus'
-require 'mongo'
-require 'json'
-
-class MongoInput
-  attr_reader :client, :db, :collection
-
-  include Rodimus::Step
-
-  def initialize
-    @client = Mongo::MongoClient.new('localhost', 27017)
-    @db = client['inventory_events']
-    @collection = db['model_events']
-    @incoming = collection.find.limit(4)
-  end
-
-  def process_row(row)
-    row.to_json
-  end
-end
-
-class TempfileOut
-  include Rodimus::Step
-
-  def initialize
-    @outgoing = File.new('output.txt', 'w')
-  end
-
-  def process_row(row)
-    JSON.parse(row).keys.join(',')
-  end
-end
-
-t = Rodimus::Transformation.new
-s1 = MongoInput.new
-s2 = TempfileOut.new
-t.steps << s1
-t.steps << s2
-t.run
-puts "Transformation to #{s2.outgoing.path} complete!"