rodimus 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3db99d55f80c320e1e3502b90562d8c2791e4f21
-  data.tar.gz: 422f9092d2a028d66d1465665310e664077ee082
+  metadata.gz: 90a48da418f9255e96fa21a4c77c4f08ad4b706e
+  data.tar.gz: 72e832c9f3dbba8f5cb969883c071153a339666f
 SHA512:
-  metadata.gz: 483942b6d0d61c3904d03a8944492353f2e1702448639ac0b21cbf76637e5f111d7a37b3cd871bbd744f78544e5b3b0d6394580e2a157444e913cb6133b91953
-  data.tar.gz: 0fdfb929397cd4fb8db4a1cd038eb73b11e13e6367356ec284973893f9945aa7ce076bb87f6ef0b30ecc3d3006091897a0825f6c631fbaf3b8d1f8f759398d9c
+  metadata.gz: 95bda7a98743242537fe2a55a7f2ec154c1e285b7069f0cc3618e80a5aac3a4df8dc356f4b968b91c7d69c5d79697c7469920895f8046b0885c65d8bc134316b
+  data.tar.gz: 0617781ea52018694d4c54c2aacdee034eb07b8cc6df98f232f620bb92f84a597633b8ac7baa0ed5f494ecd95aeb8579f4c6fe2bd99c5166d0b44a0e2691b4fb

data/.ruby-version

@@ -1 +1 @@
-ruby-2.1.2
+ruby-2.2.0

data/.travis.yml

@@ -1,8 +1,10 @@
+before_install:
+  - gem install bundler
 language: ruby
 rvm:
-  - 2.1.2
+  - 2.2.0
+  - 2.1.5
   - 2.0.0
   - 1.9.3
-  - 1.9.2
   - jruby-19mode
   - rbx-2

data/README.md

@@ -38,7 +38,7 @@ destination.
 In Rodimus, you create a transformation object, and then you add
 one or more steps to its array of steps.  You typically create steps by writing 
 your own classes that inherit from Rodimus::Step.  When the transformation is
-subsequently run, a new process is forked for each step.  On steps that support 
+subsequently run, a new process is forked for each step.  On platforms that support 
 native threads (JRuby, Rubinius), threads are used instead of forking processes. 
 All processes are connected together using pipes except for the first and last 
 steps (those being the source and destination steps).  Each step then consumes

data/Rakefile

@@ -8,7 +8,14 @@ task :default => :test
 
 Rake::TestTask.new do |t|
   t.libs.push 'lib'
+  t.libs.push 'test'
   t.test_files = FileList['test/**/*_test.rb']
   t.verbose = true
 end
 
+desc 'Run a simulation on a large data set'
+task :simulate, :rows do |t, args|
+  require 'rodimus/simulation'
+  rows = (args[:rows] || 50_000).to_i
+  Rodimus::Simulation.run(rows)
+end

data/lib/rodimus.rb

@@ -4,6 +4,7 @@ require 'rodimus/observing'
 require 'rodimus/benchmark'
 require 'rodimus/runtime_logging'
 require 'rodimus/step'
+require 'rodimus/buffered_step'
 require 'rodimus/transformation'
 require 'rodimus/version'
 

data/lib/rodimus/buffered_step.rb

@@ -0,0 +1,34 @@
+module Rodimus
+
+  class BufferedStep < Step
+    # The maximum size of the buffer
+    attr_accessor :buffer_size
+    attr_reader   :buffer
+
+    def initialize(buffer_size = 100)
+      super()
+      @buffer_size = buffer_size
+      @buffer      = []
+    end
+
+    def close_descriptors # override
+      flush if buffer.any?
+      super
+    end
+
+    def handle_output(transformed_row) # override
+      buffer << transformed_row
+
+      if buffer.length >= buffer_size
+        flush
+      end
+    end
+
+    # Flush the contents of the buffer to the outgoing data stream
+    def flush
+      outgoing.puts(buffer.join("\n"))
+      @buffer = []
+    end
+  end
+
+end

data/lib/rodimus/simulation.rb

@@ -0,0 +1,26 @@
+require 'rodimus/simulation/row_generator'
+require 'rodimus/simulation/step1'
+require 'rodimus/simulation/step2'
+
+
+module Rodimus
+  module Simulation
+    def self.run(rows = 50_000)
+      Rodimus.configure do |config|
+        config.benchmarking = true
+      end
+
+      transformation = Transformation.new
+      Rodimus.logger.info "Generating data."
+      generator = RowGenerator.new(rows)
+      step1 = Step1.new(generator)
+      step2 = Step2.new
+
+      transformation.steps << step1
+      transformation.steps << step2
+
+      Rodimus.logger.info "Running transformation."
+      transformation.run
+    end
+  end
+end

data/lib/rodimus/simulation/row_generator.rb

@@ -0,0 +1,19 @@
+require 'faker'
+require 'forwardable'
+
+module Rodimus
+  module Simulation
+
+    class RowGenerator
+      extend Forwardable
+
+      def_delegator :@rows, :each
+
+      def initialize(count = 50_000)
+        @count = count
+        @rows = count.times.map { |i| Faker::Lorem.sentence }
+      end
+    end
+
+  end
+end

data/lib/rodimus/simulation/step1.rb

@@ -0,0 +1,18 @@
+require 'json'
+
+module Rodimus
+  module Simulation
+    
+    class Step1 < Rodimus::Step
+      def initialize(generator)
+        super()
+        @incoming = generator
+      end
+
+      def process_row(row)
+        row.split.sort.to_json
+      end
+    end
+
+  end
+end

data/lib/rodimus/simulation/step2.rb

@@ -0,0 +1,19 @@
+require 'json'
+
+module Rodimus
+  module Simulation
+
+    class Step2 < Rodimus::Step
+      def before_run_set_output
+        fd = IO.sysopen('/dev/null', 'w')
+        @outgoing = IO.new(fd, 'w')
+      end
+
+      def process_row(row)
+        row = JSON.parse(row)
+        row.map { |i| i.split('').sort.join('') }
+      end
+    end
+
+  end
+end

data/lib/rodimus/version.rb

@@ -1,3 +1,3 @@
 module Rodimus
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

data/rodimus.gemspec

@@ -12,13 +12,15 @@ Gem::Specification.new do |spec|
   spec.description           = "An ETL (Extract-Transform-Load) library that uses a forking process model for concurrency."
   spec.homepage              = "https://github.com/nevern02/rodimus"
   spec.license               = "MIT"
-  spec.required_ruby_version = ">= 1.9.2"
+  spec.required_ruby_version = ">= 1.9.3"
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.5"
-  spec.add_development_dependency "rake"
+  spec.add_development_dependency "bundler", "~> 1.8"
+  spec.add_development_dependency "rake", "~> 10.4"
+  spec.add_development_dependency "faker", "~> 1.4"
+  spec.add_development_dependency "minitest", "~> 5.4"
 end

data/test/rodimus/buffered_step_test.rb

@@ -0,0 +1,33 @@
+require 'test_helper'
+
+class TestBufferedStep < MiniTest::Test
+  attr_reader :incoming, :outgoing, :step
+
+  def setup
+    rows      = "row 1\nrow 2\nrow 3"
+    @incoming = StringIO.new(rows)
+    @outgoing = TestIO.new
+    @step     = Rodimus::BufferedStep.new
+    
+    step.incoming = incoming
+    step.outgoing = outgoing
+  end
+
+  def test_buffering_rows
+    step.buffer_size     = 3
+    expected_write_count = 1 # writing 3 rows
+
+    step.run
+
+    assert_equal expected_write_count, outgoing.history.length 
+  end
+
+  def test_flushing_rows
+    step.buffer_size     = 2
+    expected_write_count = 2
+
+    step.run
+
+    assert_equal expected_write_count, outgoing.history.length
+  end
+end

data/test/rodimus/observable_test.rb

@@ -1,7 +1,6 @@
-require 'minitest/autorun'
-require 'rodimus'
+require 'test_helper'
 
-class TestObservable < MiniTest::Unit::TestCase
+class TestObservable < MiniTest::Test
   def setup
     @fake_observer = Class.new do
       attr_reader :subject, :event_type

data/test/rodimus/observing_test.rb

@@ -1,7 +1,6 @@
-require 'minitest/autorun'
-require 'rodimus'
+require 'test_helper'
 
-class TestObserving < MiniTest::Unit::TestCase
+class TestObserving < MiniTest::Test
   def setup
     @observer = Class.new do
       include Rodimus::Observing

data/test/rodimus/step_test.rb

@@ -1,25 +1,6 @@
-require 'minitest/autorun'
-require 'rodimus'
+require 'test_helper'
 
-Rodimus.configure do |config|
-  config.logger = Logger.new(nil)
-end
-
-class TestIO < IO # Because we can't read closed StringIOs
-  attr_reader :history
-
-  def initialize
-    @history = []
-  end
-
-  def close; nil; end
-
-  def puts(string)
-    history << string
-  end
-end
-
-class TestStep < MiniTest::Unit::TestCase
+class TestStep < MiniTest::Test
   def setup
     @test_string = "row 1\nrow 2"
     @incoming = StringIO.new(@test_string)

data/test/rodimus/transformation_test.rb

@@ -1,9 +1,8 @@
-require 'minitest/autorun'
-require 'rodimus'
+require 'test_helper'
 
 module Rodimus
 
-  class TestTransformation < MiniTest::Unit::TestCase
+  class TestTransformation < MiniTest::Test
     Rodimus.configure do |config|
       config.logger = Logger.new(nil)
     end

data/test/test_helper.rb

@@ -0,0 +1,20 @@
+require 'minitest/autorun'
+require 'rodimus'
+
+Rodimus.configure do |config|
+  config.logger = Logger.new(nil)
+end
+
+class TestIO < IO # Because we can't read closed StringIOs
+  attr_reader :history
+
+  def initialize
+    @history = []
+  end
+
+  def close; nil; end
+
+  def puts(string)
+    history << string
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rodimus
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Brandon Rice
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-25 00:00:00.000000000 Z
+date: 2015-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,28 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.4'
+- !ruby/object:Gem::Dependency
+  name: faker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.4'
 description: An ETL (Extract-Transform-Load) library that uses a forking process model
   for concurrency.
 email:
@@ -63,18 +91,25 @@ files:
 - examples/worldbank-sample.csv
 - lib/rodimus.rb
 - lib/rodimus/benchmark.rb
+- lib/rodimus/buffered_step.rb
 - lib/rodimus/configuration.rb
 - lib/rodimus/observable.rb
 - lib/rodimus/observing.rb
 - lib/rodimus/runtime_logging.rb
+- lib/rodimus/simulation.rb
+- lib/rodimus/simulation/row_generator.rb
+- lib/rodimus/simulation/step1.rb
+- lib/rodimus/simulation/step2.rb
 - lib/rodimus/step.rb
 - lib/rodimus/transformation.rb
 - lib/rodimus/version.rb
 - rodimus.gemspec
+- test/rodimus/buffered_step_test.rb
 - test/rodimus/observable_test.rb
 - test/rodimus/observing_test.rb
 - test/rodimus/step_test.rb
 - test/rodimus/transformation_test.rb
+- test/test_helper.rb
 homepage: https://github.com/nevern02/rodimus
 licenses:
 - MIT
@@ -87,7 +122,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.2
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -95,13 +130,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: An ETL (Extract-Transform-Load) library that uses a forking process model
   for concurrency.
 test_files:
+- test/rodimus/buffered_step_test.rb
 - test/rodimus/observable_test.rb
 - test/rodimus/observing_test.rb
 - test/rodimus/step_test.rb
 - test/rodimus/transformation_test.rb
+- test/test_helper.rb