rodimus 0.1.2 → 1.0.0

data/examples/rails_log/run.rb

@@ -0,0 +1,16 @@
+require 'rodimus'
+require_relative 'log_input'
+require_relative 'parse_connection'
+require_relative 'file_output'
+
+log = File.expand_path('../rails_example.log', __FILE__)
+t = Rodimus::Transformation.new
+step1 = LogInput.new(log)
+step2 = ParseConnection.new
+step3 = FileOutput.new
+t.steps << step1
+t.steps << step2
+t.steps << step3
+t.run
+
+puts "Transformation complete!"

data/lib/rodimus.rb

@@ -1,4 +1,8 @@
 require 'rodimus/configuration'
+require 'rodimus/observable'
+require 'rodimus/observing'
+require 'rodimus/benchmark'
+require 'rodimus/runtime_logging'
 require 'rodimus/step'
 require 'rodimus/transformation'
 require 'rodimus/version'

data/lib/rodimus/benchmark.rb

@@ -0,0 +1,48 @@
+module Rodimus
+
+  class Benchmark
+    include Observing
+
+    attr_reader :stats
+
+    def on_notify(subject, event_type)
+      case event_type
+      when :before_run
+        initialize_stats
+      when :after_run
+        finalize_stats(subject)
+      when :before_row
+        before_row
+      when :after_row
+        after_row
+      end
+    end
+
+    private
+
+    def after_row
+      row_run_time = (Time.now.to_f - @start_time).round(4)
+      stats[:total] = (stats[:total] + row_run_time).round(4)
+      stats[:min] = row_run_time if stats[:min] > row_run_time
+      stats[:max] = row_run_time if stats[:max] < row_run_time
+    end
+
+    def before_row
+      stats[:count] += 1
+      @start_time = Time.now.to_f
+    end
+
+    def finalize_stats(subject)
+      if stats[:count] > 0
+        stats[:average] = (stats[:total] / stats[:count]).round(4)
+      end
+
+      Rodimus.logger.info "#{subject} benchmarks: #{stats}"
+    end
+
+    def initialize_stats
+      @stats = {count: 0, total: 0, min: 1, max: 0, average: 0}
+    end
+  end
+
+end

data/lib/rodimus/configuration.rb

@@ -3,10 +3,11 @@ require 'logger'
 module Rodimus
 
   class Configuration
-    attr_accessor :logger
+    attr_accessor :logger, :benchmarking
 
     def initialize
       @logger = Logger.new(STDOUT)
+      @benchmarking = false
     end
   end
 

data/lib/rodimus/observable.rb

@@ -0,0 +1,17 @@
+require 'set'
+
+module Rodimus
+
+  module Observable
+    def notify(subject, event)
+      observers.each do |observer|
+        observer.on_notify(subject, event)
+      end
+    end
+
+    def observers
+      @observers ||= Set.new
+    end
+  end
+
+end

data/lib/rodimus/observing.rb

@@ -0,0 +1,17 @@
+module Rodimus
+
+  module Observing
+    def on_notify(subject, event_type)
+      discovered_hooks(event_type).each do |hook|
+        self.send(hook)
+      end
+    end
+
+    private
+
+    def discovered_hooks(matcher)
+      methods.grep(/^#{matcher}/)
+    end
+  end
+
+end

data/lib/rodimus/runtime_logging.rb

@@ -0,0 +1,22 @@
+module Rodimus
+
+  module RuntimeLogging
+    attr_reader :start_time
+
+    def before_run_record_time
+      @start_time = Time.now.to_i
+      Rodimus.logger.info "Running #{self}"
+    end
+
+    def after_run_record_time
+      run_time = Time.now.to_i - start_time
+      remaining_seconds = run_time % 3600
+      elapsed_hours = run_time / 3600
+      elapsed_minuntes = remaining_seconds / 60
+      elapsed_seconds = remaining_seconds % 60
+
+      Rodimus.logger.info "Finished #{self} after #{elapsed_hours} hours, #{elapsed_minuntes} minutes, #{elapsed_seconds} seconds."
+    end
+  end
+
+end

data/lib/rodimus/step.rb

@@ -1,6 +1,10 @@
 module Rodimus
 
-  module Step
+  class Step
+    include Observable
+    include Observing # Steps observe themselves for run hooks
+    include RuntimeLogging
+
     # The incoming data stream.  Can be anything that quacks like an IO
     attr_accessor :incoming
 
@@ -11,15 +15,17 @@ module Rodimus
     # This is initialized by the Transformation when the step begins to run.
     attr_accessor :shared_data
 
+    def initialize
+      observers << self
+      observers << Benchmark.new if Rodimus.configuration.benchmarking
+    end
+
     def close_descriptors
       [incoming, outgoing].reject(&:nil?).each do |descriptor|
         descriptor.close if descriptor.respond_to?(:close)
       end
     end
 
-    # Override this for custom cleanup functionality.
-    def finalize; end
-
     # Override this for custom output handling functionality per-row.
     def handle_output(transformed_row)
       outgoing.puts(transformed_row)
@@ -31,16 +37,17 @@ module Rodimus
     end
 
     def run
-      Rodimus.logger.info "Running #{self}"
+      notify(self, :before_run)
       @row_count = 1
       incoming.each do |row|
+        notify(self, :before_row)
         transformed_row = process_row(row)
         handle_output(transformed_row)
         Rodimus.logger.info(self) { "#{@row_count} rows processed" } if @row_count % 50000 == 0
         @row_count += 1
+        notify(self, :after_row)
       end
-      finalize
-      Rodimus.logger.info "Finished #{self}"
+      notify(self, :after_run)
     ensure
       close_descriptors
     end

data/lib/rodimus/transformation.rb

@@ -3,6 +3,10 @@ require 'drb'
 module Rodimus
 
   class Transformation
+    include Observable
+    include Observing # Transformations observe themselves for run hooks
+    include RuntimeLogging
+
     attr_reader :drb_server, :pids, :steps
 
     # User-data accessible across all running steps.
@@ -12,9 +16,11 @@ module Rodimus
       @steps = []
       @pids = []
       @shared_data = {} # TODO: This needs to be thread safe
+      observers << self
     end
 
     def run
+      notify(self, :before_run)
       @drb_server = DRb.start_service(nil, shared_data)
       pids.clear
       prepare
@@ -30,6 +36,7 @@ module Rodimus
     ensure
       Process.waitall
       drb_server.stop_service
+      notify(self, :after_run)
     end
 
     def to_s
@@ -39,7 +46,6 @@ module Rodimus
     private
 
     def prepare
-      Rodimus.logger.info "Preparing #{self}..."
       # [1, 2, 3, 4] => [1, 2], [2, 3], [3, 4]
       steps.inject do |first, second|
         read, write = IO.pipe

data/lib/rodimus/version.rb

@@ -1,3 +1,3 @@
 module Rodimus
-  VERSION = "0.1.2"
+  VERSION = "1.0.0"
 end

data/test/rodimus/observing_test.rb

@@ -0,0 +1,26 @@
+require 'minitest/autorun'
+require 'rodimus'
+
+class TestObserving < MiniTest::Unit::TestCase
+  def setup
+    @observer = Class.new do
+      include Rodimus::Observing
+
+      attr_reader :called
+
+      def initialize
+        @called = false
+      end
+
+      def before_run_test
+        @called = true
+      end
+    end.new
+  end
+
+  def test_hook_discovery
+    refute @observer.called
+    @observer.on_notify(self, :before_run)
+    assert @observer.called
+  end
+end

data/test/rodimus/step_test.rb

@@ -0,0 +1,44 @@
+require 'minitest/autorun'
+require 'rodimus'
+
+Rodimus.configure do |config|
+  config.logger = Logger.new(nil)
+end
+
+class TestIO < IO # Because we can't read closed StringIOs
+  attr_reader :history
+
+  def initialize
+    @history = []
+  end
+
+  def close; nil; end
+
+  def puts(string)
+    history << string
+  end
+end
+
+class TestStep < MiniTest::Unit::TestCase
+  def setup
+    @test_string = "row 1\nrow 2"
+    @incoming = StringIO.new(@test_string)
+    @outgoing = TestIO.new
+    @step = Rodimus::Step.new
+    @step.incoming = @incoming
+    @step.outgoing = @outgoing
+  end
+
+  def test_streaming_rows
+    @step.run
+    assert_equal @test_string, @outgoing.history.join
+  end
+
+  def test_process_row
+    @step.define_singleton_method(:process_row) do |row|
+      row.upcase
+    end
+    @step.run
+    assert_equal @test_string.upcase, @outgoing.history.join
+  end
+end

data/test/{transformation_test.rb → rodimus/transformation_test.rb}

@@ -11,16 +11,13 @@ module Rodimus
     def test_forking_processes
       incoming = StringIO.new
       transformation = Transformation.new
-      steps = []
       number_of_steps = 2 + rand(5)
-      number_of_steps.times { steps << Object.new }
-      steps.each do |step| 
-        step.extend(Rodimus::Step) 
-        transformation.steps << step
+      number_of_steps.times do 
+        transformation.steps << Rodimus::Step.new
       end
-      steps.first.incoming = incoming
+      transformation.steps.first.incoming = incoming
       transformation.run
-      assert_equal(steps.count, transformation.pids.count)
+      assert_equal(transformation.steps.count, transformation.pids.count)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rodimus
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 1.0.0
 platform: ruby
 authors:
 - Brandon Rice
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-22 00:00:00.000000000 Z
+date: 2014-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -55,15 +55,25 @@ files:
 - README.md
 - Rakefile
 - examples/csv_input_stdout.rb
+- examples/rails_log/file_output.rb
+- examples/rails_log/log_input.rb
+- examples/rails_log/parse_connection.rb
+- examples/rails_log/rails_example.log
+- examples/rails_log/run.rb
 - examples/worldbank-sample.csv
 - lib/rodimus.rb
+- lib/rodimus/benchmark.rb
 - lib/rodimus/configuration.rb
+- lib/rodimus/observable.rb
+- lib/rodimus/observing.rb
+- lib/rodimus/runtime_logging.rb
 - lib/rodimus/step.rb
 - lib/rodimus/transformation.rb
 - lib/rodimus/version.rb
 - rodimus.gemspec
-- test/step_test.rb
-- test/transformation_test.rb
+- test/rodimus/observing_test.rb
+- test/rodimus/step_test.rb
+- test/rodimus/transformation_test.rb
 homepage: https://github.com/nevern02/rodimus
 licenses:
 - MIT
@@ -90,5 +100,6 @@ specification_version: 4
 summary: An ETL (Extract-Transform-Load) library that uses a forking process model
   for concurrency.
 test_files:
-- test/step_test.rb
-- test/transformation_test.rb
+- test/rodimus/observing_test.rb
+- test/rodimus/step_test.rb
+- test/rodimus/transformation_test.rb

data/test/step_test.rb

@@ -1,48 +0,0 @@
-require 'minitest/autorun'
-require 'rodimus'
-
-module Rodimus
-  Rodimus.configure do |config|
-    config.logger = Logger.new(nil)
-  end
-
-  class TestIO < IO
-    attr_reader :history
-
-    def initialize
-      @history = []
-    end
-
-    def close; nil; end
-
-    def puts(string)
-      history << string
-    end
-  end
-  
-  class TestStep < MiniTest::Unit::TestCase
-    def setup
-      @test_string = "row 1\nrow 2"
-      @incoming = StringIO.new(@test_string)
-      @outgoing = TestIO.new
-      @step = Object.new
-      @step.extend(Rodimus::Step)
-      @step.incoming = @incoming
-      @step.outgoing = @outgoing
-    end
-
-    def test_streaming_rows
-      @step.run
-      assert_equal @test_string, @outgoing.history.join
-    end
-
-    def test_process_row
-      @step.define_singleton_method(:process_row) do |row|
-        row.upcase
-      end
-      @step.run
-      assert_equal @test_string.upcase, @outgoing.history.join
-    end
-  end
-
-end