rodimus 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b41faf166609c0301fa56b7b675f3be73b4a986d
-  data.tar.gz: dd8f0d1d9a73cb73c38dfea42e448cf36ab926ec
+  metadata.gz: 3db99d55f80c320e1e3502b90562d8c2791e4f21
+  data.tar.gz: 422f9092d2a028d66d1465665310e664077ee082
 SHA512:
-  metadata.gz: cdf745cb5fb1645602df330d5f32dc1121932c288ea1ddc8e4b8e40095ad7a1a6928b69272688a61089a6a356a2e439846c35e8abbac90700d159664a137213a
-  data.tar.gz: f7830584a6e02a39a3b82ebcd680762ff415ae7a971e832f5a5c805cf560a4b6a3f66179a50c4c2a27bc4a8d3374290dc2f1413993e5406bcda5ed7c44d00d59
+  metadata.gz: 483942b6d0d61c3904d03a8944492353f2e1702448639ac0b21cbf76637e5f111d7a37b3cd871bbd744f78544e5b3b0d6394580e2a157444e913cb6133b91953
+  data.tar.gz: 0fdfb929397cd4fb8db4a1cd038eb73b11e13e6367356ec284973893f9945aa7ce076bb87f6ef0b30ecc3d3006091897a0825f6c631fbaf3b8d1f8f759398d9c

data/.travis.yml

@@ -4,3 +4,5 @@ rvm:
   - 2.0.0
   - 1.9.3
   - 1.9.2
+  - jruby-19mode
+  - rbx-2

data/README.md

@@ -38,11 +38,12 @@ destination.
 In Rodimus, you create a transformation object, and then you add
 one or more steps to its array of steps.  You typically create steps by writing 
 your own classes that inherit from Rodimus::Step.  When the transformation is
-subsequently run, a new process is forked for each step.  All processes are
-connected together using pipes except for the first and last steps (those being the
-source and destination steps).  Each step then consumes rows of data from its
-incoming pipe and performs some operation on it before writing it to the
-outgoing pipe.  
+subsequently run, a new process is forked for each step.  On steps that support 
+native threads (JRuby, Rubinius), threads are used instead of forking processes. 
+All processes are connected together using pipes except for the first and last 
+steps (those being the source and destination steps).  Each step then consumes
+rows of data from its incoming pipe and performs some operation on it before
+writing it to the outgoing pipe.  
 
 There are several methods on the Rodimus::Step class that are able to be
 overridden for custom processing behavior before, during, or after the each

data/lib/rodimus.rb

@@ -8,8 +8,6 @@ require 'rodimus/transformation'
 require 'rodimus/version'
 
 module Rodimus
-  $SAFE = 1 # Because we're using DRb
-
   class << self
     attr_accessor :configuration
   end
@@ -22,4 +20,8 @@ module Rodimus
   def self.logger
     configuration.logger
   end
+
+  unless Rodimus.configuration.use_threads
+    $SAFE = 1 # Because we're using DRb
+  end
 end

data/lib/rodimus/configuration.rb

@@ -3,11 +3,19 @@ require 'logger'
 module Rodimus
 
   class Configuration
-    attr_accessor :logger, :benchmarking
+    attr_accessor :logger
+
+    # Set to true for extra output with step performance details
+    attr_accessor :benchmarking
+
+    # Use threads for concurrency instead of forking processes.
+    # Automatically set to true for JRuby and Rubinius
+    attr_accessor :use_threads
 
     def initialize
       @logger = Logger.new(STDOUT)
       @benchmarking = false
+      @use_threads = ['jruby', 'rbx'].include?(RUBY_ENGINE)
     end
   end
 

data/lib/rodimus/transformation.rb

@@ -7,35 +7,37 @@ module Rodimus
     include Observing # Transformations observe themselves for run hooks
     include RuntimeLogging
 
-    attr_reader :drb_server, :pids, :steps
+    attr_reader :drb_server, :steps
+
+    # Contains the thread or process identifiers currently in use
+    attr_reader :ids
 
     # User-data accessible across all running steps.
     attr_reader :shared_data
 
     def initialize
       @steps = []
-      @pids = []
+      @ids = []
       @shared_data = {} # TODO: This needs to be thread safe
       observers << self
     end
 
+    # Run the transformation
     def run
       notify(self, :before_run)
-      @drb_server = DRb.start_service(nil, shared_data)
-      pids.clear
+      @drb_server = DRb.start_service(nil, shared_data) unless using_threads?
+      ids.clear
       prepare
 
       steps.each do |step|
-        pids << fork do
-          DRb.start_service # the parent DRb thread dies across the fork
-          step.shared_data = DRbObject.new_with_uri(drb_server.uri)
+        ids << in_parallel do
+          step.shared_data = step_shared_data
           step.run
         end
-        step.close_descriptors
+        step.close_descriptors unless using_threads?
       end
     ensure
-      Process.waitall
-      drb_server.stop_service
+      cleanup
       notify(self, :after_run)
     end
 
@@ -45,6 +47,23 @@ module Rodimus
 
     private
 
+    def cleanup
+      if using_threads?
+        ids.each { |t| t.join }
+      else
+        Process.waitall
+        drb_server.stop_service
+      end
+    end
+    
+    def in_parallel 
+      if using_threads?
+        Thread.start { yield }
+      else
+        fork { yield }
+      end
+    end
+
     def prepare
       # [1, 2, 3, 4] => [1, 2], [2, 3], [3, 4]
       steps.inject do |first, second|
@@ -54,6 +73,19 @@ module Rodimus
         second
       end
     end
+
+    def step_shared_data
+      if using_threads?
+        shared_data
+      else
+        DRb.start_service # service dies across forked process
+        DRbObject.new_with_uri(drb_server.uri)
+      end
+    end
+
+    def using_threads?
+      Rodimus.configuration.use_threads
+    end
   end
 
 end

data/lib/rodimus/version.rb

@@ -1,3 +1,3 @@
 module Rodimus
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
 end

data/test/rodimus/transformation_test.rb

@@ -8,7 +8,7 @@ module Rodimus
       config.logger = Logger.new(nil)
     end
 
-    def test_forking_processes
+    def test_parallel_steps
       incoming = StringIO.new
       transformation = Transformation.new
       number_of_steps = 2 + rand(5)
@@ -17,7 +17,7 @@ module Rodimus
       end
       transformation.steps.first.incoming = incoming
       transformation.run
-      assert_equal(transformation.steps.count, transformation.pids.count)
+      assert_equal(transformation.steps.count, transformation.ids.count)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rodimus
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Brandon Rice
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-23 00:00:00.000000000 Z
+date: 2014-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler