tem_mr_search 0.3 → 0.3.1

data/CHANGELOG

@@ -1,3 +1,5 @@
+v0.3.1. Optimized Map-Reduce scheduler. 1.5x speed increase.
+
 v0.3. SECpack binding and migration. Now we're secure.
 
 v0.2.5. Symmetric encryption for partial outputs.

data/lib/tem_mr_search/client.rb

@@ -37,16 +37,25 @@ class Client
   #   server_addr:: string with the address of the Map-Reduce server's RPC port.
   #   client_query:: a ClientQuery instance expressing the Map-Reduce search
   #
-  # Returns the result of the Map-Reduce computation.
+  # Returns a hash with the following keys:
+  #   :result:: the result of the Map-Reduce computation
+  #   :timings:: timing statistics on the job's execution 
   def self.search(server_addr, client_query)
-    tem_info = get_tem server_addr
-    # TODO: check the endorsement certificate.
-    client_query.bind tem_info[:pubek]
+    tem_certs = {}
+    tem_ids = {}
+    [:mapper, :reducer, :finalizer].each do |sec|
+      tem_info = get_tem server_addr
+      tem_ids[sec] = tem_info[:id]
+      # TODO: check the endorsement certificate.
+      tem_certs[sec] = tem_info[:pubek]
+    end
+    client_query.bind tem_certs
     
     output = issue_request server_addr, :type => :search,
-                                        :root_tem => tem_info[:id],
+                                        :root_tems => tem_ids,
                                         :map_reduce => client_query.to_hash
-    output ? client_query.unpack_output(output) : nil
+    return nil unless output
+    output.merge! :result => client_query.unpack_output(output[:result])
   end
   
   # Asks for an item in the server's database.

data/lib/tem_mr_search/map_reduce_executor.rb

@@ -23,9 +23,11 @@ class MapReduceExecutor
   #   job:: the Map-Reduce job (see Tem::Mr::Search::MapReduceJob)
   #   db:: the database to run Map-Reduce over
   #   tems:: sessions to the available TEMs
-  #   root_tem:: the index of the TEM that has the root mapper and reducer
+  #   root_tems:: the indexes of the TEMs that have the initial SECpacks bound
+  #               to them (hash with the keys +:mapper+, +:reducer+ and
+  #               +:finalizer+)
   #   planner_class:: (optional) replacement for the default planner strategy
-  def initialize(job, db, tems, root_tem, planner_class = nil)
+  def initialize(job, db, tems, root_tems, planner_class = nil)
     planner_class ||= MapReducePlanner
     
     @db = db  # Writable only in main thread.
@@ -35,15 +37,20 @@ class MapReduceExecutor
     @tem_certs = Array.new @tems.length
 
     # Writable only in main thread.
-    @planner = planner_class.new job, db.length, tems.length, root_tem
+    @planner = planner_class.new job, db.length, tems.length, root_tems
     
     # Protected by @lock
-    @tem_parts = { :mapper => { root_tem => job.mapper },
-                   :reducer => { root_tem => job.reducer },
-                   :finalizer => { root_tem => job.finalizer } }
+    @tem_parts = { :mapper => { root_tems[:mapper] => job.mapper },
+                   :reducer => {root_tems[:reducer] => job.reducer },
+                   :finalizer => { root_tems[:finalizer] => job.finalizer } }
     # Protected by @lock
     @outputs = {}
     
+    # Protected by @lock
+    @timings = { :tems => Array.new(@tems.length, 0.0),
+                 :tasks => { :map => 0.0, :reduce => 0.0, :finalize => 0.0,
+                             :migrate => 0.0 } }
+    
     # Thread-safe.
     @thread_queues = tems.map { |tem| Queue.new }
     @main_queue = Queue.new
@@ -51,6 +58,10 @@ class MapReduceExecutor
   end
 
   # Executes the job.
+  #
+  # Returns a hash with the following keys:
+  #   :result:: the job's result
+  #   :timings:: timing statistics on the job's execution 
   def execute
     collect_tem_ids
     
@@ -67,8 +78,8 @@ class MapReduceExecutor
       @planner.action_done action
     end
     
-    return @outputs[@planner.output_id]
-  end
+    return { :result => @outputs[@planner.output_id], :timings => @timings }
+  end  
 
   # Collects identification information from all the TEMs.
   def collect_tem_ids
@@ -95,14 +106,22 @@ class MapReduceExecutor
   #
   # This method is called on the thread corresponding to the TEM that the action
   # is supposed to execute on.
+  #
+  # The method's return value is unspecified. 
   def execute_action(action, tem_index)
     case action[:action]
     when :migrate
       in_part = @lock.synchronize { @tem_parts[action[:secpack]][tem_index] }
       target_ecert = @tem_certs[action[:to]]
+      
+      t0 = Time.now
       out_part = in_part.migrate target_ecert, @tems[tem_index]
+      time_delta = Time.now - t0
+      
       @lock.synchronize do
         @tem_parts[action[:secpack]][action[:to]] = out_part
+        @timings[:tems][tem_index] += time_delta
+        @timings[:tasks][:migrate] += time_delta
       end
       
     when :map
@@ -111,9 +130,15 @@ class MapReduceExecutor
         mapper = @tem_parts[:mapper][tem_index]
         item = @db.item(action[:item])
       end
+
+      t0 = Time.now
       output = mapper.map_object item, @tems[tem_index]
+      time_delta = Time.now - t0
+      
       @lock.synchronize do
         @outputs[action[:output_id]] = output
+        @timings[:tems][tem_index] += time_delta
+        @timings[:tasks][:map] += time_delta
       end
       
     when :reduce
@@ -123,9 +148,15 @@ class MapReduceExecutor
         output1 = @outputs[action[:output1_id]]
         output2 = @outputs[action[:output2_id]]
       end
+      
+      t0 = Time.now
       output = reducer.reduce_outputs output1, output2, @tems[tem_index]
+      time_delta = Time.now - t0
+      
       @lock.synchronize do
         @outputs[action[:output_id]] = output
+        @timings[:tems][tem_index] += time_delta
+        @timings[:tasks][:reduce] += time_delta
       end
 
     when :finalize
@@ -134,9 +165,15 @@ class MapReduceExecutor
         finalizer = @tem_parts[:finalizer][tem_index]
         output = @outputs[action[:output_id]]
       end
+      
+      t0 = Time.now
       final_output = finalizer.finalize_output output, @tems[tem_index]
+      time_delta = Time.now - t0
+      
       @lock.synchronize do
         @outputs[action[:final_id]] = final_output
+        @timings[:tems][tem_index] += time_delta
+        @timings[:tasks][:finalize] += time_delta
       end
     end
   end

data/lib/tem_mr_search/map_reduce_job.rb

@@ -48,9 +48,16 @@ class MapReduceJob
       :finalize => @finalizer.to_plain_object }
   end
   
-  # Binds the SECpacks in this job to the given key.
-  def bind(tem_pubek)
-    [@mapper, @reducer, @finalizer].each { |part| part.bind tem_pubek }
+  # Binds the SECpacks in this job to the given keys.
+  #
+  # Args:
+  #   tem_pubeks:: the public keys to bind the SECpacks to; hash with the keys
+  #                +:mapper+, +:reducer+ and +:finalizer+; each key is mapped to
+  #                a TEM key
+  def bind(tem_pubeks)
+    @mapper.bind tem_pubeks[:mapper]
+    @reducer.bind tem_pubeks[:reducer]
+    @finalizer.bind tem_pubeks[:finalizer]
   end
 
   # Base class for the Map-Reduce SECpack wrappers.

data/lib/tem_mr_search/map_reduce_planner.rb

@@ -36,19 +36,37 @@ class MapReducePlanner
   #   job:: the Map-Reduce job (see Tem::Mr::Search::MapReduceJob)
   #   num_items: how many data items does the Map-Reduce run over
   #   num_tems:: how many TEMs are available
-  #   root_tem:: the index of the TEM that has the root mapper and reducer
-  def initialize(job, num_items, num_tems, root_tem)
+  #   root_tems:: the indexes of the TEMs that have the initial SECpacks bound
+  #               to them (hash with the keys +:mapper+, +:reducer+ and
+  #               +:finalizer+)
+  def initialize(job, num_items, num_tems, root_tems)
     @job = job
-    @root_tem = root_tem
+    @root_tems = root_tems
     
     @without = { :mapper => RBTree.new, :reducer => RBTree.new }
-    @with = { :mapper => Set.new([root_tem]),
-              :reducer => Set.new([root_tem]) }
+    @with = { :mapper => Set.new([@root_tems[:mapper]]),
+              :reducer => Set.new([@root_tems[:reducer]]) }
     @free_tems = RBTree.new
-    0.upto(num_tems - 1) do |tem|
-      @free_tems[tem] = true
-      next if tem == root_tem
-      @without.each { |k, v| v[tem] = true }
+    
+    # TEM ordering: the mapper root is first, the reducer root is last, and the
+    #               finalizer root is second
+    @ordered_tems = (0...num_tems).to_a
+    @ordered_tems -= @root_tems.values
+    @ordered_tems = [@root_tems[:mapper]] + @ordered_tems
+    unless @ordered_tems.include? @root_tems[:reducer]
+      @ordered_tems += [@root_tems[:reducer]]
+    end
+    unless @ordered_tems.include? @root_tems[:finalizer]
+      @ordered_tems = [@ordered_tems[0], @root_tems[:finalizer]] +
+                       @ordered_tems[1..-1]
+    end
+    # Reverted index for the TEM ordering.
+    @rindex_tems = Array.new(num_tems)
+    @ordered_tems.each_with_index { |t, i| @rindex_tems[t] = i }
+    
+    @ordered_tems.each_with_index do |tem, i|
+      @free_tems[[i, tem]] = true
+      @without.each { |k, v| v[[i, tem]] = true unless tem == @root_tems[k] }
     end
     
     @unmapped_items = (0...num_items).to_a.reverse
@@ -125,10 +143,11 @@ class MapReducePlanner
     return actions if @without[sec_type].length == 0
     free_tems = free_tems_with_sec sec_type
     free_tems.each do |source_tem|
-      break if @without[sec_type].length == 0
-      target_tem = @without[sec_type].min.first
-      @without[sec_type].delete target_tem
-      @free_tems.delete source_tem
+      break if @without[sec_type].length == 0      
+      target_tem = (sec_type == :mapper ? @without[sec_type].min :
+                                          @without[sec_type].max).first.last
+      @without[sec_type].delete [@rindex_tems[target_tem], target_tem]
+      @free_tems.delete [@rindex_tems[source_tem], source_tem]
       actions.push :action => :migrate, :secpack => sec_type,
                    :with => source_tem, :to => target_tem
     end
@@ -138,7 +157,7 @@ class MapReducePlanner
   
   # Informs the planner that a SECpack migration has completed.
   def done_migrating(action)
-    @free_tems[action[:with]] = true
+    @free_tems[[@rindex_tems[action[:with]], action[:with]]] = true
     @with[action[:secpack]] << action[:to]
   end
   private :done_migrating
@@ -151,7 +170,7 @@ class MapReducePlanner
     return actions if @unmapped_items.empty?
     free_tems_with_sec(:mapper).each do |tem|
       break unless item = @unmapped_items.pop
-      @free_tems.delete tem
+      @free_tems.delete [@rindex_tems[tem], tem]
       actions.push :action => :map, :item => item, :with => tem,
                    :output_id => next_output_id
     end
@@ -166,7 +185,7 @@ class MapReducePlanner
   #
   # The return value is not specified.
   def done_mapping(action)
-    @free_tems[action[:with]] = true
+    @free_tems[[@rindex_tems[action[:with]], action[:with]]] = true
     @reduce_queue[action[:output_id]] = true
   end
   private :done_mapping
@@ -177,14 +196,14 @@ class MapReducePlanner
   def reduce_actions
     actions = []
     return actions if @reduce_queue.length <= 1
-    free_tems_with_sec(:reducer).each do |tem|
+    free_tems_with_sec(:reducer).reverse.each do |tem|
       break if @reduce_queue.length <= 1
       output1_id, output2_id = *[0, 1].map do |i|
         output_id = @reduce_queue.min.first
         @reduce_queue.delete output_id
         output_id
       end
-      @free_tems.delete tem
+      @free_tems.delete [@rindex_tems[tem], tem]
       actions.push :action => :reduce, :with => tem, :output1_id => output1_id,
                    :output2_id => output2_id, :output_id => next_output_id
     end
@@ -199,7 +218,7 @@ class MapReducePlanner
   #
   # The return value is not specified.
   def done_reducing(action)
-    @free_tems[action[:with]] = true
+    @free_tems[[@rindex_tems[action[:with]], action[:with]]] = true
     if action[:output_id] == @last_reduce_id
       @done_reducing = true      
       return
@@ -212,9 +231,13 @@ class MapReducePlanner
   #
   # See next_actions! for a description of the return value.
   def finalize_actions
-    return [] unless @done_reducing and !@output_id and @free_tems[@root_tem]
+    root_tem = @root_tems[:finalizer]
+    unless @done_reducing and !@output_id and
+           @free_tems[[@rindex_tems[root_tem], root_tem]]
+      return []
+    end
     @finalize_ready = false
-    return [ :action => :finalize, :with => @root_tem,
+    return [ :action => :finalize, :with => root_tem,
              :output_id => @last_reduce_id, :final_id => next_output_id ]
   end
   private :finalize_actions
@@ -226,7 +249,7 @@ class MapReducePlanner
   #
   # The return value is not specified.
   def done_finalizing(action)
-    @free_tems[action[:with]] = true
+    @free_tems[[@rindex_tems[action[:with]], action[:with]]] = true
     @output_id = action[:final_id]    
   end
   private :done_finalizing
@@ -237,8 +260,8 @@ class MapReducePlanner
   #   sec_type:: the SECpack type (+:mapper+ or +:reducer+)
   def free_tems_with_sec(sec_type)
     tems = []
-    @free_tems.each do |tem, true_value|
-      tems << tem if @with[sec_type].include? tem
+    @free_tems.each do |index_tem, true_value|
+      tems << index_tem.last if @with[sec_type].include? index_tem.last
     end
     tems
   end

data/lib/tem_mr_search/server.rb

@@ -85,8 +85,8 @@ class Server
     when :search
       refresh_tems!
       job = MapReduceJob.new request[:map_reduce]
-      root_tem = request[:root_tem]
-      executor = MapReduceExecutor.new job, @db, @tems, root_tem
+      root_tems = request[:root_tems]
+      executor = MapReduceExecutor.new job, @db, @tems, root_tems
       executor.execute
     when :fetch
       @db.item_by_id(request[:id]) || :not_found

data/tem_mr_search.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{tem_mr_search}
-  s.version = "0.3"
+  s.version = "0.3.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Victor Costan"]

data/test/test_client_server.rb

@@ -72,12 +72,13 @@ class ClientServerTest < MrTestCase
       [$tem]
     end
     _test_request do |server_addr|
-      result = Client.search server_addr, @client_query
+      data = Client.search server_addr, @client_query
       gold_item = @db.item 5
-      assert_equal fare_id(gold_item), result[:id],
+      assert_equal fare_id(gold_item), data[:result][:id],
                    'Incorrect Map-Reduce result (ID)'
-      assert_equal fare_score(gold_item), result[:score],
+      assert_equal fare_score(gold_item), data[:result][:score],
                    'Incorrect Map-Reduce result (score)'
+      assert data[:timings], 'No timing statistics in Map-Reduce result'
     end
   end
 end

data/test/test_map_reduce_executor.rb

@@ -17,20 +17,36 @@ class MapReduceExecutorTest < MrTestCase
     super
   end
       
-  def _test_executor(tems, root_tem)
-    @client_query.bind tems[root_tem].pubek
-    executor = MRExecutor.new @client_query, @db, tems, root_tem
-    packed_output = executor.execute
-    result = @client_query.unpack_output packed_output
+  def _test_executor(tems, root_tems)
+    certs = {}
+    [:mapper, :reducer, :finalizer].each do |sec|
+      certs[sec] = tems[root_tems[sec]].pubek
+    end
+    @client_query.bind certs
+    executor = MRExecutor.new @client_query, @db, tems, root_tems
+    data = executor.execute
+    result = @client_query.unpack_output data[:result]
+    
     gold_item = @db.item 5 
     assert_equal fare_id(gold_item), result[:id],
                  'Incorrect Map-Reduce result (ID)'
     assert_equal fare_score(gold_item), result[:score],
                  'Incorrect Map-Reduce result (score)'
+  
+    assert data[:timings], 'No timings returned'
+    assert data[:timings][:tasks], 'No tasks data in the timings'
+    [:map, :reduce, :finalize, :migrate].each do |task|
+      assert data[:timings][:tasks][task], "No data on #{task} in the timings"
+    end
+    assert_operator data[:timings][:tems], :kind_of?, Array,
+                    'No per-TEM data in the timings'
+
+    # Dump timing stats to show scheduler performance.
+    p data[:timings]
   end
   
   def test_executor_with_autoconf
-    _test_executor [$tem], 0
+    _test_executor [$tem], {:mapper => 0, :reducer => 0, :finalizer => 0}
   end
   
   def test_executor_with_cluster    
@@ -39,6 +55,6 @@ class MapReduceExecutorTest < MrTestCase
     
     tems.each { |tem| tem.emit if tem.activate }
     
-    _test_executor tems, 0
+    _test_executor tems, {:mapper => 0, :reducer => 7, :finalizer => 0}
   end
 end

data/test/test_map_reduce_planner.rb

@@ -31,25 +31,30 @@ class MapReducePlannerTest < Test::Unit::TestCase
     all_actions
   end
   
-  def _test_planning(method_name, items, tems, root_tem, gold_file)
-    planner = MRPlanner.new nil, items, tems, root_tem
+  def _test_planning(method_name, items, tems, root_tems, gold_file)
+    planner = MRPlanner.new nil, items, tems, root_tems
     all_actions = self.send method_name, planner
     gold_actions = File.open(File.join(@testdata_path, gold_file), 'r') do |f|
       YAML.load f
     end
     assert_equal gold_actions, all_actions, "Failed #{method_name}: " +
-        "#{tems} tems with root #{root_tem}, #{items} items"
+        "#{tems} tems with roots #{root_tems.inspect}, #{items} items"
     assert_equal items * 2 - 1, planner.output_id, "Wrong final output_id"
   end
   
   def test_planning
-    [[:parallel_planning, 7, 4, 0, 'parallel_plan_740.yml'],
-     [:parallel_planning, 4, 3, 1, 'parallel_plan_431.yml'],
-     [:serial_planning, 4, 1, 0, 'serial_plan_410.yml'],
-     [:serial_planning, 7, 4, 0, 'serial_plan_740.yml'],
-     [:serial_planning, 4, 3, 1, 'serial_plan_431.yml'],
+    [[:parallel_planning, 7, 4, { :mapper => 0, :reducer => 3, :finalizer => 1},
+      'parallel_plan_740.yml'],
+     [:parallel_planning, 4, 3, { :mapper => 1, :reducer => 2, :finalizer => 0},
+      'parallel_plan_431.yml'],
+     [:serial_planning, 4, 1, { :mapper => 0, :reducer => 0, :finalizer => 0},
+      'serial_plan_410.yml'],
+     [:serial_planning, 7, 4, { :mapper => 0, :reducer => 3, :finalizer => 0},
+      'serial_plan_740.yml'],
+     [:serial_planning, 4, 3, { :mapper => 0, :reducer => 2, :finalizer => 1},
+      'serial_plan_431.yml'],
     ].each do |testcase|
-      _test_planning *testcase
+      _test_planning(*testcase)
     end
   end  
 end

data/testdata/parallel_plan_431.yml

@@ -1,52 +1,52 @@
 --- 
-- - :secpack: :mapper
-    :action: :migrate
+- - :action: :migrate
     :with: 1
+    :secpack: :mapper
     :to: 0
-- - :secpack: :mapper
-    :action: :migrate
-    :with: 0
-    :to: 2
-  - :secpack: :reducer
-    :action: :migrate
-    :with: 1
+  - :action: :migrate
+    :with: 2
+    :secpack: :reducer
     :to: 0
-- - :secpack: :reducer
-    :action: :migrate
-    :with: 0
-    :to: 2
-  - :output_id: 0
-    :action: :map
+- - :action: :migrate
     :with: 1
+    :secpack: :mapper
+    :to: 2
+  - :action: :migrate
+    :with: 0
+    :secpack: :reducer
+    :to: 1
+- - :action: :map
     :item: 0
-  - :output_id: 1
-    :action: :map
-    :with: 2
+    :with: 1
+    :output_id: 0
+  - :action: :map
     :item: 1
-- - :output_id: 2
-    :action: :map
     :with: 0
+    :output_id: 1
+  - :action: :map
     :item: 2
-  - :output_id: 3
-    :action: :map
-    :with: 1
+    :with: 2
+    :output_id: 2
+- - :action: :map
     :item: 3
-  - :output1_id: 0
-    :output2_id: 1
+    :with: 1
+    :output_id: 3
+  - :action: :reduce
+    :with: 2
     :output_id: 4
-    :action: :reduce
+    :output1_id: 0
+    :output2_id: 1
+- - :action: :reduce
     :with: 2
-- - :output1_id: 2
-    :output2_id: 3
     :output_id: 5
-    :action: :reduce
-    :with: 0
-- - :output1_id: 4
-    :output2_id: 5
+    :output1_id: 2
+    :output2_id: 3
+- - :action: :reduce
+    :with: 2
     :output_id: 6
-    :action: :reduce
+    :output1_id: 4
+    :output2_id: 5
+- - :action: :finalize
     :with: 0
-- - :output_id: 6
-    :action: :finalize
-    :with: 1
+    :output_id: 6
     :final_id: 7

data/testdata/parallel_plan_740.yml

@@ -1,87 +1,87 @@
 --- 
-- - :secpack: :mapper
-    :action: :migrate
+- - :action: :migrate
     :with: 0
+    :secpack: :mapper
     :to: 1
-- - :secpack: :mapper
-    :action: :migrate
+  - :action: :migrate
+    :with: 3
+    :secpack: :reducer
+    :to: 2
+- - :action: :migrate
     :with: 0
+    :secpack: :mapper
     :to: 2
-  - :secpack: :mapper
-    :action: :migrate
+  - :action: :migrate
     :with: 1
+    :secpack: :mapper
     :to: 3
-- - :secpack: :reducer
-    :action: :migrate
-    :with: 0
-    :to: 1
-  - :output_id: 0
-    :action: :map
-    :with: 1
-    :item: 0
-  - :output_id: 1
-    :action: :map
+  - :action: :migrate
     :with: 2
-    :item: 1
-  - :output_id: 2
-    :action: :map
+    :secpack: :reducer
+    :to: 1
+  - :action: :migrate
     :with: 3
-    :item: 2
-- - :secpack: :reducer
-    :action: :migrate
+    :secpack: :reducer
+    :to: 0
+- - :action: :map
+    :item: 0
     :with: 0
-    :to: 2
-  - :secpack: :reducer
-    :action: :migrate
+    :output_id: 0
+  - :action: :map
+    :item: 1
     :with: 1
-    :to: 3
-  - :output_id: 3
-    :action: :map
+    :output_id: 1
+  - :action: :map
+    :item: 2
     :with: 2
+    :output_id: 2
+  - :action: :map
     :item: 3
-  - :output_id: 4
-    :action: :map
     :with: 3
+    :output_id: 3
+- - :action: :map
     :item: 4
-- - :output_id: 5
-    :action: :map
     :with: 0
+    :output_id: 4
+  - :action: :map
     :item: 5
-  - :output_id: 6
-    :action: :map
     :with: 1
+    :output_id: 5
+  - :action: :map
     :item: 6
-  - :output1_id: 0
-    :output2_id: 1
-    :output_id: 7
-    :action: :reduce
     :with: 2
-  - :output1_id: 2
-    :output2_id: 3
-    :output_id: 8
-    :action: :reduce
+    :output_id: 6
+  - :action: :reduce
     :with: 3
-- - :output1_id: 4
-    :output2_id: 5
+    :output_id: 7
+    :output1_id: 0
+    :output2_id: 1
+- - :action: :reduce
+    :with: 3
+    :output_id: 8
+    :output1_id: 2
+    :output2_id: 3
+  - :action: :reduce
+    :with: 2
     :output_id: 9
-    :action: :reduce
-    :with: 0
-  - :output1_id: 6
-    :output2_id: 7
-    :output_id: 10
-    :action: :reduce
+    :output1_id: 4
+    :output2_id: 5
+  - :action: :reduce
     :with: 1
-- - :output1_id: 8
-    :output2_id: 9
+    :output_id: 10
+    :output1_id: 6
+    :output2_id: 7
+- - :action: :reduce
+    :with: 3
     :output_id: 11
-    :action: :reduce
-    :with: 0
-- - :output1_id: 10
+    :output1_id: 8
+    :output2_id: 9
+- - :action: :reduce
+    :with: 3
+    :output_id: 12
+    :output1_id: 10
     :output2_id: 11
+- - :action: :finalize
+    :with: 1
     :output_id: 12
-    :action: :reduce
-    :with: 0
-- - :output_id: 12
-    :action: :finalize
-    :with: 0
     :final_id: 13

data/testdata/serial_plan_410.yml

@@ -1,36 +1,36 @@
 --- 
-- - :output_id: 0
-    :action: :map
-    :with: 0
+- - :action: :map
     :item: 0
-- - :output_id: 1
-    :action: :map
     :with: 0
+    :output_id: 0
+- - :action: :map
     :item: 1
-- - :output_id: 2
-    :action: :map
     :with: 0
+    :output_id: 1
+- - :action: :map
     :item: 2
-- - :output_id: 3
-    :action: :map
     :with: 0
+    :output_id: 2
+- - :action: :map
     :item: 3
-- - :output1_id: 0
-    :output2_id: 1
+    :with: 0
+    :output_id: 3
+- - :action: :reduce
+    :with: 0
     :output_id: 4
-    :action: :reduce
+    :output1_id: 0
+    :output2_id: 1
+- - :action: :reduce
     :with: 0
-- - :output1_id: 2
-    :output2_id: 3
     :output_id: 5
-    :action: :reduce
+    :output1_id: 2
+    :output2_id: 3
+- - :action: :reduce
     :with: 0
-- - :output1_id: 4
-    :output2_id: 5
     :output_id: 6
-    :action: :reduce
-    :with: 0
-- - :output_id: 6
-    :action: :finalize
+    :output1_id: 4
+    :output2_id: 5
+- - :action: :finalize
     :with: 0
+    :output_id: 6
     :final_id: 7

data/testdata/serial_plan_431.yml

@@ -1,56 +1,56 @@
 --- 
-- - :secpack: :mapper
-    :action: :migrate
-    :with: 1
-    :to: 0
-- - :secpack: :mapper
-    :action: :migrate
+- - :action: :migrate
     :with: 0
-    :to: 2
-  - :secpack: :reducer
-    :action: :migrate
-    :with: 1
-    :to: 0
-- - :output_id: 0
-    :action: :map
+    :secpack: :mapper
+    :to: 1
+  - :action: :migrate
+    :with: 2
+    :secpack: :reducer
+    :to: 1
+- - :action: :migrate
     :with: 0
+    :secpack: :mapper
+    :to: 2
+  - :action: :map
     :item: 0
-  - :output_id: 1
-    :action: :map
+    :with: 1
+    :output_id: 0
+- - :action: :migrate
     :with: 2
+    :secpack: :reducer
+    :to: 0
+- - :action: :map
     :item: 1
-- - :secpack: :reducer
-    :action: :migrate
-    :with: 1
-    :to: 2
-- - :output_id: 2
-    :action: :map
     :with: 0
+    :output_id: 1
+- - :action: :map
     :item: 2
-- - :output_id: 3
-    :action: :map
-    :with: 2
+    :with: 1
+    :output_id: 2
+- - :action: :map
     :item: 3
-- - :output1_id: 0
-    :output2_id: 1
+    :with: 2
+    :output_id: 3
+- - :action: :reduce
+    :with: 0
     :output_id: 4
-    :action: :reduce
-    :with: 1
+    :output1_id: 0
+    :output2_id: 1
 - []
 
-- - :output1_id: 2
-    :output2_id: 3
+- - :action: :reduce
+    :with: 2
     :output_id: 5
-    :action: :reduce
-    :with: 0
+    :output1_id: 2
+    :output2_id: 3
 - []
 
-- - :output1_id: 4
-    :output2_id: 5
+- - :action: :reduce
+    :with: 2
     :output_id: 6
-    :action: :reduce
-    :with: 0
-- - :output_id: 6
-    :action: :finalize
+    :output1_id: 4
+    :output2_id: 5
+- - :action: :finalize
     :with: 1
+    :output_id: 6
     :final_id: 7

data/testdata/serial_plan_740.yml

@@ -1,93 +1,93 @@
 --- 
-- - :secpack: :mapper
-    :action: :migrate
+- - :action: :migrate
     :with: 0
+    :secpack: :mapper
     :to: 1
-- - :secpack: :mapper
-    :action: :migrate
+  - :action: :migrate
+    :with: 3
+    :secpack: :reducer
+    :to: 2
+- - :action: :migrate
     :with: 0
+    :secpack: :mapper
     :to: 2
-  - :secpack: :mapper
-    :action: :migrate
+  - :action: :migrate
     :with: 1
+    :secpack: :mapper
     :to: 3
-- - :secpack: :reducer
-    :action: :migrate
-    :with: 0
-    :to: 1
-  - :output_id: 0
-    :action: :map
+- - :action: :migrate
     :with: 2
+    :secpack: :reducer
+    :to: 1
+  - :action: :migrate
+    :with: 3
+    :secpack: :reducer
+    :to: 0
+- - :action: :map
     :item: 0
-- - :output_id: 1
-    :action: :map
-    :with: 1
+    :with: 0
+    :output_id: 0
+- - :action: :map
     :item: 1
-  - :output_id: 2
-    :action: :map
-    :with: 3
+    :with: 1
+    :output_id: 1
+- - :action: :map
     :item: 2
-- - :secpack: :reducer
-    :action: :migrate
-    :with: 0
-    :to: 2
-- - :output_id: 3
-    :action: :map
     :with: 2
+    :output_id: 2
+- - :action: :map
     :item: 3
-- - :secpack: :reducer
-    :action: :migrate
-    :with: 1
-    :to: 3
-- - :output_id: 4
-    :action: :map
     :with: 3
+    :output_id: 3
+- - :action: :map
     :item: 4
-- - :output_id: 5
-    :action: :map
     :with: 0
+    :output_id: 4
+- - :action: :map
     :item: 5
-- - :output_id: 6
-    :action: :map
-    :with: 2
+    :with: 1
+    :output_id: 5
+- - :action: :map
     :item: 6
-- - :output1_id: 0
-    :output2_id: 1
+    :with: 2
+    :output_id: 6
+- - :action: :reduce
+    :with: 3
     :output_id: 7
-    :action: :reduce
-    :with: 1
-- - :output1_id: 2
-    :output2_id: 3
+    :output1_id: 0
+    :output2_id: 1
+- - :action: :reduce
+    :with: 0
     :output_id: 8
-    :action: :reduce
-    :with: 3
-- - :output1_id: 4
-    :output2_id: 5
+    :output1_id: 2
+    :output2_id: 3
+- - :action: :reduce
+    :with: 1
     :output_id: 9
-    :action: :reduce
-    :with: 0
+    :output1_id: 4
+    :output2_id: 5
 - []
 
-- - :output1_id: 6
-    :output2_id: 7
+- - :action: :reduce
+    :with: 3
     :output_id: 10
-    :action: :reduce
-    :with: 1
+    :output1_id: 6
+    :output2_id: 7
 - []
 
-- - :output1_id: 8
-    :output2_id: 9
+- - :action: :reduce
+    :with: 2
     :output_id: 11
-    :action: :reduce
-    :with: 0
+    :output1_id: 8
+    :output2_id: 9
 - []
 
-- - :output1_id: 10
-    :output2_id: 11
+- - :action: :reduce
+    :with: 3
     :output_id: 12
-    :action: :reduce
-    :with: 0
-- - :output_id: 12
-    :action: :finalize
+    :output1_id: 10
+    :output2_id: 11
+- - :action: :finalize
     :with: 0
+    :output_id: 12
     :final_id: 13

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: tem_mr_search
 version: !ruby/object:Gem::Version 
-  version: "0.3"
+  version: 0.3.1
 platform: ruby
 authors: 
 - Victor Costan