tem_mr_search 0.1

data/test/mr_test_case.rb

@@ -0,0 +1,58 @@
+require 'test/unit'
+require 'tem_mr_search'
+
+class MrTestCase < Test::Unit::TestCase
+  include Tem::Mr::Search
+  
+  def setup
+    super
+    
+    Thread.abort_on_exception = true
+    
+    testdb_path = File.join File.dirname(__FILE__), "..", "testdata",
+                            "fares.yml"
+    @db = Db.new testdb_path
+
+    @client_query = QueryBuilder.query { |q|
+      q.attributes :price => :tem_short, :start => :tem_short,
+                   :end => :tem_short
+      q.id_attribute :flight
+      
+      # Score: 200 + start / 100 - duration - price
+      q.map { |s|
+        s.ldwc 200
+        s.ldw :start
+        s.ldbc 100
+        s.div
+        s.add
+        s.ldw :end
+        s.ldw :start
+        s.sub
+        s.sub
+        s.ldw :price
+        s.sub
+        s.stw :score
+      }
+
+      # The greater score wins.
+      q.reduce { |s|
+        s.ldw :score1
+        s.ldw :score2
+        s.cmp
+        s.stw :comparison
+      }
+    }
+  end
+  
+  def fare_score(fare)
+    200 + fare['start'] / 100 - fare['price'] - (fare['end'] - fare['start'])
+  end
+  
+  def fare_id(fare)
+    fare['flight']
+  end
+  
+  # Ensures that everything has loaded.
+  def test_smoke
+  end
+end

data/test/test_db.rb

@@ -0,0 +1,10 @@
+require 'test/mr_test_case'
+
+class DbTest < MrTestCase  
+  def test_loading    
+    assert_equal 4, @db.length, 'Number of items in the database'
+    gold_item = {'from' => 'BOS', 'to' => 'NYC', 'price' => 150, 'start' => 900,
+                 'end' => 1000, 'flight' => 15 }
+    assert_equal gold_item, @db.item(0), 'First database item'
+  end
+end

data/test/test_map_reduce_executor.rb

@@ -0,0 +1,37 @@
+require 'test/mr_test_case'
+
+class MapReduceExecutorTest < MrTestCase
+  MRExecutor = Tem::Mr::Search::MapReduceExecutor
+  
+  def setup
+    super
+    Tem.auto_conf
+    $tem.activate
+    $tem.emit
+  end
+    
+  def _test_executor(tems, root_tem)
+    executor = MRExecutor.new @client_query, @db, tems, root_tem
+    packed_output = executor.execute
+    result = @client_query.unpack_output packed_output
+    assert_equal 18, result[:id], 'Incorrect Map-Reduce result'
+  end
+  
+  def test_executor_with_autoconf
+    _test_executor [$tem], 0
+  end
+  
+  def test_executor_with_cluster
+    cluster_config = ['lightbulb2.local', 'darkbulb.local'].map { |host|
+      Tem::MultiProxy::Client.query_tems host    
+    }.flatten
+    assert_equal 8, cluster_config.length, 'Incorrect cluster setup'
+    tems = cluster_config.map do |config|
+      Tem::Session.new Tem::Transport::AutoConfigurator.try_transport(config)
+    end
+    
+    tems.each { |tem| tem.activate; tem.emit }
+    
+    _test_executor tems, 0
+  end
+end

data/test/test_map_reduce_job.rb

@@ -0,0 +1,45 @@
+require 'test/mr_test_case'
+
+class MapReduceJobTest < MrTestCase
+  def setup
+    super
+    
+    @obj1 = @db.item 0
+    @obj2 = @db.item 1
+    @output1 = (1..16).to_a
+    @output2 = (17..32).to_a
+  end
+  
+  def test_map_for_object
+    obj = @obj1.merge 'flight' => 0x12345678
+    secpack = @client_query.map_for_object obj
+    
+    assert_equal [0, 0, 0, 0, 0x12, 0x34, 0x56, 0x78],
+                 secpack.get_bytes(:_id, 8), 'Object ID embedded incorrectly'
+    assert_equal @obj1['price'], secpack.get_value(:price, :tem_short),
+                 'Price embedded incorrectly'
+    assert_equal @obj1['start'], secpack.get_value(:start, :tem_short),
+                 'Starting time embedded incorrectly'
+    assert_equal @obj1['end'], secpack.get_value(:end, :tem_short),
+                 'Ending time embedded incorrectly'
+  end
+  
+  def test_reduce_for_outputs
+    secpack = @client_query.reduce_for_outputs @output1, @output2
+    
+    assert_equal @output1, secpack.get_bytes(:_output1, 16),
+                 'Output1 embedded incorrectly'
+    assert_equal @output2, secpack.get_bytes(:_output2, 16),
+                 'Output2 embedded incorrectly'
+  end
+  
+  def test_unpack_unencrypted_output
+    packed_output = [0, 0, 0, 0, 0x12, 0x34, 0x56, 0x78, 0x31, 0x41, 0xCC, 0xCD,
+                     0xCE, 0xBE, 0xEF, 0xFE]
+    output = @client_query.unpack_decrypted_output packed_output
+              
+    assert_equal 0x12345678, output[:id], 'ID incorrectly unpacked'
+    assert_equal 0x3141, output[:score], 'Score incorrectly unpacked'
+    assert_equal [0xBE, 0xEF, 0xFE], output[:check], 'Check bytes'    
+  end
+end

data/test/test_map_reduce_planner.rb

@@ -0,0 +1,55 @@
+require 'test/mr_test_case'
+require 'yaml'
+
+class MapReducePlannerTest < Test::Unit::TestCase
+  MRPlanner = Tem::Mr::Search::MapReducePlanner
+  
+  def setup
+    @testdata_path = File.join(File.dirname(__FILE__), '..', 'testdata')
+  end
+  
+  def parallel_planning(planner)
+    all_actions = []
+    until planner.done?
+      actions = planner.next_actions!
+      all_actions << actions
+      actions.each { |action| planner.action_done action }
+    end
+    all_actions    
+  end
+  
+  def serial_planning(planner)    
+    all_actions = []
+    pending_actions = []
+    until planner.done?
+      actions = planner.next_actions!
+      all_actions << actions
+      pending_actions += actions
+      action = pending_actions.shift   
+      planner.action_done action if action 
+    end
+    all_actions
+  end
+  
+  def _test_planning(method_name, items, tems, root_tem, gold_file)
+    planner = MRPlanner.new nil, items, tems, root_tem
+    all_actions = self.send method_name, planner
+    gold_actions = File.open(File.join(@testdata_path, gold_file), 'r') do |f|
+      YAML.load f
+    end
+    assert_equal gold_actions, all_actions, "Failed #{method_name}: " +
+        "#{tems} tems with root #{root_tem}, #{items} items"
+    assert_equal items * 2 - 1, planner.output_id, "Wrong final output_id"
+  end
+  
+  def test_planning
+    [[:parallel_planning, 7, 4, 0, 'parallel_plan_740.yml'],
+     [:parallel_planning, 4, 3, 1, 'parallel_plan_431.yml'],
+     [:serial_planning, 4, 1, 0, 'serial_plan_410.yml'],
+     [:serial_planning, 7, 4, 0, 'serial_plan_740.yml'],
+     [:serial_planning, 4, 3, 1, 'serial_plan_431.yml'],
+    ].each do |testcase|
+      _test_planning *testcase
+    end
+  end  
+end

data/test/test_query_builder.rb

@@ -0,0 +1,40 @@
+require 'test/mr_test_case'
+
+class QueryBuilderTest < MrTestCase
+  def setup
+    super
+    Tem.auto_conf
+    $tem.activate
+    $tem.emit
+  end
+  
+  def _test_map_fare(fare)
+    enc_output = @client_query.map_object fare, $tem
+    output = @client_query.unpack_output enc_output
+    assert_equal fare_id(fare), output[:id], 'Object ID incorrectly encoded'
+    assert_equal fare_score(fare), output[:score],
+                 'Score incorrectly computed'
+    enc_output
+  end
+  
+  def test_map_reduce
+    fare1 = @db.item 0
+    output1 = _test_map_fare fare1
+    fare2 = @db.item 1
+    output2 = _test_map_fare fare2
+    
+    win_fare = (fare_score(fare1) > fare_score(fare2)) ? fare1 : fare2
+    # Try both permutations to ensure all branches of the reduce code work.
+    [[output1, output2], [output2, output1]].each do |o1, o2|
+      enc_output = @client_query.reduce_outputs o1, o2, $tem
+      output = @client_query.unpack_output enc_output
+      assert_equal fare_id(win_fare), output[:id], 'The wrong fare won (bad ID)'
+      assert_equal fare_score(win_fare), output[:score],
+                   'The wrong fare won (bad score)'
+      assert_equal [1, 2, 3], output[:check], 'Incorrect check bytes'
+
+      assert_not_equal enc_output, output1, 'Nonce fail'
+      assert_not_equal enc_output, output2, 'Nonce fail'
+    end
+  end
+end

data/testdata/fares.yml

@@ -0,0 +1,49 @@
+---
+- from: BOS
+  to: NYC
+  price: 150
+  start: 900
+  end: 1000
+  flight: 15
+- from: BOS
+  to: NYC
+  price: 150
+  start: 1800
+  end: 1900
+  flight: 16
+- from: BOS
+  to: NYC
+  price: 200
+  start: 1200
+  end: 1245
+  flight: 17
+- from: BOS
+  to: NYC
+  price: 200
+  start: 1700
+  end: 1745
+  flight: 18
+- from: BOS
+  to: NYC
+  price: 160
+  start: 900
+  end: 1000
+  flight: 21
+- from: BOS
+  to: NYC
+  price: 160
+  start: 1800
+  end: 1900
+  flight: 22
+- from: BOS
+  to: NYC
+  price: 210
+  start: 1200
+  end: 1245
+  flight: 23
+- from: BOS
+  to: NYC
+  price: 210
+  start: 1700
+  end: 1745
+  flight: 24

data/testdata/parallel_plan_431.yml

@@ -0,0 +1,52 @@
+--- 
+- - :secpack: :mapper
+    :action: :migrate
+    :with: 1
+    :to: 0
+- - :secpack: :mapper
+    :action: :migrate
+    :with: 0
+    :to: 2
+  - :secpack: :reducer
+    :action: :migrate
+    :with: 1
+    :to: 0
+- - :secpack: :reducer
+    :action: :migrate
+    :with: 0
+    :to: 2
+  - :output_id: 0
+    :action: :map
+    :with: 1
+    :item: 0
+  - :output_id: 1
+    :action: :map
+    :with: 2
+    :item: 1
+- - :output_id: 2
+    :action: :map
+    :with: 0
+    :item: 2
+  - :output_id: 3
+    :action: :map
+    :with: 1
+    :item: 3
+  - :output1_id: 0
+    :output2_id: 1
+    :output_id: 4
+    :action: :reduce
+    :with: 2
+- - :output1_id: 2
+    :output2_id: 3
+    :output_id: 5
+    :action: :reduce
+    :with: 0
+- - :output1_id: 4
+    :output2_id: 5
+    :output_id: 6
+    :action: :reduce
+    :with: 0
+- - :output_id: 6
+    :action: :finalize
+    :with: 1
+    :final_id: 7

data/testdata/parallel_plan_740.yml

@@ -0,0 +1,87 @@
+--- 
+- - :secpack: :mapper
+    :action: :migrate
+    :with: 0
+    :to: 1
+- - :secpack: :mapper
+    :action: :migrate
+    :with: 0
+    :to: 2
+  - :secpack: :mapper
+    :action: :migrate
+    :with: 1
+    :to: 3
+- - :secpack: :reducer
+    :action: :migrate
+    :with: 0
+    :to: 1
+  - :output_id: 0
+    :action: :map
+    :with: 1
+    :item: 0
+  - :output_id: 1
+    :action: :map
+    :with: 2
+    :item: 1
+  - :output_id: 2
+    :action: :map
+    :with: 3
+    :item: 2
+- - :secpack: :reducer
+    :action: :migrate
+    :with: 0
+    :to: 2
+  - :secpack: :reducer
+    :action: :migrate
+    :with: 1
+    :to: 3
+  - :output_id: 3
+    :action: :map
+    :with: 2
+    :item: 3
+  - :output_id: 4
+    :action: :map
+    :with: 3
+    :item: 4
+- - :output_id: 5
+    :action: :map
+    :with: 0
+    :item: 5
+  - :output_id: 6
+    :action: :map
+    :with: 1
+    :item: 6
+  - :output1_id: 0
+    :output2_id: 1
+    :output_id: 7
+    :action: :reduce
+    :with: 2
+  - :output1_id: 2
+    :output2_id: 3
+    :output_id: 8
+    :action: :reduce
+    :with: 3
+- - :output1_id: 4
+    :output2_id: 5
+    :output_id: 9
+    :action: :reduce
+    :with: 0
+  - :output1_id: 6
+    :output2_id: 7
+    :output_id: 10
+    :action: :reduce
+    :with: 1
+- - :output1_id: 8
+    :output2_id: 9
+    :output_id: 11
+    :action: :reduce
+    :with: 0
+- - :output1_id: 10
+    :output2_id: 11
+    :output_id: 12
+    :action: :reduce
+    :with: 0
+- - :output_id: 12
+    :action: :finalize
+    :with: 0
+    :final_id: 13

data/testdata/serial_plan_410.yml

@@ -0,0 +1,36 @@
+--- 
+- - :output_id: 0
+    :action: :map
+    :with: 0
+    :item: 0
+- - :output_id: 1
+    :action: :map
+    :with: 0
+    :item: 1
+- - :output_id: 2
+    :action: :map
+    :with: 0
+    :item: 2
+- - :output_id: 3
+    :action: :map
+    :with: 0
+    :item: 3
+- - :output1_id: 0
+    :output2_id: 1
+    :output_id: 4
+    :action: :reduce
+    :with: 0
+- - :output1_id: 2
+    :output2_id: 3
+    :output_id: 5
+    :action: :reduce
+    :with: 0
+- - :output1_id: 4
+    :output2_id: 5
+    :output_id: 6
+    :action: :reduce
+    :with: 0
+- - :output_id: 6
+    :action: :finalize
+    :with: 0
+    :final_id: 7

data/testdata/serial_plan_431.yml

@@ -0,0 +1,56 @@
+--- 
+- - :secpack: :mapper
+    :action: :migrate
+    :with: 1
+    :to: 0
+- - :secpack: :mapper
+    :action: :migrate
+    :with: 0
+    :to: 2
+  - :secpack: :reducer
+    :action: :migrate
+    :with: 1
+    :to: 0
+- - :output_id: 0
+    :action: :map
+    :with: 0
+    :item: 0
+  - :output_id: 1
+    :action: :map
+    :with: 2
+    :item: 1
+- - :secpack: :reducer
+    :action: :migrate
+    :with: 1
+    :to: 2
+- - :output_id: 2
+    :action: :map
+    :with: 0
+    :item: 2
+- - :output_id: 3
+    :action: :map
+    :with: 2
+    :item: 3
+- - :output1_id: 0
+    :output2_id: 1
+    :output_id: 4
+    :action: :reduce
+    :with: 1
+- []
+
+- - :output1_id: 2
+    :output2_id: 3
+    :output_id: 5
+    :action: :reduce
+    :with: 0
+- []
+
+- - :output1_id: 4
+    :output2_id: 5
+    :output_id: 6
+    :action: :reduce
+    :with: 0
+- - :output_id: 6
+    :action: :finalize
+    :with: 1
+    :final_id: 7

data/testdata/serial_plan_740.yml

@@ -0,0 +1,93 @@
+--- 
+- - :secpack: :mapper
+    :action: :migrate
+    :with: 0
+    :to: 1
+- - :secpack: :mapper
+    :action: :migrate
+    :with: 0
+    :to: 2
+  - :secpack: :mapper
+    :action: :migrate
+    :with: 1
+    :to: 3
+- - :secpack: :reducer
+    :action: :migrate
+    :with: 0
+    :to: 1
+  - :output_id: 0
+    :action: :map
+    :with: 2
+    :item: 0
+- - :output_id: 1
+    :action: :map
+    :with: 1
+    :item: 1
+  - :output_id: 2
+    :action: :map
+    :with: 3
+    :item: 2
+- - :secpack: :reducer
+    :action: :migrate
+    :with: 0
+    :to: 2
+- - :output_id: 3
+    :action: :map
+    :with: 2
+    :item: 3
+- - :secpack: :reducer
+    :action: :migrate
+    :with: 1
+    :to: 3
+- - :output_id: 4
+    :action: :map
+    :with: 3
+    :item: 4
+- - :output_id: 5
+    :action: :map
+    :with: 0
+    :item: 5
+- - :output_id: 6
+    :action: :map
+    :with: 2
+    :item: 6
+- - :output1_id: 0
+    :output2_id: 1
+    :output_id: 7
+    :action: :reduce
+    :with: 1
+- - :output1_id: 2
+    :output2_id: 3
+    :output_id: 8
+    :action: :reduce
+    :with: 3
+- - :output1_id: 4
+    :output2_id: 5
+    :output_id: 9
+    :action: :reduce
+    :with: 0
+- []
+
+- - :output1_id: 6
+    :output2_id: 7
+    :output_id: 10
+    :action: :reduce
+    :with: 1
+- []
+
+- - :output1_id: 8
+    :output2_id: 9
+    :output_id: 11
+    :action: :reduce
+    :with: 0
+- []
+
+- - :output1_id: 10
+    :output2_id: 11
+    :output_id: 12
+    :action: :reduce
+    :with: 0
+- - :output_id: 12
+    :action: :finalize
+    :with: 0
+    :final_id: 13