tem_mr_search 0.2.4 → 0.2.5

data/CHANGELOG

@@ -1,3 +1,5 @@
+v0.2.5. Symmetric encryption for partial outputs.
+
 v0.2.4. Disabled Nagle's algorithm to shave off 100ms in query times.
 
 v0.2.3. Slightly more robust communication code in the RPC server/client.

data/Rakefile

@@ -1,3 +1,9 @@
+# Rakefile for the tem_mr_search Rubygem.
+#
+# Author:: Victor Costan
+# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
+# License:: MIT
+
 require 'rubygems'
 gem 'echoe'
 require 'echoe'
@@ -10,7 +16,7 @@ Echoe.new('tem_mr_search') do |p|
   p.email = 'victor@costan.us'
   p.summary = 'Tem Map-Reduce proof of concept: database search.'
   p.url = 'http://tem.rubyforge.org'
-  p.dependencies = ['tem_ruby >=0.13.0', 'tem_multi_proxy >=0.2.6']
+  p.dependencies = ['tem_ruby >=0.14.0', 'tem_multi_proxy >=0.2.7']
   
   p.need_tar_gz = !Gem.win_platform?
   p.need_zip = !Gem.win_platform?

data/lib/tem_mr_search/client.rb

@@ -1,11 +1,25 @@
+# Client for the map-reduce RPC server.
+#
+# Author:: Victor Costan
+# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
+# License:: MIT
+
 # :nodoc: namespace
 module Tem::Mr::Search
 
+
+# Client for the map-reduce RPC server.
 class Client
   OP = Zerg::Support::Protocols::ObjectProtocol
   OPAdapter = Zerg::Support::Sockets::ProtocolAdapter.adapter_module OP
   
   # Performs a private database search using a Map-Reduce.
+  #
+  # Args:
+  #   server_addr:: string with the address of the Map-Reduce server's RPC port.
+  #   client_query:: a ClientQuery instance expressing the Map-Reduce search
+  #
+  # Returns the result of the Map-Reduce computation.
   def self.search(server_addr, client_query)
     output = issue_request server_addr, :type => :search, :root_tem => 0,
                                         :map_reduce => client_query.to_hash
@@ -34,6 +48,8 @@ class Client
   end
   
   # Issues a request against a Map-Reduce server and returns the response.
+  #
+  # This method should not be called directly.
   def self.issue_request(server_addr, request)
     socket = Zerg::Support::SocketFactory.socket :out_addr => server_addr,
         :out_port => Server::DEFAULT_PORT, :no_delay => true
@@ -43,6 +59,6 @@ class Client
     socket.close
     response
   end
-end
+end  # class Tem::Mr::Search::Client
 
 end  # namespace Tem::Mr::Search

data/lib/tem_mr_search/client_query.rb

@@ -12,8 +12,7 @@ class ClientQuery < MapReduceJob
   # This is expected to be called with the encrypted output returned by the
   # search provider.
   def unpack_output(output)
-    # TODO(costan): decrypt output once we enable encryption
-    decrypted_output = output
+    decrypted_output = @query_key.decrypt output
     unpack_decrypted_output decrypted_output
   end
 end

data/lib/tem_mr_search/db.rb

@@ -1,26 +1,50 @@
+# Mock database for the secure Map-Reduce proof of concept.
+#
+# Author:: Victor Costan
+# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
+# License:: MIT
+
 # :nodoc: namespace
 module Tem::Mr::Search
-  
+
+# Mock database for the Map-Reduce proof of concept.
 class Db
   attr_reader :data
   attr_reader :id_attribute
   
+  # Creates a new mock database.
+  #
+  # Args:
+  #   path:: filesystem path to the YAML file containing the database 
   def initialize(path)
     @data = File.open(path, 'r') { |f| YAML.load f }
     @id_attribute = 'flight'
   end
   
+  # The number of records in the database.
   def length
     @data.length
   end
   
+  # Retrieves an item in table scan order.
+  #
+  # Args:
+  #   item_index:: the item's 0-based index in table scan order
+  #
+  # Returns a hash with the item data. 
   def item(item_index)
     @data[item_index]
   end
   
+  # Retrieves an item using its primary key.
+  #
+  # Args:
+  #   item_id:: the item's primary key
+  #
+  # Returns a hash with the item data. 
   def item_by_id(item_id)
     @data.find { |item| item[@id_attribute] == item_id }
   end
-end
+end  # class Tem::Mr::Search::Db
   
-end  # namespace Tem::Mr::search
+end  # namespace Tem::Mr::Search

data/lib/tem_mr_search/map_reduce_executor.rb

@@ -1,9 +1,21 @@
+# Coordination code (executor) for performing a Map-Reduce computation.
+#
+# Author:: Victor Costan
+# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
+# License:: MIT
+
 require 'thread'
 
 
 # :nodoc: namespace
 module Tem::Mr::Search
   
+# Coordination code (executor) for performing a Map-Reduce computation.
+#
+# The executor distributes the Map-Reduce computation across multiple TEMs. The
+# strategy used to allocate tasks to TEMs is expressed by a MapReducePlanner
+# class, and the executor instantiates that class. The executor is responsible
+# for coordinating between the TEMs and the planner.
 class MapReduceExecutor
   # Creates an executor for a Map-Reduce job.
   #
@@ -61,6 +73,7 @@ class MapReduceExecutor
       @main_queue << action
     end
   end
+  private :executor_thread
   
   # Executes a Map-Reduce planner action.
   #
@@ -108,8 +121,9 @@ class MapReduceExecutor
       @lock.synchronize do
         @outputs[action[:final_id]] = final_output
       end
-    end    
+    end
   end
-end
+  private :execute_action
+end  # class Tem::Mr::Search::MapReduceExecutor
 
 end  # namespace Tem::Mr::Search

data/lib/tem_mr_search/map_reduce_job.rb

@@ -1,6 +1,22 @@
+# Expresses a Map-Reduce computation whose components are performed on TEMs.
+#
+# Author:: Victor Costan
+# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
+# License:: MIT
+
 # :nodoc: namespace
 module Tem::Mr::Search
   
+# Expresses a Map-Reduce computation whose components are performed on TEMs.
+#
+# Client code should put together Map-Reduce computations using QueryBuilder or
+# similar builder classes. In turn, builders construct and return MapReduceJob
+# instances.
+#
+# Jobs can be serialized to a hash for network transmission. To de-serialize a
+# job, pass the hash to the hash constructor:
+#     hash = job.to_hash
+#     job = MapReduceJob.new hash
 class MapReduceJob  
   attr_reader :mapper, :reducer, :finalizer, :attributes, :id_attribute
   
@@ -18,7 +34,7 @@ class MapReduceJob
     {
       :id => output[0, 8].reverse.pack('C*').unpack('q').first,
       :score => Tem::Abi.read_tem_short(output, 8),
-      :check => output[13, 3]
+      :check => output[13, 10]
     }
   end
   

data/lib/tem_mr_search/map_reduce_planner.rb

@@ -1,9 +1,34 @@
+# Allocates the individual components of a Map-Reduce job across TEMs.
+#
+# Author:: Victor Costan
+# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
+# License:: MIT
+
 require 'rbtree'
 require 'set'
 
 # :nodoc: namespace
 module Tem::Mr::Search
   
+# Allocates the individual components of a Map-Reduce job across TEMs.
+#
+# This class is instantiated and used by MapReduceExecutor. It should not be
+# used directly in client code, except for the purpose of replacing the default
+# planner.
+#
+# The Map-Reduce coordinator calls next_actions! on the planner, to obtain a
+# list of actions that can be carried out. The planner guarantees that the
+# actions are independent of each other, and that all their dependencies are
+# satisfied. When the coordinator learns about the completion of some actions,
+# it updates the planner's state by calling action_done. After action_done is
+# called, new_action should be called again to obtain new actions that can be
+# carried out.
+#
+# Partial results (outputs) in the Map-Reduce computation are identified by 
+# unique numbers starting from 0. The output IDs can be used as file names, if
+# the outputs are stored in a distributed file system. When the computation is
+# done (calling done? returns +true+), the +output_id+ attribute will contain
+# the ID of the computation's final result.
 class MapReducePlanner  
   # Creates a planner for a Map-Reduce job.
   #
@@ -32,8 +57,69 @@ class MapReducePlanner
     @last_reduce_id = 2 * num_items - 2
     @done_reducing, @output_id = false, nil
   end
+  
+  # Issues a set of actions that can be performed right now.
+  #
+  # The method alters the planner's state assuming the actions will be
+  # performed.
+  #
+  # Returns an array of hashes, with one hash per action to be performed. The
+  # +:action+ key specifies the type of action to be performed, and can be
+  # +:migrate+ +:map+, +:reduce+, or +:finalize. All the actions have the
+  # +:with+ key, which is the ID (0-based index) of the TEM that will be doing
+  # the action.
+  #
+  # Migrate actions have the following keys:
+  #   :secpack:: the type of SECpack to be migrated ( +:mapper+ or +:reducer+ )
+  #   :with:: the ID of the TEM doing the migration  
+  #   :to:: the number of the TEM that the SECpack should be migrated to
+  #
+  # Map actions have the following keys:
+  #   :item_id:: the ID of the item to be mapped (number in Table-Scan order)
+  #   :with:: the ID of the TEM doing the mapping
+  #   :output_id:: ID for the result of the map operation
+  #
+  # Reduce actions have the following keys:
+  #   :output1_id, :output2_id:: the IDs of the partial outputs to be reduced
+  #   :with:: the ID of the TEM doing the reducing
+  #   :output_id:: the ID for the result of the reduce operation
+  #
+  # The finalize action has the following keys:
+  #   :output_id:: the ID of the last partial output, which will be finalized
+  #   :with:: the ID of the TEM doing the finalization
+  #   :final_id:: the ID for the computation's final result
+  def next_actions!
+    actions = migrate_actions :mapper
+    actions += migrate_actions :reducer
+    actions += map_actions
+    actions += reduce_actions
+    actions += finalize_actions
+    actions
+  end
+  
+  # Informs the planner that an action issued by next_actions! was completed.
+  #
+  # Args:
+  #   action:: an action hash, as returned by next_actions!
+  #
+  # The return value is not specified.
+  def action_done(action)
+    dispatch = { :migrate => :done_migrating, :map => :done_mapping, :reduce =>
+                 :done_reducing, :finalize => :done_finalizing } 
+    self.send dispatch[action[:action]], action
+  end
+
+  # True when the Map-Reduce computation is complete.
+  def done?
+    !@output_id.nil?
+  end
+  
+  # The output ID of the Map-Reduce's final result.
+  attr_reader :output_id
 
   # Generates migrating actions for a SECpack type that are possible now.
+  #
+  # See next_actions! for a description of the return value.
   def migrate_actions(sec_type)
     actions = []
     return actions if @without[sec_type].length == 0
@@ -56,24 +142,10 @@ class MapReducePlanner
     @with[action[:secpack]] << action[:to]
   end
   private :done_migrating
-  
-  # A sorted array of the free TEMs that have a SECpack type.
-  def free_tems_with_sec(sec_type)
-    tems = []
-    @free_tems.each do |tem, true_value|
-      tems << tem if @with[sec_type].include? tem
-    end
-    tems
-  end
-  
-  # A unique output_id.
-  def next_output_id
-    next_id = @last_output_id
-    @last_output_id += 1
-    next_id
-  end
-  
+    
   # Generates mapping actions possible right now.
+  #
+  # See next_actions! for a description of the return value.
   def map_actions
     actions = []
     return actions if @unmapped_items.empty?
@@ -88,6 +160,11 @@ class MapReducePlanner
   private :map_actions
   
   # Informs the planner that a data mapping has completed.
+  #
+  # Args:
+  #   action:: an action hash, as returned by map_actions
+  #
+  # The return value is not specified.
   def done_mapping(action)
     @free_tems[action[:with]] = true
     @reduce_queue[action[:output_id]] = true
@@ -95,6 +172,8 @@ class MapReducePlanner
   private :done_mapping
   
   # Generates reducing actions possible right now.
+  #
+  # See next_actions! for a description of the return value.
   def reduce_actions
     actions = []
     return actions if @reduce_queue.length <= 1
@@ -114,6 +193,11 @@ class MapReducePlanner
   private :reduce_actions
   
   # Informs the planner that a data reduction has completed.
+  #
+  # Args:
+  #   action:: an action hash, as returned by reduce_actions
+  #
+  # The return value is not specified.
   def done_reducing(action)
     @free_tems[action[:with]] = true
     if action[:output_id] == @last_reduce_id
@@ -125,6 +209,8 @@ class MapReducePlanner
   private :done_reducing
   
   # Generates finalizing actions possible right now.
+  #
+  # See next_actions! for a description of the return value.
   def finalize_actions
     return [] unless @done_reducing and !@output_id and @free_tems[@root_tem]
     @finalize_ready = false
@@ -134,36 +220,40 @@ class MapReducePlanner
   private :finalize_actions
   
   # Informs the planner that an action issued by next_action was done.
+  #
+  # Args:
+  #   action:: an action hash, as returned by finalize_actions
+  #
+  # The return value is not specified.
   def done_finalizing(action)
     @free_tems[action[:with]] = true
     @output_id = action[:final_id]    
   end
   private :done_finalizing
 
-  # True when the Map-Reduce job is complete.
-  def done?
-    !@output_id.nil?
+  # A sorted array of the free TEMs that have a SECpack type migrated to them.
+  #
+  # Args:
+  #   sec_type:: the SECpack type (+:mapper+ or +:reducer+)
+  def free_tems_with_sec(sec_type)
+    tems = []
+    @free_tems.each do |tem, true_value|
+      tems << tem if @with[sec_type].include? tem
+    end
+    tems
   end
+  private :free_tems_with_sec
   
-  # The output ID of the Map-Reduce's final result.
-  attr_reader :output_id
-
-  # Informs the planner that an action issued by next_actions was completed.
-  def action_done(action)
-    dispatch = { :migrate => :done_migrating, :map => :done_mapping, :reduce =>
-                 :done_reducing, :finalize => :done_finalizing } 
-    self.send dispatch[action[:action]], action
+  # Generates a unique output ID.
+  #
+  # Returns the unique output ID, which is a non-negative integer. Future calls
+  # of this method are guaranteed to return different output IDs.
+  def next_output_id
+    next_id = @last_output_id
+    @last_output_id += 1
+    next_id
   end
+  private :next_output_id
+end  # class Tem::Mr::Search::MapReducePlanner
 
-  # Issues a set of actions that can be performed right now.
-  def next_actions!
-    actions = migrate_actions :mapper
-    actions += migrate_actions :reducer
-    actions += map_actions
-    actions += reduce_actions
-    actions += finalize_actions
-    actions
-  end  
-end
-
-end  # namespace Tem::Mr::search
+end  # namespace Tem::Mr::Search

data/lib/tem_mr_search/query_builder.rb

@@ -29,7 +29,7 @@ class QueryBuilder
     @map_secpack = Tem::Assembler.assemble do |s|
       s.label :_secret
       s.label :_key
-      s.zeros :tem_ubyte, 16
+      s.data :tem_ubyte, @query_key.to_tem_key
       s.label :_check_bytes
       s.data :tem_ubyte, @check_bytes
       
@@ -39,15 +39,21 @@ class QueryBuilder
       s.ret
       
       s.entry
-      s.ldbc 16
+      s.ldbc 24
       s.outnew
-      s.call :_ranking
+      # Compute score.
+      s.call :_ranking      
+      # Compute padding.
       s.ldbc 3
       s.ldwc :_nonce
       s.rnd
-      s.mcfxb :from => :_check_bytes, :to => :_check, :size => 3
-      # TODO(costan): encryption instead of plain dump
-      s.outfxb :from => :_id, :size => 16
+      s.mcfxb :from => :_check_bytes, :to => :_check,
+              :size => @check_bytes.length
+      
+      # Encrypt output.
+      s.ldwc :const => :_key
+      s.rdk
+      s.kefxb :from => :_id, :size => 23, :to => 0xFFFF
       s.halt
       
       s.label :_plain
@@ -68,7 +74,7 @@ class QueryBuilder
       s.zeros :tem_ubyte, 3
       # Check bytes to prevent malicious input corruption.
       s.label :_check
-      s.zeros :tem_ubyte, 3
+      s.zeros :tem_ubyte, @check_bytes.length
       
       s.stack 64
     end    
@@ -79,7 +85,7 @@ class QueryBuilder
     @reduce_secpack = Tem::Assembler.assemble do |s|
       s.label :_secret
       s.label :_key
-      s.zeros :tem_ubyte, 16
+      s.data :tem_ubyte, @query_key.to_tem_key
       s.label :_check
       s.data :tem_ubyte, @check_bytes
       
@@ -90,15 +96,19 @@ class QueryBuilder
       s.ret
       
       s.entry
-      s.ldbc 16
+      s.ldbc :const => 24
       s.outnew      
-      # Decode inputs.
+      # Decrypt inputs.
+      s.ldwc :const => :_key
+      s.rdk
+      s.stw :_key_id
       [1, 2].each do |i|
-        # TODO(costan): decrypt instead of copying
-        s.mcfxb :from => :"_output#{i}", :to => :"_id#{i}", :size => 16
+        s.ldw :_key_id
+        s.kdfxb :from => :"_output#{i}", :to => :"_id#{i}", :size => 24
         
         # Compare the check bytes and abort if the inputs were tampered with.
-        s.mcmpfxb :op1 => :"_check#{i}", :op2 => :"_check", :size => 3
+        s.mcmpfxb :op1 => :"_check#{i}", :op2 => :"_check",
+                  :size => @check_bytes.length
         s.jz :"_check_#{i}_ok"
         s.halt
         s.label :"_check_#{i}_ok"
@@ -119,20 +129,24 @@ class QueryBuilder
       s.ldbc 3
       s.ldwc :_nonce1
       s.rnd
-      # TODO(costan): encrypt instead of copying
-      s.outfxb :from => :_id1, :size => 16
+      # Encrypt output.
+      s.ldwc :const => :_key
+      s.rdk
+      s.kefxb :from => :_id1, :size => 23, :to => 0xFFFF
       s.halt
       
       s.label :_plain
       # The comparison result produced by the user comparison procedure.
       s.label :comparison
       s.zeros :tem_short, 1
+      s.label :_key_id
+      s.zeros :tem_short, 1
       
       # The two inputs to reduce.
       [1, 2].each do |i|
         # Encrypted map/reduce output.
         s.label :"_output#{i}"        
-        s.zeros :tem_ubyte, 16
+        s.zeros :tem_ubyte, 24
         # Unencrypted input (decrypted inside TEM).
         s.label :"_id#{i}"
         s.zeros :tem_ubyte, 8
@@ -141,9 +155,9 @@ class QueryBuilder
         s.label :"_nonce#{i}"
         s.zeros :tem_ubyte, 3        
         s.label :"_check#{i}"
-        s.zeros :tem_ubyte, 3
+        s.zeros :tem_ubyte, @check_bytes.length
       end
-      s.stack 8
+      s.stack 16
     end
   end
   
@@ -158,9 +172,8 @@ class QueryBuilder
   end
 
   def initialize
-    @check_bytes = [1, 2, 3]
-    # TODO(costan): generate query key
-    @query_key = nil
+    @check_bytes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+    @query_key = Tem::Keys::Symmetric.generate
   end
 end  # class QueryBuilder
 

data/lib/tem_mr_search/server.rb

@@ -1,9 +1,31 @@
+# Map-Reduce RPC server.
+#
+# Author:: Victor Costan
+# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
+# License:: MIT
+
 require 'logger'
 require 'yaml'
 
 # :nodoc: namespace
 module Tem::Mr::Search
 
+
+# Map-Reduce RPC server.
+#
+# The RPC server models the data provider in the secure Map-Reduce proof of
+# concept.
+#
+# The server accepts database queries expressed as Map-Reduce computations,
+# where each computation is enclosed in a SECpack. This makes it impossible for
+# the data provider to learn about the query. Map-Reduce computations have a 
+# single result: the ID of the record that's the query response, and its score. 
+#
+# The server also accepts direct queries for single database records, so clients
+# can retrieve the record whose ID they learn from the Map-Reduce result.
+#
+# The db_dump and shutdown requests are for demonstration and testing purposes,
+# and would not be exposed in production servers.
 class Server
   DEFAULT_PORT = 9052
   

data/lib/tem_mr_search.rb

@@ -1,3 +1,9 @@
+# Main include file for the tem_mr_search Rubygem.
+#
+# Author:: Victor Costan
+# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
+# License:: MIT
+
 require 'rubygems'
 require 'tem_multi_proxy'
 require 'tem_ruby'

data/tem_mr_search.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{tem_mr_search}
-  s.version = "0.2.4"
+  s.version = "0.2.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Victor Costan"]
-  s.date = %q{2009-11-11}
+  s.date = %q{2009-11-15}
   s.default_executable = %q{tem_mr_search_server}
   s.description = %q{Tem Map-Reduce proof of concept: database search.}
   s.email = %q{victor@costan.us}
@@ -26,14 +26,14 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<tem_ruby>, [">= 0.13.0"])
-      s.add_runtime_dependency(%q<tem_multi_proxy>, [">= 0.2.6"])
+      s.add_runtime_dependency(%q<tem_ruby>, [">= 0.14.0"])
+      s.add_runtime_dependency(%q<tem_multi_proxy>, [">= 0.2.7"])
     else
-      s.add_dependency(%q<tem_ruby>, [">= 0.13.0"])
-      s.add_dependency(%q<tem_multi_proxy>, [">= 0.2.6"])
+      s.add_dependency(%q<tem_ruby>, [">= 0.14.0"])
+      s.add_dependency(%q<tem_multi_proxy>, [">= 0.2.7"])
     end
   else
-    s.add_dependency(%q<tem_ruby>, [">= 0.13.0"])
-    s.add_dependency(%q<tem_multi_proxy>, [">= 0.2.6"])
+    s.add_dependency(%q<tem_ruby>, [">= 0.14.0"])
+    s.add_dependency(%q<tem_multi_proxy>, [">= 0.2.7"])
   end
 end

data/test/test_query_builders.rb

@@ -31,7 +31,8 @@ class QueryBuildersTest < MrTestCase
       assert_equal fare_id(win_fare), output[:id], 'The wrong fare won (bad ID)'
       assert_equal fare_score(win_fare), output[:score],
                    'The wrong fare won (bad score)'
-      assert_equal [1, 2, 3], output[:check], 'Incorrect check bytes'
+      assert_equal [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], output[:check],
+                   'Incorrect check bytes'
 
       assert_not_equal enc_output, output1, 'Nonce fail'
       assert_not_equal enc_output, output2, 'Nonce fail'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: tem_mr_search
 version: !ruby/object:Gem::Version 
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors: 
 - Victor Costan
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-11 00:00:00 -05:00
+date: 2009-11-15 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.13.0
+        version: 0.14.0
     version: 
 - !ruby/object:Gem::Dependency 
   name: tem_multi_proxy
@@ -30,7 +30,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.2.6
+        version: 0.2.7
     version: 
 description: "Tem Map-Reduce proof of concept: database search."
 email: victor@costan.us