tem_mr_search 0.2.5 → 0.3

data/CHANGELOG

@@ -1,3 +1,5 @@
+v0.3. SECpack binding and migration. Now we're secure.
+
 v0.2.5. Symmetric encryption for partial outputs.
 
 v0.2.4. Disabled Nagle's algorithm to shave off 100ms in query times.

data/Manifest

@@ -1,5 +1,10 @@
-bin/tem_mr_search_server
 CHANGELOG
+LICENSE
+Manifest
+README
+Rakefile
+bin/tem_mr_search_server
+lib/tem_mr_search.rb
 lib/tem_mr_search/client.rb
 lib/tem_mr_search/client_query.rb
 lib/tem_mr_search/db.rb
@@ -9,12 +14,6 @@ lib/tem_mr_search/map_reduce_planner.rb
 lib/tem_mr_search/query_builder.rb
 lib/tem_mr_search/server.rb
 lib/tem_mr_search/web_client_query_builder.rb
-lib/tem_mr_search.rb
-LICENSE
-Manifest
-Rakefile
-README
-tem_mr_search.gemspec
 test/mr_test_case.rb
 test/test_client_server.rb
 test/test_db.rb

data/Rakefile

@@ -16,7 +16,9 @@ Echoe.new('tem_mr_search') do |p|
   p.email = 'victor@costan.us'
   p.summary = 'Tem Map-Reduce proof of concept: database search.'
   p.url = 'http://tem.rubyforge.org'
-  p.dependencies = ['tem_ruby >=0.14.0', 'tem_multi_proxy >=0.2.7']
+  p.dependencies = ['tem_ruby >=0.15.2', 'tem_multi_proxy >=0.2.7']
+  p.development_dependencies = ["echoe >=3.2",
+                                "flexmock >=0.8.6"]
   
   p.need_tar_gz = !Gem.win_platform?
   p.need_zip = !Gem.win_platform?

data/lib/tem_mr_search/client.rb

@@ -13,6 +13,24 @@ class Client
   OP = Zerg::Support::Protocols::ObjectProtocol
   OPAdapter = Zerg::Support::Sockets::ProtocolAdapter.adapter_module OP
   
+  # Requests information for a random TEM to be used as a query's root TEM.
+  #
+  # Args:
+  #   server_addr:: string with the address of the Map-Reduce server's RPC port.
+  #
+  # Returns a hash with the following keys:
+  #   :id:: the TEM's ID (to be used as the :root_tem argument in search calls)
+  #   :ecert:: the TEM's Endorsement Certificate
+  #   :pubek:: the TEM's public Endorsement Key (from the ECert)
+  def self.get_tem(server_addr)
+    output = issue_request server_addr, :type => :get_tem
+    return nil unless output
+    
+    ecert = OpenSSL::X509::Certificate.new output[:ecert]
+    pubek = Tem::Key.new_from_ssl_key ecert.public_key
+    { :id => output[:id], :ecert => ecert, :pubek => pubek }
+  end
+  
   # Performs a private database search using a Map-Reduce.
   #
   # Args:
@@ -21,7 +39,12 @@ class Client
   #
   # Returns the result of the Map-Reduce computation.
   def self.search(server_addr, client_query)
-    output = issue_request server_addr, :type => :search, :root_tem => 0,
+    tem_info = get_tem server_addr
+    # TODO: check the endorsement certificate.
+    client_query.bind tem_info[:pubek]
+    
+    output = issue_request server_addr, :type => :search,
+                                        :root_tem => tem_info[:id],
                                         :map_reduce => client_query.to_hash
     output ? client_query.unpack_output(output) : nil
   end

data/lib/tem_mr_search/map_reduce_executor.rb

@@ -30,9 +30,12 @@ class MapReduceExecutor
     
     @db = db  # Writable only in main thread.
     @tems = tems  # Writable only in main thread.
+    
+    # Protected by @lock during collect_tem_ids, read-only during execute.
+    @tem_certs = Array.new @tems.length
 
     # Writable only in main thread.
-    @planner = planner_class.new @job, db.length, tems.length, root_tem
+    @planner = planner_class.new job, db.length, tems.length, root_tem
     
     # Protected by @lock
     @tem_parts = { :mapper => { root_tem => job.mapper },
@@ -49,6 +52,8 @@ class MapReduceExecutor
 
   # Executes the job.
   def execute
+    collect_tem_ids
+    
     # Spawn TEM threads.
     @tems.each_index { |i| Thread.new(i) { |i| executor_thread i } }
     
@@ -64,7 +69,18 @@ class MapReduceExecutor
     
     return @outputs[@planner.output_id]
   end
-  
+
+  # Collects identification information from all the TEMs.
+  def collect_tem_ids
+    threads = (0...@tems.length).map do |tem_index|
+      Thread.new(tem_index, @tems[tem_index]) do |index, tem|
+        ecert = tem.endorsement_cert
+        @lock.synchronize { @tem_certs[index] = ecert }
+      end
+    end
+    threads.each { |thread| thread.join }
+  end
+
   # Main method for thread in charge of a TEM.
   def executor_thread(tem_index)
     queue = @thread_queues[tem_index]    
@@ -83,7 +99,8 @@ class MapReduceExecutor
     case action[:action]
     when :migrate
       in_part = @lock.synchronize { @tem_parts[action[:secpack]][tem_index] }
-      out_part = in_part  # TODO(costan): actual migration
+      target_ecert = @tem_certs[action[:to]]
+      out_part = in_part.migrate target_ecert, @tems[tem_index]
       @lock.synchronize do
         @tem_parts[action[:secpack]][action[:to]] = out_part
       end

data/lib/tem_mr_search/map_reduce_job.rb

@@ -47,6 +47,11 @@ class MapReduceJob
       :map => @mapper.to_plain_object, :reduce => @reducer.to_plain_object,
       :finalize => @finalizer.to_plain_object }
   end
+  
+  # Binds the SECpacks in this job to the given key.
+  def bind(tem_pubek)
+    [@mapper, @reducer, @finalizer].each { |part| part.bind tem_pubek }
+  end
 
   # Base class for the Map-Reduce SECpack wrappers.
   class JobPart
@@ -54,7 +59,23 @@ class MapReduceJob
       unless secpack.nil? or secpack.kind_of? Tem::SecPack
         secpack = Tem::SecPack.new_from_array secpack
       end
-      @secpack = secpack      
+      @job = job
+      @secpack = secpack
+    end
+    
+    # Binds the wrapped SECpack to the given key.
+    def bind(tem_pubek)
+      @secpack.bind tem_pubek, :_secret, :_plain if @secpack
+    end
+    
+    # Creates a copy of the job part by migrating the wrapped SECpack.
+    #
+    # Args:
+    #   target_ecert:: the Endorsement Certificate of the target TEM
+    #   tem:: session to the TEM to be used for migration
+    def migrate(target_ecert, tem)
+      migrated_secpack = tem.migrate @secpack, target_ecert
+      self.class.new migrated_secpack, @job
     end
     
     def to_plain_object
@@ -71,7 +92,7 @@ class MapReduceJob
       @id_attribute = job.id_attribute
     end
     
-    # Returns a SECpack for mapping the given object data into the query.
+    # SECpack for mapping the given object data into the query.
     def map_for_object(object_data)
       return nil unless @secpack
       object_id = object_data[@id_attribute.to_s]    
@@ -93,7 +114,7 @@ class MapReduceJob
   
   # Wrapper for the reduce SECpack.
   class Reducer < JobPart
-    # Returns a SECpack for reducing two inputs coming from maps or other reduces.
+    # SECpack for reducing two inputs coming from maps or other reduces.
     def reduce_for_outputs(output1, output2)
       new_secpack = Tem::SecPack.new_from_array @secpack.to_array
       

data/lib/tem_mr_search/server.rb

@@ -87,15 +87,18 @@ class Server
       job = MapReduceJob.new request[:map_reduce]
       root_tem = request[:root_tem]
       executor = MapReduceExecutor.new job, @db, @tems, root_tem
-      return executor.execute
+      executor.execute
     when :fetch
-      return @db.item_by_id(request[:id]) || :not_found
+      @db.item_by_id(request[:id]) || :not_found
+    when :get_tem
+      tem_id = rand @tems.length
+      { :id => tem_id, :ecert => @tems[tem_id].endorsement_cert.to_pem }
     when :shutdown
-      return :shutdown
+      :shutdown
     when :db_dump
-      return (0...@db.length).map { |i| @db.item(i) }
+      (0...@db.length).map { |i| @db.item(i) }
     else
-      return :unknown
+      :unknown
     end
   end
   

data/tem_mr_search.gemspec

@@ -2,38 +2,44 @@
 
 Gem::Specification.new do |s|
   s.name = %q{tem_mr_search}
-  s.version = "0.2.5"
+  s.version = "0.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Victor Costan"]
-  s.date = %q{2009-11-15}
+  s.date = %q{2009-11-18}
   s.default_executable = %q{tem_mr_search_server}
   s.description = %q{Tem Map-Reduce proof of concept: database search.}
   s.email = %q{victor@costan.us}
   s.executables = ["tem_mr_search_server"]
-  s.extra_rdoc_files = ["bin/tem_mr_search_server", "CHANGELOG", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb", "lib/tem_mr_search.rb", "LICENSE", "README"]
-  s.files = ["bin/tem_mr_search_server", "CHANGELOG", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb", "lib/tem_mr_search.rb", "LICENSE", "Manifest", "Rakefile", "README", "tem_mr_search.gemspec", "test/mr_test_case.rb", "test/test_client_server.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builders.rb", "testdata/cluster.yml", "testdata/empty_cluster.yml", "testdata/fares.yml", "testdata/parallel_plan_431.yml", "testdata/parallel_plan_740.yml", "testdata/serial_plan_410.yml", "testdata/serial_plan_431.yml", "testdata/serial_plan_740.yml"]
+  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README", "bin/tem_mr_search_server", "lib/tem_mr_search.rb", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb"]
+  s.files = ["CHANGELOG", "LICENSE", "Manifest", "README", "Rakefile", "bin/tem_mr_search_server", "lib/tem_mr_search.rb", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb", "test/mr_test_case.rb", "test/test_client_server.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builders.rb", "testdata/cluster.yml", "testdata/empty_cluster.yml", "testdata/fares.yml", "testdata/parallel_plan_431.yml", "testdata/parallel_plan_740.yml", "testdata/serial_plan_410.yml", "testdata/serial_plan_431.yml", "testdata/serial_plan_740.yml", "tem_mr_search.gemspec"]
   s.homepage = %q{http://tem.rubyforge.org}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Tem_mr_search", "--main", "README"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{tem}
   s.rubygems_version = %q{1.3.5}
   s.summary = %q{Tem Map-Reduce proof of concept: database search.}
-  s.test_files = ["test/test_query_builders.rb", "test/test_map_reduce_job.rb", "test/test_client_server.rb", "test/test_map_reduce_planner.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb"]
+  s.test_files = ["test/test_client_server.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builders.rb"]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<tem_ruby>, [">= 0.14.0"])
+      s.add_runtime_dependency(%q<tem_ruby>, [">= 0.15.2"])
       s.add_runtime_dependency(%q<tem_multi_proxy>, [">= 0.2.7"])
+      s.add_development_dependency(%q<echoe>, [">= 3.2"])
+      s.add_development_dependency(%q<flexmock>, [">= 0.8.6"])
     else
-      s.add_dependency(%q<tem_ruby>, [">= 0.14.0"])
+      s.add_dependency(%q<tem_ruby>, [">= 0.15.2"])
       s.add_dependency(%q<tem_multi_proxy>, [">= 0.2.7"])
+      s.add_dependency(%q<echoe>, [">= 3.2"])
+      s.add_dependency(%q<flexmock>, [">= 0.8.6"])
     end
   else
-    s.add_dependency(%q<tem_ruby>, [">= 0.14.0"])
+    s.add_dependency(%q<tem_ruby>, [">= 0.15.2"])
     s.add_dependency(%q<tem_multi_proxy>, [">= 0.2.7"])
+    s.add_dependency(%q<echoe>, [">= 3.2"])
+    s.add_dependency(%q<flexmock>, [">= 0.8.6"])
   end
 end

data/test/test_client_server.rb

@@ -37,6 +37,24 @@ class ClientServerTest < MrTestCase
     end
   end
   
+  def test_get_tem
+    flexmock(Server).should_receive(:tems_from_cluster_file).
+                     with(@empty_cluster_file).and_return do |file|
+      Tem.auto_conf
+      $tem.emit if $tem.activate
+      [$tem]
+    end
+    
+    _test_request do |server_addr|
+      tem_info = Client.get_tem server_addr
+      assert_equal 0, tem_info[:id], 'Incorrect TEM id'
+      assert_equal $tem.endorsement_cert.to_pem, tem_info[:ecert].to_pem,
+                   'Incorrect ECert'
+      assert_equal $tem.pubek.ssl_key.to_pem, tem_info[:pubek].ssl_key.to_pem,
+                   'Incorrect PubEK'
+    end
+  end
+  
   def test_dump_database
     _test_request do |server_addr|
       items = Client.dump_database server_addr

data/test/test_map_reduce_executor.rb

@@ -18,6 +18,7 @@ class MapReduceExecutorTest < MrTestCase
   end
       
   def _test_executor(tems, root_tem)
+    @client_query.bind tems[root_tem].pubek
     executor = MRExecutor.new @client_query, @db, tems, root_tem
     packed_output = executor.execute
     result = @client_query.unpack_output packed_output

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: tem_mr_search
 version: !ruby/object:Gem::Version 
-  version: 0.2.5
+  version: "0.3"
 platform: ruby
 authors: 
 - Victor Costan
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-15 00:00:00 -05:00
+date: 2009-11-18 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.14.0
+        version: 0.15.2
     version: 
 - !ruby/object:Gem::Dependency 
   name: tem_multi_proxy
@@ -32,6 +32,26 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 0.2.7
     version: 
+- !ruby/object:Gem::Dependency 
+  name: echoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "3.2"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: flexmock
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.8.6
+    version: 
 description: "Tem Map-Reduce proof of concept: database search."
 email: victor@costan.us
 executables: 
@@ -39,8 +59,11 @@ executables:
 extensions: []
 
 extra_rdoc_files: 
-- bin/tem_mr_search_server
 - CHANGELOG
+- LICENSE
+- README
+- bin/tem_mr_search_server
+- lib/tem_mr_search.rb
 - lib/tem_mr_search/client.rb
 - lib/tem_mr_search/client_query.rb
 - lib/tem_mr_search/db.rb
@@ -50,12 +73,14 @@ extra_rdoc_files:
 - lib/tem_mr_search/query_builder.rb
 - lib/tem_mr_search/server.rb
 - lib/tem_mr_search/web_client_query_builder.rb
-- lib/tem_mr_search.rb
+files: 
+- CHANGELOG
 - LICENSE
+- Manifest
 - README
-files: 
+- Rakefile
 - bin/tem_mr_search_server
-- CHANGELOG
+- lib/tem_mr_search.rb
 - lib/tem_mr_search/client.rb
 - lib/tem_mr_search/client_query.rb
 - lib/tem_mr_search/db.rb
@@ -65,12 +90,6 @@ files:
 - lib/tem_mr_search/query_builder.rb
 - lib/tem_mr_search/server.rb
 - lib/tem_mr_search/web_client_query_builder.rb
-- lib/tem_mr_search.rb
-- LICENSE
-- Manifest
-- Rakefile
-- README
-- tem_mr_search.gemspec
 - test/mr_test_case.rb
 - test/test_client_server.rb
 - test/test_db.rb
@@ -86,6 +105,7 @@ files:
 - testdata/serial_plan_410.yml
 - testdata/serial_plan_431.yml
 - testdata/serial_plan_740.yml
+- tem_mr_search.gemspec
 has_rdoc: true
 homepage: http://tem.rubyforge.org
 licenses: []
@@ -120,9 +140,9 @@ signing_key:
 specification_version: 3
 summary: "Tem Map-Reduce proof of concept: database search."
 test_files: 
-- test/test_query_builders.rb
-- test/test_map_reduce_job.rb
 - test/test_client_server.rb
-- test/test_map_reduce_planner.rb
 - test/test_db.rb
 - test/test_map_reduce_executor.rb
+- test/test_map_reduce_job.rb
+- test/test_map_reduce_planner.rb
+- test/test_query_builders.rb