costan-tem_mr_search 0.2.1

data/CHANGELOG

@@ -0,0 +1,5 @@
+v0.2.1. Check for cluster configuration on each query.
+
+v0.2. Implemented remote queries.
+
+v0.1. Initial release.

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2009 Massachusetts Institute of Technology
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Manifest

@@ -0,0 +1,32 @@
+bin/tem_mr_search_server
+CHANGELOG
+lib/tem_mr_search/client.rb
+lib/tem_mr_search/client_query.rb
+lib/tem_mr_search/db.rb
+lib/tem_mr_search/map_reduce_executor.rb
+lib/tem_mr_search/map_reduce_job.rb
+lib/tem_mr_search/map_reduce_planner.rb
+lib/tem_mr_search/query_builder.rb
+lib/tem_mr_search/server.rb
+lib/tem_mr_search/web_client_query_builder.rb
+lib/tem_mr_search.rb
+LICENSE
+Manifest
+Rakefile
+README
+tem_mr_search.gemspec
+test/mr_test_case.rb
+test/test_client_server.rb
+test/test_db.rb
+test/test_map_reduce_executor.rb
+test/test_map_reduce_job.rb
+test/test_map_reduce_planner.rb
+test/test_query_builders.rb
+testdata/cluster.yml
+testdata/empty_cluster.yml
+testdata/fares.yml
+testdata/parallel_plan_431.yml
+testdata/parallel_plan_740.yml
+testdata/serial_plan_410.yml
+testdata/serial_plan_431.yml
+testdata/serial_plan_740.yml

data/README

@@ -0,0 +1,4 @@
+This is a concept proof of a Map-Reduce on a cluster of Trusted Execution
+Modules (TEMs). The Map-Reduce cluster performs queries consisting of finding
+the best item in a homogenous database using secret ranking criteria.
+ 

data/Rakefile

@@ -0,0 +1,23 @@
+require 'rubygems'
+gem 'echoe'
+require 'echoe'
+
+Echoe.new('tem_mr_search') do |p|
+  p.project = 'tem' # rubyforge project
+  p.docs_host = "costan@rubyforge.org:/var/www/gforge-projects/tem/rdoc/"
+  
+  p.author = 'Victor Costan'
+  p.email = 'victor@costan.us'
+  p.summary = 'Tem Map-Reduce proof of concept: database search.'
+  p.url = 'http://tem.rubyforge.org'
+  p.dependencies = ['tem_ruby >=0.11.2', 'tem_multi_proxy >=0.2']
+  
+  p.need_tar_gz = !Platform.windows?
+  p.need_zip = !Platform.windows?
+  p.rdoc_pattern = /^(lib|bin|tasks|ext)|^BUILD|^README|^CHANGELOG|^TODO|^LICENSE|^COPYING$/  
+end
+
+if $0 == __FILE__
+  Rake.application = Rake::Application.new
+  Rake.application.run
+end

data/bin/tem_mr_search_server

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'tem_mr_search'
+
+if ARGV.length < 2 || ARGV.length >= 3
+  print "Usage: #{$0} db_file cluster_file [server_port]\n"
+  exit
+end
+
+db_file, cluster_file, server_port = ARGV[0], ARGV[1], ARGV[2]
+server = Tem::Mr::Search::Server.new db_file, cluster_file, server_port
+server.serve_loop

data/lib/tem_mr_search/client.rb

@@ -0,0 +1,52 @@
+# :nodoc: namespace
+module Tem::Mr::Search
+
+class Client
+  OP = Zerg::Support::Protocols::ObjectProtocol
+  OPAdapter  = Zerg::Support::Sockets::ProtocolAdapter.adapter_module OP
+  
+  # Performs a private database search using a Map-Reduce.
+  def self.search(server_addr, client_query)
+    output = issue_request server_addr, :type => :search, :root_tem => 0,
+                                        :map_reduce => client_query.to_hash
+    output ? client_query.unpack_output(output) : nil
+  end
+  
+  # Asks for an item in the server's database.
+  #
+  # In production, there should be per-client rate-limiting on this request.
+  def self.fetch_item(server_addr, item_id)
+    issue_request server_addr, :type => :fetch, :id => item_id
+  end
+  
+  # Terminates the server.
+  #
+  # In production, normal clients wouldn't have access to this.
+  def self.shutdown_server(server_addr)
+    issue_request server_addr, :type => :shutdown
+  end
+  
+  # Dumps the server database.
+  #
+  # In production, normal clients wouldn't have access to this.
+  def self.dump_database(server_addr)
+    issue_request server_addr, :type => :db_dump
+  end
+  
+  # Issues a request against a Map-Reduce server and returns the response.
+  def self.issue_request(server_addr, request)
+    socket = Zerg::Support::SocketFactory.socket :out_addr => server_addr,
+        :out_port => Server::DEFAULT_PORT
+    socket.extend OPAdapter
+    begin
+      socket.send_object request
+      response = socket.recv_object response
+    rescue
+      response = nil
+    end
+    socket.close rescue nil
+    response
+  end
+end
+
+end  # namespace Tem::Mr::Search

data/lib/tem_mr_search/client_query.rb

@@ -0,0 +1,21 @@
+# :nodoc: namespace
+module Tem::Mr::Search
+
+class ClientQuery < MapReduceJob
+  def initialize(attributes)
+    super
+    @query_key = attributes[:key]
+  end
+
+  # Unpacks a reduce output into its components.
+  #
+  # This is expected to be called with the encrypted output returned by the
+  # search provider.
+  def unpack_output(output)
+    # TODO(costan): decrypt output once we enable encryption
+    decrypted_output = output
+    unpack_decrypted_output decrypted_output
+  end
+end
+  
+end  # namespace Tem::Mr::search

data/lib/tem_mr_search/db.rb

@@ -0,0 +1,26 @@
+# :nodoc: namespace
+module Tem::Mr::Search
+  
+class Db
+  attr_reader :data
+  attr_reader :id_attribute
+  
+  def initialize(path)
+    @data = File.open(path, 'r') { |f| YAML.load f }
+    @id_attribute = 'flight'
+  end
+  
+  def length
+    @data.length
+  end
+  
+  def item(item_index)
+    @data[item_index]
+  end
+  
+  def item_by_id(item_id)
+    @data.find { |item| item[@id_attribute] == item_id }
+  end
+end
+  
+end  # namespace Tem::Mr::search

data/lib/tem_mr_search/map_reduce_executor.rb

@@ -0,0 +1,115 @@
+require 'thread'
+
+
+# :nodoc: namespace
+module Tem::Mr::Search
+  
+class MapReduceExecutor
+  # Creates an executor for a Map-Reduce job.
+  #
+  # Arguments:
+  #   job:: the Map-Reduce job (see Tem::Mr::Search::MapReduceJob)
+  #   db:: the database to run Map-Reduce over
+  #   tems:: sessions to the available TEMs
+  #   root_tem:: the index of the TEM that has the root mapper and reducer
+  #   planner_class:: (optional) replacement for the default planner strategy
+  def initialize(job, db, tems, root_tem, planner_class = nil)
+    planner_class ||= MapReducePlanner
+    
+    @db = db  # Writable only in main thread.
+    @tems = tems  # Writable only in main thread.
+
+    # Writable only in main thread.
+    @planner = planner_class.new @job, db.length, tems.length, root_tem
+    
+    # Protected by @lock
+    @tem_parts = { :mapper => { root_tem => job.mapper },
+                   :reducer => { root_tem => job.reducer },
+                   :finalizer => { root_tem => job.finalizer } }
+    # Protected by @lock
+    @outputs = {}
+    
+    # Thread-safe.
+    @thread_queues = tems.map { |tem| Queue.new }
+    @main_queue = Queue.new
+    @lock = Mutex.new
+  end
+
+  # Executes the job.
+  def execute
+    # Spawn TEM threads.
+    @tems.each_index { |i| Thread.new(i) { |i| executor_thread i } }
+    
+    until @planner.done?
+      actions = @planner.next_actions!
+      @lock.synchronize do
+        actions.each { |action| @thread_queues[action[:with]] << action }
+      end
+      
+      action = @main_queue.pop
+      @planner.action_done action
+    end
+    
+    return @outputs[@planner.output_id]
+  end
+  
+  # Main method for thread in charge of a TEM.
+  def executor_thread(tem_index)
+    queue = @thread_queues[tem_index]    
+    while action = queue.pop
+      execute_action action, tem_index
+      @main_queue << action
+    end
+  end
+  
+  # Executes a Map-Reduce planner action.
+  #
+  # This method is called on the thread corresponding to the TEM that the action
+  # is supposed to execute on.
+  def execute_action(action, tem_index)
+    case action[:action]
+    when :migrate
+      in_part = @lock.synchronize { @tem_parts[action[:secpack]][tem_index] }
+      out_part = in_part  # TODO(costan): actual migration
+      @lock.synchronize do
+        @tem_parts[action[:secpack]][action[:to]] = out_part
+      end
+      
+    when :map
+      mapper, item = nil, nil
+      @lock.synchronize do
+        mapper = @tem_parts[:mapper][tem_index]
+        item = @db.item(action[:item])
+      end
+      output = mapper.map_object item, @tems[tem_index]
+      @lock.synchronize do
+        @outputs[action[:output_id]] = output
+      end
+      
+    when :reduce
+      reducer, output1, output2 = nil, nil, nil
+      @lock.synchronize do
+        reducer = @tem_parts[:reducer][tem_index]
+        output1 = @outputs[action[:output1_id]]
+        output2 = @outputs[action[:output2_id]]
+      end
+      output = reducer.reduce_outputs output1, output2, @tems[tem_index]
+      @lock.synchronize do
+        @outputs[action[:output_id]] = output
+      end
+
+    when :finalize
+      finalizer = nil
+      @lock.synchronize do
+        finalizer = @tem_parts[:finalizer][tem_index]
+        output = @outputs[action[:output_id]]
+      end
+      final_output = finalizer.finalize_output output, @tems[tem_index]
+      @lock.synchronize do
+        @outputs[action[:final_id]] = final_output
+      end
+    end    
+  end
+end
+
+end  # namespace Tem::Mr::Search

data/lib/tem_mr_search/map_reduce_job.rb

@@ -0,0 +1,108 @@
+# :nodoc: namespace
+module Tem::Mr::Search
+  
+class MapReduceJob  
+  attr_reader :mapper, :reducer, :finalizer, :attributes, :id_attribute
+  
+  def initialize(attributes)
+    @attributes = attributes[:attributes]
+    @id_attribute = attributes[:id_attribute]
+
+    @mapper = Mapper.new attributes[:map], self
+    @reducer = Reducer.new attributes[:reduce], self
+    @finalizer = Finalizer.new attributes[:finalize], self
+  end
+  
+  # Unpacks a decrypted output into its components.
+  def unpack_decrypted_output(output)
+    {
+      :id => output[0, 8].reverse.pack('C*').unpack('q').first,
+      :score => Tem::Abi.read_tem_short(output, 8),
+      :check => output[13, 3]
+    }
+  end
+  
+  # Serializes a job to a hash.
+  #
+  # Useful in conjunction with ObjectProtocol in ZergSupport, for sending jobs
+  # across the wire. De-serialize with MapReduceJob#new
+  def to_hash
+    { :attributes => @attributes, :id_attribute => @id_attribute,
+      :map => @mapper.to_plain_object, :reduce => @reducer.to_plain_object,
+      :finalize => @finalizer.to_plain_object }
+  end
+
+  # Base class for the Map-Reduce SECpack wrappers.
+  class JobPart
+    def initialize(secpack, job)
+      unless secpack.nil? or secpack.kind_of? Tem::SecPack
+        secpack = Tem::SecPack.new_from_array secpack
+      end
+      @secpack = secpack      
+    end
+    
+    def to_plain_object
+      return nil if @secpack.nil?
+      @secpack.to_array
+    end
+  end
+
+  # Wrapper for the map SECpack.
+  class Mapper < JobPart
+    def initialize(secpack, job)
+      super
+      @attributes = job.attributes
+      @id_attribute = job.id_attribute
+    end
+    
+    # Returns a SECpack for mapping the given object data into the query.
+    def map_for_object(object_data)
+      return nil unless @secpack
+      object_id = object_data[@id_attribute.to_s]    
+      new_secpack = Tem::SecPack.new_from_array @secpack.to_array
+      new_secpack.set_bytes :_id, [object_id].pack('q').unpack('C*').reverse
+      @attributes.each do |attribute|
+        name, type = attribute[:name], attribute[:type]
+        new_secpack.set_value name.to_sym, type, object_data[name.to_s]
+      end
+      new_secpack
+    end
+    
+    # Maps the given object into the query.
+    def map_object(object_data, tem)    
+      secpack = map_for_object object_data
+      secpack ? tem.execute(secpack) : object_data
+    end
+  end  
+  
+  # Wrapper for the reduce SECpack.
+  class Reducer < JobPart
+    # Returns a SECpack for reducing two inputs coming from maps or other reduces.
+    def reduce_for_outputs(output1, output2)
+      new_secpack = Tem::SecPack.new_from_array @secpack.to_array
+      
+      new_secpack.set_bytes :_output1, output1
+      new_secpack.set_bytes :_output2, output2
+      new_secpack
+    end
+
+    # Reduces two inputs coming from maps or other reduces.
+    def reduce_outputs(output1, output2, tem)
+      secpack = reduce_for_outputs output1, output2
+      tem.execute secpack
+    end
+  end
+  
+  # Wrapper for the finalize SECpack.
+  class Finalizer < JobPart
+    # Converts a map/reduce output into the final result for the operation.
+    def finalize_output(output, tem)
+      return output unless @secpack
+      secpack = Tem::SecPack.new_from_array @finalize_secpack.to_array
+      secpack.set_bytes :_output, output
+      tem.execute secpack
+    end
+  end  
+end
+
+end  # namespace Tem::Mr::search

data/lib/tem_mr_search/map_reduce_planner.rb

@@ -0,0 +1,169 @@
+require 'rbtree'
+require 'set'
+
+# :nodoc: namespace
+module Tem::Mr::Search
+  
+class MapReducePlanner  
+  # Creates a planner for a Map-Reduce job.
+  #
+  # Arguments:
+  #   job:: the Map-Reduce job (see Tem::Mr::Search::MapReduceJob)
+  #   num_items: how many data items does the Map-Reduce run over
+  #   num_tems:: how many TEMs are available
+  #   root_tem:: the index of the TEM that has the root mapper and reducer
+  def initialize(job, num_items, num_tems, root_tem)
+    @job = job
+    @root_tem = root_tem
+    
+    @without = { :mapper => RBTree.new, :reducer => RBTree.new }
+    @with = { :mapper => Set.new([root_tem]),
+              :reducer => Set.new([root_tem]) }
+    @free_tems = RBTree.new
+    0.upto(num_tems - 1) do |tem|
+      @free_tems[tem] = true
+      next if tem == root_tem
+      @without.each { |k, v| v[tem] = true }
+    end
+    
+    @unmapped_items = (0...num_items).to_a.reverse
+    @reduce_queue = RBTree.new
+    @last_output_id = 0
+    @last_reduce_id = 2 * num_items - 2
+    @done_reducing, @output_id = false, nil
+  end
+
+  # Generates migrating actions for a SECpack type that are possible now.
+  def migrate_actions(sec_type)
+    actions = []
+    return actions if @without[sec_type].length == 0
+    free_tems = free_tems_with_sec sec_type
+    free_tems.each do |source_tem|
+      break if @without[sec_type].length == 0
+      target_tem = @without[sec_type].min.first
+      @without[sec_type].delete target_tem
+      @free_tems.delete source_tem
+      actions.push :action => :migrate, :secpack => sec_type,
+                   :with => source_tem, :to => target_tem
+    end
+    actions
+  end
+  private :migrate_actions
+  
+  # Informs the planner that a SECpack migration has completed.
+  def done_migrating(action)
+    @free_tems[action[:with]] = true
+    @with[action[:secpack]] << action[:to]
+  end
+  private :done_migrating
+  
+  # A sorted array of the free TEMs that have a SECpack type.
+  def free_tems_with_sec(sec_type)
+    tems = []
+    @free_tems.each do |tem, true_value|
+      tems << tem if @with[sec_type].include? tem
+    end
+    tems
+  end
+  
+  # A unique output_id.
+  def next_output_id
+    next_id = @last_output_id
+    @last_output_id += 1
+    next_id
+  end
+  
+  # Generates mapping actions possible right now.
+  def map_actions
+    actions = []
+    return actions if @unmapped_items.empty?
+    free_tems_with_sec(:mapper).each do |tem|
+      break unless item = @unmapped_items.pop
+      @free_tems.delete tem
+      actions.push :action => :map, :item => item, :with => tem,
+                   :output_id => next_output_id
+    end
+    actions
+  end
+  private :map_actions
+  
+  # Informs the planner that a data mapping has completed.
+  def done_mapping(action)
+    @free_tems[action[:with]] = true
+    @reduce_queue[action[:output_id]] = true
+  end
+  private :done_mapping
+  
+  # Generates reducing actions possible right now.
+  def reduce_actions
+    actions = []
+    return actions if @reduce_queue.length <= 1
+    free_tems_with_sec(:reducer).each do |tem|
+      break if @reduce_queue.length <= 1
+      output1_id, output2_id = *[0, 1].map do |i|
+        output_id = @reduce_queue.min.first
+        @reduce_queue.delete output_id
+        output_id
+      end
+      @free_tems.delete tem
+      actions.push :action => :reduce, :with => tem, :output1_id => output1_id,
+                   :output2_id => output2_id, :output_id => next_output_id
+    end
+    actions
+  end
+  private :reduce_actions
+  
+  # Informs the planner that a data reduction has completed.
+  def done_reducing(action)
+    @free_tems[action[:with]] = true
+    if action[:output_id] == @last_reduce_id
+      @done_reducing = true      
+      return
+    end
+    @reduce_queue[action[:output_id]] = true
+  end
+  private :done_reducing
+  
+  # Generates finalizing actions possible right now.
+  def finalize_actions
+    return [] unless @done_reducing and !@output_id and @free_tems[@root_tem]
+    @finalize_ready = false
+    return [ :action => :finalize, :with => @root_tem,
+             :output_id => @last_reduce_id, :final_id => next_output_id ]
+  end
+  private :finalize_actions
+  
+  # Informs the planner that an action issued by next_action was done.
+  def done_finalizing(action)
+    @free_tems[action[:with]] = true
+    @output_id = action[:final_id]    
+  end
+  private :done_finalizing
+
+  # True when the Map-Reduce job is complete.
+  def done?
+    !@output_id.nil?
+  end
+  
+  # The output ID of the Map-Reduce's final result.
+  attr_reader :output_id
+
+  # Informs the planner that an action issued by next_actions was completed.
+  def action_done(action)
+    dispatch = { :migrate => :done_migrating, :map => :done_mapping, :reduce =>
+                 :done_reducing, :finalize => :done_finalizing } 
+    self.send dispatch[action[:action]], action
+  end
+
+  # Issues a set of actions that can be performed right now.
+  def next_actions!
+    actions = migrate_actions :mapper
+    actions += migrate_actions :reducer
+    actions += map_actions
+    actions += reduce_actions
+    actions += finalize_actions
+    actions
+  end  
+end
+
+end  # namespace Tem::Mr::search