tem_mr_search 0.1 → 0.2

data/CHANGELOG

@@ -1 +1,3 @@
+v0.2. Implemented remote queries.
+
 v0.1. Initial release.

data/Manifest

@@ -1,10 +1,14 @@
+bin/tem_mr_search_server
 CHANGELOG
+lib/tem_mr_search/client.rb
 lib/tem_mr_search/client_query.rb
 lib/tem_mr_search/db.rb
 lib/tem_mr_search/map_reduce_executor.rb
 lib/tem_mr_search/map_reduce_job.rb
 lib/tem_mr_search/map_reduce_planner.rb
 lib/tem_mr_search/query_builder.rb
+lib/tem_mr_search/server.rb
+lib/tem_mr_search/web_client_query_builder.rb
 lib/tem_mr_search.rb
 LICENSE
 Manifest
@@ -12,11 +16,14 @@ Rakefile
 README
 tem_mr_search.gemspec
 test/mr_test_case.rb
+test/test_client_server.rb
 test/test_db.rb
 test/test_map_reduce_executor.rb
 test/test_map_reduce_job.rb
 test/test_map_reduce_planner.rb
-test/test_query_builder.rb
+test/test_query_builders.rb
+testdata/cluster.yml
+testdata/empty_cluster.yml
 testdata/fares.yml
 testdata/parallel_plan_431.yml
 testdata/parallel_plan_740.yml

data/bin/tem_mr_search_server

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'tem_mr_search'
+
+if ARGV.length < 2 || ARGV.length >= 3
+  print "Usage: #{$0} db_file cluster_file [server_port]\n"
+  exit
+end
+
+db_file, cluster_file, server_port = ARGV[0], ARGV[1], ARGV[2]
+server = Tem::Mr::Search::Server.new db_file, cluster_file, server_port
+server.serve_loop

data/lib/tem_mr_search/client.rb

@@ -0,0 +1,44 @@
+# :nodoc: namespace
+module Tem::Mr::Search
+
+class Client
+  OP = Zerg::Support::Protocols::ObjectProtocol
+  OPAdapter  = Zerg::Support::Sockets::ProtocolAdapter.adapter_module OP
+  
+  # Performs a private database search using a Map-Reduce.
+  def self.search(server_addr, client_query)
+    output = issue_request server_addr, :type => :search, :root_tem => 0,
+                                        :map_reduce => client_query.to_hash
+    client_query.unpack_output output
+  end
+  
+  # Asks for an item in the server's database.
+  #
+  # In production, there should be per-client rate-limiting on this request.
+  def self.fetch_item(server_addr, item_id)
+    issue_request server_addr, :type => :fetch, :id => item_id
+  end
+  
+  # Terminates the server.
+  #
+  # In production, normal clients wouldn't have access to this.
+  def self.shutdown_server(server_addr)
+    issue_request server_addr, :type => :shutdown
+  end
+  
+  # Issues a request against a Map-Reduce server and returns the response.
+  def self.issue_request(server_addr, request)
+    socket = Zerg::Support::SocketFactory.socket :out_addr => server_addr
+    socket.extend OPAdapter
+    begin
+      socket.send_object request
+      response = socket.recv_object response
+    rescue
+      response = nil
+    end
+    socket.close rescue nil
+    response
+  end
+end
+
+end  # namespace Tem::Mr::Search

data/lib/tem_mr_search/db.rb

@@ -3,8 +3,11 @@ module Tem::Mr::Search
   
 class Db
   attr_reader :data
+  attr_reader :id_attribute
+  
   def initialize(path)
     @data = File.open(path, 'r') { |f| YAML.load f }
+    @id_attribute = 'flight'
   end
   
   def length
@@ -14,6 +17,10 @@ class Db
   def item(item_index)
     @data[item_index]
   end
+  
+  def item_by_id(item_id)
+    @data.find { |item| item[@id_attribute] == item_id }
+  end
 end
   
 end  # namespace Tem::Mr::search

data/lib/tem_mr_search/map_reduce_executor.rb

@@ -8,12 +8,12 @@ class MapReduceExecutor
   # Creates an executor for a Map-Reduce job.
   #
   # Arguments:
-  #   root_job:: the Map-Reduce job (see Tem::Mr::Search::MapReduceJob)
+  #   job:: the Map-Reduce job (see Tem::Mr::Search::MapReduceJob)
   #   db:: the database to run Map-Reduce over
   #   tems:: sessions to the available TEMs
   #   root_tem:: the index of the TEM that has the root mapper and reducer
   #   planner_class:: (optional) replacement for the default planner strategy
-  def initialize(root_job, db, tems, root_tem, planner_class = nil)
+  def initialize(job, db, tems, root_tem, planner_class = nil)
     planner_class ||= MapReducePlanner
     
     @db = db  # Writable only in main thread.
@@ -23,9 +23,9 @@ class MapReduceExecutor
     @planner = planner_class.new @job, db.length, tems.length, root_tem
     
     # Protected by @lock
-    @tem_jobs = { :mapper => { root_tem => root_job },
-                  :reducer => { root_tem => root_job },
-                  :finalizer => { root_tem => root_job } }
+    @tem_parts = { :mapper => { root_tem => job.mapper },
+                   :reducer => { root_tem => job.reducer },
+                   :finalizer => { root_tem => job.finalizer } }
     # Protected by @lock
     @outputs = {}
     
@@ -69,41 +69,42 @@ class MapReduceExecutor
   def execute_action(action, tem_index)
     case action[:action]
     when :migrate
-      in_job = @lock.synchronize { @tem_jobs[action[:secpack]][tem_index] }
-      out_job = in_job  # TODO(costan): actual migration
+      in_part = @lock.synchronize { @tem_parts[action[:secpack]][tem_index] }
+      out_part = in_part  # TODO(costan): actual migration
       @lock.synchronize do
-        @tem_jobs[action[:secpack]][action[:to]] = out_job
+        @tem_parts[action[:secpack]][action[:to]] = out_part
       end
       
     when :map
-      job, item = nil, nil
+      mapper, item = nil, nil
       @lock.synchronize do
-        job = @tem_jobs[:mapper][tem_index]
+        mapper = @tem_parts[:mapper][tem_index]
         item = @db.item(action[:item])
       end
-      output = job.map_object item, @tems[tem_index]
+      output = mapper.map_object item, @tems[tem_index]
       @lock.synchronize do
         @outputs[action[:output_id]] = output
       end
       
     when :reduce
-      job, output1, output2 = nil, nil, nil
+      reducer, output1, output2 = nil, nil, nil
       @lock.synchronize do
-        job = @tem_jobs[:reducer][tem_index]
+        reducer = @tem_parts[:reducer][tem_index]
         output1 = @outputs[action[:output1_id]]
         output2 = @outputs[action[:output2_id]]
       end
-      output = job.reduce_outputs output1, output2, @tems[tem_index]
+      output = reducer.reduce_outputs output1, output2, @tems[tem_index]
       @lock.synchronize do
         @outputs[action[:output_id]] = output
       end
 
     when :finalize
+      finalizer = nil
       @lock.synchronize do
-        job = @tem_jobs[:finalizer][tem_index]
+        finalizer = @tem_parts[:finalizer][tem_index]
         output = @outputs[action[:output_id]]
       end
-      final_output = job.finalize_output output, @tems[tem_index]
+      final_output = finalizer.finalize_output output, @tems[tem_index]
       @lock.synchronize do
         @outputs[action[:final_id]] = final_output
       end

data/lib/tem_mr_search/map_reduce_job.rb

@@ -2,66 +2,107 @@
 module Tem::Mr::Search
   
 class MapReduceJob  
-  attr_reader :map_secpack, :reduce_secpack, :attributes, :id_attribute
+  attr_reader :mapper, :reducer, :finalizer, :attributes, :id_attribute
   
   def initialize(attributes)
-    @map_secpack = attributes[:map]
-    @reduce_secpack = attributes[:reduce]
-    @finalize_secpack = attributes[:finalize]
     @attributes = attributes[:attributes]
     @id_attribute = attributes[:id_attribute]
+
+    @mapper = Mapper.new attributes[:map], self
+    @reducer = Reducer.new attributes[:reduce], self
+    @finalizer = Finalizer.new attributes[:finalize], self
   end
   
-  # Returns a SECpack for mapping the given object data into the query.
-  def map_for_object(object_data)
-    return nil unless @map_secpack
-    object_id = object_data[id_attribute.to_s]    
-    secpack = Tem::SecPack.new_from_array @map_secpack.to_array
-    secpack.set_bytes :_id, [object_id].pack('q').unpack('C*').reverse
-    attributes.each do |attribute|
-      name, type = attribute[:name], attribute[:type]
-      secpack.set_value name.to_sym, type, object_data[name.to_s]
-    end
-    secpack
+  # Unpacks a decrypted output into its components.
+  def unpack_decrypted_output(output)
+    {
+      :id => output[0, 8].reverse.pack('C*').unpack('q').first,
+      :score => Tem::Abi.read_tem_short(output, 8),
+      :check => output[13, 3]
+    }
   end
   
-  # Maps the given object into the query.
-  def map_object(object_data, tem)    
-    secpack = map_for_object object_data
-    secpack ? tem.execute(secpack) : object_data
+  # Serializes a job to a hash.
+  #
+  # Useful in conjunction with ObjectProtocol in ZergSupport, for sending jobs
+  # across the wire. De-serialize with MapReduceJob#new
+  def to_hash
+    { :attributes => @attributes, :id_attribute => @id_attribute,
+      :map => @mapper.to_plain_object, :reduce => @reducer.to_plain_object,
+      :finalize => @finalizer.to_plain_object }
   end
 
-  # Returns a SECpack for reducing two inputs coming from maps or other reduces.
-  def reduce_for_outputs(output1, output2)
-    secpack = Tem::SecPack.new_from_array @reduce_secpack.to_array
+  # Base class for the Map-Reduce SECpack wrappers.
+  class JobPart
+    def initialize(secpack, job)
+      unless secpack.nil? or secpack.kind_of? Tem::SecPack
+        secpack = Tem::SecPack.new_from_array secpack
+      end
+      @secpack = secpack      
+    end
     
-    secpack.set_bytes :_output1, output1
-    secpack.set_bytes :_output2, output2
-    secpack
+    def to_plain_object
+      return nil if @secpack.nil?
+      @secpack.to_array
+    end
   end
+
+  # Wrapper for the map SECpack.
+  class Mapper < JobPart
+    def initialize(secpack, job)
+      super
+      @attributes = job.attributes
+      @id_attribute = job.id_attribute
+    end
+    
+    # Returns a SECpack for mapping the given object data into the query.
+    def map_for_object(object_data)
+      return nil unless @secpack
+      object_id = object_data[@id_attribute.to_s]    
+      new_secpack = Tem::SecPack.new_from_array @secpack.to_array
+      new_secpack.set_bytes :_id, [object_id].pack('q').unpack('C*').reverse
+      @attributes.each do |attribute|
+        name, type = attribute[:name], attribute[:type]
+        new_secpack.set_value name.to_sym, type, object_data[name.to_s]
+      end
+      new_secpack
+    end
+    
+    # Maps the given object into the query.
+    def map_object(object_data, tem)    
+      secpack = map_for_object object_data
+      secpack ? tem.execute(secpack) : object_data
+    end
+  end  
   
-  # Reduces two inputs coming from maps or other reduces.
-  def reduce_outputs(output1, output2, tem)
-    secpack = reduce_for_outputs output1, output2
-    tem.execute secpack
-  end
+  # Wrapper for the reduce SECpack.
+  class Reducer < JobPart
+    # Returns a SECpack for reducing two inputs coming from maps or other reduces.
+    def reduce_for_outputs(output1, output2)
+      new_secpack = Tem::SecPack.new_from_array @secpack.to_array
+      
+      new_secpack.set_bytes :_output1, output1
+      new_secpack.set_bytes :_output2, output2
+      new_secpack
+    end
 
-  # Converts a map/reduce output into the final result for the operation.
-  def finalize_output(output, tem)
-    return output unless @finalize_secpack
-    secpack = Tem::SecPack.new_from_array @finalize_secpack.to_array
-    secpack.set_bytes :_output, output
-    tem.execute secpack
+    # Reduces two inputs coming from maps or other reduces.
+    def reduce_outputs(output1, output2, tem)
+      secpack = reduce_for_outputs output1, output2
+      tem.execute secpack
+    end
   end
   
-  # Unpacks a decrypted output into its components.
-  def unpack_decrypted_output(output)
-    {
-      :id => output[0, 8].reverse.pack('C*').unpack('q').first,
-      :score => Tem::Abi.read_tem_short(output, 8),
-      :check => output[13, 3]
-    }
-  end
+  # Wrapper for the finalize SECpack.
+  class Finalizer < JobPart
+    # Converts a map/reduce output into the final result for the operation.
+    def finalize_output(output, tem)
+      return output unless @secpack
+      secpack = Tem::SecPack.new_from_array @finalize_secpack.to_array
+      secpack.set_bytes :_output, output
+      tem.execute secpack
+    end
+  end  
 end
 
 end  # namespace Tem::Mr::search

data/lib/tem_mr_search/server.rb

@@ -0,0 +1,74 @@
+require 'logger'
+require 'yaml'
+
+# :nodoc: namespace
+module Tem::Mr::Search
+
+class Server
+  OP = Zerg::Support::Protocols::ObjectProtocol
+  OPAdapter  = Zerg::Support::Sockets::ProtocolAdapter.adapter_module OP
+  
+  # Creates a new Map-Reduce server (master).
+  def initialize(db_file, cluster_file, port)
+    @logger = Logger.new STDERR
+    @db = Db.new db_file
+    @tems = Server.tems_from_cluster_file cluster_file
+    @port = port
+  end
+
+  # This server's loop.
+  def serve_loop
+    listen_socket = Zerg::Support::SocketFactory.socket :in_port => @port
+    listen_socket.listen
+    shutdown_received = false
+    until shutdown_received
+      begin
+        client_socket, client_addr = listen_socket.accept
+        client_socket.extend OPAdapter
+        request = client_socket.recv_object
+        begin
+          response = process_request request
+        rescue Exception => e
+          @logger.error e
+          response = nil
+        end        
+        client_socket.send_object response if response
+        shutdown_received = true if response == :shutdown
+      rescue Exception => e
+        @logger.error e
+      end
+      client_socket.close rescue nil
+    end
+    listen_socket.close
+  end
+  
+  # Computes the response of a single request.
+  def process_request(request)    
+    case request[:type]
+    when :search
+      job = MapReduceJob.new request[:map_reduce]
+      root_tem = request[:root_tem]
+      executor = MapReduceExecutor.new job, @db, @tems, root_tem
+      return executor.execute
+    when :fetch
+      return @db.item_by_id(request[:id]) || :not_found
+    when :shutdown
+      return :shutdown
+    else
+      return :unknown
+    end
+  end
+  
+  # Creates sessions to all the TEMs in a cluster.
+  def self.tems_from_cluster_file(cluster_file)
+    cluster_hosts = File.open(cluster_file, 'r') { |f| YAML.load f }
+    cluster_configs = cluster_hosts.map { |host|
+      Tem::MultiProxy::Client.query_tems host
+    }.flatten
+    cluster_configs.reject { |config| config.nil? }.map do |config|
+      Tem::Session.new Tem::Transport::AutoConfigurator.try_transport(config)
+    end
+  end
+end
+
+end  # namespace Tem::Mr::Search

data/lib/tem_mr_search/web_client_query_builder.rb

@@ -0,0 +1,50 @@
+# :nodoc: namespace
+module Tem::Mr::Search
+
+class WebClientQueryBuilder < MapReduceJob
+  # Builds a client query covering preferences expressed in the Web UI.
+  #
+  # The supported (and required) options are:
+  #   layovers_cost:: the cost of each layover
+  #   start_time_cost:: the cost of the flight's departure time, in minutes
+  #   duration_cost:: the cost of each minute of flying
+  def self.query(options)
+    QueryBuilder.query { |q|
+      q.attributes :price => :tem_short, :start_time => :tem_short,
+                   :end_time => :tem_short, :layovers => :tem_short
+      q.id_attribute :flight
+      
+      # Score: 20000 - price - layover_cost * layovers -
+      #                start_time * start_time_cost -
+      #                (end_time - start_time) * duration_cost
+      q.map { |s|
+        s.ldwc 20000
+        s.ldw :price
+        s.sub
+        s.ldw :end_time
+        s.ldw :start_time
+        s.sub
+        s.ldwc options[:duration_cost]
+        s.mul
+        s.sub
+        [:start_time, :layovers].each do |factor|
+          s.ldw factor
+          s.ldwc options[:"#{factor}_cost"]
+          s.mul
+          s.sub
+        end
+        s.stw :score
+      }
+
+      # The greater score wins.
+      q.reduce { |s|
+        s.ldw :score1
+        s.ldw :score2
+        s.cmp
+        s.stw :comparison
+      }
+    }
+  end
+end
+  
+end  # namespace Tem::Mr::search

data/lib/tem_mr_search.rb

@@ -9,9 +9,12 @@ end
 module Tem::Mr::Search
 end
 
+require 'tem_mr_search/client.rb'
 require 'tem_mr_search/db.rb'
 require 'tem_mr_search/map_reduce_executor.rb'
 require 'tem_mr_search/map_reduce_job.rb'
 require 'tem_mr_search/map_reduce_planner.rb'
 require 'tem_mr_search/query_builder.rb'
 require 'tem_mr_search/client_query.rb'
+require 'tem_mr_search/server.rb'
+require 'tem_mr_search/web_client_query_builder.rb'

data/tem_mr_search.gemspec

@@ -2,22 +2,24 @@
 
 Gem::Specification.new do |s|
   s.name = %q{tem_mr_search}
-  s.version = "0.1"
+  s.version = "0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Victor Costan"]
-  s.date = %q{2009-06-02}
+  s.date = %q{2009-06-04}
+  s.default_executable = %q{tem_mr_search_server}
   s.description = %q{Tem Map-Reduce proof of concept: database search.}
   s.email = %q{victor@costan.us}
-  s.extra_rdoc_files = ["CHANGELOG", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search.rb", "LICENSE", "README"]
-  s.files = ["CHANGELOG", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search.rb", "LICENSE", "Manifest", "Rakefile", "README", "tem_mr_search.gemspec", "test/mr_test_case.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builder.rb", "testdata/fares.yml", "testdata/parallel_plan_431.yml", "testdata/parallel_plan_740.yml", "testdata/serial_plan_410.yml", "testdata/serial_plan_431.yml", "testdata/serial_plan_740.yml"]
+  s.executables = ["tem_mr_search_server"]
+  s.extra_rdoc_files = ["bin/tem_mr_search_server", "CHANGELOG", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb", "lib/tem_mr_search.rb", "LICENSE", "README"]
+  s.files = ["bin/tem_mr_search_server", "CHANGELOG", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb", "lib/tem_mr_search.rb", "LICENSE", "Manifest", "Rakefile", "README", "tem_mr_search.gemspec", "test/mr_test_case.rb", "test/test_client_server.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builders.rb", "testdata/cluster.yml", "testdata/empty_cluster.yml", "testdata/fares.yml", "testdata/parallel_plan_431.yml", "testdata/parallel_plan_740.yml", "testdata/serial_plan_410.yml", "testdata/serial_plan_431.yml", "testdata/serial_plan_740.yml"]
   s.homepage = %q{http://tem.rubyforge.org}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Tem_mr_search", "--main", "README"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{tem}
   s.rubygems_version = %q{1.3.4}
   s.summary = %q{Tem Map-Reduce proof of concept: database search.}
-  s.test_files = ["test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builder.rb"]
+  s.test_files = ["test/test_client_server.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builders.rb"]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION

data/test/mr_test_case.rb

@@ -9,43 +9,21 @@ class MrTestCase < Test::Unit::TestCase
     
     Thread.abort_on_exception = true
     
-    testdb_path = File.join File.dirname(__FILE__), "..", "testdata",
-                            "fares.yml"
-    @db = Db.new testdb_path
+    @db_path = File.join File.dirname(__FILE__), "..", "testdata", "fares.yml"
+    @cluster_file = File.join File.dirname(__FILE__), "..", "testdata",
+                            "cluster.yml"
+    @empty_cluster_file = File.join File.dirname(__FILE__), "..", "testdata",
+                                    "empty_cluster.yml"
+    @db = Db.new @db_path
 
-    @client_query = QueryBuilder.query { |q|
-      q.attributes :price => :tem_short, :start => :tem_short,
-                   :end => :tem_short
-      q.id_attribute :flight
-      
-      # Score: 200 + start / 100 - duration - price
-      q.map { |s|
-        s.ldwc 200
-        s.ldw :start
-        s.ldbc 100
-        s.div
-        s.add
-        s.ldw :end
-        s.ldw :start
-        s.sub
-        s.sub
-        s.ldw :price
-        s.sub
-        s.stw :score
-      }
-
-      # The greater score wins.
-      q.reduce { |s|
-        s.ldw :score1
-        s.ldw :score2
-        s.cmp
-        s.stw :comparison
-      }
-    }
+    @client_query = WebClientQueryBuilder.query :layovers_cost => 1000,
+                                                :start_time_cost => -1,
+                                                :duration_cost => 1
   end
   
   def fare_score(fare)
-    200 + fare['start'] / 100 - fare['price'] - (fare['end'] - fare['start'])
+    20000 + fare['start_time'] - fare['price'] - (fare['end_time'] -
+        fare['start_time']) - fare['layovers'] * 1000
   end
   
   def fare_id(fare)

data/test/test_client_server.rb

@@ -0,0 +1,44 @@
+require 'test/mr_test_case'
+require 'flexmock/test_unit'
+
+
+class ClientServerTest < MrTestCase
+  Client = Tem::Mr::Search::Client
+  Server = Tem::Mr::Search::Server
+    
+  def setup
+    super
+    @server_port = 29552
+  end
+    
+  def _test_request
+    Thread.new do
+      Server.new(@db_path, @empty_cluster_file, @server_port).serve_loop
+    end
+    sleep 0.1
+    yield "localhost:#{@server_port}"
+    Client.shutdown_server "localhost:#{@server_port}"
+  end
+  
+  def test_fetch_item
+    _test_request do |server_addr|
+      fetched_item = Client.fetch_item server_addr, fare_id(@db.item(3))
+      assert_equal @db.item(3), fetched_item, 'Fetch fail' 
+    end
+  end
+  
+  def test_query
+    Tem.auto_conf
+    @server_port = 29553
+    flexmock(Server).should_receive(:tems_from_cluster_file).
+                     with(@empty_cluster_file).and_return [$tem]
+    _test_request do |server_addr|
+      result = Client.search server_addr, @client_query
+      gold_item = @db.item 5
+      assert_equal fare_id(gold_item), result[:id],
+                   'Incorrect Map-Reduce result (ID)'
+      assert_equal fare_score(gold_item), result[:score],
+                   'Incorrect Map-Reduce result (score)'
+    end
+  end
+end

data/test/test_db.rb

@@ -2,9 +2,15 @@ require 'test/mr_test_case'
 
 class DbTest < MrTestCase  
   def test_loading    
-    assert_equal 4, @db.length, 'Number of items in the database'
-    gold_item = {'from' => 'BOS', 'to' => 'NYC', 'price' => 150, 'start' => 900,
-                 'end' => 1000, 'flight' => 15 }
+    assert_equal 8, @db.length, 'Number of items in the database'
+    gold_item = {"price" => 2500, "from" => "BOS", "to" => "TPE",
+                 "flight" => 15, "layovers"=>2, "end_time"=>2100,
+                 "start_time"=>900}
     assert_equal gold_item, @db.item(0), 'First database item'
   end
+  
+  def test_by_id
+    assert_equal 18, @db.item_by_id(18)['flight'], 'Finding existing item by ID'
+    assert_equal nil, @db.item_by_id(5), 'Finding non-existing item'
+  end
 end

data/test/test_map_reduce_executor.rb

@@ -14,21 +14,20 @@ class MapReduceExecutorTest < MrTestCase
     executor = MRExecutor.new @client_query, @db, tems, root_tem
     packed_output = executor.execute
     result = @client_query.unpack_output packed_output
-    assert_equal 18, result[:id], 'Incorrect Map-Reduce result'
+    gold_item = @db.item 5 
+    assert_equal fare_id(gold_item), result[:id],
+                 'Incorrect Map-Reduce result (ID)'
+    assert_equal fare_score(gold_item), result[:score],
+                 'Incorrect Map-Reduce result (score)'
   end
   
   def test_executor_with_autoconf
     _test_executor [$tem], 0
   end
   
-  def test_executor_with_cluster
-    cluster_config = ['lightbulb2.local', 'darkbulb.local'].map { |host|
-      Tem::MultiProxy::Client.query_tems host    
-    }.flatten
-    assert_equal 8, cluster_config.length, 'Incorrect cluster setup'
-    tems = cluster_config.map do |config|
-      Tem::Session.new Tem::Transport::AutoConfigurator.try_transport(config)
-    end
+  def test_executor_with_cluster    
+    tems = Tem::Mr::Search::Server.tems_from_cluster_file @cluster_file
+    assert_equal 8, tems.length, 'Incorrect cluster setup'
     
     tems.each { |tem| tem.activate; tem.emit }
     

data/test/test_map_reduce_job.rb

@@ -1,6 +1,8 @@
 require 'test/mr_test_case'
 
 class MapReduceJobTest < MrTestCase
+  MapReduceJob = Tem::Mr::Search::MapReduceJob
+  
   def setup
     super
     
@@ -12,20 +14,24 @@ class MapReduceJobTest < MrTestCase
   
   def test_map_for_object
     obj = @obj1.merge 'flight' => 0x12345678
-    secpack = @client_query.map_for_object obj
+    secpack = @client_query.mapper.map_for_object obj
     
     assert_equal [0, 0, 0, 0, 0x12, 0x34, 0x56, 0x78],
                  secpack.get_bytes(:_id, 8), 'Object ID embedded incorrectly'
     assert_equal @obj1['price'], secpack.get_value(:price, :tem_short),
                  'Price embedded incorrectly'
-    assert_equal @obj1['start'], secpack.get_value(:start, :tem_short),
+    assert_equal @obj1['layovers'], secpack.get_value(:layovers, :tem_short),
+                 'Layover count embedded incorrectly'
+    assert_equal @obj1['start_time'],
+                 secpack.get_value(:start_time, :tem_short),
                  'Starting time embedded incorrectly'
-    assert_equal @obj1['end'], secpack.get_value(:end, :tem_short),
+    assert_equal @obj1['end_time'],
+                 secpack.get_value(:end_time, :tem_short),
                  'Ending time embedded incorrectly'
   end
   
   def test_reduce_for_outputs
-    secpack = @client_query.reduce_for_outputs @output1, @output2
+    secpack = @client_query.reducer.reduce_for_outputs @output1, @output2
     
     assert_equal @output1, secpack.get_bytes(:_output1, 16),
                  'Output1 embedded incorrectly'
@@ -41,5 +47,19 @@ class MapReduceJobTest < MrTestCase
     assert_equal 0x12345678, output[:id], 'ID incorrectly unpacked'
     assert_equal 0x3141, output[:score], 'Score incorrectly unpacked'
     assert_equal [0xBE, 0xEF, 0xFE], output[:check], 'Check bytes'    
+  end  
+end
+
+class SerializedMapReduceJobTest < MapReduceJobTest
+  # Serialization is successful if a de-serialized job can pass all the tests.
+  
+  def setup
+    super
+    @hash = @client_query.to_hash
+    @client_query = MapReduceJob.new @hash
+  end
+  
+  def test_serialized_form
+    assert_equal Hash, @hash.class, 'Serialization did not produce a plain hash'    
   end
 end

data/test/{test_query_builder.rb → test_query_builders.rb}

@@ -1,6 +1,6 @@
 require 'test/mr_test_case'
 
-class QueryBuilderTest < MrTestCase
+class QueryBuildersTest < MrTestCase
   def setup
     super
     Tem.auto_conf
@@ -9,7 +9,7 @@ class QueryBuilderTest < MrTestCase
   end
   
   def _test_map_fare(fare)
-    enc_output = @client_query.map_object fare, $tem
+    enc_output = @client_query.mapper.map_object fare, $tem
     output = @client_query.unpack_output enc_output
     assert_equal fare_id(fare), output[:id], 'Object ID incorrectly encoded'
     assert_equal fare_score(fare), output[:score],
@@ -26,7 +26,7 @@ class QueryBuilderTest < MrTestCase
     win_fare = (fare_score(fare1) > fare_score(fare2)) ? fare1 : fare2
     # Try both permutations to ensure all branches of the reduce code work.
     [[output1, output2], [output2, output1]].each do |o1, o2|
-      enc_output = @client_query.reduce_outputs o1, o2, $tem
+      enc_output = @client_query.reducer.reduce_outputs o1, o2, $tem
       output = @client_query.unpack_output enc_output
       assert_equal fare_id(win_fare), output[:id], 'The wrong fare won (bad ID)'
       assert_equal fare_score(win_fare), output[:score],

data/testdata/cluster.yml

@@ -0,0 +1,3 @@
+---
+- darkbulb2.local
+- lightbulb2.local

data/testdata/empty_cluster.yml

@@ -0,0 +1,2 @@
+--- []
+

data/testdata/fares.yml

@@ -1,49 +1,57 @@
 ---
 - from: BOS
-  to: NYC
-  price: 150
-  start: 900
-  end: 1000
+  to: TPE
+  price: 2500
+  start_time: 900
+  end_time: 2100
+  layovers: 2
   flight: 15
 - from: BOS
-  to: NYC
-  price: 150
-  start: 1800
-  end: 1900
+  to: TPE
+  price: 2500
+  start_time: 1100
+  end_time: 2300
+  layovers: 2
   flight: 16
 - from: BOS
-  to: NYC
-  price: 200
-  start: 1200
-  end: 1245
+  to: TPE
+  price: 3000
+  start_time: 900
+  end_time: 1900
+  layovers: 1
   flight: 17
 - from: BOS
-  to: NYC
-  price: 200
-  start: 1700
-  end: 1745
+  to: TPE
+  price: 3000
+  start_time: 1100
+  end_time: 2100
+  layovers: 1
   flight: 18
 - from: BOS
-  to: NYC
-  price: 160
-  start: 900
-  end: 1000
+  to: TPE
+  price: 4000
+  start_time: 900
+  end_time: 1700
+  layovers: 0
+  flight: 19
+- from: BOS
+  to: TPE
+  price: 4000
+  start_time: 1100
+  end_time: 1900
+  layovers: 0
+  flight: 20
+- from: BOS
+  to: TPE
+  price: 2400
+  start_time: 900
+  end_time: 2100
+  layovers: 2
   flight: 21
 - from: BOS
-  to: NYC
-  price: 160
-  start: 1800
-  end: 1900
+  to: TPE
+  price: 2400
+  start_time: 1100
+  end_time: 2300
+  layovers: 2
   flight: 22
-- from: BOS
-  to: NYC
-  price: 210
-  start: 1200
-  end: 1245
-  flight: 23
-- from: BOS
-  to: NYC
-  price: 210
-  start: 1700
-  end: 1745
-  flight: 24

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: tem_mr_search
 version: !ruby/object:Gem::Version 
-  version: "0.1"
+  version: "0.2"
 platform: ruby
 authors: 
 - Victor Costan
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-02 00:00:00 -04:00
+date: 2009-06-04 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -34,29 +34,37 @@ dependencies:
     version: 
 description: "Tem Map-Reduce proof of concept: database search."
 email: victor@costan.us
-executables: []
-
+executables: 
+- tem_mr_search_server
 extensions: []
 
 extra_rdoc_files: 
+- bin/tem_mr_search_server
 - CHANGELOG
+- lib/tem_mr_search/client.rb
 - lib/tem_mr_search/client_query.rb
 - lib/tem_mr_search/db.rb
 - lib/tem_mr_search/map_reduce_executor.rb
 - lib/tem_mr_search/map_reduce_job.rb
 - lib/tem_mr_search/map_reduce_planner.rb
 - lib/tem_mr_search/query_builder.rb
+- lib/tem_mr_search/server.rb
+- lib/tem_mr_search/web_client_query_builder.rb
 - lib/tem_mr_search.rb
 - LICENSE
 - README
 files: 
+- bin/tem_mr_search_server
 - CHANGELOG
+- lib/tem_mr_search/client.rb
 - lib/tem_mr_search/client_query.rb
 - lib/tem_mr_search/db.rb
 - lib/tem_mr_search/map_reduce_executor.rb
 - lib/tem_mr_search/map_reduce_job.rb
 - lib/tem_mr_search/map_reduce_planner.rb
 - lib/tem_mr_search/query_builder.rb
+- lib/tem_mr_search/server.rb
+- lib/tem_mr_search/web_client_query_builder.rb
 - lib/tem_mr_search.rb
 - LICENSE
 - Manifest
@@ -64,11 +72,14 @@ files:
 - README
 - tem_mr_search.gemspec
 - test/mr_test_case.rb
+- test/test_client_server.rb
 - test/test_db.rb
 - test/test_map_reduce_executor.rb
 - test/test_map_reduce_job.rb
 - test/test_map_reduce_planner.rb
-- test/test_query_builder.rb
+- test/test_query_builders.rb
+- testdata/cluster.yml
+- testdata/empty_cluster.yml
 - testdata/fares.yml
 - testdata/parallel_plan_431.yml
 - testdata/parallel_plan_740.yml
@@ -109,8 +120,9 @@ signing_key:
 specification_version: 3
 summary: "Tem Map-Reduce proof of concept: database search."
 test_files: 
+- test/test_client_server.rb
 - test/test_db.rb
 - test/test_map_reduce_executor.rb
 - test/test_map_reduce_job.rb
 - test/test_map_reduce_planner.rb
-- test/test_query_builder.rb
+- test/test_query_builders.rb