tem_mr_search 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +2 -0
- data/Rakefile +7 -1
- data/lib/tem_mr_search/client.rb +17 -1
- data/lib/tem_mr_search/client_query.rb +1 -2
- data/lib/tem_mr_search/db.rb +27 -3
- data/lib/tem_mr_search/map_reduce_executor.rb +16 -2
- data/lib/tem_mr_search/map_reduce_job.rb +17 -1
- data/lib/tem_mr_search/map_reduce_planner.rb +130 -40
- data/lib/tem_mr_search/query_builder.rb +34 -21
- data/lib/tem_mr_search/server.rb +22 -0
- data/lib/tem_mr_search.rb +6 -0
- data/tem_mr_search.gemspec +8 -8
- data/test/test_query_builders.rb +2 -1
- metadata +4 -4
data/CHANGELOG
CHANGED
data/Rakefile
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
# Rakefile for the tem_mr_search Rubygem.
|
2
|
+
#
|
3
|
+
# Author:: Victor Costan
|
4
|
+
# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
|
5
|
+
# License:: MIT
|
6
|
+
|
1
7
|
require 'rubygems'
|
2
8
|
gem 'echoe'
|
3
9
|
require 'echoe'
|
@@ -10,7 +16,7 @@ Echoe.new('tem_mr_search') do |p|
|
|
10
16
|
p.email = 'victor@costan.us'
|
11
17
|
p.summary = 'Tem Map-Reduce proof of concept: database search.'
|
12
18
|
p.url = 'http://tem.rubyforge.org'
|
13
|
-
p.dependencies = ['tem_ruby >=0.
|
19
|
+
p.dependencies = ['tem_ruby >=0.14.0', 'tem_multi_proxy >=0.2.7']
|
14
20
|
|
15
21
|
p.need_tar_gz = !Gem.win_platform?
|
16
22
|
p.need_zip = !Gem.win_platform?
|
data/lib/tem_mr_search/client.rb
CHANGED
@@ -1,11 +1,25 @@
|
|
1
|
+
# Client for the map-reduce RPC server.
|
2
|
+
#
|
3
|
+
# Author:: Victor Costan
|
4
|
+
# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
|
5
|
+
# License:: MIT
|
6
|
+
|
1
7
|
# :nodoc: namespace
|
2
8
|
module Tem::Mr::Search
|
3
9
|
|
10
|
+
|
11
|
+
# Client for the map-reduce RPC server.
|
4
12
|
class Client
|
5
13
|
OP = Zerg::Support::Protocols::ObjectProtocol
|
6
14
|
OPAdapter = Zerg::Support::Sockets::ProtocolAdapter.adapter_module OP
|
7
15
|
|
8
16
|
# Performs a private database search using a Map-Reduce.
|
17
|
+
#
|
18
|
+
# Args:
|
19
|
+
# server_addr:: string with the address of the Map-Reduce server's RPC port.
|
20
|
+
# client_query:: a ClientQuery instance expressing the Map-Reduce search
|
21
|
+
#
|
22
|
+
# Returns the result of the Map-Reduce computation.
|
9
23
|
def self.search(server_addr, client_query)
|
10
24
|
output = issue_request server_addr, :type => :search, :root_tem => 0,
|
11
25
|
:map_reduce => client_query.to_hash
|
@@ -34,6 +48,8 @@ class Client
|
|
34
48
|
end
|
35
49
|
|
36
50
|
# Issues a request against a Map-Reduce server and returns the response.
|
51
|
+
#
|
52
|
+
# This method should not be called directly.
|
37
53
|
def self.issue_request(server_addr, request)
|
38
54
|
socket = Zerg::Support::SocketFactory.socket :out_addr => server_addr,
|
39
55
|
:out_port => Server::DEFAULT_PORT, :no_delay => true
|
@@ -43,6 +59,6 @@ class Client
|
|
43
59
|
socket.close
|
44
60
|
response
|
45
61
|
end
|
46
|
-
end
|
62
|
+
end # class Tem::Mr::Search::Client
|
47
63
|
|
48
64
|
end # namespace Tem::Mr::Search
|
@@ -12,8 +12,7 @@ class ClientQuery < MapReduceJob
|
|
12
12
|
# This is expected to be called with the encrypted output returned by the
|
13
13
|
# search provider.
|
14
14
|
def unpack_output(output)
|
15
|
-
|
16
|
-
decrypted_output = output
|
15
|
+
decrypted_output = @query_key.decrypt output
|
17
16
|
unpack_decrypted_output decrypted_output
|
18
17
|
end
|
19
18
|
end
|
data/lib/tem_mr_search/db.rb
CHANGED
@@ -1,26 +1,50 @@
|
|
1
|
+
# Mock database for the secure Map-Reduce proof of concept.
|
2
|
+
#
|
3
|
+
# Author:: Victor Costan
|
4
|
+
# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
|
5
|
+
# License:: MIT
|
6
|
+
|
1
7
|
# :nodoc: namespace
|
2
8
|
module Tem::Mr::Search
|
3
|
-
|
9
|
+
|
10
|
+
# Mock database for the Map-Reduce proof of concept.
|
4
11
|
class Db
|
5
12
|
attr_reader :data
|
6
13
|
attr_reader :id_attribute
|
7
14
|
|
15
|
+
# Creates a new mock database.
|
16
|
+
#
|
17
|
+
# Args:
|
18
|
+
# path:: filesystem path to the YAML file containing the database
|
8
19
|
def initialize(path)
|
9
20
|
@data = File.open(path, 'r') { |f| YAML.load f }
|
10
21
|
@id_attribute = 'flight'
|
11
22
|
end
|
12
23
|
|
24
|
+
# The number of records in the database.
|
13
25
|
def length
|
14
26
|
@data.length
|
15
27
|
end
|
16
28
|
|
29
|
+
# Retrieves an item in table scan order.
|
30
|
+
#
|
31
|
+
# Args:
|
32
|
+
# item_index:: the item's 0-based index in table scan order
|
33
|
+
#
|
34
|
+
# Returns a hash with the item data.
|
17
35
|
def item(item_index)
|
18
36
|
@data[item_index]
|
19
37
|
end
|
20
38
|
|
39
|
+
# Retrieves an item using its primary key.
|
40
|
+
#
|
41
|
+
# Args:
|
42
|
+
# item_id:: the item's primary key
|
43
|
+
#
|
44
|
+
# Returns a hash with the item data.
|
21
45
|
def item_by_id(item_id)
|
22
46
|
@data.find { |item| item[@id_attribute] == item_id }
|
23
47
|
end
|
24
|
-
end
|
48
|
+
end # class Tem::Mr::Search::Db
|
25
49
|
|
26
|
-
end # namespace Tem::Mr::
|
50
|
+
end # namespace Tem::Mr::Search
|
@@ -1,9 +1,21 @@
|
|
1
|
+
# Coordination code (executor) for performing a Map-Reduce computation.
|
2
|
+
#
|
3
|
+
# Author:: Victor Costan
|
4
|
+
# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
|
5
|
+
# License:: MIT
|
6
|
+
|
1
7
|
require 'thread'
|
2
8
|
|
3
9
|
|
4
10
|
# :nodoc: namespace
|
5
11
|
module Tem::Mr::Search
|
6
12
|
|
13
|
+
# Coordination code (executor) for performing a Map-Reduce computation.
|
14
|
+
#
|
15
|
+
# The executor distributes the Map-Reduce computation across multiple TEMs. The
|
16
|
+
# strategy used to allocate tasks to TEMs is expressed by a MapReducePlanner
|
17
|
+
# class, and the executor instantiates that class. The executor is responsible
|
18
|
+
# for coordinating between the TEMs and the planner.
|
7
19
|
class MapReduceExecutor
|
8
20
|
# Creates an executor for a Map-Reduce job.
|
9
21
|
#
|
@@ -61,6 +73,7 @@ class MapReduceExecutor
|
|
61
73
|
@main_queue << action
|
62
74
|
end
|
63
75
|
end
|
76
|
+
private :executor_thread
|
64
77
|
|
65
78
|
# Executes a Map-Reduce planner action.
|
66
79
|
#
|
@@ -108,8 +121,9 @@ class MapReduceExecutor
|
|
108
121
|
@lock.synchronize do
|
109
122
|
@outputs[action[:final_id]] = final_output
|
110
123
|
end
|
111
|
-
end
|
124
|
+
end
|
112
125
|
end
|
113
|
-
|
126
|
+
private :execute_action
|
127
|
+
end # class Tem::Mr::Search::MapReduceExecutor
|
114
128
|
|
115
129
|
end # namespace Tem::Mr::Search
|
@@ -1,6 +1,22 @@
|
|
1
|
+
# Expresses a Map-Reduce computation whose components are performed on TEMs.
|
2
|
+
#
|
3
|
+
# Author:: Victor Costan
|
4
|
+
# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
|
5
|
+
# License:: MIT
|
6
|
+
|
1
7
|
# :nodoc: namespace
|
2
8
|
module Tem::Mr::Search
|
3
9
|
|
10
|
+
# Expresses a Map-Reduce computation whose components are performed on TEMs.
|
11
|
+
#
|
12
|
+
# Client code should put together Map-Reduce computations using QueryBuilder or
|
13
|
+
# similar builder classes. In turn, builders construct and return MapReduceJob
|
14
|
+
# instances.
|
15
|
+
#
|
16
|
+
# Jobs can be serialized to a hash for network transmission. To de-serialize a
|
17
|
+
# job, pass the hash to the hash constructor:
|
18
|
+
# hash = job.to_hash
|
19
|
+
# job = MapReduceJob.new hash
|
4
20
|
class MapReduceJob
|
5
21
|
attr_reader :mapper, :reducer, :finalizer, :attributes, :id_attribute
|
6
22
|
|
@@ -18,7 +34,7 @@ class MapReduceJob
|
|
18
34
|
{
|
19
35
|
:id => output[0, 8].reverse.pack('C*').unpack('q').first,
|
20
36
|
:score => Tem::Abi.read_tem_short(output, 8),
|
21
|
-
:check => output[13,
|
37
|
+
:check => output[13, 10]
|
22
38
|
}
|
23
39
|
end
|
24
40
|
|
@@ -1,9 +1,34 @@
|
|
1
|
+
# Allocates the individual components of a Map-Reduce job across TEMs.
|
2
|
+
#
|
3
|
+
# Author:: Victor Costan
|
4
|
+
# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
|
5
|
+
# License:: MIT
|
6
|
+
|
1
7
|
require 'rbtree'
|
2
8
|
require 'set'
|
3
9
|
|
4
10
|
# :nodoc: namespace
|
5
11
|
module Tem::Mr::Search
|
6
12
|
|
13
|
+
# Allocates the individual components of a Map-Reduce job across TEMs.
|
14
|
+
#
|
15
|
+
# This class is instantiated and used by MapReduceExecutor. It should not be
|
16
|
+
# used directly in client code, except for the purpose of replacing the default
|
17
|
+
# planner.
|
18
|
+
#
|
19
|
+
# The Map-Reduce coordinator calls next_actions! on the planner, to obtain a
|
20
|
+
# list of actions that can be carried out. The planner guarantees that the
|
21
|
+
# actions are independent of each other, and that all their dependencies are
|
22
|
+
# satisfied. When the coordinator learns about the completion of some actions,
|
23
|
+
# it updates the planner's state by calling action_done. After action_done is
|
24
|
+
# called, new_action should be called again to obtain new actions that can be
|
25
|
+
# carried out.
|
26
|
+
#
|
27
|
+
# Partial results (outputs) in the Map-Reduce computation are identified by
|
28
|
+
# unique numbers starting from 0. The output IDs can be used as file names, if
|
29
|
+
# the outputs are stored in a distributed file system. When the computation is
|
30
|
+
# done (calling done? returns +true+), the +output_id+ attribute will contain
|
31
|
+
# the ID of the computation's final result.
|
7
32
|
class MapReducePlanner
|
8
33
|
# Creates a planner for a Map-Reduce job.
|
9
34
|
#
|
@@ -32,8 +57,69 @@ class MapReducePlanner
|
|
32
57
|
@last_reduce_id = 2 * num_items - 2
|
33
58
|
@done_reducing, @output_id = false, nil
|
34
59
|
end
|
60
|
+
|
61
|
+
# Issues a set of actions that can be performed right now.
|
62
|
+
#
|
63
|
+
# The method alters the planner's state assuming the actions will be
|
64
|
+
# performed.
|
65
|
+
#
|
66
|
+
# Returns an array of hashes, with one hash per action to be performed. The
|
67
|
+
# +:action+ key specifies the type of action to be performed, and can be
|
68
|
+
# +:migrate+ +:map+, +:reduce+, or +:finalize. All the actions have the
|
69
|
+
# +:with+ key, which is the ID (0-based index) of the TEM that will be doing
|
70
|
+
# the action.
|
71
|
+
#
|
72
|
+
# Migrate actions have the following keys:
|
73
|
+
# :secpack:: the type of SECpack to be migrated ( +:mapper+ or +:reducer+ )
|
74
|
+
# :with:: the ID of the TEM doing the migration
|
75
|
+
# :to:: the number of the TEM that the SECpack should be migrated to
|
76
|
+
#
|
77
|
+
# Map actions have the following keys:
|
78
|
+
# :item_id:: the ID of the item to be mapped (number in Table-Scan order)
|
79
|
+
# :with:: the ID of the TEM doing the mapping
|
80
|
+
# :output_id:: ID for the result of the map operation
|
81
|
+
#
|
82
|
+
# Reduce actions have the following keys:
|
83
|
+
# :output1_id, :output2_id:: the IDs of the partial outputs to be reduced
|
84
|
+
# :with:: the ID of the TEM doing the reducing
|
85
|
+
# :output_id:: the ID for the result of the reduce operation
|
86
|
+
#
|
87
|
+
# The finalize action has the following keys:
|
88
|
+
# :output_id:: the ID of the last partial output, which will be finalized
|
89
|
+
# :with:: the ID of the TEM doing the finalization
|
90
|
+
# :final_id:: the ID for the computation's final result
|
91
|
+
def next_actions!
|
92
|
+
actions = migrate_actions :mapper
|
93
|
+
actions += migrate_actions :reducer
|
94
|
+
actions += map_actions
|
95
|
+
actions += reduce_actions
|
96
|
+
actions += finalize_actions
|
97
|
+
actions
|
98
|
+
end
|
99
|
+
|
100
|
+
# Informs the planner that an action issued by next_actions! was completed.
|
101
|
+
#
|
102
|
+
# Args:
|
103
|
+
# action:: an action hash, as returned by next_actions!
|
104
|
+
#
|
105
|
+
# The return value is not specified.
|
106
|
+
def action_done(action)
|
107
|
+
dispatch = { :migrate => :done_migrating, :map => :done_mapping, :reduce =>
|
108
|
+
:done_reducing, :finalize => :done_finalizing }
|
109
|
+
self.send dispatch[action[:action]], action
|
110
|
+
end
|
111
|
+
|
112
|
+
# True when the Map-Reduce computation is complete.
|
113
|
+
def done?
|
114
|
+
!@output_id.nil?
|
115
|
+
end
|
116
|
+
|
117
|
+
# The output ID of the Map-Reduce's final result.
|
118
|
+
attr_reader :output_id
|
35
119
|
|
36
120
|
# Generates migrating actions for a SECpack type that are possible now.
|
121
|
+
#
|
122
|
+
# See next_actions! for a description of the return value.
|
37
123
|
def migrate_actions(sec_type)
|
38
124
|
actions = []
|
39
125
|
return actions if @without[sec_type].length == 0
|
@@ -56,24 +142,10 @@ class MapReducePlanner
|
|
56
142
|
@with[action[:secpack]] << action[:to]
|
57
143
|
end
|
58
144
|
private :done_migrating
|
59
|
-
|
60
|
-
# A sorted array of the free TEMs that have a SECpack type.
|
61
|
-
def free_tems_with_sec(sec_type)
|
62
|
-
tems = []
|
63
|
-
@free_tems.each do |tem, true_value|
|
64
|
-
tems << tem if @with[sec_type].include? tem
|
65
|
-
end
|
66
|
-
tems
|
67
|
-
end
|
68
|
-
|
69
|
-
# A unique output_id.
|
70
|
-
def next_output_id
|
71
|
-
next_id = @last_output_id
|
72
|
-
@last_output_id += 1
|
73
|
-
next_id
|
74
|
-
end
|
75
|
-
|
145
|
+
|
76
146
|
# Generates mapping actions possible right now.
|
147
|
+
#
|
148
|
+
# See next_actions! for a description of the return value.
|
77
149
|
def map_actions
|
78
150
|
actions = []
|
79
151
|
return actions if @unmapped_items.empty?
|
@@ -88,6 +160,11 @@ class MapReducePlanner
|
|
88
160
|
private :map_actions
|
89
161
|
|
90
162
|
# Informs the planner that a data mapping has completed.
|
163
|
+
#
|
164
|
+
# Args:
|
165
|
+
# action:: an action hash, as returned by map_actions
|
166
|
+
#
|
167
|
+
# The return value is not specified.
|
91
168
|
def done_mapping(action)
|
92
169
|
@free_tems[action[:with]] = true
|
93
170
|
@reduce_queue[action[:output_id]] = true
|
@@ -95,6 +172,8 @@ class MapReducePlanner
|
|
95
172
|
private :done_mapping
|
96
173
|
|
97
174
|
# Generates reducing actions possible right now.
|
175
|
+
#
|
176
|
+
# See next_actions! for a description of the return value.
|
98
177
|
def reduce_actions
|
99
178
|
actions = []
|
100
179
|
return actions if @reduce_queue.length <= 1
|
@@ -114,6 +193,11 @@ class MapReducePlanner
|
|
114
193
|
private :reduce_actions
|
115
194
|
|
116
195
|
# Informs the planner that a data reduction has completed.
|
196
|
+
#
|
197
|
+
# Args:
|
198
|
+
# action:: an action hash, as returned by reduce_actions
|
199
|
+
#
|
200
|
+
# The return value is not specified.
|
117
201
|
def done_reducing(action)
|
118
202
|
@free_tems[action[:with]] = true
|
119
203
|
if action[:output_id] == @last_reduce_id
|
@@ -125,6 +209,8 @@ class MapReducePlanner
|
|
125
209
|
private :done_reducing
|
126
210
|
|
127
211
|
# Generates finalizing actions possible right now.
|
212
|
+
#
|
213
|
+
# See next_actions! for a description of the return value.
|
128
214
|
def finalize_actions
|
129
215
|
return [] unless @done_reducing and !@output_id and @free_tems[@root_tem]
|
130
216
|
@finalize_ready = false
|
@@ -134,36 +220,40 @@ class MapReducePlanner
|
|
134
220
|
private :finalize_actions
|
135
221
|
|
136
222
|
# Informs the planner that an action issued by next_action was done.
|
223
|
+
#
|
224
|
+
# Args:
|
225
|
+
# action:: an action hash, as returned by finalize_actions
|
226
|
+
#
|
227
|
+
# The return value is not specified.
|
137
228
|
def done_finalizing(action)
|
138
229
|
@free_tems[action[:with]] = true
|
139
230
|
@output_id = action[:final_id]
|
140
231
|
end
|
141
232
|
private :done_finalizing
|
142
233
|
|
143
|
-
#
|
144
|
-
|
145
|
-
|
234
|
+
# A sorted array of the free TEMs that have a SECpack type migrated to them.
|
235
|
+
#
|
236
|
+
# Args:
|
237
|
+
# sec_type:: the SECpack type (+:mapper+ or +:reducer+)
|
238
|
+
def free_tems_with_sec(sec_type)
|
239
|
+
tems = []
|
240
|
+
@free_tems.each do |tem, true_value|
|
241
|
+
tems << tem if @with[sec_type].include? tem
|
242
|
+
end
|
243
|
+
tems
|
146
244
|
end
|
245
|
+
private :free_tems_with_sec
|
147
246
|
|
148
|
-
#
|
149
|
-
|
150
|
-
|
151
|
-
#
|
152
|
-
def
|
153
|
-
|
154
|
-
|
155
|
-
|
247
|
+
# Generates a unique output ID.
|
248
|
+
#
|
249
|
+
# Returns the unique output ID, which is a non-negative integer. Future calls
|
250
|
+
# of this method are guaranteed to return different output IDs.
|
251
|
+
def next_output_id
|
252
|
+
next_id = @last_output_id
|
253
|
+
@last_output_id += 1
|
254
|
+
next_id
|
156
255
|
end
|
256
|
+
private :next_output_id
|
257
|
+
end # class Tem::Mr::Search::MapReducePlanner
|
157
258
|
|
158
|
-
#
|
159
|
-
def next_actions!
|
160
|
-
actions = migrate_actions :mapper
|
161
|
-
actions += migrate_actions :reducer
|
162
|
-
actions += map_actions
|
163
|
-
actions += reduce_actions
|
164
|
-
actions += finalize_actions
|
165
|
-
actions
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
end # namespace Tem::Mr::search
|
259
|
+
end # namespace Tem::Mr::Search
|
@@ -29,7 +29,7 @@ class QueryBuilder
|
|
29
29
|
@map_secpack = Tem::Assembler.assemble do |s|
|
30
30
|
s.label :_secret
|
31
31
|
s.label :_key
|
32
|
-
s.
|
32
|
+
s.data :tem_ubyte, @query_key.to_tem_key
|
33
33
|
s.label :_check_bytes
|
34
34
|
s.data :tem_ubyte, @check_bytes
|
35
35
|
|
@@ -39,15 +39,21 @@ class QueryBuilder
|
|
39
39
|
s.ret
|
40
40
|
|
41
41
|
s.entry
|
42
|
-
s.ldbc
|
42
|
+
s.ldbc 24
|
43
43
|
s.outnew
|
44
|
-
|
44
|
+
# Compute score.
|
45
|
+
s.call :_ranking
|
46
|
+
# Compute padding.
|
45
47
|
s.ldbc 3
|
46
48
|
s.ldwc :_nonce
|
47
49
|
s.rnd
|
48
|
-
s.mcfxb :from => :_check_bytes, :to => :_check,
|
49
|
-
|
50
|
-
|
50
|
+
s.mcfxb :from => :_check_bytes, :to => :_check,
|
51
|
+
:size => @check_bytes.length
|
52
|
+
|
53
|
+
# Encrypt output.
|
54
|
+
s.ldwc :const => :_key
|
55
|
+
s.rdk
|
56
|
+
s.kefxb :from => :_id, :size => 23, :to => 0xFFFF
|
51
57
|
s.halt
|
52
58
|
|
53
59
|
s.label :_plain
|
@@ -68,7 +74,7 @@ class QueryBuilder
|
|
68
74
|
s.zeros :tem_ubyte, 3
|
69
75
|
# Check bytes to prevent malicious input corruption.
|
70
76
|
s.label :_check
|
71
|
-
s.zeros :tem_ubyte,
|
77
|
+
s.zeros :tem_ubyte, @check_bytes.length
|
72
78
|
|
73
79
|
s.stack 64
|
74
80
|
end
|
@@ -79,7 +85,7 @@ class QueryBuilder
|
|
79
85
|
@reduce_secpack = Tem::Assembler.assemble do |s|
|
80
86
|
s.label :_secret
|
81
87
|
s.label :_key
|
82
|
-
s.
|
88
|
+
s.data :tem_ubyte, @query_key.to_tem_key
|
83
89
|
s.label :_check
|
84
90
|
s.data :tem_ubyte, @check_bytes
|
85
91
|
|
@@ -90,15 +96,19 @@ class QueryBuilder
|
|
90
96
|
s.ret
|
91
97
|
|
92
98
|
s.entry
|
93
|
-
s.ldbc
|
99
|
+
s.ldbc :const => 24
|
94
100
|
s.outnew
|
95
|
-
#
|
101
|
+
# Decrypt inputs.
|
102
|
+
s.ldwc :const => :_key
|
103
|
+
s.rdk
|
104
|
+
s.stw :_key_id
|
96
105
|
[1, 2].each do |i|
|
97
|
-
|
98
|
-
s.
|
106
|
+
s.ldw :_key_id
|
107
|
+
s.kdfxb :from => :"_output#{i}", :to => :"_id#{i}", :size => 24
|
99
108
|
|
100
109
|
# Compare the check bytes and abort if the inputs were tampered with.
|
101
|
-
s.mcmpfxb :op1 => :"_check#{i}", :op2 => :"_check",
|
110
|
+
s.mcmpfxb :op1 => :"_check#{i}", :op2 => :"_check",
|
111
|
+
:size => @check_bytes.length
|
102
112
|
s.jz :"_check_#{i}_ok"
|
103
113
|
s.halt
|
104
114
|
s.label :"_check_#{i}_ok"
|
@@ -119,20 +129,24 @@ class QueryBuilder
|
|
119
129
|
s.ldbc 3
|
120
130
|
s.ldwc :_nonce1
|
121
131
|
s.rnd
|
122
|
-
#
|
123
|
-
s.
|
132
|
+
# Encrypt output.
|
133
|
+
s.ldwc :const => :_key
|
134
|
+
s.rdk
|
135
|
+
s.kefxb :from => :_id1, :size => 23, :to => 0xFFFF
|
124
136
|
s.halt
|
125
137
|
|
126
138
|
s.label :_plain
|
127
139
|
# The comparison result produced by the user comparison procedure.
|
128
140
|
s.label :comparison
|
129
141
|
s.zeros :tem_short, 1
|
142
|
+
s.label :_key_id
|
143
|
+
s.zeros :tem_short, 1
|
130
144
|
|
131
145
|
# The two inputs to reduce.
|
132
146
|
[1, 2].each do |i|
|
133
147
|
# Encrypted map/reduce output.
|
134
148
|
s.label :"_output#{i}"
|
135
|
-
s.zeros :tem_ubyte,
|
149
|
+
s.zeros :tem_ubyte, 24
|
136
150
|
# Unencrypted input (decrypted inside TEM).
|
137
151
|
s.label :"_id#{i}"
|
138
152
|
s.zeros :tem_ubyte, 8
|
@@ -141,9 +155,9 @@ class QueryBuilder
|
|
141
155
|
s.label :"_nonce#{i}"
|
142
156
|
s.zeros :tem_ubyte, 3
|
143
157
|
s.label :"_check#{i}"
|
144
|
-
s.zeros :tem_ubyte,
|
158
|
+
s.zeros :tem_ubyte, @check_bytes.length
|
145
159
|
end
|
146
|
-
s.stack
|
160
|
+
s.stack 16
|
147
161
|
end
|
148
162
|
end
|
149
163
|
|
@@ -158,9 +172,8 @@ class QueryBuilder
|
|
158
172
|
end
|
159
173
|
|
160
174
|
def initialize
|
161
|
-
@check_bytes = [1, 2, 3]
|
162
|
-
|
163
|
-
@query_key = nil
|
175
|
+
@check_bytes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
176
|
+
@query_key = Tem::Keys::Symmetric.generate
|
164
177
|
end
|
165
178
|
end # class QueryBuilder
|
166
179
|
|
data/lib/tem_mr_search/server.rb
CHANGED
@@ -1,9 +1,31 @@
|
|
1
|
+
# Map-Reduce RPC server.
|
2
|
+
#
|
3
|
+
# Author:: Victor Costan
|
4
|
+
# Copyright:: Copyright (C) 2009 Massachusetts Institute of Technology
|
5
|
+
# License:: MIT
|
6
|
+
|
1
7
|
require 'logger'
|
2
8
|
require 'yaml'
|
3
9
|
|
4
10
|
# :nodoc: namespace
|
5
11
|
module Tem::Mr::Search
|
6
12
|
|
13
|
+
|
14
|
+
# Map-Reduce RPC server.
|
15
|
+
#
|
16
|
+
# The RPC server models the data provider in the secure Map-Reduce proof of
|
17
|
+
# concept.
|
18
|
+
#
|
19
|
+
# The server accepts database queries expressed as Map-Reduce computations,
|
20
|
+
# where each computation is enclosed in a SECpack. This makes it impossible for
|
21
|
+
# the data provider to learn about the query. Map-Reduce computations have a
|
22
|
+
# single result: the ID of the record that's the query response, and its score.
|
23
|
+
#
|
24
|
+
# The server also accepts direct queries for single database records, so clients
|
25
|
+
# can retrieve the record whose ID they learn from the Map-Reduce result.
|
26
|
+
#
|
27
|
+
# The db_dump and shutdown requests are for demonstration and testing purposes,
|
28
|
+
# and would not be exposed in production servers.
|
7
29
|
class Server
|
8
30
|
DEFAULT_PORT = 9052
|
9
31
|
|
data/lib/tem_mr_search.rb
CHANGED
data/tem_mr_search.gemspec
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{tem_mr_search}
|
5
|
-
s.version = "0.2.
|
5
|
+
s.version = "0.2.5"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Victor Costan"]
|
9
|
-
s.date = %q{2009-11-
|
9
|
+
s.date = %q{2009-11-15}
|
10
10
|
s.default_executable = %q{tem_mr_search_server}
|
11
11
|
s.description = %q{Tem Map-Reduce proof of concept: database search.}
|
12
12
|
s.email = %q{victor@costan.us}
|
@@ -26,14 +26,14 @@ Gem::Specification.new do |s|
|
|
26
26
|
s.specification_version = 3
|
27
27
|
|
28
28
|
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
29
|
-
s.add_runtime_dependency(%q<tem_ruby>, [">= 0.
|
30
|
-
s.add_runtime_dependency(%q<tem_multi_proxy>, [">= 0.2.
|
29
|
+
s.add_runtime_dependency(%q<tem_ruby>, [">= 0.14.0"])
|
30
|
+
s.add_runtime_dependency(%q<tem_multi_proxy>, [">= 0.2.7"])
|
31
31
|
else
|
32
|
-
s.add_dependency(%q<tem_ruby>, [">= 0.
|
33
|
-
s.add_dependency(%q<tem_multi_proxy>, [">= 0.2.
|
32
|
+
s.add_dependency(%q<tem_ruby>, [">= 0.14.0"])
|
33
|
+
s.add_dependency(%q<tem_multi_proxy>, [">= 0.2.7"])
|
34
34
|
end
|
35
35
|
else
|
36
|
-
s.add_dependency(%q<tem_ruby>, [">= 0.
|
37
|
-
s.add_dependency(%q<tem_multi_proxy>, [">= 0.2.
|
36
|
+
s.add_dependency(%q<tem_ruby>, [">= 0.14.0"])
|
37
|
+
s.add_dependency(%q<tem_multi_proxy>, [">= 0.2.7"])
|
38
38
|
end
|
39
39
|
end
|
data/test/test_query_builders.rb
CHANGED
@@ -31,7 +31,8 @@ class QueryBuildersTest < MrTestCase
|
|
31
31
|
assert_equal fare_id(win_fare), output[:id], 'The wrong fare won (bad ID)'
|
32
32
|
assert_equal fare_score(win_fare), output[:score],
|
33
33
|
'The wrong fare won (bad score)'
|
34
|
-
assert_equal [1, 2, 3], output[:check],
|
34
|
+
assert_equal [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], output[:check],
|
35
|
+
'Incorrect check bytes'
|
35
36
|
|
36
37
|
assert_not_equal enc_output, output1, 'Nonce fail'
|
37
38
|
assert_not_equal enc_output, output2, 'Nonce fail'
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tem_mr_search
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Victor Costan
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-11-
|
12
|
+
date: 2009-11-15 00:00:00 -05:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -20,7 +20,7 @@ dependencies:
|
|
20
20
|
requirements:
|
21
21
|
- - ">="
|
22
22
|
- !ruby/object:Gem::Version
|
23
|
-
version: 0.
|
23
|
+
version: 0.14.0
|
24
24
|
version:
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: tem_multi_proxy
|
@@ -30,7 +30,7 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 0.2.
|
33
|
+
version: 0.2.7
|
34
34
|
version:
|
35
35
|
description: "Tem Map-Reduce proof of concept: database search."
|
36
36
|
email: victor@costan.us
|