timocratic-skynet 0.9.4
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +152 -0
- data/License.txt +20 -0
- data/Manifest.txt +144 -0
- data/README.txt +178 -0
- data/Rakefile +5 -0
- data/app_generators/skynet_install/USAGE +5 -0
- data/app_generators/skynet_install/skynet_install_generator.rb +94 -0
- data/app_generators/skynet_install/templates/migration.rb +43 -0
- data/app_generators/skynet_install/templates/skynet_config.rb +50 -0
- data/app_generators/skynet_install/templates/skynet_initializer.rb +1 -0
- data/app_generators/skynet_install/templates/skynet_mysql_schema.sql +33 -0
- data/bin/skynet +71 -0
- data/bin/skynet_install +36 -0
- data/bin/skynet_tuplespace_server +74 -0
- data/config/hoe.rb +75 -0
- data/config/requirements.rb +17 -0
- data/examples/dgrep/README +70 -0
- data/examples/dgrep/config/skynet_config.rb +26 -0
- data/examples/dgrep/data/shakespeare/README +2 -0
- data/examples/dgrep/data/shakespeare/poetry/loverscomplaint +381 -0
- data/examples/dgrep/data/shakespeare/poetry/rapeoflucrece +2199 -0
- data/examples/dgrep/data/shakespeare/poetry/sonnets +2633 -0
- data/examples/dgrep/data/shakespeare/poetry/various +640 -0
- data/examples/dgrep/data/shakespeare/poetry/venusandadonis +1423 -0
- data/examples/dgrep/data/testfile1.txt +1 -0
- data/examples/dgrep/data/testfile2.txt +1 -0
- data/examples/dgrep/data/testfile3.txt +1 -0
- data/examples/dgrep/data/testfile4.txt +1 -0
- data/examples/dgrep/lib/dgrep.rb +59 -0
- data/examples/dgrep/lib/mapreduce_test.rb +32 -0
- data/examples/dgrep/lib/most_common_words.rb +45 -0
- data/examples/dgrep/script/dgrep +75 -0
- data/examples/rails_mysql_example/README +66 -0
- data/examples/rails_mysql_example/Rakefile +10 -0
- data/examples/rails_mysql_example/app/controllers/application.rb +10 -0
- data/examples/rails_mysql_example/app/helpers/application_helper.rb +3 -0
- data/examples/rails_mysql_example/app/models/user.rb +21 -0
- data/examples/rails_mysql_example/app/models/user_favorite.rb +5 -0
- data/examples/rails_mysql_example/app/models/user_mailer.rb +12 -0
- data/examples/rails_mysql_example/app/views/user_mailer/welcome.erb +5 -0
- data/examples/rails_mysql_example/config/boot.rb +109 -0
- data/examples/rails_mysql_example/config/database.yml +42 -0
- data/examples/rails_mysql_example/config/environment.rb +59 -0
- data/examples/rails_mysql_example/config/environments/development.rb +18 -0
- data/examples/rails_mysql_example/config/environments/production.rb +19 -0
- data/examples/rails_mysql_example/config/environments/test.rb +22 -0
- data/examples/rails_mysql_example/config/initializers/inflections.rb +10 -0
- data/examples/rails_mysql_example/config/initializers/mime_types.rb +5 -0
- data/examples/rails_mysql_example/config/initializers/skynet.rb +1 -0
- data/examples/rails_mysql_example/config/routes.rb +35 -0
- data/examples/rails_mysql_example/config/skynet_config.rb +36 -0
- data/examples/rails_mysql_example/db/migrate/001_create_skynet_tables.rb +43 -0
- data/examples/rails_mysql_example/db/migrate/002_create_users.rb +16 -0
- data/examples/rails_mysql_example/db/migrate/003_create_user_favorites.rb +14 -0
- data/examples/rails_mysql_example/db/schema.rb +85 -0
- data/examples/rails_mysql_example/db/skynet_mysql_schema.sql +33 -0
- data/examples/rails_mysql_example/doc/README_FOR_APP +2 -0
- data/examples/rails_mysql_example/lib/tasks/rails_mysql_example.rake +20 -0
- data/examples/rails_mysql_example/public/.htaccess +40 -0
- data/examples/rails_mysql_example/public/404.html +30 -0
- data/examples/rails_mysql_example/public/422.html +30 -0
- data/examples/rails_mysql_example/public/500.html +30 -0
- data/examples/rails_mysql_example/public/dispatch.cgi +10 -0
- data/examples/rails_mysql_example/public/dispatch.fcgi +24 -0
- data/examples/rails_mysql_example/public/dispatch.rb +10 -0
- data/examples/rails_mysql_example/public/favicon.ico +0 -0
- data/examples/rails_mysql_example/public/images/rails.png +0 -0
- data/examples/rails_mysql_example/public/index.html +277 -0
- data/examples/rails_mysql_example/public/javascripts/application.js +2 -0
- data/examples/rails_mysql_example/public/javascripts/controls.js +963 -0
- data/examples/rails_mysql_example/public/javascripts/dragdrop.js +972 -0
- data/examples/rails_mysql_example/public/javascripts/effects.js +1120 -0
- data/examples/rails_mysql_example/public/javascripts/prototype.js +4225 -0
- data/examples/rails_mysql_example/public/robots.txt +5 -0
- data/examples/rails_mysql_example/script/about +3 -0
- data/examples/rails_mysql_example/script/console +3 -0
- data/examples/rails_mysql_example/script/destroy +3 -0
- data/examples/rails_mysql_example/script/generate +3 -0
- data/examples/rails_mysql_example/script/performance/benchmarker +3 -0
- data/examples/rails_mysql_example/script/performance/profiler +3 -0
- data/examples/rails_mysql_example/script/performance/request +3 -0
- data/examples/rails_mysql_example/script/plugin +3 -0
- data/examples/rails_mysql_example/script/process/inspector +3 -0
- data/examples/rails_mysql_example/script/process/reaper +3 -0
- data/examples/rails_mysql_example/script/process/spawner +3 -0
- data/examples/rails_mysql_example/script/runner +3 -0
- data/examples/rails_mysql_example/script/server +3 -0
- data/examples/rails_mysql_example/test/fixtures/user_favorites.yml +9 -0
- data/examples/rails_mysql_example/test/fixtures/users.yml +11 -0
- data/examples/rails_mysql_example/test/test_helper.rb +38 -0
- data/examples/rails_mysql_example/test/unit/user_favorite_test.rb +8 -0
- data/examples/rails_mysql_example/test/unit/user_test.rb +8 -0
- data/extras/README +7 -0
- data/extras/init.d/skynet +87 -0
- data/extras/nagios/check_skynet.sh +121 -0
- data/extras/rails/controllers/skynet_controller.rb +43 -0
- data/extras/rails/views/skynet/index.rhtml +137 -0
- data/lib/skynet.rb +95 -0
- data/lib/skynet/mapreduce_helper.rb +74 -0
- data/lib/skynet/mapreduce_test.rb +56 -0
- data/lib/skynet/message_queue_adapters/message_queue_adapter.rb +70 -0
- data/lib/skynet/message_queue_adapters/mysql.rb +509 -0
- data/lib/skynet/message_queue_adapters/tuple_space.rb +316 -0
- data/lib/skynet/skynet_active_record_extensions.rb +280 -0
- data/lib/skynet/skynet_config.rb +232 -0
- data/lib/skynet/skynet_console.rb +50 -0
- data/lib/skynet/skynet_console_helper.rb +66 -0
- data/lib/skynet/skynet_debugger.rb +138 -0
- data/lib/skynet/skynet_guid_generator.rb +68 -0
- data/lib/skynet/skynet_job.rb +892 -0
- data/lib/skynet/skynet_launcher.rb +40 -0
- data/lib/skynet/skynet_logger.rb +62 -0
- data/lib/skynet/skynet_manager.rb +706 -0
- data/lib/skynet/skynet_message.rb +359 -0
- data/lib/skynet/skynet_message_queue.rb +136 -0
- data/lib/skynet/skynet_partitioners.rb +96 -0
- data/lib/skynet/skynet_ruby_extensions.rb +53 -0
- data/lib/skynet/skynet_task.rb +118 -0
- data/lib/skynet/skynet_tuplespace_server.rb +83 -0
- data/lib/skynet/skynet_worker.rb +451 -0
- data/lib/skynet/version.rb +9 -0
- data/script/destroy +14 -0
- data/script/generate +14 -0
- data/script/txt2html +74 -0
- data/setup.rb +1585 -0
- data/tasks/deployment.rake +34 -0
- data/tasks/environment.rake +7 -0
- data/tasks/website.rake +17 -0
- data/test/test_active_record_extensions.rb +138 -0
- data/test/test_generator_helper.rb +20 -0
- data/test/test_helper.rb +10 -0
- data/test/test_mysql_message_queue_adapter.rb +263 -0
- data/test/test_skynet.rb +19 -0
- data/test/test_skynet_install_generator.rb +49 -0
- data/test/test_skynet_job.rb +717 -0
- data/test/test_skynet_manager.rb +157 -0
- data/test/test_skynet_message.rb +229 -0
- data/test/test_skynet_task.rb +24 -0
- data/test/test_tuplespace_message_queue.rb +174 -0
- data/website/index.html +181 -0
- data/website/index.txt +98 -0
- data/website/javascripts/rounded_corners_lite.inc.js +285 -0
- data/website/stylesheets/screen.css +138 -0
- data/website/template.rhtml +48 -0
- metadata +247 -0
@@ -0,0 +1,316 @@
|
|
1
|
+
require 'rinda/tuplespace'
|
2
|
+
class Rinda::TupleSpaceProxy
|
3
|
+
def take(tuple, sec=nil, &block)
|
4
|
+
port = []
|
5
|
+
port.push @ts.move(nil, tuple, sec, &block)
|
6
|
+
port[0]
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
class Skynet
|
11
|
+
class Error < StandardError
|
12
|
+
end
|
13
|
+
|
14
|
+
class RequestExpiredError < Skynet::Error
|
15
|
+
end
|
16
|
+
|
17
|
+
class InvalidMessage < Skynet::Error
|
18
|
+
end
|
19
|
+
|
20
|
+
class MessageQueueAdapter
|
21
|
+
|
22
|
+
class TupleSpace < Skynet::MessageQueueAdapter
|
23
|
+
|
24
|
+
include SkynetDebugger
|
25
|
+
|
26
|
+
USE_FALLBACK_TASKS = true
|
27
|
+
|
28
|
+
@@ts = nil
|
29
|
+
@@curhostidx = 0
|
30
|
+
|
31
|
+
def self.debug_class_desc
|
32
|
+
"TUPLESPACE"
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.adapter
|
36
|
+
:tuplespace
|
37
|
+
end
|
38
|
+
|
39
|
+
attr_accessor :start_options
|
40
|
+
|
41
|
+
def initialize(options={})
|
42
|
+
@start_options = options
|
43
|
+
@ts = self.class.get_tuple_space(options)
|
44
|
+
end
|
45
|
+
|
46
|
+
def take_next_task(curver,timeout=nil,payload_type=nil,queue_id=0)
|
47
|
+
message = Skynet::Message.new(take(Skynet::Message.next_task_template(curver,payload_type, queue_id),timeout))
|
48
|
+
write_fallback_task(message)
|
49
|
+
message
|
50
|
+
end
|
51
|
+
|
52
|
+
def write_message(message,timeout=nil)
|
53
|
+
timeout ||= message.expiry
|
54
|
+
write(message,timeout)
|
55
|
+
end
|
56
|
+
|
57
|
+
def write_result(message,result=[],timeout=nil)
|
58
|
+
result_message = message.result_message(result).to_a
|
59
|
+
timeout ||= result_message.expiry
|
60
|
+
write(result_message,timeout)
|
61
|
+
take_fallback_message(message)
|
62
|
+
result_message
|
63
|
+
end
|
64
|
+
|
65
|
+
def take_result(job_id,timeout=nil)
|
66
|
+
Skynet::Message.new(take(Skynet::Message.result_template(job_id),timeout))
|
67
|
+
end
|
68
|
+
|
69
|
+
def write_error(message,error='',timeout=nil)
|
70
|
+
timeout ||= message.expiry
|
71
|
+
write(message.error_message(error),timeout)
|
72
|
+
take_fallback_message(message)
|
73
|
+
end
|
74
|
+
|
75
|
+
def list_tasks(iteration=nil,queue_id=0)
|
76
|
+
read_all(Skynet::Message.outstanding_tasks_template(iteration,queue_id))
|
77
|
+
end
|
78
|
+
|
79
|
+
def list_results
|
80
|
+
read_all(Skynet::Message.outstanding_results_template)
|
81
|
+
end
|
82
|
+
|
83
|
+
def version_active?(curver=nil, queue_id= 0)
|
84
|
+
return true unless curver
|
85
|
+
begin
|
86
|
+
message_row = read(Skynet::Message.next_task_template(curver, nil, queue_id),0.00001)
|
87
|
+
true
|
88
|
+
rescue Skynet::RequestExpiredError
|
89
|
+
return true if curver.to_i == get_worker_version.to_i
|
90
|
+
false
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def get_worker_version
|
95
|
+
begin
|
96
|
+
message = Skynet::WorkerVersionMessage.new(read(Skynet::WorkerVersionMessage.template,0.00001))
|
97
|
+
if message
|
98
|
+
curver = message.version
|
99
|
+
else
|
100
|
+
curver=0
|
101
|
+
end
|
102
|
+
rescue Skynet::RequestExpiredError => e
|
103
|
+
curver = 0
|
104
|
+
end
|
105
|
+
curver
|
106
|
+
end
|
107
|
+
|
108
|
+
def set_worker_version(ver=nil)
|
109
|
+
begin
|
110
|
+
messages = read_all(Skynet::WorkerVersionMessage.template).collect {|ret| Skynet::WorkerVersionMessage.new(ret)}
|
111
|
+
curver = 0
|
112
|
+
messages.each do |message|
|
113
|
+
curver = message.version
|
114
|
+
debug "CURRENT WORKER VERSION #{curver}"
|
115
|
+
curvmessage = Skynet::WorkerVersionMessage.new(take(message.template,0.00001))
|
116
|
+
if curvmessage
|
117
|
+
curver = curvmessage.version
|
118
|
+
else
|
119
|
+
curver=0
|
120
|
+
end
|
121
|
+
end
|
122
|
+
rescue Skynet::RequestExpiredError => e
|
123
|
+
curver = 0
|
124
|
+
end
|
125
|
+
|
126
|
+
newver = ver ? ver : curver + 1
|
127
|
+
debug "WRITING CURRENT WORKER REV #{newver}"
|
128
|
+
write(Skynet::WorkerVersionMessage.new(:version=>newver))
|
129
|
+
newver
|
130
|
+
end
|
131
|
+
|
132
|
+
def clear_outstanding_tasks
|
133
|
+
begin
|
134
|
+
tasks = read_all(Skynet::Message.outstanding_tasks_template)
|
135
|
+
rescue DRb::DRbConnError, Errno::ECONNREFUSED => e
|
136
|
+
error "ERROR #{e.inspect}", caller
|
137
|
+
end
|
138
|
+
|
139
|
+
tasks.size.times do |ii|
|
140
|
+
take(Skynet::Message.outstanding_tasks_template,0.00001)
|
141
|
+
end
|
142
|
+
|
143
|
+
results = read_all(Skynet::Message.outstanding_results_template)
|
144
|
+
results.size.times do |ii|
|
145
|
+
take(Skynet::Message.outstanding_results_template,0.00001)
|
146
|
+
end
|
147
|
+
|
148
|
+
task_tuples = read_all(Skynet::Message.outstanding_tasks_template)
|
149
|
+
result_tuples = read_all(Skynet::Message.outstanding_results_template)
|
150
|
+
return task_tuples + result_tuples
|
151
|
+
end
|
152
|
+
|
153
|
+
def stats
|
154
|
+
t1 = Time.now
|
155
|
+
tasks = list_tasks
|
156
|
+
results = list_results
|
157
|
+
t2 = Time.now - t1
|
158
|
+
p_tasks = tasks.partition {|task| task[9] == 0}
|
159
|
+
{:taken_tasks => p_tasks[1].size, :untaken_tasks => p_tasks[0].size, :results => list_results.size, :time => t2.to_f}
|
160
|
+
end
|
161
|
+
|
162
|
+
private
|
163
|
+
|
164
|
+
attr_accessor :ts
|
165
|
+
|
166
|
+
def write(tuple,timeout=nil)
|
167
|
+
ts_command(:write,tuple,timeout)
|
168
|
+
end
|
169
|
+
|
170
|
+
def take(template,timeout=nil)
|
171
|
+
ts_command(:take,template,timeout)
|
172
|
+
end
|
173
|
+
|
174
|
+
def read(template,timeout=nil)
|
175
|
+
ts_command(:read,template,timeout)
|
176
|
+
end
|
177
|
+
|
178
|
+
def read_all(template)
|
179
|
+
ts_command(:read_all,template)
|
180
|
+
end
|
181
|
+
|
182
|
+
###### FALLBACK METHODS
|
183
|
+
def write_fallback_task(message)
|
184
|
+
return unless USE_FALLBACK_TASKS
|
185
|
+
debug "4 WRITING BACKUP TASK #{message.task_id}", message.to_h
|
186
|
+
ftm = message.fallback_task_message
|
187
|
+
debug "WRITE FALLBACK TASK", ftm.to_h
|
188
|
+
timeout = message.expiry * 8
|
189
|
+
write(ftm,timeout) unless ftm.iteration == -1
|
190
|
+
ftm
|
191
|
+
end
|
192
|
+
|
193
|
+
def take_fallback_message(message,timeout=0.01)
|
194
|
+
return unless USE_FALLBACK_TASKS
|
195
|
+
return if message.retry <= message.iteration
|
196
|
+
begin
|
197
|
+
# debug "LOOKING FOR FALLBACK TEMPLATE", message.fallback_template
|
198
|
+
fb_message = Skynet::Message.new(take(message.fallback_template,timeout))
|
199
|
+
# debug "TOOK FALLBACK MESSAGE for TASKID: #{fb_message.task_id}"
|
200
|
+
rescue Skynet::RequestExpiredError => e
|
201
|
+
error "Couldn't find expected FALLBACK MESSAGE", Skynet::Message.new(message.fallback_template).to_h
|
202
|
+
end
|
203
|
+
end
|
204
|
+
## END FALLBACK METHODS
|
205
|
+
|
206
|
+
def ts_command(command,message,timeout=nil)
|
207
|
+
# tries = 0
|
208
|
+
# until(tries > 3)
|
209
|
+
if message.is_a?(Skynet::Message)
|
210
|
+
tuple = message.to_a
|
211
|
+
elsif message.is_a?(Array)
|
212
|
+
tuple = message
|
213
|
+
else
|
214
|
+
raise InvalidMessage.new("You must provide a valid Skynet::Message object when calling #{command}. You passed #{message.inspect}.")
|
215
|
+
end
|
216
|
+
|
217
|
+
begin
|
218
|
+
if command==:read_all
|
219
|
+
return ts.send(command,tuple)
|
220
|
+
else
|
221
|
+
return ts.send(command,tuple,timeout)
|
222
|
+
end
|
223
|
+
|
224
|
+
rescue Rinda::RequestExpiredError
|
225
|
+
raise Skynet::RequestExpiredError.new
|
226
|
+
rescue DRb::DRbConnError => e
|
227
|
+
begin
|
228
|
+
error "Couldnt run command [#{command}] on tuplespace. start options: #{@start_options.inspect}"
|
229
|
+
@ts = self.class.get_tuple_space(@start_options)
|
230
|
+
raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect} #{@start_options.inspect}")
|
231
|
+
# tries += 1
|
232
|
+
# next
|
233
|
+
rescue Skynet::ConnectionError => e
|
234
|
+
raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect} #{@start_options.inspect}")
|
235
|
+
# rescue RuntimeError => e
|
236
|
+
# raise Skynet::ConnectionError.new("Can't find ring finger. #{}")
|
237
|
+
rescue DRb::DRbConnError, Errno::ECONNREFUSED => e
|
238
|
+
raise Skynet::ConnectionError.new("There was a problem conected to the #{self.class} #{e.class} #{e.message}")
|
239
|
+
end
|
240
|
+
end
|
241
|
+
# end
|
242
|
+
end
|
243
|
+
|
244
|
+
####################################
|
245
|
+
######## CLASS METHODS #############
|
246
|
+
####################################
|
247
|
+
|
248
|
+
### XXX ACCEPT MULTIPLE TUPLE SPACES and a flag whether to use replication or failover.
|
249
|
+
|
250
|
+
def self.get_tuple_space(options = {})
|
251
|
+
use_ringserver = options[:use_ringserver]
|
252
|
+
ringserver_hosts = options[:ringserver_hosts]
|
253
|
+
drburi = options[:drburi]
|
254
|
+
|
255
|
+
return @@ts if valid_tuplespace?(@@ts)
|
256
|
+
loop do
|
257
|
+
begin
|
258
|
+
DRb.start_service
|
259
|
+
if use_ringserver
|
260
|
+
ringserver_hosts[@@curhostidx] =~ /(.+):(\d+)/
|
261
|
+
host = $1
|
262
|
+
port = $2.to_i
|
263
|
+
@@ts = connect_to_tuple_space(host,port,use_ringserver)
|
264
|
+
else
|
265
|
+
drburi = "druby://#{drburi}" unless drburi =~ %r{druby://}
|
266
|
+
@@ts = get_tuple_space_from_drburi(drburi)
|
267
|
+
raise DRb::DRbConnError.new unless valid_tuplespace?(@@ts)
|
268
|
+
info "#{self} CONNECTED TO #{drburi}"
|
269
|
+
end
|
270
|
+
return @@ts
|
271
|
+
rescue RuntimeError => e
|
272
|
+
if ringserver_hosts[@@curhostidx + 1]
|
273
|
+
error "#{self} Couldn't connect to #{ringserver_hosts[@@curhostidx]} trying #{ringserver_hosts[@@curhostidx+1]}"
|
274
|
+
@@curhostidx += 1
|
275
|
+
next
|
276
|
+
else
|
277
|
+
raise Skynet::ConnectionError.new("Can't find ring finger @ #{ringserver_hosts[@@curhostidx]}. #{e.class} #{e.message}")
|
278
|
+
end
|
279
|
+
rescue Exception => e
|
280
|
+
raise Skynet::ConnectionError.new("Error getting tuplespace @ #{ringserver_hosts[@@curhostidx]}. #{e.class} #{e.message}")
|
281
|
+
end
|
282
|
+
end
|
283
|
+
return @@ts
|
284
|
+
end
|
285
|
+
|
286
|
+
def self.connect_to_tuple_space(host,port,use_ringserver=Skynet::CONFIG[:TS_USE_RINGSERVER])
|
287
|
+
info "#{self} trying to connect to #{host}:#{port}"
|
288
|
+
if use_ringserver
|
289
|
+
ring_finger = Rinda::RingFinger.new(host,port)
|
290
|
+
ring_server = ring_finger.lookup_ring_any(0.5)
|
291
|
+
|
292
|
+
ringts = ring_server.read([:name, :TupleSpace, nil, nil],0.00005)[2]
|
293
|
+
ts = Rinda::TupleSpaceProxy.new(ringts)
|
294
|
+
else
|
295
|
+
ts = get_tuple_space_from_drburi("druby://#{host}:#{port}")
|
296
|
+
end
|
297
|
+
info "#{self} CONNECTED TO #{host}:#{port}"
|
298
|
+
ts
|
299
|
+
end
|
300
|
+
|
301
|
+
def self.get_tuple_space_from_drburi(drburi)
|
302
|
+
DRbObject.new(nil, drburi)
|
303
|
+
end
|
304
|
+
|
305
|
+
def self.valid_tuplespace?(ts)
|
306
|
+
return false unless ts
|
307
|
+
begin
|
308
|
+
ts.read_all([:valid])
|
309
|
+
return true
|
310
|
+
rescue DRb::DRbConnError, RuntimeError, Errno::ECONNREFUSED => e
|
311
|
+
return false
|
312
|
+
end
|
313
|
+
end
|
314
|
+
end
|
315
|
+
end
|
316
|
+
end
|
@@ -0,0 +1,280 @@
|
|
1
|
+
class ActiveRecord::Base
|
2
|
+
def send_later(method,opts=nil,save=nil)
|
3
|
+
raise NoMethodError.new("Method: #{method} doesn't exist in #{self.class}") unless self.respond_to?(method)
|
4
|
+
data = {
|
5
|
+
:model_class => self.class.to_s,
|
6
|
+
:model_id => self.id,
|
7
|
+
:method => method,
|
8
|
+
}
|
9
|
+
data[:save] = 1 if save
|
10
|
+
data[:opts] = opts.to_yaml if opts
|
11
|
+
|
12
|
+
jobopts = {
|
13
|
+
:single => true,
|
14
|
+
:mappers => 1,
|
15
|
+
:map_data => [data],
|
16
|
+
:name => "send_later #{self.class}##{method}",
|
17
|
+
:map_name => "",
|
18
|
+
:map_timeout => 60,
|
19
|
+
:reduce_timeout => 60,
|
20
|
+
:master_timeout => 60,
|
21
|
+
:master_result_timeout => 1.minute,
|
22
|
+
:map_reduce_class => Skynet::ActiveRecordAsync,
|
23
|
+
:master_retry => 0,
|
24
|
+
:map_retry => 0
|
25
|
+
}
|
26
|
+
job = Skynet::AsyncJob.new(jobopts)
|
27
|
+
job.run
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class <<ActiveRecord::Base
|
32
|
+
def distributed_find(*args)
|
33
|
+
all = ActiveRecord::Mapreduce.find(*args)
|
34
|
+
all.model_class = self
|
35
|
+
all
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class Skynet::ActiveRecordAsync
|
40
|
+
include SkynetDebugger
|
41
|
+
|
42
|
+
def self.map(datas)
|
43
|
+
datas.each do |data|
|
44
|
+
begin
|
45
|
+
model = data[:model_class].constantize.find(data[:model_id])
|
46
|
+
if data[:opts]
|
47
|
+
model.send(data[:method].to_sym, YAML.load(data[:opts]))
|
48
|
+
else
|
49
|
+
model.send(data[:method].to_sym)
|
50
|
+
end
|
51
|
+
model.save if data[:save]
|
52
|
+
rescue Exception => e
|
53
|
+
error "Error in #{self} #{e.inspect}"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
return
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
module ActiveRecord
|
61
|
+
|
62
|
+
|
63
|
+
class Mapreduce
|
64
|
+
BATCH_SIZE=1000 unless defined?(BATCH_SIZE)
|
65
|
+
MAX_BATCHES_PER_JOB = 1000 unless defined?(MAX_BATCHES_PER_JOB)
|
66
|
+
attr_accessor :find_args, :batch_size
|
67
|
+
attr_reader :model_class
|
68
|
+
|
69
|
+
delegate :primary_key, :table_name, :to => :model_klass
|
70
|
+
delegate :execute, :select_all, :to => 'model_klass.connection'
|
71
|
+
|
72
|
+
def initialize(options = {})
|
73
|
+
@find_args = options[:find_args]
|
74
|
+
@batch_size = options[:batch_size] || BATCH_SIZE
|
75
|
+
@model_class = options[:model_class]
|
76
|
+
end
|
77
|
+
|
78
|
+
def model_klass
|
79
|
+
@model_klass ||= model_class.constantize
|
80
|
+
end
|
81
|
+
|
82
|
+
def model_class=(model_c)
|
83
|
+
@model_class = model_c.to_s
|
84
|
+
end
|
85
|
+
|
86
|
+
def self.find(*args)
|
87
|
+
if not args.first.is_a?(Hash)
|
88
|
+
args.shift
|
89
|
+
end
|
90
|
+
if args.nil? or args.empty?
|
91
|
+
args = {}
|
92
|
+
else
|
93
|
+
args = *args
|
94
|
+
end
|
95
|
+
new(:find_args => args, :batch_size => args.delete(:batch_size), :model_class => args.delete(:model_class))
|
96
|
+
end
|
97
|
+
|
98
|
+
def log
|
99
|
+
Skynet::Logger.get
|
100
|
+
end
|
101
|
+
|
102
|
+
def each_range(opts={})
|
103
|
+
opts = opts.clone
|
104
|
+
opts[:id] || opts[:id] = 0
|
105
|
+
count = model_klass.count(:all,:conditions => opts[:conditions], :joins => opts[:joins])
|
106
|
+
if count <= batch_size
|
107
|
+
return yield({"first" => 0, "last" => nil, "cnt" => 0}, 0)
|
108
|
+
end
|
109
|
+
|
110
|
+
rows = chunk_query(opts)
|
111
|
+
# log.error "ROWS, #{rows.pretty_print_inspect}"
|
112
|
+
|
113
|
+
|
114
|
+
ii = 0
|
115
|
+
if rows.empty?
|
116
|
+
rows = [{"first" => 0, "last" => nil, "cnt" => ii}]
|
117
|
+
end
|
118
|
+
last_row = nil
|
119
|
+
while rows.any?
|
120
|
+
rows.each do |record|
|
121
|
+
last_row = record
|
122
|
+
yield record, ii
|
123
|
+
end
|
124
|
+
ii +=1
|
125
|
+
return if last_row["last"].nil?
|
126
|
+
rows = chunk_query(opts.merge(:id => rows.last["last"]))
|
127
|
+
end
|
128
|
+
|
129
|
+
if last_row["last"] and (last_row["last"].to_i - last_row["first"].to_i) >= batch_size
|
130
|
+
catchall_row = {"first" => last_row["last"].to_i+1, "last" => nil, "cnt" => ii}
|
131
|
+
yield catchall_row, ii
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def chunk_query(opts={})
|
136
|
+
|
137
|
+
conditions = "#{table_name}.#{primary_key} > #{opts[:id]} AND ((@t1:=(@t1+1) % #{batch_size})=0)"
|
138
|
+
opts = opts.clone
|
139
|
+
if opts[:conditions].nil? or opts[:conditions].empty?
|
140
|
+
opts[:conditions] = conditions
|
141
|
+
else
|
142
|
+
opts[:conditions] += " AND " unless opts[:conditions].empty?
|
143
|
+
opts[:conditions] += conditions
|
144
|
+
end
|
145
|
+
limit = opts[:limit] ? "LIMIT #{opts[:limit]}" : nil
|
146
|
+
# select @t2:=(@t2+1), @t3:=@t4, @t4:=id from profiles where ( ((@t1:=(@t1+1) % 1000)=0) or (((@t1+1) % 1000)=0) ) order by id LIMIT 100;
|
147
|
+
|
148
|
+
# BEST
|
149
|
+
# select @t1:=0, @t2:=0, @t3:=0, @t4:=0;
|
150
|
+
# select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
|
151
|
+
# select (@t2:=(@t2+1) % 2) as evenodd, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
|
152
|
+
|
153
|
+
execute('select @t1:=0, @t2:=-1, @t3:=0, @t4:=0')
|
154
|
+
sql = "select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=#{table_name}.#{primary_key} as last from #{table_name} #{opts[:joins]} where #{opts[:conditions]} ORDER BY #{table_name}.#{primary_key} #{limit}"
|
155
|
+
# log.error "SQL #{sql}"
|
156
|
+
select_all(sql)
|
157
|
+
|
158
|
+
# mc.connection.select_values(mc.send(:construct_finder_sql, :select => "#{mc.table_name}.id", :joins => opts[:joins], :conditions => conditions, :limit => opts[:limit], :order => :id))
|
159
|
+
end
|
160
|
+
|
161
|
+
def run_job_for_batch(batches,&block)
|
162
|
+
jobopts = {
|
163
|
+
:mappers => 20000,
|
164
|
+
:map_data => batches,
|
165
|
+
:name => "each #{model_class} MASTER",
|
166
|
+
:map_name => "each #{model_class} MAP",
|
167
|
+
:map_timeout => 60,
|
168
|
+
:master_timeout => 12.hours,
|
169
|
+
:master_result_timeout => 60,
|
170
|
+
:master_retry => 0,
|
171
|
+
:map_retry => 0
|
172
|
+
}
|
173
|
+
|
174
|
+
job = nil
|
175
|
+
if block_given?
|
176
|
+
job = Skynet::Job.new(jobopts.merge(:map => block), :local_master => true)
|
177
|
+
else
|
178
|
+
job = Skynet::AsyncJob.new(jobopts.merge(:map_reduce_class => "#{self.class}"))
|
179
|
+
end
|
180
|
+
job.run
|
181
|
+
end
|
182
|
+
|
183
|
+
def map(klass_or_method=nil,&block)
|
184
|
+
klass_or_method ||= model_class
|
185
|
+
log = Skynet::Logger.get
|
186
|
+
|
187
|
+
batches = []
|
188
|
+
each_range(find_args) do |ids,ii|
|
189
|
+
batch_item = [
|
190
|
+
ids['first'].to_i,
|
191
|
+
ids['last'].to_i,
|
192
|
+
find_args.clone,
|
193
|
+
model_class
|
194
|
+
]
|
195
|
+
if block_given?
|
196
|
+
batch_item << block
|
197
|
+
else
|
198
|
+
batch_item << "#{klass_or_method}"
|
199
|
+
end
|
200
|
+
batches << batch_item
|
201
|
+
if batches.size >= MAX_BATCHES_PER_JOB
|
202
|
+
log.error "MAX BATCH SIZE EXCEEDED RUNNING: #{batches.size}"
|
203
|
+
run_job_for_batch(batches)
|
204
|
+
batches = []
|
205
|
+
end
|
206
|
+
end
|
207
|
+
run_job_for_batch(batches)
|
208
|
+
end
|
209
|
+
|
210
|
+
alias_method :each, :map
|
211
|
+
alias_method :mapreduce, :map
|
212
|
+
|
213
|
+
def model_class
|
214
|
+
@model_class || self.class.model_class
|
215
|
+
end
|
216
|
+
|
217
|
+
def self.model_class(model_class)
|
218
|
+
(class << self; self; end).module_eval do
|
219
|
+
define_method(:model_class) {model_class}
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
def self.log
|
224
|
+
Skynet::Logger.get
|
225
|
+
end
|
226
|
+
|
227
|
+
def self.map(datas)
|
228
|
+
return unless datas and not datas.empty?
|
229
|
+
datas.each do |data|
|
230
|
+
next if (not data.is_a?(Array))
|
231
|
+
next if data.empty?
|
232
|
+
model_class = data[3].constantize
|
233
|
+
table_name = model_class.table_name
|
234
|
+
conditions = "#{table_name}.#{model_class.primary_key} >= #{data[0]}"
|
235
|
+
conditions += " AND #{table_name}.#{model_class.primary_key} <= #{data[1]}" if data[1] > data[0]
|
236
|
+
conditions = "(#{conditions})"
|
237
|
+
# conditions = "ID BETWEEN #{data[0]} and #{data[1]}"
|
238
|
+
if not data[2]
|
239
|
+
data[2] = {:conditions => conditions}
|
240
|
+
elsif data[2].is_a?(Hash) and data[2].empty?
|
241
|
+
data[2] = {:conditions => conditions}
|
242
|
+
elsif data[2].is_a?(Hash) and (not data[2][:conditions] or data[2][:conditions].empty?)
|
243
|
+
data[2][:conditions] = conditions
|
244
|
+
else
|
245
|
+
data[2][:conditions] += " AND #{conditions}"
|
246
|
+
end
|
247
|
+
data[2][:select] = "#{table_name}.*"
|
248
|
+
|
249
|
+
# log.error "GETTING #{data.pretty_print_inspect}"
|
250
|
+
models = model_class.find(:all, data[2])
|
251
|
+
# log.error "GOT MODELS: #{models.size}"
|
252
|
+
models.each do |ar_object|
|
253
|
+
begin
|
254
|
+
if data[4].kind_of?(String)
|
255
|
+
if ar_object.respond_to?(data[4].to_sym)
|
256
|
+
# log.error "CALLING #{data[4]} on #{ar_object.class}:#{ar_object.id}"
|
257
|
+
ar_object.send(data[4].to_sym)
|
258
|
+
else
|
259
|
+
begin
|
260
|
+
data[4].constantize.each(ar_object)
|
261
|
+
rescue NameError
|
262
|
+
raise NameError.new("#{data[4]} is not a class or an instance method in #{model_class}")
|
263
|
+
end
|
264
|
+
end
|
265
|
+
else
|
266
|
+
data[4].call(ar_object)
|
267
|
+
end
|
268
|
+
rescue Exception => e
|
269
|
+
if data[4].kind_of?(String)
|
270
|
+
log.error("Error in #{data[4]} #{e.inspect} #{e.backtrace.join("\n")}")
|
271
|
+
else
|
272
|
+
log.error("Error in #{self} with given block #{e.inspect} #{e.backtrace.join("\n")}")
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|
276
|
+
end
|
277
|
+
nil
|
278
|
+
end
|
279
|
+
end
|
280
|
+
end
|