skynet 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +4 -0
- data/License.txt +20 -0
- data/Manifest.txt +65 -0
- data/README.txt +100 -0
- data/Rakefile +4 -0
- data/app_generators/skynet_install/USAGE +5 -0
- data/app_generators/skynet_install/skynet_install_generator.rb +84 -0
- data/app_generators/skynet_install/templates/migration.rb +60 -0
- data/app_generators/skynet_install/templates/skynet +33 -0
- data/app_generators/skynet_install/templates/skynet_console +16 -0
- data/bin/skynet +20 -0
- data/bin/skynet_console +9 -0
- data/bin/skynet_install +12 -0
- data/bin/skynet_tuplespace_server +53 -0
- data/config/hoe.rb +74 -0
- data/config/requirements.rb +17 -0
- data/lib/skynet.rb +34 -0
- data/lib/skynet/mapreduce_test.rb +25 -0
- data/lib/skynet/message_queue_adapters/message_queue_adapter.rb +70 -0
- data/lib/skynet/message_queue_adapters/mysql.rb +573 -0
- data/lib/skynet/message_queue_adapters/tuple_space.rb +327 -0
- data/lib/skynet/skynet_active_record_extensions.rb +237 -0
- data/lib/skynet/skynet_config.rb +59 -0
- data/lib/skynet/skynet_console.rb +34 -0
- data/lib/skynet/skynet_console_helper.rb +59 -0
- data/lib/skynet/skynet_debugger.rb +84 -0
- data/lib/skynet/skynet_guid_generator.rb +68 -0
- data/lib/skynet/skynet_job.rb +607 -0
- data/lib/skynet/skynet_launcher.rb +10 -0
- data/lib/skynet/skynet_logger.rb +52 -0
- data/lib/skynet/skynet_manager.rb +486 -0
- data/lib/skynet/skynet_message.rb +366 -0
- data/lib/skynet/skynet_message_queue.rb +100 -0
- data/lib/skynet/skynet_ruby_extensions.rb +36 -0
- data/lib/skynet/skynet_task.rb +76 -0
- data/lib/skynet/skynet_tuplespace_server.rb +82 -0
- data/lib/skynet/skynet_worker.rb +395 -0
- data/lib/skynet/version.rb +9 -0
- data/log/debug.log +0 -0
- data/log/skynet.log +29 -0
- data/log/skynet_tuplespace_server.log +7 -0
- data/log/skynet_worker.pid +1 -0
- data/script/destroy +14 -0
- data/script/generate +14 -0
- data/script/txt2html +74 -0
- data/setup.rb +1585 -0
- data/sometest.rb +23 -0
- data/tasks/deployment.rake +34 -0
- data/tasks/environment.rake +7 -0
- data/tasks/website.rake +17 -0
- data/test/all_models_test.rb +139 -0
- data/test/mysql_message_queue_adaptor_test.rb +199 -0
- data/test/skynet_manager_test.rb +107 -0
- data/test/skynet_message_test.rb +42 -0
- data/test/test_generator_helper.rb +20 -0
- data/test/test_helper.rb +2 -0
- data/test/test_skynet.rb +11 -0
- data/test/test_skynet_install_generator.rb +53 -0
- data/test/tuplespace_message_queue_test.rb +179 -0
- data/tmtags +1242 -0
- data/website/index.html +93 -0
- data/website/index.txt +39 -0
- data/website/javascripts/rounded_corners_lite.inc.js +285 -0
- data/website/stylesheets/screen.css +138 -0
- data/website/template.rhtml +48 -0
- metadata +129 -0
@@ -0,0 +1,327 @@
|
|
1
|
+
require 'rinda/tuplespace'
|
2
|
+
class Rinda::TupleSpaceProxy
|
3
|
+
def take(tuple, sec=nil, &block)
|
4
|
+
port = []
|
5
|
+
port.push @ts.move(nil, tuple, sec, &block)
|
6
|
+
port[0]
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
class Skynet
|
11
|
+
class Error < StandardError
|
12
|
+
end
|
13
|
+
|
14
|
+
class RequestExpiredError < Skynet::Error
|
15
|
+
end
|
16
|
+
|
17
|
+
class InvalidMessage < Skynet::Error
|
18
|
+
end
|
19
|
+
|
20
|
+
class MessageQueueAdapter
|
21
|
+
|
22
|
+
class TupleSpace < Skynet::MessageQueueAdapter
|
23
|
+
|
24
|
+
include SkynetDebugger
|
25
|
+
|
26
|
+
USE_FALLBACK_TASKS = true
|
27
|
+
|
28
|
+
@@ts = nil
|
29
|
+
@@curhostidx = 0
|
30
|
+
|
31
|
+
def self.debug_class_desc
|
32
|
+
"TUPLESPACE"
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.adapter
|
36
|
+
:tuplespace
|
37
|
+
end
|
38
|
+
|
39
|
+
def initialize(ts=nil)
|
40
|
+
if not ts
|
41
|
+
ts = self.class.get_tuple_space
|
42
|
+
end
|
43
|
+
@ts = ts
|
44
|
+
end
|
45
|
+
|
46
|
+
def take_next_task(curver,timeout=nil,payload_type=nil)
|
47
|
+
message = Skynet::Message.new(take(Skynet::Message.next_task_template(curver,payload_type),timeout))
|
48
|
+
write_fallback_task(message)
|
49
|
+
message
|
50
|
+
end
|
51
|
+
|
52
|
+
def write_message(message,timeout=nil)
|
53
|
+
write(message,timeout)
|
54
|
+
end
|
55
|
+
|
56
|
+
def write_result(message,result=[],timeout=nil)
|
57
|
+
result_message = message.result_message(result).to_a
|
58
|
+
write(result_message,timeout)
|
59
|
+
take_fallback_message(message)
|
60
|
+
result_message
|
61
|
+
end
|
62
|
+
|
63
|
+
def take_result(job_id,timeout=nil)
|
64
|
+
Skynet::Message.new(take(Skynet::Message.result_template(job_id),timeout))
|
65
|
+
end
|
66
|
+
|
67
|
+
def write_error(message,error='',timeout=nil)
|
68
|
+
write(message.error_message(error),timeout)
|
69
|
+
take_fallback_message(message)
|
70
|
+
end
|
71
|
+
|
72
|
+
|
73
|
+
def write_worker_status(task, timeout=nil)
|
74
|
+
begin
|
75
|
+
take_worker_status(task,0.00001)
|
76
|
+
rescue Skynet::RequestExpiredError
|
77
|
+
end
|
78
|
+
write(Skynet::WorkerStatusMessage.new(task), timeout)
|
79
|
+
end
|
80
|
+
|
81
|
+
def take_worker_status(task, timeout=nil)
|
82
|
+
Skynet::WorkerStatusMessage.new(take(Skynet::WorkerStatusMessage.worker_status_template(task), timeout))
|
83
|
+
end
|
84
|
+
|
85
|
+
def read_all_worker_statuses(hostname=nil)
|
86
|
+
ws = Skynet::WorkerStatusMessage.all_workers_template(hostname)
|
87
|
+
workers = read_all(ws).collect{ |w| Skynet::WorkerStatusMessage.new(w) }#.sort{ |a,b| a.process_id <=> b.process_id }
|
88
|
+
end
|
89
|
+
|
90
|
+
def clear_worker_status(hostname=nil)
|
91
|
+
cnt = 0
|
92
|
+
begin
|
93
|
+
loop do
|
94
|
+
take(Skynet::WorkerStatusMessage.new([:status, :worker, hostname, nil, nil]),0.01)
|
95
|
+
cnt += 1
|
96
|
+
end
|
97
|
+
rescue Skynet::RequestExpiredError
|
98
|
+
end
|
99
|
+
cnt
|
100
|
+
end
|
101
|
+
|
102
|
+
def list_tasks(iteration=nil)
|
103
|
+
read_all(Skynet::Message.outstanding_tasks_template(iteration))
|
104
|
+
end
|
105
|
+
|
106
|
+
def list_results
|
107
|
+
read_all(Skynet::Message.outstanding_results_template)
|
108
|
+
end
|
109
|
+
|
110
|
+
def get_worker_version
|
111
|
+
begin
|
112
|
+
message = Skynet::WorkerVersionMessage.new(read(Skynet::WorkerVersionMessage.template,0.00001))
|
113
|
+
if message
|
114
|
+
curver = message.version
|
115
|
+
else
|
116
|
+
curver=0
|
117
|
+
end
|
118
|
+
rescue Skynet::RequestExpiredError => e
|
119
|
+
curver = 0
|
120
|
+
end
|
121
|
+
curver
|
122
|
+
end
|
123
|
+
|
124
|
+
def set_worker_version(ver=nil)
|
125
|
+
begin
|
126
|
+
messages = read_all(Skynet::WorkerVersionMessage.template).collect {|ret| Skynet::WorkerVersionMessage.new(ret)}
|
127
|
+
curver = 0
|
128
|
+
messages.each do |message|
|
129
|
+
curver = message.version
|
130
|
+
debug "CURRENT WORKER VERSION #{curver}"
|
131
|
+
curvmessage = Skynet::WorkerVersionMessage.new(take(message.template,0.00001))
|
132
|
+
if curvmessage
|
133
|
+
curver = curvmessage.version
|
134
|
+
else
|
135
|
+
curver=0
|
136
|
+
end
|
137
|
+
end
|
138
|
+
rescue Skynet::RequestExpiredError => e
|
139
|
+
curver = 0
|
140
|
+
end
|
141
|
+
|
142
|
+
newver = ver ? ver : curver + 1
|
143
|
+
debug "WRITING CURRENT WORKER REV #{newver}"
|
144
|
+
write(Skynet::WorkerVersionMessage.new(:version=>newver))
|
145
|
+
newver
|
146
|
+
end
|
147
|
+
|
148
|
+
def clear_outstanding_tasks
|
149
|
+
begin
|
150
|
+
tasks = read_all(Skynet::Message.outstanding_tasks_template)
|
151
|
+
rescue DRb::DRbConnError, Errno::ECONNREFUSED => e
|
152
|
+
error "ERROR #{e.inspect}", caller
|
153
|
+
end
|
154
|
+
|
155
|
+
tasks.size.times do |ii|
|
156
|
+
take(Skynet::Message.outstanding_tasks_template,0.00001)
|
157
|
+
end
|
158
|
+
|
159
|
+
results = read_all(Skynet::Message.outstanding_results_template)
|
160
|
+
results.size.times do |ii|
|
161
|
+
take(Skynet::Message.outstanding_results_template,0.00001)
|
162
|
+
end
|
163
|
+
|
164
|
+
task_tuples = read_all(Skynet::Message.outstanding_tasks_template)
|
165
|
+
result_tuples = read_all(Skynet::Message.outstanding_results_template)
|
166
|
+
return task_tuples + result_tuples
|
167
|
+
end
|
168
|
+
|
169
|
+
def stats
|
170
|
+
t1 = Time.now
|
171
|
+
tasks = list_tasks
|
172
|
+
results = list_results
|
173
|
+
t2 = Time.now - t1
|
174
|
+
p_tasks = tasks.partition {|task| task[9] == 0}
|
175
|
+
{:taken_tasks => p_tasks[1].size, :untaken_tasks => p_tasks[0].size, :results => list_results.size, :time => t2.to_f}
|
176
|
+
end
|
177
|
+
|
178
|
+
private
|
179
|
+
|
180
|
+
attr_accessor :ts
|
181
|
+
|
182
|
+
def write(tuple,timeout=nil)
|
183
|
+
ts_command(:write,tuple,timeout)
|
184
|
+
end
|
185
|
+
|
186
|
+
def take(template,timeout=nil)
|
187
|
+
ts_command(:take,template,timeout)
|
188
|
+
end
|
189
|
+
|
190
|
+
def read(template,timeout=nil)
|
191
|
+
ts_command(:read,template,timeout)
|
192
|
+
end
|
193
|
+
|
194
|
+
def read_all(template)
|
195
|
+
ts_command(:read_all,template)
|
196
|
+
end
|
197
|
+
|
198
|
+
###### FALLBACK METHODS
|
199
|
+
def write_fallback_task(message)
|
200
|
+
return unless USE_FALLBACK_TASKS
|
201
|
+
debug "4 WRITING BACKUP TASK #{message.task_id}", @fallback_worker_message
|
202
|
+
ftm = message.fallback_task_message
|
203
|
+
debug "WRITE FALLBACK TASK", ftm.to_a
|
204
|
+
timeout = message.expiry * 8
|
205
|
+
write(ftm,timeout)
|
206
|
+
ftm
|
207
|
+
end
|
208
|
+
|
209
|
+
def take_fallback_message(message,timeout=0.01)
|
210
|
+
return unless USE_FALLBACK_TASKS
|
211
|
+
begin
|
212
|
+
# debug "LOOKING FOR FALLBACK TEMPLATE", message.fallback_template
|
213
|
+
fb_message = Skynet::Message.new(take(message.fallback_template,timeout))
|
214
|
+
# debug "TOOK FALLBACK MESSAGE for TASKID: #{fb_message.task_id}"
|
215
|
+
rescue Skynet::RequestExpiredError => e
|
216
|
+
error "Couldn't find expected FALLBACK MESSAGE"
|
217
|
+
end
|
218
|
+
end
|
219
|
+
## END FALLBACK METHODS
|
220
|
+
|
221
|
+
def ts_command(command,message,timeout=nil)
|
222
|
+
# tries = 0
|
223
|
+
# until(tries > 3)
|
224
|
+
if message.is_a?(Skynet::Message)
|
225
|
+
tuple = message.to_a
|
226
|
+
elsif message.is_a?(Array)
|
227
|
+
tuple = message
|
228
|
+
else
|
229
|
+
raise InvalidMessage.new("You must provide a valid Skynet::Message object when calling #{command}. You passed #{message.inspect}.")
|
230
|
+
end
|
231
|
+
|
232
|
+
begin
|
233
|
+
if command==:read_all
|
234
|
+
return ts.send(command,tuple)
|
235
|
+
else
|
236
|
+
return ts.send(command,tuple,timeout)
|
237
|
+
end
|
238
|
+
|
239
|
+
rescue Rinda::RequestExpiredError
|
240
|
+
raise Skynet::RequestExpiredError.new
|
241
|
+
rescue DRb::DRbConnError => e
|
242
|
+
begin
|
243
|
+
log.error "Couldnt run command [#{command}] on tuplespace."
|
244
|
+
@ts = self.class.get_tuple_space
|
245
|
+
raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect}")
|
246
|
+
# tries += 1
|
247
|
+
# next
|
248
|
+
rescue Skynet::ConnectionError => e
|
249
|
+
raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect}")
|
250
|
+
# rescue RuntimeError => e
|
251
|
+
# raise Skynet::ConnectionError.new("Can't find ring finger. #{}")
|
252
|
+
rescue DRb::DRbConnError, Errno::ECONNREFUSED => e
|
253
|
+
raise Skynet::ConnectionError.new("There was a problem conected to the #{self.class} #{e.class} #{e.message}")
|
254
|
+
end
|
255
|
+
end
|
256
|
+
# end
|
257
|
+
end
|
258
|
+
|
259
|
+
####################################
|
260
|
+
######## CLASS METHODS #############
|
261
|
+
####################################
|
262
|
+
|
263
|
+
### XXX ACCEPT MULTIPLE TUPLE SPACES and a flag whether to use replication or failover.
|
264
|
+
|
265
|
+
def self.get_tuple_space
|
266
|
+
return @@ts if is_valid_tuplespace?(@@ts)
|
267
|
+
loop do
|
268
|
+
begin
|
269
|
+
DRb.start_service
|
270
|
+
if Skynet::CONFIG[:USE_RINGSERVER]
|
271
|
+
Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx] =~ /(.+):(\d+)/
|
272
|
+
host = $1
|
273
|
+
port = $2.to_i
|
274
|
+
@@ts = connect_to_tuple_space(host,port)
|
275
|
+
else
|
276
|
+
drburi = Skynet::CONFIG[:TUPLESPACE_DRBURIS].first
|
277
|
+
drburi = "druby://#{drburi}" unless drburi =~ %r{druby://}
|
278
|
+
@@ts = get_tuple_space_from_drburi(drburi)
|
279
|
+
log.info "#{self} CONNECTED TO #{drburi}"
|
280
|
+
end
|
281
|
+
return @@ts
|
282
|
+
rescue RuntimeError => e
|
283
|
+
if Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx + 1]
|
284
|
+
log.error "#{self} Couldn't connect to #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx]} trying #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx+1]}"
|
285
|
+
@@curhostidx += 1
|
286
|
+
next
|
287
|
+
else
|
288
|
+
raise Skynet::ConnectionError.new("Can't find ring finger @ #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx]}. #{e.class} #{e.message}")
|
289
|
+
end
|
290
|
+
rescue Exception => e
|
291
|
+
raise Skynet::ConnectionError.new("Error getting tuplespace @ #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx]}. #{e.class} #{e.message}")
|
292
|
+
end
|
293
|
+
end
|
294
|
+
return @@ts
|
295
|
+
end
|
296
|
+
|
297
|
+
def self.connect_to_tuple_space(host,port)
|
298
|
+
log.info "#{self} trying to connect to #{host}:#{port}"
|
299
|
+
if Skynet::CONFIG[:USE_RINGSERVER]
|
300
|
+
ring_finger = Rinda::RingFinger.new(host,port)
|
301
|
+
ring_server = ring_finger.lookup_ring_any(0.5)
|
302
|
+
|
303
|
+
ringts = ring_server.read([:name, :TupleSpace, nil, nil],0.00005)[2]
|
304
|
+
ts = Rinda::TupleSpaceProxy.new(ringts)
|
305
|
+
else
|
306
|
+
ts = get_tuple_space_from_drburi("druby://#{host}:#{port}")
|
307
|
+
end
|
308
|
+
log.info "#{self} CONNECTED TO #{host}:#{port}"
|
309
|
+
ts
|
310
|
+
end
|
311
|
+
|
312
|
+
def self.get_tuple_space_from_drburi(drburi)
|
313
|
+
DRbObject.new(nil, drburi)
|
314
|
+
end
|
315
|
+
|
316
|
+
def self.is_valid_tuplespace?(ts)
|
317
|
+
return false unless ts
|
318
|
+
begin
|
319
|
+
ts.read_all([:valid])
|
320
|
+
return true
|
321
|
+
rescue DRb::DRbConnError, RuntimeError, Errno::ECONNREFUSED => e
|
322
|
+
return false
|
323
|
+
end
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
327
|
+
end
|
@@ -0,0 +1,237 @@
|
|
1
|
+
class ActiveRecord::Base
|
2
|
+
def send_later(method,opts=nil,save=nil)
|
3
|
+
raise NoMethodError.new("Method: #{method} doesn't exist in #{self.class}") unless self.respond_to?(method)
|
4
|
+
data = {
|
5
|
+
:model_class => self.class.to_s,
|
6
|
+
:model_id => self.id,
|
7
|
+
:method => method,
|
8
|
+
}
|
9
|
+
data[:save] = 1 if save
|
10
|
+
data[:opts] = opts.to_yaml if opts
|
11
|
+
|
12
|
+
jobopts = {
|
13
|
+
:single => true,
|
14
|
+
:map_tasks => 1,
|
15
|
+
:map_data => [data],
|
16
|
+
:name => "send_later #{self.class}##{method}",
|
17
|
+
:map_name => "",
|
18
|
+
:map_timeout => 1.hour,
|
19
|
+
:reduce_timeout => 1.hour,
|
20
|
+
:master_timeout => 8.hours,
|
21
|
+
:master_result_timeout => 1.minute,
|
22
|
+
:map_reduce_class => Skynet::ActiveRecordAsync
|
23
|
+
}
|
24
|
+
job = Skynet::AsyncJob.new(jobopts)
|
25
|
+
job.run_master
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
class <<ActiveRecord::Base
|
30
|
+
def distributed_find(*args)
|
31
|
+
all = ActiveRecord::Mapreduce.find(*args)
|
32
|
+
all.model_class = self
|
33
|
+
all
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
class Skynet::ActiveRecordAsync
|
38
|
+
include SkynetDebugger
|
39
|
+
|
40
|
+
def self.map(datas)
|
41
|
+
datas.each do |data|
|
42
|
+
begin
|
43
|
+
model = data[:model_class].constantize.find(data[:model_id])
|
44
|
+
if data[:opts]
|
45
|
+
model.send(data[:method].to_sym, YAML.load(data[:opts]))
|
46
|
+
else
|
47
|
+
model.send(data[:method].to_sym)
|
48
|
+
end
|
49
|
+
model.save if data[:save]
|
50
|
+
rescue Exception => e
|
51
|
+
error "Error in #{self} #{e.inspect}"
|
52
|
+
end
|
53
|
+
end
|
54
|
+
return
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
module ActiveRecord
|
59
|
+
|
60
|
+
|
61
|
+
class Mapreduce
|
62
|
+
BATCH_SIZE=1000 unless defined?(BATCH_SIZE)
|
63
|
+
|
64
|
+
attr_accessor :find_args, :batch_size
|
65
|
+
attr_reader :model_class
|
66
|
+
|
67
|
+
def initialize(options = {})
|
68
|
+
@find_args = options[:find_args]
|
69
|
+
@batch_size = options[:batch_size] || BATCH_SIZE
|
70
|
+
@model_class = options[:model_class]
|
71
|
+
end
|
72
|
+
|
73
|
+
def model_class=(model_c)
|
74
|
+
@model_class = model_c.to_s
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.find(*args)
|
78
|
+
if not args.first.is_a?(Hash)
|
79
|
+
args.shift
|
80
|
+
end
|
81
|
+
if args.nil? or args.empty?
|
82
|
+
args = {}
|
83
|
+
else
|
84
|
+
args = *args
|
85
|
+
end
|
86
|
+
new(:find_args => args, :batch_size => args.delete(:batch_size), :model_class => args.delete(:model_class))
|
87
|
+
end
|
88
|
+
|
89
|
+
def each_range(opts={})
|
90
|
+
opts = opts.clone
|
91
|
+
opts[:id] || opts[:id] = 0
|
92
|
+
rows = chunk_query(opts)
|
93
|
+
|
94
|
+
ii = 0
|
95
|
+
if rows.empty?
|
96
|
+
rows = [{"first" => 0, "last" => nil, "cnt" => ii}]
|
97
|
+
end
|
98
|
+
last_row = nil
|
99
|
+
while rows.any?
|
100
|
+
rows.each do |record|
|
101
|
+
last_row = record
|
102
|
+
yield record, ii
|
103
|
+
end
|
104
|
+
ii +=1
|
105
|
+
return if last_row["last"].nil?
|
106
|
+
rows = chunk_query(opts.merge(:id => rows.last["last"]))
|
107
|
+
end
|
108
|
+
|
109
|
+
if last_row["last"] and (last_row["last"].to_i - last_row["first"].to_i) >= batch_size
|
110
|
+
catchall_row = {"first" => last_row["last"].to_i+1, "last" => nil, "cnt" => ii}
|
111
|
+
yield catchall_row, ii
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def chunk_query(opts={})
|
116
|
+
mc = model_class.constantize
|
117
|
+
table_name = mc.table_name
|
118
|
+
|
119
|
+
conditions = "#{table_name}.id > #{opts[:id]} AND ((@t1:=(@t1+1) % #{batch_size})=0)"
|
120
|
+
opts = opts.clone
|
121
|
+
if opts[:conditions].nil? or opts[:conditions].empty?
|
122
|
+
opts[:conditions] = conditions
|
123
|
+
else
|
124
|
+
opts[:conditions] += " AND " unless opts[:conditions].empty?
|
125
|
+
opts[:conditions] += conditions
|
126
|
+
end
|
127
|
+
limit = opts[:limit] ? "LIMIT #{opts[:limit]}" : nil
|
128
|
+
# select @t2:=(@t2+1), @t3:=@t4, @t4:=id from profiles where ( ((@t1:=(@t1+1) % 1000)=0) or (((@t1+1) % 1000)=0) ) order by id LIMIT 100;
|
129
|
+
|
130
|
+
# BEST
|
131
|
+
# select @t1:=0, @t2:=0, @t3:=0, @t4:=0;
|
132
|
+
# select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
|
133
|
+
# select (@t2:=(@t2+1) % 2) as evenodd, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
|
134
|
+
|
135
|
+
mc.connection.execute('select @t1:=0, @t2:=-1, @t3:=0, @t4:=0')
|
136
|
+
mc.connection.select_all("select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=#{table_name}.id as last from #{table_name} #{opts[:joins]} where #{opts[:conditions]} ORDER BY #{table_name}.id #{limit}")
|
137
|
+
|
138
|
+
# mc.connection.select_values(mc.send(:construct_finder_sql, :select => "#{mc.table_name}.id", :joins => opts[:joins], :conditions => conditions, :limit => opts[:limit], :order => :id))
|
139
|
+
end
|
140
|
+
|
141
|
+
|
142
|
+
def each(klass_or_method=nil,&block)
|
143
|
+
klass_or_method ||= model_class
|
144
|
+
batches = []
|
145
|
+
each_range(find_args) do |ids,ii|
|
146
|
+
batch_count = ids["cnt"].to_i
|
147
|
+
batches[batch_count] = [
|
148
|
+
ids['first'].to_i,
|
149
|
+
ids['last'].to_i,
|
150
|
+
find_args.clone,
|
151
|
+
model_class
|
152
|
+
]
|
153
|
+
if block_given?
|
154
|
+
batches[batch_count][4] = block
|
155
|
+
else
|
156
|
+
batches[batch_count][4] = "#{klass_or_method}"
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
job = nil
|
161
|
+
jobopts = {
|
162
|
+
:map_tasks => 20000,
|
163
|
+
:map_data => batches,
|
164
|
+
:name => "each #{model_class} MASTER",
|
165
|
+
:map_name => "each #{model_class} MAP",
|
166
|
+
:map_timeout => 1.hour,
|
167
|
+
:reduce_timeout => 1.hour,
|
168
|
+
:master_timeout => 8.hours,
|
169
|
+
:master_result_timeout => 1.minute
|
170
|
+
|
171
|
+
}
|
172
|
+
if block_given?
|
173
|
+
job = Skynet::Job.new(jobopts.merge(:map_reduce_class => "#{self.class}"))
|
174
|
+
else
|
175
|
+
job = Skynet::AsyncJob.new(jobopts.merge(:map_reduce_class => "#{self.class}"))
|
176
|
+
end
|
177
|
+
job.run
|
178
|
+
end
|
179
|
+
|
180
|
+
alias_method :mapreduce, :each
|
181
|
+
|
182
|
+
def model_class
|
183
|
+
@model_class || self.class.model_class
|
184
|
+
end
|
185
|
+
|
186
|
+
def self.model_class(model_class)
|
187
|
+
(class << self; self; end).module_eval do
|
188
|
+
define_method(:model_class) {model_class}
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
def self.log
|
193
|
+
Skynet::Logger.get
|
194
|
+
end
|
195
|
+
|
196
|
+
def self.map(datas)
|
197
|
+
datas.each do |data|
|
198
|
+
model_class = data[3].constantize
|
199
|
+
table_name = model_class.table_name
|
200
|
+
conditions = "#{table_name}.id >= #{data[0]}"
|
201
|
+
conditions += " AND #{table_name}.id <= #{data[1]}" if data[1] > data[0]
|
202
|
+
conditions = "(#{conditions})"
|
203
|
+
# conditions = "ID BETWEEN #{data[0]} and #{data[1]}"
|
204
|
+
if data[2].empty? or data[2][:conditions].empty?
|
205
|
+
data[2] = {:conditions => conditions}
|
206
|
+
else
|
207
|
+
data[2][:conditions] += " AND #{conditions}"
|
208
|
+
end
|
209
|
+
data[2][:select] = "#{table_name}.*"
|
210
|
+
model_class.find(:all, data[2]).each do |ar_object|
|
211
|
+
begin
|
212
|
+
if data[4].kind_of?(String)
|
213
|
+
begin
|
214
|
+
data[4].constantize.each(ar_object)
|
215
|
+
rescue NameError
|
216
|
+
if ar_object.respond_to?(data[4].to_sym)
|
217
|
+
ar_object.send(data[4])
|
218
|
+
else
|
219
|
+
raise NameError.new("#{data[4]} is not a class or an instance method in #{model_class}")
|
220
|
+
end
|
221
|
+
end
|
222
|
+
else
|
223
|
+
data[4].call(ar_object)
|
224
|
+
end
|
225
|
+
rescue Exception => e
|
226
|
+
if data[4].kind_of?(String)
|
227
|
+
log.error("Error in #{data[4]} #{e.inspect} #{e.backtrace.join("\n")}")
|
228
|
+
else
|
229
|
+
log.error("Error in #{self} with given block #{e.inspect} #{e.backtrace.join("\n")}")
|
230
|
+
end
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end
|
234
|
+
nil
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|