pampa 2.0.29 → 2.0.31

Sign up to get free protection for your applications and to get access to all the features.
data/lib/pampa.rb CHANGED
@@ -1,86 +1,19 @@
1
- require 'sequel'
2
1
  require 'blackstack-core'
2
+ require 'blackstack-db'
3
3
  require 'blackstack-nodes'
4
4
  require 'simple_command_line_parser'
5
5
  require 'simple_cloud_logging'
6
+ require 'colorize'
7
+ require 'sinatra'
6
8
 
7
9
  module BlackStack
8
10
  module Pampa
9
- # setup custom locations for config and worker files.
10
- @@config_filename = "config.rb"
11
- @@worker_filename = "worker.rb"
12
- # setu the directory where the worker.rb file will be lauched, and the log files will be stored.
13
- @@working_directory = "$HOME/pampa"
14
11
  # arrays of workers, nodes, and jobs.
15
12
  @@nodes = []
16
13
  @@jobs = []
17
- # logger configuration
18
- @@log_filename = nil
19
14
  @@logger = BlackStack::DummyLogger.new(nil)
20
- # Connection string to the database. Example: mysql2://user:password@localhost:3306/database
21
- @@connection_string = nil
22
-
23
- def self.now()
24
- tz = 'America/Argentina/Buenos_Aires' #DB["SELECT current_setting('TIMEZONE') AS tz"].first[:tz]
25
- DB["SELECT current_timestamp at TIME ZONE '#{tz}' AS now"].first[:now]
26
- end
27
-
28
- # @@config_filename
29
- def self.config_filename()
30
- @@config_filename
31
- end
32
-
33
- def self.set_config_filename(s)
34
- @@config_filename = s
35
- end
36
-
37
- # @@worker_filename
38
- def self.worker_filename()
39
- @@worker_filename
40
- end
41
-
42
- def self.set_worker_filename(s)
43
- @@worker_filename = s
44
- end
45
-
46
- ## @@working_directory
47
- def self.working_directory()
48
- @@working_directory
49
- end
50
-
51
- def self.set_working_directory(s)
52
- @@working_directory = s
53
- end
54
-
55
- # define a filename for the log file.
56
- def self.set_log_filename(s)
57
- @@log_filename = s
58
- @@logger = BlackStack::LocalLogger.new(s)
59
- end
60
-
61
- # return the logger.
62
- def self.logger()
63
- @@logger
64
- end
65
-
66
- def self.set_logger(l)
67
- @@logger = l
68
- end
69
-
70
- # return the log filename.
71
- def self.log_filename()
72
- @@log_filename
73
- end
74
-
75
- # define a connection string to the database.
76
- def self.set_connection_string(s)
77
- @@connection_string = s
78
- end
79
-
80
- # return connection string to the database. Example: mysql2://user:password@localhost:3306/database
81
- def self.connection_string()
82
- @@connection_string
83
- end
15
+ @@dispatcher_function = nil
16
+ @@worker_function = nil
84
17
 
85
18
  # add a node to the cluster.
86
19
  def self.add_node(h)
@@ -105,7 +38,7 @@ module BlackStack
105
38
 
106
39
  # return the array of all workers, beloning all nodes.
107
40
  def self.workers()
108
- @@nodes.map { |node| node.workers }.flatten
41
+ @@nodes.map { |node| node.workers }.flatten
109
42
  end
110
43
 
111
44
  # add a job to the cluster.
@@ -129,6 +62,30 @@ module BlackStack
129
62
  @@jobs
130
63
  end
131
64
 
65
+ # get and set logger
66
+ def self.logger()
67
+ @@logger
68
+ end
69
+
70
+ def self.set_logger(l)
71
+ @@logger = l
72
+ end
73
+
74
+
75
+ def self.set_snippets(h)
76
+ @@dispatcher_function = h[:dispatcher_function]
77
+ @@worker_function = h[:worker_function]
78
+ end
79
+
80
+ def self.dispatcher_function
81
+ @@dispatcher_function
82
+ end
83
+
84
+ def self.worker_function
85
+ @@worker_function
86
+ end
87
+
88
+
132
89
  # get attached and unassigned workers.
133
90
  # assign and unassign workers to jobs.
134
91
  #
@@ -137,70 +94,63 @@ module BlackStack
137
94
  # - worker: relative path of the worker.rb file. Example: '../worker.rb'
138
95
  #
139
96
  def self.stretch()
140
- # validate: the connection string is not nil
141
- raise "The connection string is nil" if @@connection_string.nil?
142
- # validate: the connection string is not empty
143
- raise "The connection string is empty" if @@connection_string.empty?
144
- # validate: the connection string is not blank
145
- raise "The connection string is blank" if @@connection_string.strip.empty?
146
97
  # getting logger
147
98
  l = self.logger()
148
99
  # get the job this worker is working with
149
100
  BlackStack::Pampa.jobs.each { |job|
150
- l.log ''
151
101
  l.logs "job #{job.name}... "
152
102
  # get attached and unassigned workers
153
103
  l.logs "Getting attached and unassigned workers... "
154
104
  workers = BlackStack::Pampa.workers.select { |w| w.attached && w.assigned_job.nil? }
155
- l.logf "done (#{workers.size.to_s})"
105
+ l.logf 'done'.green + " (#{workers.size.to_s.blue})"
156
106
  # get the workers that match the filter
157
107
  l.logs "Getting workers that match the filter... "
158
108
  workers = workers.select { |w| w.id =~ job.filter_worker_id }
159
- l.logf "done (#{workers.size.to_s})"
109
+ l.logf "done".green + " (#{workers.size.to_s.blue})"
160
110
  # if theere are workers
161
111
  if workers.size > 0
162
112
  l.logs("Gettting assigned workers... ")
163
113
  assigned = BlackStack::Pampa.workers.select { |worker| worker.attached && worker.assigned_job.to_s == job.name.to_s }
164
- l.logf("done (#{assigned.size.to_s})")
114
+ l.logf "done ".green + " (#{assigned.size.to_s.blue})"
165
115
 
166
116
  l.logs("Getting total pending (pending) tasks... ")
167
117
  pendings = job.pending
168
- l.logf("done (#{pendings.to_s})")
118
+ l.logf "done".green + " (#{pendings.to_s.blue})"
169
119
 
170
120
  l.logs("0 pending tasks?.... ")
171
121
  if pendings.size == 0
172
- l.logf("yes")
122
+ l.logf "yes".green
173
123
 
174
124
  l.logs("Unassigning all assigned workers... ")
175
125
  assigned.each { |w|
176
126
  l.logs("Unassigning worker... ")
177
127
  w.assigned_job = nil
178
128
  workers << w # add worker back to the list of unassigned
179
- l.logf "done (#{w.id})"
129
+ l.logf "done".green + " (#{w.id.to_s.blue})"
180
130
  }
181
131
  l.done
182
132
  else
183
- l.logf("no")
133
+ l.logf "no".red
184
134
 
185
135
  l.logs("Under :max_pending_tasks (#{job.max_pending_tasks}) and more than 1 assigned workers ?... ")
186
136
  if pendings.size < job.max_pending_tasks && assigned.size > 1
187
- l.logf("yes")
137
+ l.logf "yes".green
188
138
 
189
139
  while assigned.size > 1
190
140
  l.logs("Unassigning worker... ")
191
141
  w = assigned.pop # TODO: find a worker with no pending tasks
192
142
  w.assigned_job = nil
193
143
  workers << w # add worker back to the array of unassigned workers
194
- l.logf "done (#{w.id})"
144
+ l.logf "done".green + " (#{w.id.to_s.blue})"
195
145
  end
196
146
  else
197
- l.logf("no")
147
+ l.logf "no".red
198
148
 
199
- l.logs("Over :max_assigned_workers (#{job.max_assigned_workers}) and more than 1 assigned workers?... ")
149
+ l.logs("Over :max_assigned_workers (#{job.max_assigned_workers.to_s.blue}) and more than 1 assigned workers?... ")
200
150
  if assigned.size >= job.max_assigned_workers && assigned.size > 1
201
- l.logf("yes")
151
+ l.logf("yes".green)
202
152
  else
203
- l.logf("no")
153
+ l.logf("no".red)
204
154
 
205
155
  i = assigned.size
206
156
  while i < job.max_assigned_workers
@@ -208,11 +158,11 @@ module BlackStack
208
158
  l.logs("Assigning worker... ")
209
159
  w = workers.pop
210
160
  if w.nil?
211
- l.logf("no more workers")
161
+ l.logf("no more workers".yellow)
212
162
  break
213
163
  else
214
164
  w.assigned_job = job.name.to_sym
215
- l.logf "done (#{w.id})"
165
+ l.logf "done".green + " (#{w.id.to_s.blue})"
216
166
  end
217
167
  end # while i < job.max_assigned_workers
218
168
  end # if assigned.size >= job.max_assigned_workers && assigned.size > 0
@@ -221,7 +171,7 @@ module BlackStack
221
171
  end # if workers.size > 0
222
172
  l.done
223
173
  }
224
- end
174
+ end # def self.stretch()
225
175
 
226
176
  # iterate the jobs.
227
177
  # for each job, get all the tasks to relaunch.
@@ -232,12 +182,6 @@ module BlackStack
232
182
  # - worker: relative path of the worker.rb file. Example: '../worker.rb'
233
183
  #
234
184
  def self.relaunch(n=10000)
235
- # validate: the connection string is not nil
236
- raise "The connection string is nil" if @@connection_string.nil?
237
- # validate: the connection string is not empty
238
- raise "The connection string is empty" if @@connection_string.empty?
239
- # validate: the connection string is not blank
240
- raise "The connection string is blank" if @@connection_string.strip.empty?
241
185
  # getting logger
242
186
  l = self.logger()
243
187
  # iterate the workers
@@ -245,7 +189,7 @@ module BlackStack
245
189
  l.logs("job:#{job.name}... ")
246
190
  l.logs("Gettting tasks to relaunch (max #{n})... ")
247
191
  tasks = job.relaunching(n)
248
- l.logf("done (#{tasks.size.to_s})")
192
+ l.logf("done".green + " (#{tasks.size.to_s.blue})")
249
193
 
250
194
  tasks.each { |task|
251
195
  l.logs("Relaunching task #{task[job.field_primary_key.to_sym]}... ")
@@ -255,7 +199,7 @@ module BlackStack
255
199
 
256
200
  l.done
257
201
  }
258
- end
202
+ end # def self.relaunch(n=10000)
259
203
 
260
204
  # iterate the workers.
261
205
  # for each worker, iterate the job.
@@ -265,245 +209,28 @@ module BlackStack
265
209
  # - worker: relative path of the worker.rb file. Example: '../worker.rb'
266
210
  #
267
211
  def self.dispatch()
268
- # validate: the connection string is not nil
269
- raise "The connection string is nil" if @@connection_string.nil?
270
- # validate: the connection string is not empty
271
- raise "The connection string is empty" if @@connection_string.empty?
272
- # validate: the connection string is not blank
273
- raise "The connection string is blank" if @@connection_string.strip.empty?
274
212
  # getting logger
275
213
  l = self.logger()
276
214
  # iterate the workers
277
215
  BlackStack::Pampa.workers.each { |worker|
278
216
  l.logs("worker:#{worker.id} (job:#{worker.assigned_job.to_s})... ")
279
217
  if !worker.attached
280
- l.logf("detached")
218
+ l.logf("detached".green)
281
219
  else
282
220
  if worker.assigned_job.nil?
283
- l.logf("unassigned")
221
+ l.logf("unassigned".yellow)
284
222
  else
285
223
  # get the job this worker is assigned to
286
224
  job = BlackStack::Pampa.jobs.select { |j| j.name.to_s == worker.assigned_job.to_s }.first
287
225
  if job.nil?
288
- l.logf("job #{job.name} not found")
226
+ l.logf("job #{job.name} not found".red)
289
227
  else
290
- l.logf("done (#{job.run_dispatch(worker).to_s})")
228
+ l.logf("done".green + " (#{job.run_dispatch(worker).to_s.blue})")
291
229
  end
292
230
  end
293
231
  end
294
232
  } # @@nodes.each do |node|
295
- end
296
-
297
- # connect the nodes via ssh.
298
- # kill all Ruby processes except this one.
299
- # rename any existing folder $HOME/pampa to $HOME/pampa.<current timestamp>.
300
- # create a new folder $HOME/pampa.
301
- # build the file $HOME/pampa/config.rb in the remote node.
302
- # copy the file $HOME/pampa/worker.rb to the remote node.
303
- # run the number of workers specified in the configuration of the Pampa module.
304
- # return an array with the IDs of the workers.
305
- #
306
- # Parameters:
307
- # - config: relative path of the configuration file. Example: '../config.rb'
308
- # - worker: relative path of the worker.rb file. Example: '../worker.rb'
309
- #
310
- def self.deploy()
311
- # validate: the connection string is not nil
312
- raise "The connection string is nil" if @@connection_string.nil?
313
- # validate: the connection string is not empty
314
- raise "The connection string is empty" if @@connection_string.empty?
315
- # validate: the connection string is not blank
316
- raise "The connection string is blank" if @@connection_string.strip.empty?
317
- # getting logger
318
- l = self.logger()
319
- # iterate the nodes
320
- @@nodes.each { |node|
321
- l.logs("node:#{node.name()}... ")
322
- # connect the node
323
- l.logs("Connecting... ")
324
- node.connect()
325
- l.done
326
- # kill all ruby processes except this one
327
- l.logs("Killing all Ruby processes except this one... ")
328
- node.kill_workers()
329
- l.done
330
- # rename any existing folder ~/code/pampa to ~/code/pampa.<current timestamp>.
331
- l.logs("Renaming old folder... ")
332
- node.exec("mv #{BlackStack::Pampa.working_directory} #{BlackStack::Pampa.working_directory}.#{Time.now().to_i.to_s}", false);
333
- l.done
334
- # create a new folder ~/code. - ignore if it already exists.
335
- l.logs("Creating new folder... ")
336
- node.exec("mkdir #{BlackStack::Pampa.working_directory}", false);
337
- l.done
338
- # build the file $HOME/pampa/config.rb in the remote node. - Be sure the BlackStack::Pampa.to_hash.to_s don't have single-quotes (') in the string.
339
- l.logs("Building config file... ")
340
- s = "echo \"#{File.read(config_filename)}\" > #{BlackStack::Pampa.working_directory}/#{BlackStack::Pampa.config_filename}"
341
- node.exec("#{s}", false);
342
- l.done
343
- # copy the file $HOME/pampa/worker.rb to the remote node. - Be sure the script don't have single-quotes (') in the string.
344
- l.logs("Copying worker file... ")
345
- s = "echo \"#{File.read(worker_filename)}\" > #{BlackStack::Pampa.working_directory}/#{BlackStack::Pampa.worker_filename}"
346
- node.exec("#{s}", false);
347
- l.done
348
- # run the number of workers specified in the configuration of the Pampa module.
349
- node.workers.each { |worker|
350
- # run the worker
351
- # add these parameters for debug: debug=yes pampa=~/code/pampa/lib/pampa.rb
352
- l.logs "Running worker #{worker.id}... "
353
-
354
- # write bash command to initialize bash file
355
- s = "echo \"
356
- export RUBYLIB=$HOME/code/mysaas;
357
- source $HOME/.profile;
358
- source /usr/local/rvm/scripts/rvm;
359
- cd ~/code/mysaas; rvm install 3.1.2;
360
- rvm --default use 3.1.2;
361
- cd #{BlackStack::Pampa.working_directory};
362
- nohup ruby #{worker_filename} id=#{worker.id} config=#{self.config_filename} >/dev/null 2>&1 &
363
- \" > #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
364
- node.exec(s, false);
365
-
366
- #s = "nohup bash #{BlackStack::Pampa.working_directory}/worker.sh >/dev/null 2>&1 &"
367
- s = "bash #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
368
- node.exec(s, false);
369
-
370
- l.done
371
- }
372
- # disconnect the node
373
- l.logs("Disconnecting... ")
374
- node.disconnect()
375
- l.done
376
- l.done
377
- } # @@nodes.each do |node|
378
- end
379
-
380
- # connect the nodes via ssh.
381
- # kill all Ruby processes except this one.
382
- # run the number of workers specified in the configuration of the Pampa module.
383
- # return an array with the IDs of the workers.
384
- #
385
- def self.start()
386
- # validate: the connection string is not nil
387
- raise "The connection string is nil" if @@connection_string.nil?
388
- # validate: the connection string is not empty
389
- raise "The connection string is empty" if @@connection_string.empty?
390
- # validate: the connection string is not blank
391
- raise "The connection string is blank" if @@connection_string.strip.empty?
392
- # getting logger
393
- l = self.logger()
394
- # iterate the nodes
395
- @@nodes.each { |node|
396
- l.logs("node:#{node.name()}... ")
397
- # connect the node
398
- l.logs("Connecting... ")
399
- node.connect()
400
- l.done
401
- # kill all ruby processes except this one
402
- l.logs("Killing all Ruby processes except this one... ")
403
- node.kill_workers()
404
- l.done
405
- # run the number of workers specified in the configuration of the Pampa module.
406
- node.workers.each { |worker|
407
- # run the worker
408
- # add these parameters for debug: debug=yes pampa=~/code/pampa/lib/pampa.rb
409
- # run a bash command that sources the .profile file and runs the ruby script in the background, returning immediatelly.
410
-
411
- l.logs "Running worker #{worker.id}... "
412
-
413
- # write bash command to initialize bash file
414
- s = "echo \"
415
- export RUBYLIB=$HOME/code/mysaas;
416
- source $HOME/.profile;
417
- source /usr/local/rvm/scripts/rvm;
418
- cd ~/code/mysaas; rvm install 3.1.2;
419
- rvm --default use 3.1.2;
420
- cd #{BlackStack::Pampa.working_directory};
421
- nohup ruby #{worker_filename} id=#{worker.id} config=#{self.config_filename} >/dev/null 2>&1 &
422
- \" > #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
423
- node.exec(s, false);
424
- s = "nohup bash #{BlackStack::Pampa.working_directory}/#{worker.id}.sh >/dev/null 2>&1 &"
425
- node.exec(s, false);
426
-
427
- l.done
428
- }
429
- # disconnect the node
430
- l.logs("Disconnecting... ")
431
- node.disconnect()
432
- l.done
433
- l.done
434
- } # @@nodes.each do |node|
435
- end
436
-
437
- # connect the nodes via ssh.
438
- # kill all Ruby processes except this one.
439
- #
440
- # Parameters:
441
- # - config: relative path of the configuration file. Example: '../config.rb'
442
- #
443
- def self.stop()
444
- # validate: the connection string is not nil
445
- raise "The connection string is nil" if @@connection_string.nil?
446
- # validate: the connection string is not empty
447
- raise "The connection string is empty" if @@connection_string.empty?
448
- # validate: the connection string is not blank
449
- raise "The connection string is blank" if @@connection_string.strip.empty?
450
- # getting logger
451
- l = self.logger()
452
- # iterate the nodes
453
- @@nodes.each { |node|
454
- l.logs("node:#{node.name()}... ")
455
- # connect the node
456
- l.logs("Connecting... ")
457
- node.connect()
458
- l.done
459
- # kill all ruby processes except this one
460
- l.logs("Killing all Ruby processes except this one... ")
461
- node.kill_workers()
462
- l.done
463
- # disconnect the node
464
- l.logs("Disconnecting... ")
465
- node.disconnect()
466
- l.done
467
- l.done
468
- } # @@nodes.each do |node|
469
- end
470
-
471
- # get the node by `node_name`
472
- # connect the nodes via ssh.
473
- # get how many minutes the worker wrote the log file
474
- # close the connection
475
- #
476
- # DEPRECATED. Use `ps aux | grep "..."` to know if a process is running or not.
477
- #
478
- def self.log_minutes_ago(node_name, worker_id)
479
- # get the node
480
- n = self.nodes.select { |n| n.name == node_name }.first
481
- return nil if !n
482
- # connect the node
483
- n.connect()
484
- # get the time of the last time the worker wrote the log file
485
- code = "cat #{BlackStack::Pampa.working_directory}/worker.#{worker_id}.log | tail -n 1 | cut -b1-19"
486
- s = n.exec(code, false).to_s.strip
487
- # run bash command to get the difference in minutes beteen now and the last time the worker wrote the log file
488
- s = n.exec("echo \"$(($(date +%s) - $(date -d '#{s}' +%s))) / 60\" | bc", false).to_s.strip
489
- # disconnect the node
490
- n.disconnect
491
- # return the number of minutes
492
- s
493
- end # log_minutes_ago
494
-
495
- # get the node usage of CPU, RAM, DISK, and NETWORK
496
- # return a hash with the usage of CPU, RAM, DISK, and NETWORK
497
- #
498
- # sudo apt install sysstat
499
- #
500
- def self.node_usage(node_name)
501
- ret = {}
502
- # get the node
503
- n = self.nodes.select { |n| n.name == node_name }.first
504
- return nil if !n
505
- n.usage
506
- end # node_usage
233
+ end # def self.dispatch()
507
234
 
508
235
  # stub worker class
509
236
  class Worker
@@ -537,10 +264,6 @@ module BlackStack
537
264
  def detach()
538
265
  self.attached = false
539
266
  end
540
- # get the latest n lines of the log of this worker
541
- def tail(n=10)
542
- self.node.tail("#{BlackStack::Pampa.working_directory}/worker.#{self.id}.log", n)
543
- end
544
267
  end
545
268
 
546
269
  # stub node class
@@ -811,7 +534,7 @@ module BlackStack
811
534
  q = "
812
535
  SELECT *
813
536
  FROM #{self.table.to_s}
814
- WHERE COALESCE(#{self.field_time.to_s}, '1900-01-01') < CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) - INTERVAL '#{self.max_job_duration_minutes.to_i} minutes'
537
+ WHERE COALESCE(#{self.field_time.to_s}, '1900-01-01') < CAST('#{now}' AS TIMESTAMP) - INTERVAL '#{self.max_job_duration_minutes.to_i} minutes'
815
538
  AND #{self.field_id.to_s} IS NOT NULL
816
539
  AND #{self.field_end_time.to_s} IS NULL
817
540
  --AND COALESCE(#{self.field_times.to_s},0) < #{self.max_try_times.to_i}
@@ -856,7 +579,7 @@ module BlackStack
856
579
 
857
580
  def start(o)
858
581
  if self.starter_function.nil?
859
- o[self.field_start_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_start_time.nil? # IMPORTANT: use DB location to get current time.
582
+ o[self.field_start_time.to_sym] = DB["SELECT CAST('#{now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_start_time.nil? # IMPORTANT: use DB location to get current time.
860
583
  o[self.field_times.to_sym] = o[self.field_times.to_sym].to_i + 1
861
584
  self.update(o)
862
585
  else
@@ -866,7 +589,7 @@ module BlackStack
866
589
 
867
590
  def finish(o, e=nil)
868
591
  if self.finisher_function.nil?
869
- o[self.field_end_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_end_time.nil? && e.nil? # IMPORTANT: use DB location to get current time.
592
+ o[self.field_end_time.to_sym] = DB["SELECT CAST('#{now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_end_time.nil? && e.nil? # IMPORTANT: use DB location to get current time.
870
593
  o[self.field_success.to_sym] = e.nil?
871
594
  o[self.field_error_description.to_sym] = e.to_console if !e.nil?
872
595
  self.update(o)
@@ -899,7 +622,7 @@ module BlackStack
899
622
  # dispatching n pending records
900
623
  i = 0
901
624
  if n>0
902
- ids = self.selecting(n).map { |h| h[:id] }
625
+ ids = self.selecting(n).map { |h| h[self.field_primary_key.to_sym] }
903
626
 
904
627
  i = ids.size
905
628
 
@@ -923,7 +646,7 @@ module BlackStack
923
646
  end
924
647
 
925
648
  q += "
926
- #{self.field_time.to_s} = CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP)
649
+ #{self.field_time.to_s} = CAST('#{now}' AS TIMESTAMP)
927
650
  WHERE #{self.field_primary_key.to_s} IN ('#{ids.join("','")}')
928
651
  "
929
652
 
@@ -1006,55 +729,7 @@ module BlackStack
1006
729
  else
1007
730
  return self.failed_function.call
1008
731
  end
1009
- end # def falsed
1010
-
1011
- # reporting method: timeline
1012
- # Return an array of hashes with the number of successfull processed taasks in the last period.
1013
- # The period is defined by the `scale_unit` and `scale_points` parameters.
1014
- # The `scale_unit` can be `minutes`, `hours`, `days`, `weeks`, `months`, `years`.
1015
- # The `scale_points` is the number of `scale_unit` to be reported, and it must be an integer higer than 0.
1016
- # if the numbr if running tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
1017
- def timeline(scale_unit='minutes', scale_points=60)
1018
- j = self
1019
- a = []
1020
- # validate: The period is defined by the `scale_unit` and `scale_points` parameters.
1021
- if !['minutes', 'hours', 'days', 'weeks', 'months', 'years'].include?(scale_unit)
1022
- raise "Invalid scale_unit: #{scale_unit}"
1023
- end
1024
- # validate: The `scale_points` is the number of `scale_unit` to be reported, and it must be an integer higer than 0.
1025
- if !scale_points.is_a?(Integer) || scale_points<=0
1026
- raise "Invalid scale_points: #{scale_points}"
1027
- end
1028
- # generate report
1029
- point = 0
1030
- while point<scale_points
1031
- point += 1
1032
- q = "
1033
- SELECT COUNT(*) AS n
1034
- FROM #{j.table.to_s}
1035
- WHERE COALESCE(#{j.field_success.to_s},false)=true
1036
- AND #{j.field_time.to_s} >= CAST('#{BlackStack::Pampa.now - point.send(scale_unit)}' AS TIMESTAMP)
1037
- AND #{j.field_time.to_s} < CAST('#{BlackStack::Pampa.now - (point-1).send(scale_unit)}' AS TIMESTAMP)
1038
- "
1039
- a << { :time => BlackStack::Pampa.now - (point-1).send(scale_unit), :n => DB[q].first[:n].to_i }
1040
- end # while point<scale_points
1041
- # return
1042
- a
1043
- end # def timeline
1044
-
1045
- # reporting method: error_descriptions
1046
- # return an array of hashes { :id, :error_description } with the tasks that have an the success flag in false, error description.
1047
- # if the numbr if running tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show` errors.
1048
- def error_descriptions(max_tasks_to_show=25)
1049
- j = self
1050
- q = "
1051
- SELECT #{j.field_primary_key.to_s} as id, #{j.field_error_description.to_s} as description
1052
- FROM #{j.table.to_s}
1053
- WHERE COALESCE(#{j.field_success.to_s},true)=false
1054
- LIMIT #{max_tasks_to_show}
1055
- "
1056
- DB[q].all
1057
- end
732
+ end # def failed
1058
733
  end # class Job
1059
734
  end # module Pampa
1060
735
  end # module BlackStack
data/pampa.gemspec ADDED
@@ -0,0 +1,43 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'pampa'
3
+ s.version = '2.0.31'
4
+ s.date = '2023-10-07'
5
+ s.summary = "Ruby library for async & distributed computing, supporting dynamic reconfiguration, distribution of the computation jobs, error handling, job-retry and fault tolerance, and fast (non-direct) communication to ensure real-time capabilities."
6
+ s.description = "Pampa is a Ruby library for async & distributing computing providing the following features:
7
+
8
+ - cluster-management with dynamic reconfiguration (joining and leaving nodes);
9
+ - distribution of the computation jobs to the (active) nodes;
10
+ - error handling, job-retry and fault tolerance;
11
+ - fast (non-direct) communication to ensure realtime capabilities.
12
+
13
+ The Pampa framework may be widely used for:
14
+
15
+ - large scale web scraping with what we call a \"bot-farm\";
16
+ - payments processing for large-scale ecommerce websites;
17
+ - reports generation for high demanded SaaS platforms;
18
+ - heavy mathematical model computing;
19
+
20
+ and any other tasks that requires a virtually infinite amount of CPU computing and memory resources.
21
+
22
+ Find documentation here: https://github.com/leandrosardi/pampa
23
+ "
24
+ s.authors = ["Leandro Daniel Sardi"]
25
+ s.email = 'leandro.sardi@expandedventure.com'
26
+ s.files = [
27
+ 'lib/pampa/worker.rb',
28
+ 'lib/pampa/dispatcher.rb',
29
+ 'lib/pampa/app.rb',
30
+ 'lib/pampa.rb',
31
+ 'pampa.gemspec'
32
+ ]
33
+ s.homepage = 'https://github.com/leandrosardi/pampa'
34
+ s.license = 'MIT'
35
+ s.add_runtime_dependency 'rubygems-bundler', '~> 1.4.5', '>= 1.4.5'
36
+ s.add_runtime_dependency 'sinatra', '~> 2.2.4', '>= 2.2.4'
37
+ s.add_runtime_dependency 'colorize', '~> 0.8.1', '>= 0.8.1'
38
+ s.add_runtime_dependency 'blackstack-core', '~> 1.2.15', '>= 1.2.15'
39
+ s.add_runtime_dependency 'blackstack-db', '~> 1.0.1', '>= 1.0.1'
40
+ s.add_runtime_dependency 'blackstack-nodes', '~> 1.2.12', '>= 1.2.12'
41
+ s.add_runtime_dependency 'simple_command_line_parser', '~> 1.1.2', '>= 1.1.2'
42
+ s.add_runtime_dependency 'simple_cloud_logging', '~> 1.2.2', '>= 1.2.2'
43
+ end