pampa 2.0.29 → 2.0.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/pampa.rb CHANGED
@@ -1,86 +1,17 @@
1
- require 'sequel'
2
1
  require 'blackstack-core'
2
+ require 'blackstack-db'
3
3
  require 'blackstack-nodes'
4
4
  require 'simple_command_line_parser'
5
5
  require 'simple_cloud_logging'
6
+ require 'colorize'
7
+ require 'sinatra'
6
8
 
7
9
  module BlackStack
8
10
  module Pampa
9
- # setup custom locations for config and worker files.
10
- @@config_filename = "config.rb"
11
- @@worker_filename = "worker.rb"
12
- # setu the directory where the worker.rb file will be lauched, and the log files will be stored.
13
- @@working_directory = "$HOME/pampa"
14
11
  # arrays of workers, nodes, and jobs.
15
12
  @@nodes = []
16
13
  @@jobs = []
17
- # logger configuration
18
- @@log_filename = nil
19
14
  @@logger = BlackStack::DummyLogger.new(nil)
20
- # Connection string to the database. Example: mysql2://user:password@localhost:3306/database
21
- @@connection_string = nil
22
-
23
- def self.now()
24
- tz = 'America/Argentina/Buenos_Aires' #DB["SELECT current_setting('TIMEZONE') AS tz"].first[:tz]
25
- DB["SELECT current_timestamp at TIME ZONE '#{tz}' AS now"].first[:now]
26
- end
27
-
28
- # @@config_filename
29
- def self.config_filename()
30
- @@config_filename
31
- end
32
-
33
- def self.set_config_filename(s)
34
- @@config_filename = s
35
- end
36
-
37
- # @@worker_filename
38
- def self.worker_filename()
39
- @@worker_filename
40
- end
41
-
42
- def self.set_worker_filename(s)
43
- @@worker_filename = s
44
- end
45
-
46
- ## @@working_directory
47
- def self.working_directory()
48
- @@working_directory
49
- end
50
-
51
- def self.set_working_directory(s)
52
- @@working_directory = s
53
- end
54
-
55
- # define a filename for the log file.
56
- def self.set_log_filename(s)
57
- @@log_filename = s
58
- @@logger = BlackStack::LocalLogger.new(s)
59
- end
60
-
61
- # return the logger.
62
- def self.logger()
63
- @@logger
64
- end
65
-
66
- def self.set_logger(l)
67
- @@logger = l
68
- end
69
-
70
- # return the log filename.
71
- def self.log_filename()
72
- @@log_filename
73
- end
74
-
75
- # define a connection string to the database.
76
- def self.set_connection_string(s)
77
- @@connection_string = s
78
- end
79
-
80
- # return connection string to the database. Example: mysql2://user:password@localhost:3306/database
81
- def self.connection_string()
82
- @@connection_string
83
- end
84
15
 
85
16
  # add a node to the cluster.
86
17
  def self.add_node(h)
@@ -105,7 +36,7 @@ module BlackStack
105
36
 
106
37
  # return the array of all workers, beloning all nodes.
107
38
  def self.workers()
108
- @@nodes.map { |node| node.workers }.flatten
39
+ @@nodes.map { |node| node.workers }.flatten
109
40
  end
110
41
 
111
42
  # add a job to the cluster.
@@ -129,6 +60,15 @@ module BlackStack
129
60
  @@jobs
130
61
  end
131
62
 
63
+ # get and set logger
64
+ def self.logger()
65
+ @@logger
66
+ end
67
+
68
+ def self.set_logger(l)
69
+ @@logger = l
70
+ end
71
+
132
72
  # get attached and unassigned workers.
133
73
  # assign and unassign workers to jobs.
134
74
  #
@@ -137,70 +77,63 @@ module BlackStack
137
77
  # - worker: relative path of the worker.rb file. Example: '../worker.rb'
138
78
  #
139
79
  def self.stretch()
140
- # validate: the connection string is not nil
141
- raise "The connection string is nil" if @@connection_string.nil?
142
- # validate: the connection string is not empty
143
- raise "The connection string is empty" if @@connection_string.empty?
144
- # validate: the connection string is not blank
145
- raise "The connection string is blank" if @@connection_string.strip.empty?
146
80
  # getting logger
147
81
  l = self.logger()
148
82
  # get the job this worker is working with
149
83
  BlackStack::Pampa.jobs.each { |job|
150
- l.log ''
151
84
  l.logs "job #{job.name}... "
152
85
  # get attached and unassigned workers
153
86
  l.logs "Getting attached and unassigned workers... "
154
87
  workers = BlackStack::Pampa.workers.select { |w| w.attached && w.assigned_job.nil? }
155
- l.logf "done (#{workers.size.to_s})"
88
+ l.logf 'done'.green + " (#{workers.size.to_s.blue})"
156
89
  # get the workers that match the filter
157
90
  l.logs "Getting workers that match the filter... "
158
91
  workers = workers.select { |w| w.id =~ job.filter_worker_id }
159
- l.logf "done (#{workers.size.to_s})"
92
+ l.logf "done".green + " (#{workers.size.to_s.blue})"
160
93
  # if theere are workers
161
94
  if workers.size > 0
162
95
  l.logs("Gettting assigned workers... ")
163
96
  assigned = BlackStack::Pampa.workers.select { |worker| worker.attached && worker.assigned_job.to_s == job.name.to_s }
164
- l.logf("done (#{assigned.size.to_s})")
97
+ l.logf "done ".green + " (#{assigned.size.to_s.blue})"
165
98
 
166
99
  l.logs("Getting total pending (pending) tasks... ")
167
100
  pendings = job.pending
168
- l.logf("done (#{pendings.to_s})")
101
+ l.logf "done".green + " (#{pendings.to_s.blue})"
169
102
 
170
103
  l.logs("0 pending tasks?.... ")
171
104
  if pendings.size == 0
172
- l.logf("yes")
105
+ l.logf "yes".green
173
106
 
174
107
  l.logs("Unassigning all assigned workers... ")
175
108
  assigned.each { |w|
176
109
  l.logs("Unassigning worker... ")
177
110
  w.assigned_job = nil
178
111
  workers << w # add worker back to the list of unassigned
179
- l.logf "done (#{w.id})"
112
+ l.logf "done".green + " (#{w.id.to_s.blue})"
180
113
  }
181
114
  l.done
182
115
  else
183
- l.logf("no")
116
+ l.logf "no".red
184
117
 
185
118
  l.logs("Under :max_pending_tasks (#{job.max_pending_tasks}) and more than 1 assigned workers ?... ")
186
119
  if pendings.size < job.max_pending_tasks && assigned.size > 1
187
- l.logf("yes")
120
+ l.logf "yes".green
188
121
 
189
122
  while assigned.size > 1
190
123
  l.logs("Unassigning worker... ")
191
124
  w = assigned.pop # TODO: find a worker with no pending tasks
192
125
  w.assigned_job = nil
193
126
  workers << w # add worker back to the array of unassigned workers
194
- l.logf "done (#{w.id})"
127
+ l.logf "done".green + " (#{w.id.to_s.blue})"
195
128
  end
196
129
  else
197
- l.logf("no")
130
+ l.logf "no".red
198
131
 
199
- l.logs("Over :max_assigned_workers (#{job.max_assigned_workers}) and more than 1 assigned workers?... ")
132
+ l.logs("Over :max_assigned_workers (#{job.max_assigned_workers.to_s.blue}) and more than 1 assigned workers?... ")
200
133
  if assigned.size >= job.max_assigned_workers && assigned.size > 1
201
- l.logf("yes")
134
+ l.logf("yes".green)
202
135
  else
203
- l.logf("no")
136
+ l.logf("no".red)
204
137
 
205
138
  i = assigned.size
206
139
  while i < job.max_assigned_workers
@@ -208,11 +141,11 @@ module BlackStack
208
141
  l.logs("Assigning worker... ")
209
142
  w = workers.pop
210
143
  if w.nil?
211
- l.logf("no more workers")
144
+ l.logf("no more workers".yellow)
212
145
  break
213
146
  else
214
147
  w.assigned_job = job.name.to_sym
215
- l.logf "done (#{w.id})"
148
+ l.logf "done".green + " (#{w.id.to_s.blue})"
216
149
  end
217
150
  end # while i < job.max_assigned_workers
218
151
  end # if assigned.size >= job.max_assigned_workers && assigned.size > 0
@@ -221,7 +154,7 @@ module BlackStack
221
154
  end # if workers.size > 0
222
155
  l.done
223
156
  }
224
- end
157
+ end # def self.stretch()
225
158
 
226
159
  # iterate the jobs.
227
160
  # for each job, get all the tasks to relaunch.
@@ -232,12 +165,6 @@ module BlackStack
232
165
  # - worker: relative path of the worker.rb file. Example: '../worker.rb'
233
166
  #
234
167
  def self.relaunch(n=10000)
235
- # validate: the connection string is not nil
236
- raise "The connection string is nil" if @@connection_string.nil?
237
- # validate: the connection string is not empty
238
- raise "The connection string is empty" if @@connection_string.empty?
239
- # validate: the connection string is not blank
240
- raise "The connection string is blank" if @@connection_string.strip.empty?
241
168
  # getting logger
242
169
  l = self.logger()
243
170
  # iterate the workers
@@ -245,7 +172,7 @@ module BlackStack
245
172
  l.logs("job:#{job.name}... ")
246
173
  l.logs("Gettting tasks to relaunch (max #{n})... ")
247
174
  tasks = job.relaunching(n)
248
- l.logf("done (#{tasks.size.to_s})")
175
+ l.logf("done".green + " (#{tasks.size.to_s.blue})")
249
176
 
250
177
  tasks.each { |task|
251
178
  l.logs("Relaunching task #{task[job.field_primary_key.to_sym]}... ")
@@ -255,7 +182,7 @@ module BlackStack
255
182
 
256
183
  l.done
257
184
  }
258
- end
185
+ end # def self.relaunch(n=10000)
259
186
 
260
187
  # iterate the workers.
261
188
  # for each worker, iterate the job.
@@ -265,245 +192,28 @@ module BlackStack
265
192
  # - worker: relative path of the worker.rb file. Example: '../worker.rb'
266
193
  #
267
194
  def self.dispatch()
268
- # validate: the connection string is not nil
269
- raise "The connection string is nil" if @@connection_string.nil?
270
- # validate: the connection string is not empty
271
- raise "The connection string is empty" if @@connection_string.empty?
272
- # validate: the connection string is not blank
273
- raise "The connection string is blank" if @@connection_string.strip.empty?
274
195
  # getting logger
275
196
  l = self.logger()
276
197
  # iterate the workers
277
198
  BlackStack::Pampa.workers.each { |worker|
278
199
  l.logs("worker:#{worker.id} (job:#{worker.assigned_job.to_s})... ")
279
200
  if !worker.attached
280
- l.logf("detached")
201
+ l.logf("detached".green)
281
202
  else
282
203
  if worker.assigned_job.nil?
283
- l.logf("unassigned")
204
+ l.logf("unassigned".yellow)
284
205
  else
285
206
  # get the job this worker is assigned to
286
207
  job = BlackStack::Pampa.jobs.select { |j| j.name.to_s == worker.assigned_job.to_s }.first
287
208
  if job.nil?
288
- l.logf("job #{job.name} not found")
209
+ l.logf("job #{job.name} not found".red)
289
210
  else
290
- l.logf("done (#{job.run_dispatch(worker).to_s})")
211
+ l.logf("done".green + " (#{job.run_dispatch(worker).to_s.blue})")
291
212
  end
292
213
  end
293
214
  end
294
215
  } # @@nodes.each do |node|
295
- end
296
-
297
- # connect the nodes via ssh.
298
- # kill all Ruby processes except this one.
299
- # rename any existing folder $HOME/pampa to $HOME/pampa.<current timestamp>.
300
- # create a new folder $HOME/pampa.
301
- # build the file $HOME/pampa/config.rb in the remote node.
302
- # copy the file $HOME/pampa/worker.rb to the remote node.
303
- # run the number of workers specified in the configuration of the Pampa module.
304
- # return an array with the IDs of the workers.
305
- #
306
- # Parameters:
307
- # - config: relative path of the configuration file. Example: '../config.rb'
308
- # - worker: relative path of the worker.rb file. Example: '../worker.rb'
309
- #
310
- def self.deploy()
311
- # validate: the connection string is not nil
312
- raise "The connection string is nil" if @@connection_string.nil?
313
- # validate: the connection string is not empty
314
- raise "The connection string is empty" if @@connection_string.empty?
315
- # validate: the connection string is not blank
316
- raise "The connection string is blank" if @@connection_string.strip.empty?
317
- # getting logger
318
- l = self.logger()
319
- # iterate the nodes
320
- @@nodes.each { |node|
321
- l.logs("node:#{node.name()}... ")
322
- # connect the node
323
- l.logs("Connecting... ")
324
- node.connect()
325
- l.done
326
- # kill all ruby processes except this one
327
- l.logs("Killing all Ruby processes except this one... ")
328
- node.kill_workers()
329
- l.done
330
- # rename any existing folder ~/code/pampa to ~/code/pampa.<current timestamp>.
331
- l.logs("Renaming old folder... ")
332
- node.exec("mv #{BlackStack::Pampa.working_directory} #{BlackStack::Pampa.working_directory}.#{Time.now().to_i.to_s}", false);
333
- l.done
334
- # create a new folder ~/code. - ignore if it already exists.
335
- l.logs("Creating new folder... ")
336
- node.exec("mkdir #{BlackStack::Pampa.working_directory}", false);
337
- l.done
338
- # build the file $HOME/pampa/config.rb in the remote node. - Be sure the BlackStack::Pampa.to_hash.to_s don't have single-quotes (') in the string.
339
- l.logs("Building config file... ")
340
- s = "echo \"#{File.read(config_filename)}\" > #{BlackStack::Pampa.working_directory}/#{BlackStack::Pampa.config_filename}"
341
- node.exec("#{s}", false);
342
- l.done
343
- # copy the file $HOME/pampa/worker.rb to the remote node. - Be sure the script don't have single-quotes (') in the string.
344
- l.logs("Copying worker file... ")
345
- s = "echo \"#{File.read(worker_filename)}\" > #{BlackStack::Pampa.working_directory}/#{BlackStack::Pampa.worker_filename}"
346
- node.exec("#{s}", false);
347
- l.done
348
- # run the number of workers specified in the configuration of the Pampa module.
349
- node.workers.each { |worker|
350
- # run the worker
351
- # add these parameters for debug: debug=yes pampa=~/code/pampa/lib/pampa.rb
352
- l.logs "Running worker #{worker.id}... "
353
-
354
- # write bash command to initialize bash file
355
- s = "echo \"
356
- export RUBYLIB=$HOME/code/mysaas;
357
- source $HOME/.profile;
358
- source /usr/local/rvm/scripts/rvm;
359
- cd ~/code/mysaas; rvm install 3.1.2;
360
- rvm --default use 3.1.2;
361
- cd #{BlackStack::Pampa.working_directory};
362
- nohup ruby #{worker_filename} id=#{worker.id} config=#{self.config_filename} >/dev/null 2>&1 &
363
- \" > #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
364
- node.exec(s, false);
365
-
366
- #s = "nohup bash #{BlackStack::Pampa.working_directory}/worker.sh >/dev/null 2>&1 &"
367
- s = "bash #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
368
- node.exec(s, false);
369
-
370
- l.done
371
- }
372
- # disconnect the node
373
- l.logs("Disconnecting... ")
374
- node.disconnect()
375
- l.done
376
- l.done
377
- } # @@nodes.each do |node|
378
- end
379
-
380
- # connect the nodes via ssh.
381
- # kill all Ruby processes except this one.
382
- # run the number of workers specified in the configuration of the Pampa module.
383
- # return an array with the IDs of the workers.
384
- #
385
- def self.start()
386
- # validate: the connection string is not nil
387
- raise "The connection string is nil" if @@connection_string.nil?
388
- # validate: the connection string is not empty
389
- raise "The connection string is empty" if @@connection_string.empty?
390
- # validate: the connection string is not blank
391
- raise "The connection string is blank" if @@connection_string.strip.empty?
392
- # getting logger
393
- l = self.logger()
394
- # iterate the nodes
395
- @@nodes.each { |node|
396
- l.logs("node:#{node.name()}... ")
397
- # connect the node
398
- l.logs("Connecting... ")
399
- node.connect()
400
- l.done
401
- # kill all ruby processes except this one
402
- l.logs("Killing all Ruby processes except this one... ")
403
- node.kill_workers()
404
- l.done
405
- # run the number of workers specified in the configuration of the Pampa module.
406
- node.workers.each { |worker|
407
- # run the worker
408
- # add these parameters for debug: debug=yes pampa=~/code/pampa/lib/pampa.rb
409
- # run a bash command that sources the .profile file and runs the ruby script in the background, returning immediatelly.
410
-
411
- l.logs "Running worker #{worker.id}... "
412
-
413
- # write bash command to initialize bash file
414
- s = "echo \"
415
- export RUBYLIB=$HOME/code/mysaas;
416
- source $HOME/.profile;
417
- source /usr/local/rvm/scripts/rvm;
418
- cd ~/code/mysaas; rvm install 3.1.2;
419
- rvm --default use 3.1.2;
420
- cd #{BlackStack::Pampa.working_directory};
421
- nohup ruby #{worker_filename} id=#{worker.id} config=#{self.config_filename} >/dev/null 2>&1 &
422
- \" > #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
423
- node.exec(s, false);
424
- s = "nohup bash #{BlackStack::Pampa.working_directory}/#{worker.id}.sh >/dev/null 2>&1 &"
425
- node.exec(s, false);
426
-
427
- l.done
428
- }
429
- # disconnect the node
430
- l.logs("Disconnecting... ")
431
- node.disconnect()
432
- l.done
433
- l.done
434
- } # @@nodes.each do |node|
435
- end
436
-
437
- # connect the nodes via ssh.
438
- # kill all Ruby processes except this one.
439
- #
440
- # Parameters:
441
- # - config: relative path of the configuration file. Example: '../config.rb'
442
- #
443
- def self.stop()
444
- # validate: the connection string is not nil
445
- raise "The connection string is nil" if @@connection_string.nil?
446
- # validate: the connection string is not empty
447
- raise "The connection string is empty" if @@connection_string.empty?
448
- # validate: the connection string is not blank
449
- raise "The connection string is blank" if @@connection_string.strip.empty?
450
- # getting logger
451
- l = self.logger()
452
- # iterate the nodes
453
- @@nodes.each { |node|
454
- l.logs("node:#{node.name()}... ")
455
- # connect the node
456
- l.logs("Connecting... ")
457
- node.connect()
458
- l.done
459
- # kill all ruby processes except this one
460
- l.logs("Killing all Ruby processes except this one... ")
461
- node.kill_workers()
462
- l.done
463
- # disconnect the node
464
- l.logs("Disconnecting... ")
465
- node.disconnect()
466
- l.done
467
- l.done
468
- } # @@nodes.each do |node|
469
- end
470
-
471
- # get the node by `node_name`
472
- # connect the nodes via ssh.
473
- # get how many minutes the worker wrote the log file
474
- # close the connection
475
- #
476
- # DEPRECATED. Use `ps aux | grep "..."` to know if a process is running or not.
477
- #
478
- def self.log_minutes_ago(node_name, worker_id)
479
- # get the node
480
- n = self.nodes.select { |n| n.name == node_name }.first
481
- return nil if !n
482
- # connect the node
483
- n.connect()
484
- # get the time of the last time the worker wrote the log file
485
- code = "cat #{BlackStack::Pampa.working_directory}/worker.#{worker_id}.log | tail -n 1 | cut -b1-19"
486
- s = n.exec(code, false).to_s.strip
487
- # run bash command to get the difference in minutes beteen now and the last time the worker wrote the log file
488
- s = n.exec("echo \"$(($(date +%s) - $(date -d '#{s}' +%s))) / 60\" | bc", false).to_s.strip
489
- # disconnect the node
490
- n.disconnect
491
- # return the number of minutes
492
- s
493
- end # log_minutes_ago
494
-
495
- # get the node usage of CPU, RAM, DISK, and NETWORK
496
- # return a hash with the usage of CPU, RAM, DISK, and NETWORK
497
- #
498
- # sudo apt install sysstat
499
- #
500
- def self.node_usage(node_name)
501
- ret = {}
502
- # get the node
503
- n = self.nodes.select { |n| n.name == node_name }.first
504
- return nil if !n
505
- n.usage
506
- end # node_usage
216
+ end # def self.dispatch()
507
217
 
508
218
  # stub worker class
509
219
  class Worker
@@ -537,10 +247,6 @@ module BlackStack
537
247
  def detach()
538
248
  self.attached = false
539
249
  end
540
- # get the latest n lines of the log of this worker
541
- def tail(n=10)
542
- self.node.tail("#{BlackStack::Pampa.working_directory}/worker.#{self.id}.log", n)
543
- end
544
250
  end
545
251
 
546
252
  # stub node class
@@ -811,7 +517,7 @@ module BlackStack
811
517
  q = "
812
518
  SELECT *
813
519
  FROM #{self.table.to_s}
814
- WHERE COALESCE(#{self.field_time.to_s}, '1900-01-01') < CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) - INTERVAL '#{self.max_job_duration_minutes.to_i} minutes'
520
+ WHERE COALESCE(#{self.field_time.to_s}, '1900-01-01') < CAST('#{now}' AS TIMESTAMP) - INTERVAL '#{self.max_job_duration_minutes.to_i} minutes'
815
521
  AND #{self.field_id.to_s} IS NOT NULL
816
522
  AND #{self.field_end_time.to_s} IS NULL
817
523
  --AND COALESCE(#{self.field_times.to_s},0) < #{self.max_try_times.to_i}
@@ -856,7 +562,7 @@ module BlackStack
856
562
 
857
563
  def start(o)
858
564
  if self.starter_function.nil?
859
- o[self.field_start_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_start_time.nil? # IMPORTANT: use DB location to get current time.
565
+ o[self.field_start_time.to_sym] = DB["SELECT CAST('#{now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_start_time.nil? # IMPORTANT: use DB location to get current time.
860
566
  o[self.field_times.to_sym] = o[self.field_times.to_sym].to_i + 1
861
567
  self.update(o)
862
568
  else
@@ -866,7 +572,7 @@ module BlackStack
866
572
 
867
573
  def finish(o, e=nil)
868
574
  if self.finisher_function.nil?
869
- o[self.field_end_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_end_time.nil? && e.nil? # IMPORTANT: use DB location to get current time.
575
+ o[self.field_end_time.to_sym] = DB["SELECT CAST('#{now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_end_time.nil? && e.nil? # IMPORTANT: use DB location to get current time.
870
576
  o[self.field_success.to_sym] = e.nil?
871
577
  o[self.field_error_description.to_sym] = e.to_console if !e.nil?
872
578
  self.update(o)
@@ -899,7 +605,7 @@ module BlackStack
899
605
  # dispatching n pending records
900
606
  i = 0
901
607
  if n>0
902
- ids = self.selecting(n).map { |h| h[:id] }
608
+ ids = self.selecting(n).map { |h| h[self.field_primary_key.to_sym] }
903
609
 
904
610
  i = ids.size
905
611
 
@@ -923,7 +629,7 @@ module BlackStack
923
629
  end
924
630
 
925
631
  q += "
926
- #{self.field_time.to_s} = CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP)
632
+ #{self.field_time.to_s} = CAST('#{now}' AS TIMESTAMP)
927
633
  WHERE #{self.field_primary_key.to_s} IN ('#{ids.join("','")}')
928
634
  "
929
635
 
@@ -1006,55 +712,7 @@ module BlackStack
1006
712
  else
1007
713
  return self.failed_function.call
1008
714
  end
1009
- end # def falsed
1010
-
1011
- # reporting method: timeline
1012
- # Return an array of hashes with the number of successfull processed taasks in the last period.
1013
- # The period is defined by the `scale_unit` and `scale_points` parameters.
1014
- # The `scale_unit` can be `minutes`, `hours`, `days`, `weeks`, `months`, `years`.
1015
- # The `scale_points` is the number of `scale_unit` to be reported, and it must be an integer higer than 0.
1016
- # if the numbr if running tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
1017
- def timeline(scale_unit='minutes', scale_points=60)
1018
- j = self
1019
- a = []
1020
- # validate: The period is defined by the `scale_unit` and `scale_points` parameters.
1021
- if !['minutes', 'hours', 'days', 'weeks', 'months', 'years'].include?(scale_unit)
1022
- raise "Invalid scale_unit: #{scale_unit}"
1023
- end
1024
- # validate: The `scale_points` is the number of `scale_unit` to be reported, and it must be an integer higer than 0.
1025
- if !scale_points.is_a?(Integer) || scale_points<=0
1026
- raise "Invalid scale_points: #{scale_points}"
1027
- end
1028
- # generate report
1029
- point = 0
1030
- while point<scale_points
1031
- point += 1
1032
- q = "
1033
- SELECT COUNT(*) AS n
1034
- FROM #{j.table.to_s}
1035
- WHERE COALESCE(#{j.field_success.to_s},false)=true
1036
- AND #{j.field_time.to_s} >= CAST('#{BlackStack::Pampa.now - point.send(scale_unit)}' AS TIMESTAMP)
1037
- AND #{j.field_time.to_s} < CAST('#{BlackStack::Pampa.now - (point-1).send(scale_unit)}' AS TIMESTAMP)
1038
- "
1039
- a << { :time => BlackStack::Pampa.now - (point-1).send(scale_unit), :n => DB[q].first[:n].to_i }
1040
- end # while point<scale_points
1041
- # return
1042
- a
1043
- end # def timeline
1044
-
1045
- # reporting method: error_descriptions
1046
- # return an array of hashes { :id, :error_description } with the tasks that have an the success flag in false, error description.
1047
- # if the numbr if running tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show` errors.
1048
- def error_descriptions(max_tasks_to_show=25)
1049
- j = self
1050
- q = "
1051
- SELECT #{j.field_primary_key.to_s} as id, #{j.field_error_description.to_s} as description
1052
- FROM #{j.table.to_s}
1053
- WHERE COALESCE(#{j.field_success.to_s},true)=false
1054
- LIMIT #{max_tasks_to_show}
1055
- "
1056
- DB[q].all
1057
- end
715
+ end # def failed
1058
716
  end # class Job
1059
717
  end # module Pampa
1060
718
  end # module BlackStack
data/pampa.gemspec ADDED
@@ -0,0 +1,43 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'pampa'
3
+ s.version = '2.0.30'
4
+ s.date = '2023-10-06'
5
+ s.summary = "Ruby library for async & distributed computing, supporting dynamic reconfiguration, distribution of the computation jobs, error handling, job-retry and fault tolerance, and fast (non-direct) communication to ensure real-time capabilities."
6
+ s.description = "Pampa is a Ruby library for async & distributing computing providing the following features:
7
+
8
+ - cluster-management with dynamic reconfiguration (joining and leaving nodes);
9
+ - distribution of the computation jobs to the (active) nodes;
10
+ - error handling, job-retry and fault tolerance;
11
+ - fast (non-direct) communication to ensure realtime capabilities.
12
+
13
+ The Pampa framework may be widely used for:
14
+
15
+ - large scale web scraping with what we call a \"bot-farm\";
16
+ - payments processing for large-scale ecommerce websites;
17
+ - reports generation for high demanded SaaS platforms;
18
+ - heavy mathematical model computing;
19
+
20
+ and any other tasks that requires a virtually infinite amount of CPU computing and memory resources.
21
+
22
+ Find documentation here: https://github.com/leandrosardi/pampa
23
+ "
24
+ s.authors = ["Leandro Daniel Sardi"]
25
+ s.email = 'leandro.sardi@expandedventure.com'
26
+ s.files = [
27
+ 'lib/pampa/worker.rb',
28
+ 'lib/pampa/dispatcher.rb',
29
+ 'lib/pampa/app.rb',
30
+ 'lib/pampa.rb',
31
+ 'pampa.gemspec'
32
+ ]
33
+ s.homepage = 'https://github.com/leandrosardi/pampa'
34
+ s.license = 'MIT'
35
+ s.add_runtime_dependency 'rubygems-bundler', '~> 1.4.5', '>= 1.4.5'
36
+ s.add_runtime_dependency 'sinatra', '~> 2.2.4', '>= 2.2.4'
37
+ s.add_runtime_dependency 'colorize', '~> 0.8.1', '>= 0.8.1'
38
+ s.add_runtime_dependency 'blackstack-core', '~> 1.2.15', '>= 1.2.15'
39
+ s.add_runtime_dependency 'blackstack-db', '~> 1.0.1', '>= 1.0.1'
40
+ s.add_runtime_dependency 'blackstack-nodes', '~> 1.2.12', '>= 1.2.12'
41
+ s.add_runtime_dependency 'simple_command_line_parser', '~> 1.1.2', '>= 1.1.2'
42
+ s.add_runtime_dependency 'simple_cloud_logging', '~> 1.2.2', '>= 1.2.2'
43
+ end