xpflow 0.1b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. data/bin/xpflow +96 -0
  2. data/lib/colorado.rb +198 -0
  3. data/lib/json/add/core.rb +243 -0
  4. data/lib/json/add/rails.rb +8 -0
  5. data/lib/json/common.rb +423 -0
  6. data/lib/json/editor.rb +1369 -0
  7. data/lib/json/ext.rb +28 -0
  8. data/lib/json/pure/generator.rb +442 -0
  9. data/lib/json/pure/parser.rb +320 -0
  10. data/lib/json/pure.rb +15 -0
  11. data/lib/json/version.rb +8 -0
  12. data/lib/json.rb +62 -0
  13. data/lib/mime/types.rb +881 -0
  14. data/lib/mime-types.rb +3 -0
  15. data/lib/restclient/abstract_response.rb +106 -0
  16. data/lib/restclient/exceptions.rb +193 -0
  17. data/lib/restclient/net_http_ext.rb +55 -0
  18. data/lib/restclient/payload.rb +235 -0
  19. data/lib/restclient/raw_response.rb +34 -0
  20. data/lib/restclient/request.rb +316 -0
  21. data/lib/restclient/resource.rb +169 -0
  22. data/lib/restclient/response.rb +24 -0
  23. data/lib/restclient.rb +174 -0
  24. data/lib/xpflow/bash.rb +341 -0
  25. data/lib/xpflow/bundle.rb +113 -0
  26. data/lib/xpflow/cmdline.rb +249 -0
  27. data/lib/xpflow/collection.rb +122 -0
  28. data/lib/xpflow/concurrency.rb +79 -0
  29. data/lib/xpflow/data.rb +393 -0
  30. data/lib/xpflow/dsl.rb +816 -0
  31. data/lib/xpflow/engine.rb +574 -0
  32. data/lib/xpflow/ensemble.rb +135 -0
  33. data/lib/xpflow/events.rb +56 -0
  34. data/lib/xpflow/experiment.rb +65 -0
  35. data/lib/xpflow/exts/facter.rb +30 -0
  36. data/lib/xpflow/exts/g5k.rb +931 -0
  37. data/lib/xpflow/exts/g5k_use.rb +50 -0
  38. data/lib/xpflow/exts/gui.rb +140 -0
  39. data/lib/xpflow/exts/model.rb +155 -0
  40. data/lib/xpflow/graph.rb +1603 -0
  41. data/lib/xpflow/graph_xpflow.rb +251 -0
  42. data/lib/xpflow/import.rb +196 -0
  43. data/lib/xpflow/library.rb +349 -0
  44. data/lib/xpflow/logging.rb +153 -0
  45. data/lib/xpflow/manager.rb +147 -0
  46. data/lib/xpflow/nodes.rb +1250 -0
  47. data/lib/xpflow/runs.rb +773 -0
  48. data/lib/xpflow/runtime.rb +125 -0
  49. data/lib/xpflow/scope.rb +168 -0
  50. data/lib/xpflow/ssh.rb +186 -0
  51. data/lib/xpflow/stat.rb +50 -0
  52. data/lib/xpflow/stdlib.rb +381 -0
  53. data/lib/xpflow/structs.rb +369 -0
  54. data/lib/xpflow/taktuk.rb +193 -0
  55. data/lib/xpflow/templates/ssh-config.basic +14 -0
  56. data/lib/xpflow/templates/ssh-config.inria +18 -0
  57. data/lib/xpflow/templates/ssh-config.proxy +13 -0
  58. data/lib/xpflow/templates/taktuk +6590 -0
  59. data/lib/xpflow/templates/utils/batch +4 -0
  60. data/lib/xpflow/templates/utils/bootstrap +12 -0
  61. data/lib/xpflow/templates/utils/hostname +3 -0
  62. data/lib/xpflow/templates/utils/ping +3 -0
  63. data/lib/xpflow/templates/utils/rsync +12 -0
  64. data/lib/xpflow/templates/utils/scp +17 -0
  65. data/lib/xpflow/templates/utils/scp_many +8 -0
  66. data/lib/xpflow/templates/utils/ssh +3 -0
  67. data/lib/xpflow/templates/utils/ssh-interactive +4 -0
  68. data/lib/xpflow/templates/utils/taktuk +19 -0
  69. data/lib/xpflow/threads.rb +187 -0
  70. data/lib/xpflow/utils.rb +569 -0
  71. data/lib/xpflow/visual.rb +230 -0
  72. data/lib/xpflow/with_g5k.rb +7 -0
  73. data/lib/xpflow.rb +349 -0
  74. metadata +135 -0
@@ -0,0 +1,569 @@
1
+
2
+ # encoding: UTF-8
3
+
4
+ #
5
+ # Implements useful routines and some nasty/ugly/ingenious/beautiful tricks.
6
+ #
7
+
8
+ require 'thread'
9
+
10
+ class Class
11
+
12
+ def __build_constructor__(*fields)
13
+ attrs = []
14
+ supers = []
15
+ inits = Hash.new
16
+
17
+ if fields.first.is_a?(Array)
18
+ supers = fields.first
19
+ attrs = fields[1..-1]
20
+ else
21
+ attrs = fields
22
+ end
23
+
24
+ if attrs.last.is_a?(Hash)
25
+ inits = attrs.last
26
+ attrs.pop
27
+ end
28
+
29
+ s = "def initialize("
30
+ s += (supers + attrs).map { |x| x.to_s }.join(", ")
31
+ s += ")\n"
32
+ s += "super("
33
+ s += supers.map { |x| x.to_s }.join(", ")
34
+ s += ")\n"
35
+
36
+ attrs.each do |x|
37
+ s += "@#{x} = #{x}\n"
38
+ end
39
+
40
+ inits.each_pair do |k, v|
41
+ s += "@#{k} = #{v.inspect}\n"
42
+ end
43
+
44
+ s += "self.init if self.respond_to?(:init)\n"
45
+
46
+ s += "end\n"
47
+
48
+ return s
49
+ end
50
+
51
+ # Generates constructor on-the-fly from a specification.
52
+ # See 'test_tricks' in tests or classes derived from AbstractRun.
53
+
54
+ def constructor(*fields)
55
+ s1 = __build_constructor__(*fields)
56
+ class_eval(s1)
57
+ end
58
+
59
+ # Declares a list of objects that the object consists of.
60
+ # Used to recursively traverse the structure of the workflow.
61
+ # See classes derived from AbstractRun.
62
+
63
+ def children(*fields)
64
+ list = fields.map { |x| "@#{x}" }.join(', ')
65
+ s = "def __children__\n"
66
+ s += " return XPFlow::resolve_children([ #{list} ])\n"
67
+ s += "end\n"
68
+ h = fields.map { |x| ":#{x} => @#{x}" }.join(", ")
69
+ s += "def __children_hash__\n"
70
+ s += " return { #{h} }\n"
71
+ s += "end\n"
72
+ class_eval(s)
73
+ end
74
+
75
+ # Declares additional variables declared in a run.
76
+ def declares(*fields)
77
+ hash = fields.map { |f| "@#{f} => self" }.join(", ")
78
+ s = "def __declarations__\n"
79
+ s += " return { #{hash} }\n"
80
+ s += "end\n"
81
+ class_eval(s)
82
+ end
83
+
84
+ def activities(*methods)
85
+ maps = {}
86
+ methods.each do |m|
87
+ if m.is_a?(Hash)
88
+ m.each_pair { |k, v| maps[k] = v }
89
+ elsif m.is_a?(Symbol)
90
+ maps[m] = m
91
+ else
92
+ raise
93
+ end
94
+ end
95
+ s = "def __activities__\n"
96
+ s += " return #{maps.inspect.gsub('=>',' => ')}\n"
97
+ s += "end\n"
98
+ class_eval(s)
99
+ end
100
+
101
+ end
102
+
103
+ class Object
104
+
105
+ attr_writer :__repr__
106
+
107
+ def __repr__
108
+ return instance_exec(&@__repr__) if @__repr__.is_a?(Proc)
109
+ return @__repr__ unless @__repr__.nil?
110
+ return to_s
111
+ end
112
+
113
+ def instance_variables_compat
114
+ # Ruby 1.8 returns strings, but version 1.9 returns symbols
115
+ return instance_variables.map { |x| x.to_sym }.sort
116
+ end
117
+
118
+ def inject_method(name, &block)
119
+ if $ruby19
120
+ self.define_singleton_method(name, &block)
121
+ else
122
+ (class << self; self; end).send(:define_method, name, &block)
123
+ end
124
+ end
125
+
126
+ end
127
+
128
+ class String
129
+
130
+ def extract(exp)
131
+ return exp.match(self).captures.first
132
+ end
133
+
134
+ end
135
+
136
+ class Array
137
+
138
+ def tail
139
+ self[1..-1]
140
+ end
141
+
142
+ def same(x)
143
+ return ((x - self == []) && (self - x == []))
144
+ end
145
+
146
+ def split_into(n)
147
+ # splits into n arrays of the same size
148
+ this = self
149
+ raise "Impossible to split #{this.length} into #{n} groups" \
150
+ if this.length % n != 0
151
+ chunk = this.length / n
152
+ return n.times.map { |i| this.slice(i*chunk, chunk) }
153
+ end
154
+
155
+ end
156
+
157
+ class Hash
158
+
159
+ alias :old_select :select
160
+
161
+ def select(*args, &block)
162
+ return Hash[old_select(*args, &block)]
163
+ end if $ruby18
164
+ end
165
+
166
+ class IO
167
+
168
+ def self.write(name, content)
169
+ File.open(name, 'wb') do |f|
170
+ f.write(content)
171
+ end
172
+ end
173
+
174
+ end
175
+
176
+
177
+ module XPFlow
178
+
179
+ # parse comments next to the invocation of the
180
+ # function higher in the stack
181
+
182
+ def self.realpath(filename)
183
+ return Pathname.new(filename).realpath.to_s
184
+ end
185
+
186
+ def self.stack_array
187
+ stack = Kernel.caller()
188
+ stack = stack.map do |frame|
189
+ m = frame.match(/^(.+):(\d+)$/)
190
+ m = frame.match(/^(.+):(\d+):in .+$/) if m.nil?
191
+ raise "Could not parse stack '#{frame}'" if m.nil?
192
+ filename, lineno = m.captures
193
+
194
+ filename = realpath(filename)
195
+ [ filename, lineno.to_i ]
196
+ end
197
+ return stack
198
+ end
199
+
200
+ def self.parse_comment_opts(source)
201
+ source = realpath(source)
202
+ stack = stack_array()
203
+ while stack.first.first != source
204
+ # remove all possible non-dsl files on the stack
205
+ stack.shift
206
+ end
207
+ while stack.first.first == source
208
+ # get all possible dsl files on the stack
209
+ stack.shift
210
+ end
211
+ # now the first frame *SHOULD* be the one that entered DSL
212
+ filename, line = stack.first
213
+ line = IO.read(filename).lines.to_a[line - 1]
214
+ if !line.include?('#!')
215
+ return { }
216
+ else
217
+ comment = line.split('#!').last.strip
218
+ opts = { }
219
+ comment.split(",").each do |pair|
220
+ pair = pair.strip
221
+ if !pair.include?('=')
222
+ opts[pair.to_sym] = true
223
+ else
224
+ k, v = pair.split('=', 2).map(&:strip)
225
+ opts[k.to_sym] = eval(v)
226
+ end
227
+ end
228
+ return opts
229
+ end
230
+ end
231
+
232
+ # Removes nodes that are not important from
233
+ # graphing point of view
234
+
235
+ def self.block_info(block)
236
+ # gives a string repr. of arguments to this block
237
+ s = []
238
+ arity = block.arity
239
+ if !block.respond_to?(:parameters) # Ruby 1.8
240
+ is_neg = (arity < 0)
241
+ arity = (-arity - 1) if is_neg
242
+ args = arity.times.map { |i| "arg#{i+1}" }
243
+ args.push('[args...]')
244
+ return args.join(', ')
245
+ end
246
+ for t, name in to_lambda(block).parameters
247
+ name = "[#{name}]" if t == :opt
248
+ name = "[#{name}...]" if t == :rest
249
+ s.push(name)
250
+ end
251
+ return s.join(', ')
252
+ end
253
+
254
+ def self.to_lambda(block)
255
+ # converts a block to lambda (see http://stackoverflow.com/questions/2946603)
256
+ obj = Object.new
257
+ obj.define_singleton_method(:_, &block)
258
+ return obj.method(:_).to_proc
259
+ end
260
+
261
+
262
+ # Used by 'children' above.
263
+ # Interprets dependant objects of the object
264
+ # and flattens them to one large list.
265
+
266
+ def self.resolve_children(obj)
267
+ raise unless obj.is_a?(Array)
268
+ res = []
269
+ for x in obj
270
+ if x.is_a?(Array)
271
+ res += resolve_children(x)
272
+ elsif x.is_a?(Hash)
273
+ res += resolve_children(x.values)
274
+ else
275
+ res.push(x)
276
+ end
277
+ end
278
+ return res
279
+ end
280
+
281
+ # Exception thrown if the execution of the workflow failed.
282
+ # Possibly encapsulates many inner exceptions.
283
+
284
+ class RunError < StandardError
285
+
286
+ attr_reader :run
287
+ alias :old_to_s :to_s
288
+
289
+ def initialize(run, children, msg = nil)
290
+ super(msg)
291
+ children = [ children ] unless children.is_a?(Array)
292
+ @run = run
293
+ @children = children
294
+ end
295
+
296
+ def self.trace(x)
297
+ if x.is_a?(RunError)
298
+ return x.stacktrace
299
+ else
300
+ frame = x.backtrace.first
301
+ file, line = /^(.+):(\d+)/.match(frame).captures
302
+ return [ [x.to_s, [ Frame.new(file, line.to_i) ]] ]
303
+ end
304
+ end
305
+
306
+ def stacktrace
307
+ elements = @children.map { |c| RunError.trace(c) }.reduce([], :+)
308
+ elements.each do |reason, stack|
309
+ stack.push(@run.meta)
310
+ end
311
+ return elements
312
+ end
313
+
314
+ # Gives one line summary of the error.
315
+
316
+ def summary
317
+ s = stacktrace()
318
+ if s.length == 1
319
+ reason, stack = s.first
320
+ frame = stack.first
321
+ return "'#{reason}' (#{frame.location})"
322
+ else
323
+ return "#{s.length} errors"
324
+ end
325
+ end
326
+
327
+ def to_s
328
+ summary
329
+ end
330
+
331
+ end
332
+
333
+ # A special version of RunError exception
334
+ # that simply throws an error message.
335
+
336
+ class RunMsgError < RunError
337
+
338
+ def initialize(run, msg)
339
+ super(run, nil, msg)
340
+ end
341
+
342
+ def stacktrace
343
+ return [ [self.to_s, [ @run.meta ] ] ]
344
+ end
345
+
346
+ def to_s
347
+ return old_to_s
348
+ end
349
+
350
+ end
351
+
352
+ # Measures execution time (use 'Timer.measure')
353
+ # and returns useful information.
354
+ # For example:
355
+ # t = Timer.measure do
356
+ # sleep 1
357
+ # end
358
+ # puts t.with_ms
359
+
360
+ class Timer
361
+
362
+ def self.measure
363
+ start = Time.now
364
+ x = yield
365
+ done = Time.now
366
+ return Timer.new(done - start, x)
367
+ end
368
+
369
+ def initialize(t, v)
370
+ @t = t
371
+ @v = v
372
+ end
373
+
374
+ def value
375
+ return @v
376
+ end
377
+
378
+ def to_s(digits = nil)
379
+ return @t.to_s if digits.nil?
380
+ return "%.#{digits}f" % @t
381
+ end
382
+
383
+ def with_ms
384
+ return to_s(3)
385
+ end
386
+
387
+ def secs
388
+ return @t
389
+ end
390
+ end
391
+
392
+ class AbstractDumper
393
+
394
+ def digest(meta)
395
+ # convert a meta-hash to deterministic string
396
+ fingerprint = meta.each.map.to_a.sort
397
+ return Digest::SHA256.hexdigest(fingerprint.inspect)
398
+ end
399
+
400
+ def dump(obj, opts = {})
401
+ obj = obj.clone
402
+ validity = Timespan.to_secs(opts.fetch(:valid, Infinity))
403
+ obj['valid'] = Time.now.to_f + validity
404
+ obj['time_string'] = Time.now.to_s
405
+ obj['time_float'] = Time.now.to_f
406
+ set(digest(obj['meta']), obj.to_yaml)
407
+ end
408
+
409
+ def load(meta)
410
+ s = get(digest(meta))
411
+ return nil if s.nil?
412
+ obj = YAML::load(s)
413
+ return nil if obj['meta'] != meta # collision
414
+ return nil if Time.now.to_f > obj['valid'] # expired
415
+ return obj
416
+ end
417
+
418
+ def set(key, value)
419
+ raise 'Not implemented'
420
+ end
421
+
422
+ def get(key)
423
+ raise 'Not implemented'
424
+ end
425
+
426
+ end
427
+
428
+ class FileDumper < AbstractDumper
429
+
430
+ @@prefix = ".xpflow-cp-"
431
+
432
+ def filename(key)
433
+ return "#{@@prefix}#{key}"
434
+ end
435
+
436
+ def set(key, value)
437
+ File.open(filename(key), 'w') do |f|
438
+ f.write(value)
439
+ end
440
+ end
441
+
442
+ def get(key)
443
+ name = filename(key)
444
+ return nil unless File.exists?(name)
445
+ File.open(name) do |f|
446
+ f.read
447
+ end
448
+ end
449
+
450
+ def list
451
+ # lists checkpoints
452
+ # TODO: relegate to AbstractDumper somehow
453
+ files = Dir.glob("./#{@@prefix}*")
454
+ h = Hash.new { |h, k| h[k] = [] }
455
+ files.each do |f|
456
+ contents = IO.read(f)
457
+ cp = YAML.load(contents)
458
+ name = cp["meta"][:name]
459
+ h[name].push(cp)
460
+ end
461
+
462
+ h2 = { }
463
+
464
+ h.each_pair do |name, cps|
465
+ begin
466
+ h2[name] = cps.sort { |x, y| x['time_float'] <=> y['time_float'] }
467
+ rescue
468
+ # if cp style changed
469
+ end
470
+ end
471
+
472
+ return h2
473
+ end
474
+
475
+ end
476
+
477
+ class MemoryDumper < AbstractDumper
478
+
479
+ def initialize
480
+ super
481
+ @lock = Mutex.new
482
+ @store = {}
483
+ end
484
+
485
+ def get(key)
486
+ @lock.synchronize do
487
+ @store[key]
488
+ end
489
+ end
490
+
491
+ def set(key, value)
492
+ @lock.synchronize do
493
+ @store[key] = value
494
+ end
495
+ end
496
+
497
+ end
498
+
499
+
500
+ class Cache
501
+
502
+ # TODO: avoid cache stampede
503
+
504
+ def initialize
505
+ @lock = Mutex.new
506
+ @store = {}
507
+ end
508
+
509
+ def get(label)
510
+ @lock.synchronize do
511
+ @store[label]
512
+ end
513
+ end
514
+
515
+ def set(label, value)
516
+ @lock.synchronize do
517
+ @store[label] = value
518
+ end
519
+ end
520
+
521
+ def fetch(label)
522
+ x = get(label)
523
+ return x unless x.nil?
524
+ v = yield
525
+ set(label, v)
526
+ return v
527
+ end
528
+
529
+ end
530
+
531
+ class Timespan
532
+ # parses various timespan formats
533
+
534
+ def self.to_secs(s)
535
+ return s.to_f if s.is_a?(Numeric)
536
+ return Infinity if [ 'always', 'forever', 'infinitely' ].include?(s.to_s)
537
+ parts = s.to_s.split(':').map { |x| Integer(x) rescue nil }
538
+ if parts.all? && [ 2, 3 ].include?(parts.length)
539
+ secs = parts.zip([ 3600, 60, 1 ]).map { |x, y| x * y }.reduce(:+)
540
+ return secs
541
+ end
542
+ m = /^(\d+|\d+\.\d*)\s*(\w*)?$/.match(s)
543
+ num, unit = m.captures
544
+ mul = case unit
545
+ when '' then 1
546
+ when 's' then 1
547
+ when 'm' then 60
548
+ when 'h' then 60 * 60
549
+ when 'd' then 24 * 60 * 60
550
+ else nil
551
+ end
552
+ raise "Unknown timespan unit: '#{unit}' in #{s}" if mul.nil?
553
+ return num.to_f * mul
554
+ end
555
+
556
+ def self.to_time(s)
557
+ secs = to_secs(s).to_i
558
+ minutes = secs / 60; secs %= 60
559
+ hours = minutes / 60; minutes %= 60
560
+ minutes += 1 if secs > 0
561
+ return '%.02d:%.02d' % [ hours, minutes ]
562
+ end
563
+
564
+ end
565
+
566
+ Infinity = 1.0/0.0
567
+
568
+ end
569
+