xpflow 0.1b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. data/bin/xpflow +96 -0
  2. data/lib/colorado.rb +198 -0
  3. data/lib/json/add/core.rb +243 -0
  4. data/lib/json/add/rails.rb +8 -0
  5. data/lib/json/common.rb +423 -0
  6. data/lib/json/editor.rb +1369 -0
  7. data/lib/json/ext.rb +28 -0
  8. data/lib/json/pure/generator.rb +442 -0
  9. data/lib/json/pure/parser.rb +320 -0
  10. data/lib/json/pure.rb +15 -0
  11. data/lib/json/version.rb +8 -0
  12. data/lib/json.rb +62 -0
  13. data/lib/mime/types.rb +881 -0
  14. data/lib/mime-types.rb +3 -0
  15. data/lib/restclient/abstract_response.rb +106 -0
  16. data/lib/restclient/exceptions.rb +193 -0
  17. data/lib/restclient/net_http_ext.rb +55 -0
  18. data/lib/restclient/payload.rb +235 -0
  19. data/lib/restclient/raw_response.rb +34 -0
  20. data/lib/restclient/request.rb +316 -0
  21. data/lib/restclient/resource.rb +169 -0
  22. data/lib/restclient/response.rb +24 -0
  23. data/lib/restclient.rb +174 -0
  24. data/lib/xpflow/bash.rb +341 -0
  25. data/lib/xpflow/bundle.rb +113 -0
  26. data/lib/xpflow/cmdline.rb +249 -0
  27. data/lib/xpflow/collection.rb +122 -0
  28. data/lib/xpflow/concurrency.rb +79 -0
  29. data/lib/xpflow/data.rb +393 -0
  30. data/lib/xpflow/dsl.rb +816 -0
  31. data/lib/xpflow/engine.rb +574 -0
  32. data/lib/xpflow/ensemble.rb +135 -0
  33. data/lib/xpflow/events.rb +56 -0
  34. data/lib/xpflow/experiment.rb +65 -0
  35. data/lib/xpflow/exts/facter.rb +30 -0
  36. data/lib/xpflow/exts/g5k.rb +931 -0
  37. data/lib/xpflow/exts/g5k_use.rb +50 -0
  38. data/lib/xpflow/exts/gui.rb +140 -0
  39. data/lib/xpflow/exts/model.rb +155 -0
  40. data/lib/xpflow/graph.rb +1603 -0
  41. data/lib/xpflow/graph_xpflow.rb +251 -0
  42. data/lib/xpflow/import.rb +196 -0
  43. data/lib/xpflow/library.rb +349 -0
  44. data/lib/xpflow/logging.rb +153 -0
  45. data/lib/xpflow/manager.rb +147 -0
  46. data/lib/xpflow/nodes.rb +1250 -0
  47. data/lib/xpflow/runs.rb +773 -0
  48. data/lib/xpflow/runtime.rb +125 -0
  49. data/lib/xpflow/scope.rb +168 -0
  50. data/lib/xpflow/ssh.rb +186 -0
  51. data/lib/xpflow/stat.rb +50 -0
  52. data/lib/xpflow/stdlib.rb +381 -0
  53. data/lib/xpflow/structs.rb +369 -0
  54. data/lib/xpflow/taktuk.rb +193 -0
  55. data/lib/xpflow/templates/ssh-config.basic +14 -0
  56. data/lib/xpflow/templates/ssh-config.inria +18 -0
  57. data/lib/xpflow/templates/ssh-config.proxy +13 -0
  58. data/lib/xpflow/templates/taktuk +6590 -0
  59. data/lib/xpflow/templates/utils/batch +4 -0
  60. data/lib/xpflow/templates/utils/bootstrap +12 -0
  61. data/lib/xpflow/templates/utils/hostname +3 -0
  62. data/lib/xpflow/templates/utils/ping +3 -0
  63. data/lib/xpflow/templates/utils/rsync +12 -0
  64. data/lib/xpflow/templates/utils/scp +17 -0
  65. data/lib/xpflow/templates/utils/scp_many +8 -0
  66. data/lib/xpflow/templates/utils/ssh +3 -0
  67. data/lib/xpflow/templates/utils/ssh-interactive +4 -0
  68. data/lib/xpflow/templates/utils/taktuk +19 -0
  69. data/lib/xpflow/threads.rb +187 -0
  70. data/lib/xpflow/utils.rb +569 -0
  71. data/lib/xpflow/visual.rb +230 -0
  72. data/lib/xpflow/with_g5k.rb +7 -0
  73. data/lib/xpflow.rb +349 -0
  74. metadata +135 -0
@@ -0,0 +1,125 @@
1
+
2
+ # some runtime related
3
+ # activities
4
+
5
+ module XPFlow
6
+
7
+ # TODO
8
+ # I have to decide what is a model of an event
9
+
10
+ class RuntimeEvent
11
+
12
+ attr_reader :type
13
+ attr_reader :meta
14
+
15
+ constructor :type, :meta
16
+
17
+ def to_s
18
+ "<Event: type = #{@type.inspect}, meta = #{@meta.inspect}>"
19
+ end
20
+
21
+ end
22
+
23
+ class RuntimeLibrary < MonitoredActivityLibrary
24
+
25
+ activities :activities,
26
+ :run_group, :show_gantt, :save_gantt,
27
+ :start, :finish, :event, :dump_events, :events,
28
+ :list_checkpoints,
29
+ :list => :activities
30
+
31
+ def setup
32
+ @events = []
33
+ end
34
+
35
+ def checkpoint
36
+ return @events
37
+ end
38
+
39
+ def restore(state)
40
+ @events = state
41
+ end
42
+
43
+ # these are routed as well from stdlib.rb
44
+ def event(type, meta = {})
45
+ ev = RuntimeEvent.new(type, {}.merge(meta))
46
+ @events.push(ev)
47
+ end
48
+
49
+ def start(name, meta = {})
50
+ event(:start, meta.merge({ :name => name }))
51
+ end
52
+
53
+ def finish(name, meta = {})
54
+ event(:finish, meta.merge({ :name => name }))
55
+ end
56
+
57
+ def dump_events
58
+ # debug activity
59
+ pp @events
60
+ end
61
+
62
+ def events(type = nil, &block)
63
+ if type.nil?
64
+ events = @events
65
+ else
66
+ events = @events.select { |e| ev.type == type }
67
+ end
68
+ events = events.select(&block) if block_given?
69
+ return events.dup
70
+ end
71
+
72
+ def activities
73
+ puts "List of activities:"
74
+ ns = proxy.engine.get_global_namespace()
75
+ lines = ns.map { |k, v| " * #{v.info}" }.sort
76
+ puts lines.join("\n")
77
+ end
78
+
79
+ def self.save_workflow(filename, engine, name)
80
+ p = engine.get_activity_object(name)
81
+ ns = name.to_s.split(".")[0...-1]
82
+ opts = { :engine => engine }
83
+ flow = Graphing::XPFlowConverter.from_process(p, ns, opts)
84
+
85
+ ext = File.extname(filename).downcase
86
+
87
+ ropts = { :debug => false }
88
+ case ext
89
+ when '.tikz' then Graphing.to_tikz(flow, filename, ropts)
90
+ when '.pdf' then Graphing.to_pdf(flow, filename, ropts)
91
+ when '.png' then Graphing.to_png(flow, filename, ropts)
92
+ when '.tex' then Graphing.to_latex(flow, filename, ropts)
93
+ else
94
+ raise "Unsupported format '#{ext}'."
95
+ end
96
+
97
+ puts "Workflow of '#{name.to_s.green}' process saved to '#{filename.green}'."
98
+ puts "Check out '#{"tikz_template.tex".green}' template in examples directory." if ext == '.tikz'
99
+ end
100
+
101
+ def run_group(infos)
102
+ infos.each do |i|
103
+ proxy.run i.name, *i.args
104
+ end
105
+ end
106
+
107
+ def list_checkpoints
108
+ cps = proxy.engine.dumper.list
109
+ puts "List of checkpoints:"
110
+ cps.each_pair do |k, v|
111
+ puts " #{k} => #{v.first['time_string']}"
112
+ end
113
+ end
114
+
115
+ def show_gantt
116
+ Visual.show_gantt(events())
117
+ end
118
+
119
+ def save_gantt(filename)
120
+ Visual.save_gantt(filename, events())
121
+ end
122
+
123
+ end
124
+
125
+ end
@@ -0,0 +1,168 @@
1
+
2
+ require 'thread'
3
+
4
+ module XPFlow
5
+
6
+ $scope_mutex = Mutex.new
7
+ $scope_cv = ConditionVariable.new
8
+
9
+ class Scope
10
+
11
+ constructor :parent, :vars => { }
12
+
13
+ attr_reader :parent
14
+ attr_reader :vars
15
+
16
+ def push
17
+ return Scope.new(self)
18
+ end
19
+
20
+ def vars
21
+ return @vars
22
+ end
23
+
24
+ def [](key)
25
+ return get(key)
26
+ end
27
+
28
+ def get(key, return_nil = false)
29
+ found = false
30
+ val = nil
31
+ $scope_mutex.synchronize do
32
+ while true
33
+ found, val = try_get(key)
34
+ break if (found or return_nil)
35
+ $scope_cv.wait($scope_mutex)
36
+ end
37
+ end
38
+ return val
39
+ end
40
+
41
+ def get_now(key, default = nil)
42
+ v = get(key, true)
43
+ return (v.nil?) ? default : v
44
+ end
45
+
46
+ def []=(key, value)
47
+ $scope_mutex.synchronize do
48
+ @vars[key] = value
49
+ $scope_cv.broadcast
50
+ end
51
+ end
52
+
53
+ def merge!(hash)
54
+ $scope_mutex.synchronize do
55
+ @vars.merge!(hash)
56
+ $scope_cv.broadcast
57
+ end
58
+ end
59
+
60
+ # do not use methods below!
61
+
62
+ def try_get(key)
63
+ curr = self
64
+ found = false
65
+ val = nil
66
+ while (not found) and (not curr.nil?)
67
+ found, val = curr.query(key)
68
+ curr = curr.parent
69
+ end
70
+ return [ found, val ]
71
+ end
72
+
73
+ def query(key)
74
+ return [ @vars.key?(key), @vars[key] ]
75
+ end
76
+
77
+ def to_s
78
+ @vars.inspect + @parent.to_s
79
+ end
80
+
81
+ def to_a
82
+ arr = [ @vars ]
83
+ arr += @parent.to_a unless @parent.nil?
84
+ return arr
85
+ end
86
+
87
+ def to_keys
88
+ return to_a.map(&:keys)
89
+ end
90
+
91
+ def containing(key)
92
+ if @vars.key?(key)
93
+ return self
94
+ else
95
+ return @parent.containing(key)
96
+ end
97
+ end
98
+
99
+ def current_activity(levels = 1)
100
+ this = Scope.current
101
+ while levels > 0
102
+ if this.vars.key?(:__activity__)
103
+ levels -= 1
104
+ end
105
+ break if levels == 0
106
+ this = this.parent
107
+ end
108
+ return this
109
+ end
110
+
111
+ def parent_activity
112
+ return current_activity(2)
113
+ end
114
+
115
+ def engine
116
+ value = get(:__engine__, nil)
117
+ raise "No engine!" if value.nil?
118
+ return value
119
+ end
120
+
121
+ def experiment
122
+ value = get(:__experiment__, nil)
123
+ raise "No experiment!" if value.nil?
124
+ return value
125
+ end
126
+
127
+ # static methods
128
+ # it's okay to use them
129
+
130
+ def self.reset()
131
+ Scope.set(Scope.new(nil))
132
+ end
133
+
134
+ def self.set(scope, hash = {})
135
+ Thread.current[:__scope__] = scope
136
+ scope.merge!(hash)
137
+ end
138
+
139
+ def self.current()
140
+ return Thread.current[:__scope__]
141
+ end
142
+
143
+ def self.engine()
144
+ return current().engine
145
+ end
146
+
147
+ def self.push()
148
+ return current().push
149
+ end
150
+
151
+ def self.region(&block)
152
+ previous = self.current()
153
+ result = nil
154
+ begin
155
+ new_scope = previous.push()
156
+ Scope.set(new_scope)
157
+ result = block.call(new_scope)
158
+ ensure
159
+ Scope.set(previous)
160
+ end
161
+ return result
162
+ end
163
+
164
+ end
165
+
166
+ Scope.reset() # initialize the global scope
167
+
168
+ end
data/lib/xpflow/ssh.rb ADDED
@@ -0,0 +1,186 @@
1
+ # encoding: UTF-8
2
+
3
+ #
4
+ # Extensions to NetSSH library.
5
+ # * implementation of transparent connection tunneling (Net::SSH.tunnel)
6
+ #
7
+
8
+ module Net; module SSH; module Service
9
+
10
+ class LoopThread
11
+
12
+ def initialize
13
+ @lock = Mutex.new
14
+ @active = true
15
+ @t = Thread.new do
16
+ yield Proc.new { active? }
17
+ end
18
+ end
19
+
20
+ def active?
21
+ @lock.synchronize do
22
+ @active
23
+ end
24
+ end
25
+
26
+ def join
27
+ @lock.synchronize do
28
+ @active = false
29
+ end
30
+ @t.join
31
+ end
32
+
33
+ end
34
+
35
+ # additional forwarding methods
36
+ class Forward
37
+
38
+ def direct(socket, host, port)
39
+
40
+ debug { "direct-redirect on #{socket}" }
41
+
42
+ channel = session.open_channel("direct-tcpip", :string, host, :long,
43
+ port, :string, "127.0.0.1", :long, 22334) do |ch|
44
+ ch.info { "Channel established" }
45
+ end
46
+
47
+ prepare_client(socket, channel, :local)
48
+
49
+ channel.on_open_failed do |ch, code, desc|
50
+ channel.error { "error: #{desc} (#{code})" }
51
+ channel[:socket].close
52
+ end
53
+ end
54
+
55
+ def connect(host, port)
56
+ # in Ruby 1.9, one can write: Socket.pair(:UNIX, :STREAM, 0)
57
+ client, server = Socket.pair('AF_UNIX', 'SOCK_STREAM', 0)
58
+ direct(server, host, port)
59
+ return client
60
+ end
61
+
62
+ def ssh(host, user, options = {})
63
+ port = options.fetch(:port, 22)
64
+ fd = connect(host, port)
65
+ class << fd
66
+ def peer_ip
67
+ "<faked IP>"
68
+ end
69
+ end
70
+ t = LoopThread.new do |active|
71
+ session.loop(0.1) do
72
+ active.call
73
+ end
74
+ end
75
+ options[:proxy] = FakeFactory.new(fd)
76
+ begin
77
+ s = Net::SSH.start(host, user, options)
78
+ rescue
79
+ t.join # stop the thread
80
+ raise
81
+ end
82
+ class << s
83
+ attr_accessor :loop_thread
84
+ alias old_close close
85
+
86
+ def close
87
+ old_close # let this socket to close
88
+ loop_thread.join # abandon upper loop
89
+ end
90
+ end
91
+ s.loop_thread = t
92
+
93
+ if block_given?
94
+ begin
95
+ yield s
96
+ ensure
97
+ s.close
98
+ end
99
+ else
100
+ return s
101
+ end
102
+ end
103
+
104
+ end
105
+
106
+ class FakeFactory
107
+
108
+ def initialize(socket)
109
+ @socket = socket
110
+ end
111
+
112
+ def open(*args)
113
+ return @socket
114
+ end
115
+ end
116
+
117
+ end; end; end
118
+
119
+ module Net; module SSH
120
+
121
+ class SSHLogin
122
+
123
+ attr_reader :user
124
+ attr_reader :host
125
+ attr_reader :port
126
+
127
+ def initialize(spec)
128
+ @user, @host, @port = parse(spec)
129
+ end
130
+
131
+ def parse(spec)
132
+ m = /(?:(.+)@)?(.+?)(?::(\d+))?$/.match(spec)
133
+ raise 'wrong login specification' if m.nil?
134
+ user, host, port = m.captures
135
+ port = 22 if port.nil?
136
+ user = Etc.getlogin if user.nil?
137
+ return [user, host, port.to_i]
138
+ end
139
+
140
+ end
141
+
142
+ def self.tunnel(hosts, opts = {})
143
+ hosts = hosts.map { |h| SSHLogin.new(h) }
144
+
145
+ hops = []
146
+ begin
147
+ proxy = hosts.first
148
+ opts[:port] = proxy.port
149
+ hops.push(Net::SSH.start(proxy.host, proxy.user, opts))
150
+ for node in hosts[1..-1] do
151
+ opts[:port] = node.port
152
+ next_hop = hops.last.forward.ssh(node.host, node.user, opts)
153
+ hops.push(next_hop)
154
+ end
155
+ rescue
156
+ while hops.length != 0 do
157
+ hops.pop.close
158
+ end
159
+ raise
160
+ end
161
+
162
+ session = hops.last
163
+ class << session
164
+ alias older_close close
165
+ attr_accessor :masters
166
+ def close
167
+ older_close
168
+ while masters.length != 0 do
169
+ masters.pop.close
170
+ end
171
+ end
172
+ end
173
+ session.masters = hops[0...-1]
174
+
175
+ if block_given?
176
+ begin
177
+ return(yield session)
178
+ ensure
179
+ session.close
180
+ end
181
+ else
182
+ return session
183
+ end
184
+ end
185
+
186
+ end; end
@@ -0,0 +1,50 @@
1
+ #Credit for cdf_inverse : http://home.online.no/~pjacklam/notes/invnorm/
2
+ #
3
+ module NormalDistribution
4
+ # inverse standard normal cumulative distribution function
5
+ def cdf_inverse(p)
6
+ a = [0, -3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00]
7
+ b = [0, -5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, 6.680131188771972e+01, -1.328068155288572e+01]
8
+ c = [0, -7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00]
9
+ d = [0, 7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00]
10
+ #Define break-points.
11
+ p_low = 0.02425
12
+ p_high = 1.0 - p_low
13
+
14
+ x = 0.0
15
+ q = 0.0
16
+ #Rational approximation for lower region.
17
+ if 0.0 < p && p < p_low
18
+ q = Math.sqrt(-2.0*Math.log(p))
19
+ x = (((((c[1]*q+c[2])*q+c[3])*q+c[4])*q+c[5])*q+c[6]) / ((((d[1]*q+d[2])*q+d[3])*q+d[4])*q+1.0)
20
+
21
+ #Rational approximation for central region.
22
+ elsif p_low <= p && p <= p_high
23
+ q = p - 0.5
24
+ r = q*q
25
+ x = (((((a[1]*r+a[2])*r+a[3])*r+a[4])*r+a[5])*r+a[6])*q / (((((b[1]*r+b[2])*r+b[3])*r+b[4])*r+b[5])*r+1.0)
26
+
27
+ #Rational approximation for upper region.
28
+ elsif p_high < p && p < 1.0
29
+ q = Math.sqrt(-2.0*Math.log(1.0-p))
30
+ x = -(((((c[1]*q+c[2])*q+c[3])*q+c[4])*q+c[5])*q+c[6]) / ((((d[1]*q+d[2])*q+d[3])*q+d[4])*q+1.0)
31
+ end
32
+
33
+ #The relative error of the approximation has
34
+ #absolute value less than 1.15 × 10−9. One iteration of
35
+ #Halley’s rational method (third order) gives full machine precision.
36
+ if 0 < p && p < 1
37
+ e = 0.5 * Math.erfc(-x/Math.sqrt(2.0)) - p
38
+ u = e * Math.sqrt(2.0*Math::PI) * Math.exp((x**2.0)/2.0)
39
+ x = x - u/(1.0 + x*u/2.0)
40
+ end
41
+ x
42
+ end
43
+ #vectors is values and prec and confidence is in percentage
44
+ def estimate_samples(rel,vector,precision,confidance)
45
+ avg=vector.average
46
+ real_prec=avg*precision if rel
47
+ critical_value=cdf_inverce((1-confidance)/2)
48
+ ((critical_value*vector.stddev / real_prec) ** 2).to_i + 1
49
+ end
50
+ end