xpflow 0.1b

data/lib/xpflow/visual.rb

@@ -0,0 +1,230 @@
+# encoding: UTF-8
+
+#
+# Visualization of engine execution/data.
+#
+
+def dict(h)
+    return XPFlow::Visual::Dict.new(h)
+end
+
+module XPFlow
+
+    module Visual
+
+        class Dict
+
+            def initialize(h)
+                @h = h
+            end
+
+            def to_h
+                return @h
+            end
+
+            def method_missing(method)
+                return @h[method]
+            end
+
+            def [](key)
+                return @h[key]
+            end
+
+            def to_s
+                return "d(#{@h.to_s})"
+            end
+
+        end
+
+        def self.pair_activities(events)
+            # returns a list with pairs named
+            # after the activities, and values
+            # being start, finish
+
+            starts = events.select { |x| x.type == :start_activity }
+            starts = starts.sort { |x, y| x.meta[:time] <=> y.meta[:time] }
+            ends = events.select { |x| x.type == :finish_activity }
+            contents = ends.map { |x| [ x.meta[:name], x.meta[:time] ] }
+            ends = Hash[ contents ]
+
+            return starts.map do |ev|
+                name = ev.meta[:name]
+                start = ev.meta[:time]
+                finish = ends[name]
+                dict({ :name => name, :start => start, :finish => finish, :meta => ev.meta })
+            end
+        end
+
+        def self.get_times(activities)
+            t_start = activities.map(&:start).min
+            t_finish = activities.map(&:finish).compact.max
+            return [ t_start, t_finish, t_finish - t_start ]
+        end
+
+        def self.terminal_size
+            begin
+                return `tput cols`.to_i - 14
+            rescue Errno::ENOENT
+                return nil
+            end
+        end
+
+        def self.show_gantt(events)
+            
+            if events.empty?
+                puts
+                puts "=== GANTT DIAGRAM IS EMPTY ===".red
+                puts
+                return
+            end
+
+            times = compute_times(events)
+            activities = times[:activities]
+            t_start, t_finish, t_span = times[:time_start], times[:time_finish], times[:time_span]
+            left_size = activities.map { |x| x[:name].length }.max # length of names
+
+            width = terminal_size()
+            width = width - left_size unless width.nil?
+            width = 80 if (width.nil? or width < 80)
+
+            puts
+            puts "=== ACTIVITY GANTT DIAGRAM ===".red
+            puts
+            activities.each do |a|
+                a = dict(a)
+                chars = (width * a.graph_span).to_i
+                margin = (width * a.graph_start).to_i
+                bar = ('=' * chars).yellow
+                indent = (' ' * margin)
+                status = (a.finished ? '%.3f s' % a.secs_span : 'running')
+                puts "%s  %s%s (%s)" % \
+                    [ a.name.ljust(left_size).white!, indent, bar, status ]
+            end
+        end
+
+        def self.compute_times(events)
+            activities = pair_activities(events)
+            activities = activities.select { |x| !x.name.start_with?("__") }
+            t_start, t_finish, t_span = get_times(activities)
+            activities = activities.map do |x|
+                finished = (!x.finish.nil?)
+                v = {
+                    :name => x.name,
+                    :finished => finished,
+                    :time_start => x.start,
+                    :time_finish => x.finish,
+                    :time_span => finished ? x.finish - x.start : nil,
+                    :secs_start => x.start - t_start,
+                    :secs_finish =>  finished ? x.finish - t_start : t_span,
+                    :secs_span => finished ? x.finish - x.start : t_finish - x.start,
+                }
+                v[:graph_start] = v[:secs_start] / t_span
+                v[:graph_finish] = v[:secs_finish] / t_span
+                v[:graph_span] = v[:secs_span] / t_span
+                v
+            end
+            return {
+                :time_start => t_start,
+                :time_finish => t_finish,
+                :time_span => t_span,
+                :activities => activities
+            }
+        end
+
+        def self.save_gantt(filename, events)
+            times = compute_times(events)
+            IO.write(filename, times.to_yaml)
+        end
+
+        def self.show_activities()
+            events = $engine.runtime.events()
+            activities = pair_activities(events)
+            activities = activities.select { |x| x.finish.nil? }
+            $console.synchronize do
+                bar = ("=" * 80).red
+                tab = ' ' * 4
+                puts; puts bar
+                puts "CURRENT ACTIVITIES".blue
+                activities.each do |x|
+                    puts "#{tab}#{x.meta[:name].green} #{x.meta.inspect}"
+                end
+                puts bar
+            end
+        end
+    end
+
+    def self.show_stacktrace(e)
+        trace = e.stacktrace
+
+        puts
+        puts "=== ERRORS ===".red
+
+        tab = ' ' * 4
+        for error in trace
+            msg, frames = error
+            puts tab + "#{msg}".yellow
+            for f in frames
+                line = (tab * 2) + "#{f.location_long}"
+                line += " (at #{f.obj.class})" if f.obj
+                puts line
+            end
+        end
+    end
+
+    class TerminalThread
+        # a thread that listens on the terminal
+        # to collect events
+        # assumes stdin is tty
+
+        def initialize
+            @config = %x(stty --save).strip
+            @r, @w = IO.pipe
+            @t = nil
+        end
+
+        def cols
+            size = %x(stty size).split
+            return size.last.to_i
+        end
+
+        def start
+            system("stty -icanon -echo")
+            @t = Thread.new do
+                loop do
+                    ready = IO.select([ STDIN, @r ], [], []).first
+                    if ready.include?(STDIN)
+                        char = STDIN.readpartial(1)
+                        c = cols()
+                        case char
+                            when 'i' then Visual.show_activities()
+                            else begin
+                                $console.synchronize do
+                                    puts "Key #{char.inspect} pressed.".rjust(c).blue
+                                end
+                        end
+                            
+                        end
+                    end
+                    break if ready.include?(@r)
+                end
+            end
+        end
+
+        def stop
+            system("stty #{@config}")
+            @w.close
+            @t.join(1.0)
+        end
+
+        def self.start_thread
+            if STDIN.tty? and not ENV.key?("BATCH")
+                $__terminal_thread__ = TerminalThread.new
+                $__terminal_thread__.start
+                at_exit { $__terminal_thread__.stop }
+            end
+        end
+
+    end
+
+end
+

data/lib/xpflow/with_g5k.rb

@@ -0,0 +1,7 @@
+
+# this is a shortcut to ease the pain
+
+require 'xpflow'
+require 'xpflow/exts/g5k'
+
+$engine.import_library(:g5k, XPFlow::G5K::Library.new)

data/lib/xpflow.rb

@@ -0,0 +1,349 @@
+
+require 'thread'
+require 'digest'
+require 'etc'
+require 'open3'
+require 'yaml'
+require 'optparse'
+require 'monitor'
+require 'colorado'
+require 'pathname'
+require 'highline'
+
+def __get_my_requires__
+    me = File.dirname(__FILE__)
+    files = Dir.entries(File.join(me, 'xpflow'))
+    files = files.select { |f| File.extname(f) == '.rb' }
+    files = files.select { |f| !f.start_with?('with') }
+    modules = files.map { |f| File.basename(f, File.extname(f)) }
+    return modules
+end
+
+# some Ruby version trickery
+
+_, $version = RUBY_VERSION.split('.')
+$version = $version.to_i
+$ruby18 = ($version == 8)
+$ruby19 = ($version == 9)
+
+class Symbol
+
+    def <=>(x)
+        return (self.to_s <=> x.to_s)
+    end
+
+end if $ruby18
+
+if $bundled.nil?
+    # necessary includes
+    require('xpflow/utils')
+    require('xpflow/structs')
+    require('xpflow/library')
+    require('xpflow/scope')
+
+    for m in __get_my_requires__ do
+        require('xpflow/' + m)
+    end
+end
+
+Engine = $engine = XPFlow::Engine.new
+
+def engine(&block)
+    $engine.instance_exec(&block)
+end
+
+def process(*args, &block)
+    engine { process(*args, &block) }
+end
+
+def activity(*args, &block)
+    engine { activity(*args, &block) }
+end
+
+def macro(*args, &block)
+    engine { macro(*args, &block) }
+end
+
+def dsl(name, *args, &block)
+    return XPFlow::ProcessDSL.new(name, $engine, &block)
+end
+
+def empty_activity(*args)
+    args.each do |name|
+        activity(name) do
+        end
+    end
+end
+
+def script(name, path)
+    # TODO: put more magic
+    process(name) do |nodes|
+        execute_many(nodes, path)
+    end
+    single = "#{name}_seq".to_sym
+    process(single) do |node|
+        execute(node, path)
+    end
+end
+
+def activity_visibility(name)
+    return true
+end
+
+def __parse_variables__()
+    opts = XPFlow::Options.new(ARGV.dup)
+    $variables = opts.vars
+end
+
+def parse_g5k_query(query, opts = { :max => 1 })
+    lib = XPFlow::G5K::Library.new
+    lib.logging = proc { |x| puts x }
+    sites = lib.sites.map { |x| x['uid'] }
+    best_sites = [ "nancy", "rennes", "sophia" ]
+    sites = best_sites + (sites - best_sites)
+
+    max = opts[:max]
+
+    uniq_jobs = proc { |js| Hash[js.map { |j| [ [j['uid'], j['site']], j ] }].values }
+    jobs_at_site = proc do |site|
+        js = lib.jobs(site)
+        js = js.select { |x| x['state'] == 'running' }
+        js.select { |x| x['user_uid'] == lib.g5k.user }
+    end
+    site_from_job = proc { |j| j['links'].select { |l| l['rel'] == 'parent' }.first['href'].split("/").last }
+
+    jobs = []
+
+    query.split(":").each do |word|
+        if word == '*'
+            sites.each do |s|
+                jobs = uniq_jobs.call(jobs + jobs_at_site.call(s))
+                break if jobs.length >= max
+            end
+            break if jobs.length >= max
+        elsif (m = word.match(/^([a-z]+)\/(.+)/)) and (sites.include?(m.captures.first))
+            query_site, query_str = m.captures
+            site_jobs = jobs_at_site.call(query_site)
+            if query_str == '*'
+                jobs = uniq_jobs.call(jobs + site_jobs)
+                break if jobs.length >= max
+            elsif query_str.match(/^[0-9]+$/)
+                job_uid = query_str.to_i
+                site_job = site_jobs.select { |x| x['uid'] == job_uid }
+                if site_job.length == 0
+                    raise "No job with id = #{job_uid} in '#{query_site}'."
+                end
+                jobs = uniq_jobs.call(jobs + [ site_job.first ])
+                break if jobs.length >= max
+            else
+                raise "Cannot parse '#{query_str}'"
+            end
+        else
+            raise "Cannot parse '#{word}'"
+        end
+    end
+
+    grouped_jobs = Hash.new { |h, k| h[k] = [] }
+    jobs.each { |j| grouped_jobs[site_from_job.call(j)].push(j) }
+    jobs_info = grouped_jobs.each_pair.map { |site, jobs| "{" + jobs.map { |x| x['uid'] }.join(",") + "}@#{site}" }.join(" ")
+
+    puts "Found #{jobs.length} jobs: #{jobs_info} (but :max is #{max})"
+
+    jobs = jobs[0...max]
+    return jobs
+end
+
+def parse_g5k_query_one_wait(query)
+    lib = XPFlow::G5K::Library.new
+    lib.logging = proc { |x| puts x }
+    jobs = parse_g5k_query(query, :max => 1)
+    if jobs.length == 0
+        raise "Could not find any jobs."
+    end
+    job = jobs.first
+    j = lib.g5k.get_json_raw(job.rel_self)
+    j = lib.wait_for_job(j)
+    return j
+end
+
+def _bool_var(v)
+    return v if v.is_a?(TrueClass) or v.is_a?(FalseClass)
+    v = v.strip
+    if (v.to_i == 1) or ([ "true", "yes" ].include?(v.downcase))
+        return true
+    end
+    return false
+end
+
+class Seq
+
+    def initialize(arr)
+        @arr = arr
+    end
+
+    def range
+        return Seq.new(@arr.uniq.sort)
+    end
+
+    def self.parse(spec)
+        nums = []
+        parts = spec.strip.split("+").map(&:strip)
+        parts.each do |p|
+            if p.match(/^(\d+)$/)
+                nums += [ p.to_i ]
+            elsif (m = p.match(/^(\d+)(:|-|\.\.)(\d+)$/))
+                s, _, e = m.captures.map(&:to_i)
+                nums += (s..e).to_a
+            elsif (m = p.match(/^(\d+)(:|-|\.\.)(\d+)\/(\d+)$/))
+                s, _, e, div = m.captures.map(&:to_i)
+                raise "Dividing by zero here" if div == 0
+                raise "Division must be > 1" if div == 1
+                if (e - s) % (div - 1) == 0
+                    d = (e - s) / (div - 1)  # use integer arithmetic if possible
+                else
+                    d = (e - s).to_f / (div - 1).to_f
+                end
+                pts = []
+                (1..(div - 2)).each do |i|
+                    p = i * d + s
+                    pts.push(p.to_i)
+                end
+                nums += ([ s ] + pts + [ e ])
+            elsif (m = p.match(/^(\d+):(\d+):(\d+)$/))
+                s, d, e = m.captures.map(&:to_i)
+                if d == 0
+                    raise "Zero range increment (#{p})"
+                end
+                while s <= e
+                    nums.push(s)
+                    s += d
+                end
+            else
+                raise "Wrong range specification"
+            end
+        end
+        return Seq.new(nums)
+    end
+
+    def to_s
+        return "seq(#{@arr.inspect})"
+    end
+
+    def to_list
+        return @arr.dup
+    end
+
+    def max
+        return @arr.max
+    end
+
+end
+
+$__highline__ = HighLine.new
+
+def ask_for_type(name, type, opts)
+    text = "Provide value of :#{name} (type :#{type}): "
+    text = opts[:text] if opts[:text]
+    if [ :seq, :str, :range, :pass ].include?(type)
+        value = $__highline__.ask(text) do |q|
+            q.echo = "*" if type == :pass
+        end
+        return value
+    elsif type == :int
+        value = $__highline__.ask(text, Integer)
+        return value
+    elsif type == :bool
+        value = $__highline__.agree(text) { |q| q.default = "no" }
+        return value
+    else
+        # TODO, support more variables
+        raise "Unknown value for variable :#{name} (type :#{type})"
+    end
+end
+
+def var(name, type = :str, default = nil, opts = {})
+    # TODO
+    if default.is_a?(Hash)
+        opts = default
+        default = opts[:default]
+    end
+    v = $variables[name.to_s]
+    if v.nil?
+        if default.nil?
+            # try to ask for a variable
+            if !STDIN.tty?
+                raise "Unknown value for variable :#{name} (type :#{type})"
+            end
+            v = $variables[name.to_s] = ask_for_type(name, type, opts)
+            if opts[:callback]
+                opts[:callback].call(v)
+            end
+        else
+            return default
+        end
+    end
+
+    result = case type
+        when :str then v
+        when :int then v.to_i
+        when :float then v.to_f
+        when :bool then _bool_var(v)
+        when :range then Seq.parse(v).range
+        when :seq then Seq.parse(v)
+        when :pass then v
+        when :g5k then parse_g5k_query_one_wait(v)
+        else
+            raise "Unknown type '#{type}'"
+    end
+    return result
+end
+
+def set_var(name, value)
+    $variables[name.to_s] = value
+    return value
+end
+
+def experiment(name, &block)
+    body_name = :"#{name}/body"
+    before_name = :"#{name}/before"
+    after_name = :"#{name}/after"
+    $engine.process(body_name, &block)
+    $engine.process(before_name) { }
+    $engine.process(after_name) { }
+    $engine.process(name) do |*args|
+        run :"/new_experiment"  # create a new experiment
+        log "Starting experiment '#{name}'"
+        run(before_name, *args)
+        result = run(body_name, *args)
+        run(after_name, *args)
+        log "Finished experiment '#{name}'"
+        value(result)
+    end
+end
+
+def after_activity(name, &block)
+    # we replace the given activity with
+    # a one that executes the previous one and then
+    # the one given
+
+    after_block = block
+    activity(name) do |*args|
+        value = parent(*args)
+        self.set_result(value)
+        value = self.instance_exec(*args, &after_block)
+        value
+    end
+end
+
+def before_activity(name, &block)
+    # similarly to after_activity
+    before_block = block
+    activity(name) do |*args|
+        self.instance_exec(*args, &before_block)
+        parent(*args)
+    end
+end
+
+def realize(path)
+    return Pathname.new(path).realpath.to_s
+end