xpflow 0.1b

data/lib/xpflow/exts/g5k_use.rb

@@ -0,0 +1,50 @@
+
+# just reuse existing thing
+
+def __check_g5k_credentials_path__
+    home = Etc.getpwuid.dir
+    return File.join(home, '.g5k.yaml')
+end
+
+def __check_g5k_credentials__
+    path = __check_g5k_credentials_path__()
+    if File.exist?(path)
+        h = YAML.load_file(path)
+        mode = File::Stat.new(path).mode
+        raise "#{path} is accessible by other users" if (mode & 077 != 0)
+    else
+        return nil
+    end
+    if !h.is_a?(Hash) or h["user"].nil? or h["pass"].nil?
+        raise "bad credentials"
+    end
+    [ h["user"], h["pass"] ]
+end
+
+def __save_g5k_credentials__
+    path = __check_g5k_credentials_path__()
+    File.open(path, "w") do |f|
+        f.puts({
+            "user" => $g5k_user,
+            "pass" => $g5k_pass
+        }.to_yaml)
+    end
+    File.chmod(0600, path)
+end
+
+$g5k_creds = __check_g5k_credentials__()
+
+if $g5k_creds.nil?
+    # no credentials
+    $g5k_user = var(:g5k_user, :str, :text => "Provide G5K username: ")
+    $g5k_pass = var(:g5k_pass, :pass, :text => "Provide G5K password: ")
+    $g5k_save = var(:g5k_save, :bool, :callback => proc { |x| __save_g5k_credentials__ if x }, :text => "Do you want to save these credentials?")
+else
+    $g5k_user = set_var(:g5k_user, $g5k_creds.first)
+    $g5k_pass = set_var(:g5k_pass, $g5k_creds.last)
+end
+
+# hide the password
+set_var(:g5k_pass, "*" * $g5k_pass.length)
+
+require 'xpflow/with_g5k'

data/lib/xpflow/exts/gui.rb

@@ -0,0 +1,140 @@
+
+require 'sinatra/base'
+require 'json'
+
+class << Sinatra::Base
+
+    def quit!(server, handler_name)
+        server.respond_to?(:stop!) ? server.stop! : server.stop
+    end
+
+    # copy-pasted-adjusted from sinatra code (1.3.2-2 in Debian)
+    def _run_custom!(options={})
+        set options
+        set :logging, nil
+        handler      = detect_rack_handler
+        handler_name = handler.name.gsub(/.*::/, '')
+        handler.run self, :Host => bind, :Port => port do |server|
+            @__quit__ = proc { || quit!(server, handler_name) }
+            server.threaded = settings.threaded if server.respond_to? :threaded=
+            set :running, true
+            yield server if block_given?
+        end
+    end
+
+    def run_custom!(options={})
+        # it may result in a race
+        # small note here:
+        # THERE MUST BE A SPECIAL PLACE 
+        #     IN HELL FOR PEOPLE DOING THINGS LIKE THIS!!!
+        handler = trap(:INT, 'DEFAULT')  # take previous handler
+        trap(:INT, handler)
+        t = Thread.new(handler) do |h|
+            x = h
+            while x == h do
+                sleep 0.1
+                x = trap(:INT, &h)
+            end
+            # x is sinatra wicked handler
+            # let's overwrite it back
+            trap(:INT, &h)
+        end
+        _run_custom!(options)
+    end
+
+    def quit_custom!
+        return @__quit__.call()
+    end
+
+    public
+
+        # sets xpflow engine
+        def set_engine(engine)
+            $__gui_engine__ = engine
+        end
+
+end
+
+
+class Rest
+
+    attr_reader :engine
+
+    def initialize(engine)
+        @engine = engine
+    end
+
+    def activities
+        ns = @engine.get_global_namespace()
+        return ns.each.map do |name, o|
+            { :name => name, :info => o.info }
+        end.to_a
+    end
+
+    def active
+        h = @engine.spectator.info
+        h2 = []
+        h.each_pair do |k, v|
+            h2.push({ :name => k, :info => v })
+        end
+        return h2
+    end
+
+end
+
+
+class Controller < Sinatra::Base
+
+    set :port, 8080
+    set :public_folder, File.join(File.dirname(__FILE__), 'gui-files')
+
+    def rest
+        return Rest.new($__gui_engine__)
+    end
+
+    def json(obj)
+        content_type 'application/json', :charset => 'utf-8'
+        return obj.to_json
+    end
+
+    get '/__rest__/:method' do |m|
+        x = rest.send(m.to_sym)
+        json x
+    end
+
+    get '/' do
+        erb :index
+    end
+
+end
+
+if $0 == __FILE__
+
+    require 'xpflow'
+
+    process :main do
+        run :whatever
+        run :whatever
+    end
+
+    activity :whatever do |x|
+        sleep 10
+        log "cze"
+    end
+
+    Controller.set_engine($engine)
+
+    trap(:INT) do
+        puts 'Shutting everything down...'
+        Controller.quit_custom!
+    end
+
+    t = Thread.new do
+        Controller.run_custom!
+    end
+
+    $engine.execute_from_argv :main
+
+    t.join
+
+end

data/lib/xpflow/exts/model.rb

@@ -0,0 +1,155 @@
+
+# this is pulled in by the main xpflow command line tool
+
+# a standard model for an experiment
+
+NODES_OPTS = { :doc => "Resource acquisition" }
+DEPLOYMENT_OPTS = { :doc => "Deployment", :idempotent => true }
+COLLECTION_OPTS = { :doc => "Collection of results" }
+ANALYSIS_OPTS = { :doc => "Data analysis" }
+EXPERIMENT_OPTS = { :doc => "Experiment" }
+STANDARD_OPTS = { :doc => "Experiment model" }
+
+$engine.process :__reservation__, NODES_OPTS do |nodes|
+    value(nodes)
+end
+
+$engine.process :__deployment__, DEPLOYMENT_OPTS do |nodes|
+    value(nodes)
+end
+
+$engine.process :__collection__, COLLECTION_OPTS do
+    debug "Collecting results from nodes..."
+    save_all_results
+end
+
+$engine.process :__analysis__, ANALYSIS_OPTS do |exp_result|
+    value(exp_result)
+end
+
+$engine.activity :__install_node__, :doc => "Install a given node", :log_level => :none do |node|
+    node.install()
+end
+
+$engine.process :__install_nodes__, :doc => "Initial node setup", :log_level => :none do |nodes|
+    foreach nodes do |node|
+        debug "Installing ", node
+        run :__install_node__, node
+    end
+    count = (length_of nodes)
+    on(count > 0) do
+        bootstrap_taktuk(nodes)
+    end
+    # TODO: more parallel and more scalable
+    forall nodes, :pool => 10 do |node|
+        debug "Bootstrapping ", node
+        code(node) { |node| node.bootstrap }
+    end
+
+    # checking nodes - run a simple command everywhere
+    # be idempotent - if some nodes fail, retry them
+
+    run :__check_nodes__, nodes
+    value(nodes)
+end
+
+$engine.process :__check_nodes__, :doc => "Checking all nodes", :log_level => :none do |nodes|
+    results = execute_many nodes, "true", :idempotent => true
+    length = length_of results
+    debug "All nodes (#{length}) checked"
+end
+
+$engine.activity :__fix_node_list__ do |list|
+    (list.nil?) ? [] : list
+end
+
+$engine.process :__standard__, STANDARD_OPTS do |node_list|
+
+    flattened_node_list = run :__fix_node_list__, node_list
+
+    nodes = cache :__reservation__ do
+        run :__reservation__, flattened_node_list
+    end
+
+    nodes_list = run :__install_nodes__, nodes
+
+    # replace __reservation__ with a new set
+    set_scope :__nodes__, nodes_list
+
+    deployed = cache :__deployment__ do
+        run :__deployment__, nodes
+    end
+
+    exp_result = run :__experiment__, deployed
+    run :__collection__
+    run :__analysis__, exp_result
+
+end
+
+# helpers
+
+def nodes(&block)
+    return $engine.process(:__reservation__, NODES_OPTS, &block)
+end
+
+def deployment(name = nil, &block)
+    if name.nil?
+        return $engine.process(:__deployment__, DEPLOYMENT_OPTS, &block)
+    else
+        return $engine.process(:__deployment__, DEPLOYMENT_OPTS) { |nodes| run(name, nodes) }
+    end
+end
+
+def body(&block)
+    return $engine.process(:__experiment__, EXPERIMENT_OPTS,  &block)
+end
+
+def analysis(&block)
+    return $engine.process(:__analysis__, ANALYSIS_OPTS, &block)
+end
+
+
+# importer of subexperiments
+# TODO: this is rather ugly!
+
+def with_sublibrary()
+    exp = nil
+    begin
+        old_engine = $engine
+        exp = $engine = XPFlow::BasicLibrary.new
+        yield
+    ensure
+        $engine = old_engine
+    end
+    return exp
+end
+
+def import(name, path = nil, opts = {})
+    if path.nil?
+        return import_file(name)
+    end
+    lib = with_sublibrary do
+        Kernel.load(__FILE__, true)
+        Kernel.load(path)
+    end
+    $engine.import_library(name, lib)
+    return lib
+end
+
+def library(name, &block)
+    lib = with_sublibrary do
+        Kernel.load(__FILE__, true)
+        block.call
+    end
+    $engine.import_library(name, lib)
+    return lib
+end
+
+# includes the file as-it-is
+def import_file(path)
+    return Kernel.load(path)
+end
+
+def experiment_entry(&block)
+    return $engine.process(:__standard__, &block)
+end