xpflow 0.1b

data/lib/xpflow/bash.rb

@@ -0,0 +1,341 @@
+
+#
+# Features a cool class to interface with Bash.
+#
+
+# exceptionally the requirements are present since
+# this module can be nicely used separately
+
+require 'digest'
+require 'open3'
+
+module Bash
+
+    class BashError < StandardError; end
+    class BashTimeout < BashError; end
+    class BashPaddingError < BashError; end
+
+    class StatusError < BashError
+
+        attr_reader :status
+        attr_reader :output
+
+        def initialize(cmd, status, output)
+            super("'#{cmd}' returned with status = #{status}")
+            @status = status
+            @cmd = cmd
+            @output = output
+        end
+
+    end
+
+    class Bash
+
+        def initialize(stdin, stdout, opts = {})
+            @stdin = stdin
+            @stdout = stdout
+            @opts = { 
+                :debug => false,
+                :timeout => 120.0
+            }.merge(opts)
+            @buff = ''
+            @debug = @opts[:debug]
+            @timeout = @opts[:timeout]
+        end
+
+        def _loop
+            while true do
+                x = IO::select([@stdout], [], [], @timeout)
+                raise BashTimeout.new if x.nil?
+                bytes = @stdout.sysread(1024)
+                $stderr.write("\nBASH IN: #{bytes}\n") if @debug
+                @buff << bytes
+                break if yield
+            end
+        end
+
+        def parse(&block)
+            return self.instance_exec(&block)
+        end
+
+        def _nonce
+            randee = 4.times.map { rand().to_s }.join('|')
+            return Digest::SHA512.hexdigest(randee).to_s
+        end
+
+        def _run(cmd, opts)
+            # it's a kind of magic
+            $stderr.write("\nBASH CMD: #{cmd}\n") if @debug
+            nonce = _nonce()
+            @stdin.write("#{cmd}; printf '%04d#{nonce}' $?\n")
+            @stdin.flush
+            _loop do
+                @buff.include?(nonce)
+            end
+            raise BashPaddingError.new if !@buff.end_with?(nonce)
+            output = @buff
+            @buff = ''
+            # treat the output
+            output.slice!(-nonce.length..-1)
+            status = output.slice!(-4..-1)
+            raise "Status #{status} > 255?" if status.slice(0..0) != '0'
+            return output, status.to_i
+        end
+
+        def _run_block(cmd, opts)
+            @stdin.write("#{cmd}; printf '%04d#{nonce}' $?\n")
+        end
+
+        def _extend(path, suffix)
+            path = path.chomp('/') if path.end_with?('/')
+            return path + suffix
+        end
+
+        def _unlines(s)
+            return s.lines.map { |l| l.chomp("\n") }
+        end
+
+        def _escape(args)
+            return args.map { |x| "'#{x}'" }.join(' ')
+        end
+
+        def export(name, value)
+            run("export #{name}=#{value}")
+        end
+
+        def run(cmd, opts = {})
+            out, status = _run(cmd, opts)
+            raise StatusError.new(cmd, status, out) if status != 0
+            return out
+        end
+
+        def run_status(cmd, opts = {})
+            out, status = _run(cmd, opts)
+            return status
+        end
+
+        def cd(path)
+            run("cd #{path}")
+        end
+
+        def ls
+            run("ls -1")
+        end
+
+        def sleep(t)
+            run("sleep #{t.to_i}")
+        end
+
+        def pwd
+            run("pwd").strip
+        end
+
+        def untar(name, where = nil)
+            if where.nil?
+                run("tar xvf #{name}")
+            else
+                run("tar -C #{where} -xvf #{name}")
+            end
+        end
+
+        def echo(text)
+            run("echo #{text}")
+        end
+
+        def bc(text)
+            run("echo '#{text}' | bc").strip
+        end
+
+        def cp(a, b)
+            run("cp #{a} #{b}")
+        end
+
+        def mv(a, b)
+            run("mv #{a} #{b}")
+        end
+
+        def rm(*args)
+            run("rm #{_escape(args)}")
+        end
+
+        def remove_dirs(path)
+            rm "-rf", path
+        end
+
+        def build
+            # builds a standard Unix software
+            run("./configure")
+            run("make")
+        end
+
+        def abspath(path)
+            run("readlink -f #{path}").strip
+        end
+
+        def build_tarball(tarball, path)
+            # builds a tarball containing a std Unix software
+            tarball = abspath(tarball)
+            path = abspath(path)
+            remove_dirs(path)
+            tmp = _extend(path, '-tmp')
+            remove_dirs(tmp)
+            make_dirs(tmp)
+            untar(tarball, tmp)
+            cd tmp
+            # we are in the temp dir
+            if exists('./configure')
+                cd '/'
+                mv tmp, path
+            else
+                ds = dirs()
+                raise 'Too many dirs?' if ds.length != 1
+                mv ds.first, path
+                cd '/'
+                remove_dirs(tmp)
+            end
+            cd path
+            build
+        end
+
+        def tmp_file
+            return run('mktemp').strip
+        end
+
+        def save_machines(machines, path = nil)
+            path = tmp_file if path.nil?
+            append_lines(path, machines.map { |m| m.to_s })
+            return path
+        end
+
+        def mpirun(machines, params)
+            machines = save_machines(machines) if machines.is_a?(Array)
+            return run("mpirun --mca btl ^openib -machinefile #{machines} #{params}")
+        end
+
+        def join(*args)
+            return File.join(*args)
+        end
+
+        def exists(path)
+            run_status("[[ -e #{path} ]]") == 0
+        end
+
+        def make_dirs(path)
+            run("mkdir -p #{path}")
+        end
+
+        def mkdir(path)
+            run("mkdir #{path}") unless exists(path)
+        end
+
+        def files(ignore = true, type = 'f')
+            fs = run("find . -maxdepth 1 -type #{type}")
+            fs = _unlines(fs).reject { |f| f == '.' }.map { |f| f[2..-1] }
+            fs = fs.reject { |f| f.end_with?('~') or f.start_with?('.') } if ignore
+            return fs
+        end
+
+        def which(prog)
+            return run("which #{prog}").strip
+        end
+
+        def dirs(ignore = true)
+            return files(ignore, 'd')
+        end
+
+        def get_type(name)
+            return :dir if run_status("[[ -d #{name} ]]") == 0
+            return :file if run_status("[[ -f #{name} ]]") == 0
+            raise "'#{name}' is neither file nor directory"
+        end
+
+        def expand_path(path)
+            return run("echo #{path}").strip
+        end
+
+        def append_line(path, line)
+            return run("echo '#{line}' >> #{path}")
+        end
+
+        def append_lines(path, lines)
+            lines.each { |line|
+                append_line(path, line)
+            }
+        end
+
+        def packages
+            list = run("dpkg -l")
+            list = _unlines(list).map do |p|
+                s, n, v = p.split
+                { :status => s, :name => n, :version => v }
+            end
+        end
+
+        def aptget(*args)
+            raise 'Command not given' if args.length == 0
+            cmd = args.first.to_sym
+            args = args.map { |x| x.to_s }.join(' ')
+            status = run_status("DEBIAN_FRONTEND=noninteractive apt-get -y #{args}")
+            if cmd == :purge and status == 100
+                return # ugly hack for the case when the package is not installed
+            end
+            raise StatusError.new('aptget', status, 'none') if status != 0
+        end
+
+        def contents(name)
+            run("cat #{name}")
+        end
+
+        alias cat contents
+
+        def hostname
+            return run('hostname').strip
+        end
+
+        def touch(path)
+            run("touch #{path}")
+        end
+
+        def trunc(path)
+            run("cat /dev/null > #{path}")
+        end
+
+        def distribute(path, dest, *nodes)
+            # distributes a file to the given nodes
+            nodes.flatten.each { |node|
+                run("scp -o 'StrictHostKeyChecking no' #{path} #{node}:#{dest}")
+            }
+        end
+
+        def glob(pattern)
+            out, status = _run("ls -1 #{pattern}", {})
+            return [] if status != 0
+            return out.strip.lines.map { |x| x.strip }
+        end
+
+    end
+
+
+    def self.bash(cmd = 'bash', opts = {}, &block)
+        if not block_given?
+            sin, sout, serr, thr = Open3.popen3(cmd)
+            return Bash.new(sin, sout, opts)
+        end
+        # run bash interpreter using this command
+        result = nil
+        Open3.popen3(cmd) do |sin, sout, serr, thr|
+            dsl = Bash.new(sin, sout, opts)
+            dsl.cd('~')   # go to the home dir
+            result = dsl.parse(&block)
+        end
+        return result
+    end
+
+end
+
+if __FILE__ == $0
+    Bash.bash("ssh localhost bash") do
+        cd '/tmp'
+        puts glob("/tmp/*.gz").inspect
+    end
+end

data/lib/xpflow/bundle.rb

@@ -0,0 +1,113 @@
+
+#
+# builds a one-file-bundle of xpflow
+# 
+# TODO: it's not finished:
+#   * it won't bundle various external files (e.g., node templates)
+#   * __FILE__ usage should be standarized somehow to make everything work
+#
+# We roughly do the following (bundle method):
+#
+#   - take all files from lib directory
+#   - discard some files that are not used
+#   - pull ../xpflow (entry point for the library) as well
+#   - remove "require" uses all over the place, checking if
+#     we know about them
+#   - then merge them all and wrap in an envelope that
+#     temporarily replaces $0 (so that inline code won't run)
+#   - for interested parties, we set $bundled = true
+#   - finally we append model.rb (TODO: it needs fixing)
+#   - the order of files *must* not matter!
+#
+# For executable method, we also:
+#  
+#   - pull main bin/xpflow script
+#   - replace a specially marked region with the result
+#     of bundle() method
+#
+# Therefore, a todo list:
+#   1) Fix __FILE__ uses so that model.rb works
+#   2) Bundle external files as well (pack them, serialize
+#      and then extract them on-the-fly?)
+#   3) make everything more robust and less hackish
+#
+
+module XPFlow
+
+    class Bundler
+
+        def initialize
+            basic = [ '../colorado', 'utils', 'structs', 'scope', 'library' ]
+            discard = [ 'ensemble', 'ssh', 'bash' ]
+            files = basic + (__get_my_requires__.sort - basic - discard)
+            @here = File.dirname(__FILE__)
+            files += [ '../xpflow']
+
+            @files = files.map { |x| File.join(@here, "#{x}.rb") }
+        end
+
+        def bundle
+            ignored = [ "xpflow/exts/g5k", "colorado" ]
+            libs = [ "thread", "monitor", "pp", "digest", "open3", "tmpdir",
+                "fileutils", "monitor", "erb", "ostruct", "yaml", "shellwords",
+                 "etc", "optparse", "pathname" ] # TODO: g5k should be pulled in
+            parts = []
+            puts "Bundling files:"
+            @files.each do |f|
+                puts " - #{f}"
+                contents = IO.read(f)
+                new_cont = contents.gsub(/require\s+(\S+)/) do |req|
+                    lib = req.split("'")[1]
+                    if (libs + ignored).include?(lib) or lib == "xpflow"
+                        ""
+                    else
+                        puts "Warning: untreated library (#{lib})"
+                    end
+                end
+                parts.push(new_cont)
+            end
+
+            s = parts.join("\n")
+
+            lib_lines = libs.map { |x| "require('#{x}')" }
+            model = File.join(@here, "exts", "model.rb")
+
+            final = ([ "# encoding: UTF-8" ] + lib_lines + [
+                "$tmp_0 = $0; $0 = 'faked_name'; $bundled = true",
+                s,
+                "$0 = $tmp_0",
+                IO.read(model)
+            ]).join("\n")
+
+            return final
+        end
+
+        def executable
+            b = bundle()
+            main_file = File.join(@here, "..", "..", "bin", "xpflow")
+            contents = IO.read(main_file)
+
+            exp = Regexp.new("(\#XSTART.+\#XEND)", Regexp::MULTILINE)
+            main = contents.gsub(exp, b)
+
+            return main
+        end
+
+    end
+end
+
+if __FILE__ == $0
+    require 'xpflow'
+
+    bundle = ARGV.first
+
+    if bundle.nil?
+        raise "Please provide an output file."
+    end
+
+    bundler = XPFlow::Bundler.new
+    output = bundler.executable()
+    
+    IO.write(bundle, output)
+
+end

data/lib/xpflow/cmdline.rb

@@ -0,0 +1,249 @@
+# encoding: UTF-8
+
+#
+# Takes care of command line interface.
+#
+
+module XPFlow
+
+    class RunInfo
+
+        attr_reader :name
+        attr_reader :args
+
+        def initialize(name, *args)
+            @name = name
+            @args = args
+        end
+
+    end
+
+    class CmdlineError < Exception
+
+        def ignore?
+            return false
+        end
+
+    end
+
+    class UnknownCommandError < CmdlineError
+
+        def initialize(cmd)
+            super("unknown command `#{cmd}'")
+        end
+
+    end
+
+    class CmdlineNoFileError < CmdlineError
+
+        def initialize(path)
+            super("file `#{path}' does not exist")
+        end
+    end
+
+    class CmdlineIgnoreError < CmdlineError
+
+        def ignore?
+            return true
+        end
+    end
+
+    class Options
+
+        attr_reader :args
+        attr_reader :config
+        attr_reader :includes
+        attr_reader :command
+        attr_reader :entry
+
+        def initialize(args)
+            @args = args.clone
+            @command = nil
+            @config = Options.defaults
+            @includes = []
+            @entry = nil
+            @verbose = false
+            parse
+        end
+
+        def verbose?
+            return @verbose
+        end
+
+        def self.defaults
+            # Define a sane default configuration
+            {
+                :labels => [],
+                :ignore_checkpoints => false,
+                :checkpoint => nil,
+                :activity => nil,
+                :instead => [],
+                :after => [],
+                :vars => {}
+            }
+        end
+
+        def load_config(conffile)
+            if !File.exist?(conffile)
+                raise CmdlineNoFileError.new(conffile)
+            end
+            type = File.extname(conffile).downcase
+            contents = File.read(conffile)
+
+            data = case type
+                when '.json' then JSON.parse(contents)
+                when '.yaml' then YAML.load(contents)
+                else raise CmdlineError.new("unknown config file type (#{type})")
+            end
+
+            raise CmdlineError.new("bad config file format") unless data.is_a?(Hash)
+
+            return data
+        end
+
+        # Parses options from the command line
+
+        def parse
+            parser = OptionParser.new do |opts|
+                opts.banner = [
+                    "Usage: #{$0} <command> [options] ARGUMENTS",
+                    "",
+                    "Available commands:",
+                    "",
+                    "  * help -- show help",
+                    "  * run <files*> -- run a workflow from a file",
+                    "  * workflow <files*> -o <output> -- generate a workflow from a file",
+                    "",
+                    "Options:"
+                ].join("\n")
+
+                opts.on("-v", "--verbose", "Run verbosely") do
+                    @config[:labels] += [ :verbose ]
+                    @verbose = true
+                end
+                opts.on("-q", "--quiet", "Run quietly") do
+                    @config[:labels] = [ :normal ]
+                end
+                opts.on("-p", "--paranoiac", "Run paranoically") do
+                    @config[:labels] += [ :verbose, :paranoic ]
+                end
+                opts.on("-l", "--labels LABELS", "Log messages labeled with LABELS") do |labels|
+                    @config[:labels] += labels.split(',').map { |x| x.downcase.to_sym }
+                end
+                
+                opts.on("-I", "--ignore-checkpoints", "Ignore automatically saved checkpoints") do
+                    @config[:ignore_checkpoints] = true
+                end
+                # opts.on("-C", "--checkpoint NAME", "Jump to checkpoint NAME (if exists)") do |name|
+                #     @config[:checkpoint] = name
+                # end
+                # opts.on("-c", "--list-checkpoints", "List available checkpoints") do
+                #    @config[:instead] += [ RunInfo.new(:list_checkpoints) ]
+                # end
+                # opts.on("-i", "--info NAME", "Show information NAME") do |name|
+                #     @config[:instead] += [ RunInfo.new(name.to_sym) ]
+                # end
+                # opts.on("-L", "--list", "List declared activities") do
+                #     @config[:instead] += [ RunInfo.new(:activities) ]
+                # end
+                opts.on("-g", "--gantt", "Show Gantt diagram after execution") do
+                    @config[:after] += [ RunInfo.new(:show_gantt) ]
+                end
+                opts.on("-G", "--save-gantt FILE", "Save Gantt diagram information to FILE") do |name|
+                    @config[:after] += [ RunInfo.new(:save_gantt, name) ]
+                end
+                opts.on("-V", "--vars SPEC", "Set variables from the cmdline") do |spec|
+                    pairs = spec.split(',').map { |x| x.split('=') }
+                    unless pairs.all? { |x| x.length == 2 }
+                        raise CmdlineError.new("Wrong vars syntax - <name>=<value> expected.")
+                    end
+                    @config[:vars].merge!(Hash[*pairs.flatten])
+                end
+                opts.on("-f", "--file FILE", "Set variables from a YAML file") do |file|
+                    params = load_config(file)
+                    @config[:vars].merge!(params)
+                end
+                opts.on("-o", "--output FILE", "Set output file for some commands") do |file|
+                    @config[:output] = file
+                end
+            end
+
+            parser.parse!(@args)
+            
+            @config[:labels] += [ :normal ]
+            @config[:labels].uniq!
+
+            @command = @args.first
+            @params = @args.tail
+
+            if command == "help" || command.nil?
+                show_usage(parser)
+                raise CmdlineIgnoreError.new
+            elsif command == "version"
+                raise CmdlineError.new("Unsupported.")
+            elsif command == "run"
+                load_files()
+            elsif command == "workflow"
+                if @config[:output].nil?
+                    raise CmdlineError.new("output file must be provided with -o switch")
+                end
+                load_files()
+            else
+                # try to parse command as a filename
+                begin
+                    @entry = parse_activity_spec(command)
+                rescue CmdlineNoFileError
+                    raise UnknownCommandError.new(command)
+                end
+                @command = "run"
+                @params = @args
+                load_files()
+            end
+        end
+
+        def load_files
+            # loads files from command line
+            if @params.length == 0
+                raise CmdlineError.new("at least one file must be given")
+            end
+            @entry = parse_activity_spec(@params.first)
+            @params.each do |spec|
+                x = parse_activity_spec(spec)
+                @includes.push(x.first)
+            end
+        end
+
+        def parse_activity_spec(path)
+            parts = path.split(":", 2)
+            activity = nil
+            if parts.length == 1
+                activity = "main"  # by default use 'main' activity
+            else
+                path, activity = parts
+            end
+            if !File.exist?(path)
+                raise CmdlineNoFileError.new(path)
+            end
+            return [ path, activity ]
+        end
+
+        def vars
+            return @config[:vars]
+        end
+
+        def dispatch(obj)
+            if @command == "run"
+                return obj.execute_run(*@entry)
+            elsif @command == "workflow"
+                return obj.execute_workflow(*@entry)
+            end
+        end
+
+        def show_usage(parser)
+            Kernel.puts(parser.banner)
+            Kernel.puts(parser.summarize); Kernel.puts
+        end
+
+    end
+
+end