xpflow 0.1b

data/lib/xpflow/library.rb

@@ -0,0 +1,349 @@
+# encoding: UTF-8
+
+#
+# Implementation of libraries, i.e., sets of activities
+# that can be imported or injected into a namespace.
+#
+
+require 'monitor'
+
+module XPFlow
+
+    class ResolutionError < StandardError
+
+    end
+
+    class Library
+        # implements .activity and .process methods
+        # + some kind of activity management
+
+        def initialize
+            @names = { }
+            @lock = Mutex.new
+        end
+
+        def synchronize
+            @lock.synchronize do
+                yield
+            end
+        end
+
+        def [](key)
+            return @names[key]
+        end
+
+        def []=(key, value)
+            key = stringize(key)
+            raise "'#{key}' already exists here" if @names.key?(key)
+            raise "'#{key}' contains a dot" if key.include?(".")
+            @names[key] = value
+        end
+
+        def set_force(key, value)
+            @names[key] = value
+        end
+
+        def into_parts(name)
+            array = name.split(".")
+            return [ array[0...-1], array.last ]
+        end
+
+        def resolve_name(name)
+            name = stringize(name)
+            return resolve_parts(*into_parts(name))
+        end
+
+        def resolve_libs(libs)
+            libs = libs + [] # copy
+            path = libs.join(".")
+            current = self
+            while libs.length > 0
+                name = libs.shift
+                current = current[name]
+                if !current.is_a?(Library)
+                    raise ResolutionError.new("No such library '#{path}'")
+                end
+            end
+            return current
+        end
+
+        def resolve_parts(libs, name)
+            library = resolve_libs(libs)
+            object = library[name]
+            if object.nil?
+                path = libs.join(".")
+                raise ResolutionError.new("No such object '#{name}' in '#{path}'")
+            end
+            return object
+        end
+
+        def resolve_activity(libs, name)
+            activity = resolve_parts(libs, name)
+            if activity.is_a?(Library)
+                path = (libs + [ name ]).join(".")
+                raise ResolutionError.new("'#{path}' is not an activity")
+            end
+        end
+
+        def create_libraries(libs)
+            # creates all intermediate libraries if they don't exist
+            current = self
+            nesting = []
+            while libs.length > 0
+                name = libs.shift
+                nesting.push(name)
+                object = current[name]
+                if object.nil?
+                    current[name] = BasicLibrary.new
+                elsif !object.is_a?(Library)
+                    activity = nesting.join(".")
+                    raise ResolutionError.new("Overwriting an activity '#{activity}'")
+                end
+                current = current[name]
+            end
+            return current
+        end
+
+        def get_library_or_nil(name)
+            object = @names[name]
+            return object
+        end
+
+        def get_activity_or_error(name)
+            object = @names[name]
+            if object.is_a?(Library)
+                raise ResolutionError.new("No such activity '#{name}'")
+            end
+            return object
+        end
+
+        def get_activity_or_nil(name)
+            previous = nil
+            begin
+                previous = get_activity_or_error(name)
+            rescue ResolutionError => e
+                previous = nil # there is no previous activity
+            end
+            return previous
+        end
+
+
+        def activity_alias(full_name, old_name)
+            # creates an alias between full_name and another_name
+            # please don't make it cyclic! :D
+            object = get_object(old_name)
+            raise "Object '#{old_name}' does not exist" if object.nil?
+            libs, name = into_parts(full_name)
+            library = resolve_libs(libs)
+            library[name] = object
+        end
+
+        def traversal_graph
+            g = {}
+            @names.each do |k, v|
+                r = case 
+                    when v.is_a?(Library) then v.traversal_graph
+                    when v.is_a?(XPFlow::BlockActivity) then "block"
+                    when v.is_a?(XPFlow::ProcessActivity) then "process"
+                    else
+                        raise "Error: #{k} #{v} (#{v.class})"
+                end
+                g[k] = r
+            end
+            return g
+        end
+
+        def stringize(o)
+            raise "Wrong name type: #{o.class}" if !(o.is_a?(String) || o.is_a?(Symbol))
+            return o.to_s
+        end
+
+        # ===PUBLIC API=== #
+
+        def activity(full_name, opts = {}, &block)
+            name, library = _create_entry(full_name)
+
+            opts[:__parent__] = library.get_activity_or_nil(name)
+
+            block_activity = XPFlow::BlockActivity.new(name, opts, block)
+            block_activity.doc = opts[:doc]
+
+            library.set_force(name, block_activity)
+
+            return block_activity
+        end
+
+        def constant(full_name, value)
+            activity full_name do
+                value
+            end
+        end
+
+        def _create_entry(full_name)
+            full_name = stringize(full_name)
+            libs, name = into_parts(full_name)
+            library = create_libraries(libs)
+            return [ name, library ]
+        end
+
+        def process(full_name, opts = {}, &block)
+            name, library = _create_entry(full_name)
+
+            parent = opts[:__parent__] = library.get_activity_or_nil(name)
+
+            process_activity = XPFlow::ProcessDSL.new(name, self, &block).as_process
+            process_activity.doc = parent.doc unless parent.nil?
+            process_activity.doc = opts[:doc] if opts[:doc]
+            process_activity.opts = opts
+            process_activity.collect_meta(2)
+
+            library.set_force(name, process_activity)
+
+            return process_activity
+        end
+
+        def macro(full_name, opts = {}, &block)
+            name, library = _create_entry(full_name)
+
+            m = XPFlow::MacroDSL.new(name, self, block)
+            library.set_force(name, m)
+            return m
+        end
+
+        def get_object(name)
+            begin
+                return resolve_name(name)
+            rescue ResolutionError => e
+                return nil
+            end
+        end
+
+        def get_activity_object(name)
+            object = get_object(name)
+            if object.is_a?(Library)
+                raise "'#{name}' is a library"
+            end
+            return object
+        end
+
+        def get_names
+            return @names.keys
+        end
+
+        def get_libraries
+            h = @names.select { |k, v| v.is_a?(Library) }
+            return h
+        end
+
+        def import_library(name, library)
+            self[name] = library
+        end
+
+        def inject_library(name, library)
+            # this will inject all activities from 'library' into this library
+            # this may overwrite some of them
+            import_library(name, library)
+            library.get_names.each do |object|
+                activity_alias(object, "#{name}.#{object}")
+            end
+        end
+
+        def setup
+            # empty
+        end
+
+    end
+
+    if __FILE__ == $0
+        require 'pp'
+        require 'xpflow'
+        l = Library.new
+        l.constant :a1, 1
+        l.process :"czesc.siema" do; end
+        g = Library.new
+        g.activity :cze
+        l.activity_alias("alejazda", "czesc")
+        pp l.traversal_graph
+    end
+
+    # a library that does some nice tricks
+
+    class ActivityLibrary < Library
+
+        def initialize
+            super
+            setup()
+            this = self
+            __activities__.each_pair do |name, realname|
+                activity(name, get_option(name)) do |*args|
+                    this.invoke(realname, args, self, &self.__block__)
+                end
+            end
+        end
+
+        def invoke(method, args, proxy = nil, &block)
+            Thread.current[:__proxy__] = proxy
+            return self.send(method, *args, &block)
+        end
+
+        def proxy
+            return Thread.current[:__proxy__]
+        end
+
+        def get_option(name)
+            return { }
+        end
+
+    end
+
+    class HiddenActivityLibrary < ActivityLibrary
+
+        def get_option(name)
+            return { :log_level => :none }
+        end
+
+    end
+
+    class SyncedActivityLibrary < ActivityLibrary
+
+        def invoke(method, args, proxy = nil, &block)
+            synchronize do
+                super
+            end
+        end
+
+    end
+
+    class MonitoredActivityLibrary < ActivityLibrary
+
+        # with reentrant lock
+        def initialize
+            super
+            @lock = Monitor.new
+        end
+
+    end
+
+    IGNORED_LIBRARY_VARIABLES = Library.new.instance_variables
+
+    module SerializableLibrary
+
+        def checkpoint
+            vars = self.instance_variables - IGNORED_LIBRARY_VARIABLES
+            state = {}
+            vars.each do |v|
+                value = self.instance_variable_get(v)
+                state[v] = value
+            end
+            return state
+        end
+
+        def restore(state)
+            state.each_pair do |k, v|
+                self.instance_variable_set(k, v)
+            end
+        end
+
+    end # Serializable
+
+end

data/lib/xpflow/logging.rb

@@ -0,0 +1,153 @@
+
+require 'thread'
+
+module XPFlow
+
+    class AbstractLog
+
+        def initialize
+            @mutex = Mutex.new
+        end
+
+        def open(*args)
+            raise "Not implemented!"
+        end
+
+        def log(*args)
+            raise "Not implemented!"
+        end
+
+        def close(*args)
+            raise "Not implemented!"
+        end
+
+        def synchronize(&block)
+            return @mutex.synchronize(&block)
+        end
+
+    end
+
+    class ConsoleLog < AbstractLog
+
+        def initialize
+            super
+        end
+
+        def log(*msgs)
+            synchronize do
+                msgs.each do |x|
+                    x = x.plain unless STDOUT.tty?
+                    puts x
+                end
+                STDOUT.flush
+            end
+        end
+
+        def open; end
+        def close; end
+
+    end
+
+    class FileLog < AbstractLog
+
+        def initialize(fname)
+            super()
+            @fname = fname
+            @f = nil
+        end
+
+        def open
+            @f = File.open(@fname, "w")
+            return self
+        end
+
+        def log(*msgs)
+            synchronize do
+                msgs.each do |x|
+                    @f.write("#{x.plain}\n")
+                end
+                @f.flush
+            end
+        end
+
+        def close
+            @f.close
+            @f = nil
+        end
+
+    end
+
+    class Logging
+
+        def initialize
+            @loggers = {}
+        end
+
+        def add(logger, label)
+            @loggers[label] = logger
+        end
+
+        def prefix
+            t = Time.now
+            s = t.strftime('%Y-%m-%d %H:%M:%S.%L')
+            ms = t.usec / 1000
+            return s.gsub('%L', "%03d" % ms) # Ruby 1.8 compat
+        end
+
+        def colorize(s, label)
+            colors = {
+                :normal => :green,
+                :verbose => :yellow,
+                :paranoic => :red,
+                :default => :red!
+            }
+            c = colors[:default]
+            c = colors[label] if colors.key?(label)
+            return s.send(c)
+        end
+
+        def log(msg, label = :normal)
+            pre = colorize(prefix, label)
+            msg = "[ %s ] %s" % [ pre, msg ]
+            @loggers.each_pair do |k, v|
+                v.log(msg)
+            end
+            # TODO: kind of ugly
+            Scope.current[:__experiment__].log(msg)  # log to the experiment
+        end
+
+        def get(label)
+            return @loggers[label]
+        end
+
+        def using(&block)
+            ls = @loggers.values
+            opened = []
+            x = nil?
+            begin
+                ls.each { |x| x.open(); opened.push(x) }
+                x = block.call
+            ensure
+                opened.each { |x| x.close } # TODO
+            end
+            return x
+        end
+
+    end
+
+    $console = ConsoleLog.new  # globally accessible, should be used by everybody
+
+end
+
+if __FILE__ == $0
+    require 'colorado'
+    s = "cze".green + "yo".red
+    x = XPFlow::Logging.new
+    x.add(XPFlow::ConsoleLog.new, :console)
+    x.add(XPFlow::FileLog.new("test.log"), :file)
+
+    x.using do
+        x.log(s)
+    end
+
+end

data/lib/xpflow/manager.rb

@@ -0,0 +1,147 @@
+
+# this is a temporary directory that contains
+# files generated on-the-fly to simplify things
+
+require 'tmpdir'
+require 'thread'
+require 'fileutils'
+require 'monitor'
+require 'erb'
+require 'ostruct'
+require 'yaml'
+
+module XPFlow
+
+    class ExecutionError < StandardError
+
+    end
+
+    class LocalExecutionResult
+
+        def initialize(cmd, stdout, stderr)
+            @cmd = cmd
+            @stdout = stdout
+            @stderr = stderr
+        end
+
+        def stdout
+            return IO.read(@stdout)
+        end
+
+        def stderr
+            return IO.read(@stderr)
+        end
+
+        def stdout_file
+            return @stdout
+        end
+
+        def stderr_file
+            return @stderr
+        end
+
+        def command
+            return @cmd
+        end
+
+        def to_s
+            return "LocalExecutionResult(#{@cmd}, out => #{@stdout}, err => #{@stderr})"
+        end
+
+    end
+
+    class DirectoryManager
+
+        attr_reader :path
+
+        def initialize(path)
+            @mutex = Monitor.new
+            if path.nil?
+                @path = Dir.mktmpdir()  # TODO: remove while exiting
+                # use remove_entry_secure
+            else
+                if File.directory?(path)
+                    FileUtils.remove_entry_secure(path)
+                end
+                Dir.mkdir(path)
+                @path = path
+            end
+            @counter = 0
+        end
+
+        def synchronize(&block)
+            return @mutex.synchronize(&block)
+        end
+
+        def mktemp(&block)
+            fname = synchronize do
+                @counter += 1
+                File.join(@path, "tmpfile-#{@counter}")
+            end
+            if block_given?
+                File.open(fname, "wb", &block)
+            end
+            return fname
+        end
+
+        def join(name)
+            return File.join(@path, name)
+        end
+
+        def open(fname, flags, &block)
+            path = File.join(@path, fname)
+            return synchronize { File.open(path, flags, &block) }
+        end
+
+        def run_with_files(name, stdout, stderr, opts = {})
+            # assumes that name is properly escaped!!!
+            name = join(name)
+            out = `#{name} 2> #{stderr} > #{stdout}`
+            raise ExecutionError.new("Command #{name} returned error (see #{stdout} and #{stderr})!") if $?.exitstatus != 0
+            return LocalExecutionResult.new(name, stdout, stderr)
+        end
+
+        def run(name, opts = {})
+            return run_with_files(name, self.mktemp(), self.mktemp(), opts)
+        end
+
+        def run_ssh(cmd, opts = {})
+            # runs a command remotely
+            # the form is ".../ssh 'ENV cmd args' "
+            # TODO: Escaping yo!
+
+            stdout = (opts[:out].nil?) ? self.mktemp() : opts[:out]
+            stderr = (opts[:err].nil?) ? self.mktemp() : opts[:err]
+            stdin  = opts.fetch(:in, "/dev/null")
+
+            if opts[:env] and opts[:env] != {}
+                prefix = opts[:env].each_pair.map { |k, v| "#{k}=#{v}" }.join(" ")
+                cmd = "#{prefix} #{cmd}"
+            end
+
+            if opts[:wd]
+                cmd = "cd #{opts[:wd]}; #{cmd}"
+            end
+
+            cmd = Shellwords.escape(cmd)
+            cmd = "ssh #{cmd}"
+            real_cmd = join(cmd)
+            out = `#{real_cmd} < #{stdin} 2> #{stderr} 1> #{stdout}`
+
+            raise ExecutionError.new("Command #{real_cmd} failed (see #{stdout} and #{stderr})") if $?.exitstatus != 0
+            return LocalExecutionResult.new(cmd, stdout, stderr)
+        end
+
+        def _subdir(name)
+            path = join(name)
+            Dir.mkdir(path) # TODO: what if exists?
+            return DirectoryManager.new(path)
+        end
+
+        def subdir(name)
+            synchronize { _subdir(name) }
+        end
+
+    end
+
+end # module