pione 0.1.0

data/lib/pione/resource/local.rb

@@ -0,0 +1,113 @@
+module Pione
+  module Resource
+    # Local represents local path resources.
+    class Local < BasicResource
+      # Creates a local resource handler with URI.
+      # @param [String, URI] uri
+      #   URI of a local path
+      def initialize(uri)
+        @uri = uri.kind_of?(::URI::Generic) ? uri : ::URI.parse(uri)
+        raise ArgumentError unless @uri.kind_of?(URIScheme::LocalScheme)
+        @path = Pathname.new(uri.path)
+      end
+
+      # (see BasicResource#create)
+      def create(data)
+        @path.dirname.mkpath unless @path.dirname.exist?
+        @path.open("w+"){|file| file.write(data)}
+      end
+
+      # (see BasicResource#read)
+      def read
+        @path.exist? ? @path.read : (raise NotFound.new(@uri))
+      end
+
+      # (see BasicResource#update)
+      def update(data)
+        if @path.exist?
+          @path.open("w+"){|file| file.write(data)}
+        else
+          raise NotFound.new(@uri)
+        end
+      end
+
+      # (see BasicResource#delete)
+      def delete
+        @path.delete if @path.exist?
+      end
+
+      # (see BasicResource#mtime)
+      def mtime
+        @path.mtime
+      end
+
+      # (see BasicResource#entries)
+      def entries
+        @path.entries.select{|entry| (@path + entry).file?}.map do |entry|
+          Resource[::URI.parse("local:%s" % (@path + entry).expand_path)]
+        end
+      rescue Errno::ENOENT
+        raise NotFound.new(self)
+      end
+
+      # (see BasicResource#basename)
+      def basename
+        @path.basename.to_s
+      end
+
+      # (see BasicResource#exist?)
+      def exist?
+        @path.exist?
+      end
+
+      # Makes symbolic link from the resource to the destination.
+      # @param [String] dest
+      #   destination path
+      # @return [void]
+      def link_to(dest)
+        dir = File.dirname(dest)
+        FileUtils.makedirs(dir) unless Dir.exist?(dir)
+        FileUtils.symlink(@path, dest)
+      end
+
+      # Moves the source to the resource and makes reverse link.
+      # @param [String] src
+      #   source path
+      # @return [void]
+      def link_from(src)
+        swap(src)
+      end
+
+      # Swaps the source file and the resource file. This method moves the
+      # other file to the resource path and creates a symbolic link from
+      # distination to source.
+      # @param [String] other
+      #   swap target
+      # @return [void]
+      def swap(other)
+        unless File.ftype(other) == "file"
+          raise ArgumentError.new(other)
+        end
+        dir = @path.dirname
+        dir.mkpath unless dir.exist?
+        FileUtils.mv(other, @path)
+        FileUtils.symlink(@path, other)
+      end
+
+      # Swaps the resouce file and the other resouce file.
+      # @param [Local] other
+      #   swap target
+      def shift_from(other)
+        raise ArgumentError.new(other) unless other.kind_of?(self.class)
+        unless File.ftype(other.path) == "file"
+          raise ArgumentError.new(other)
+        end
+        dir = @path.dirname
+        dir.mkpath unless dir.exist?
+        FileUtils.mv(other.path, @path)
+      end
+    end
+
+    @@schemes['local'] = Local
+  end
+end

data/lib/pione/rule-handler/action-handler.rb

@@ -0,0 +1,184 @@
+module Pione
+  module RuleHandler
+    # ActionHandler handles ActionRule.
+    class ActionHandler < BasicHandler
+      def self.message_name
+        "Action"
+      end
+
+      attr_reader :working_directory
+
+      def initialize(*args)
+        super(*args)
+        @working_directory = make_working_directory
+        setup_variable_table
+      end
+
+      # Execute the action.
+      def execute
+        # prepare input files
+        setup_working_directory
+        # prepare shell script
+        write_shell_script {|path| call_shell_script(path) }
+        # collect outputs
+        collect_outputs
+        # write resouces
+        write_output_resources
+        # write tuples
+        write_output_tuples
+        # write environment info
+        write_env_info
+        # write other resources
+        write_other_resources
+
+        # return tuples
+        return @outputs
+      end
+
+      private
+
+      # Setup the variable table with working directory in addition.
+      def setup_variable_table
+        unless @working_directory
+          super
+        else
+          @variable_table.set(
+            Variable.new("WORKING_DIRECTORY"),
+            PioneString.new(@working_directory)
+          )
+          @variable_table.set(
+            Variable.new("PWD"),
+            PioneString.new(@working_directory)
+          )
+        end
+      end
+
+      # Make a working directory.
+      def make_working_directory
+        # build directory path
+        task_dirname = ID.domain_id(
+          @rule.expr.package.name,
+          @rule.expr.name,
+          @inputs,
+          @original_params
+        )
+
+        # create a directory
+        path = Global.working_directory + task_dirname
+        FileUtils.makedirs(path)
+
+        return path
+      end
+
+      # Synchronize input data into working directory.
+      def setup_working_directory
+        @inputs.flatten.each do |input|
+          # get file path in working directory
+          path = File.join(@working_directory, input.name)
+          # create a link to cache
+          cache_path = FileCache.get(input.uri)
+          FileUtils.symlink(cache_path, path, {:force => true})
+        end
+      end
+
+      # Write the data to the tempfile as shell script.
+      def write_shell_script(&b)
+        file = File.open(
+          File.join(@working_directory,"__pione-action__.sh"),
+          "w+"
+        )
+        if @dry_run
+          @rule.outputs.flatten.each do |output|
+            file.puts("touch %s" % output.eval(@variable_table).name)
+          end
+        else
+          file.print(@rule.body.eval(@variable_table).content)
+        end
+        debug_message("Action #{file.path}")
+        user_message("-"*60, 0, "SH")
+        @rule.body.eval(@variable_table).content.split("\n").each do |line|
+          user_message(line, 0, "SH")
+        end
+        user_message("-"*60, 0, "SH")
+        file.close
+        FileUtils.chmod(0700, file.path)
+        return b.call(file.path)
+      end
+
+      # Call shell script of the path.
+      def call_shell_script(path)
+        scriptname = File.basename(path)
+
+        # stdout & stderr
+        stdout = @rule.outputs.map {|output|
+          output.eval(@variable_table)
+        }.find {|output| output.stdout?}
+        out = stdout ? stdout.name : ".stdout"
+        err = ".stderr"
+
+        # execute command
+        `cd #{@working_directory}; ./#{scriptname} > #{out} 2> #{err}`
+      end
+
+      # Make output tuple by name.
+      def make_output_tuple_with_time(name)
+        time = File.mtime(File.join(@working_directory, name))
+        uri = make_output_resource_uri(name).to_s
+        Tuple[:data].new(name: name, domain: @domain, uri: uri, time: time)
+      end
+
+      # Collect output data by names from working directory.
+      def collect_outputs
+        list = Dir.entries(@working_directory)
+        @rule.outputs.each_with_index do |output, i|
+          output = output.eval(@variable_table)
+          case output.modifier
+          when :all
+            @outputs[i] = list.select{|name| output.match(name)}.map do |name|
+              make_output_tuple_with_time(name)
+            end
+          when :each
+            if name = list.find {|name| output.match(name)}
+              @outputs[i] = make_output_tuple_with_time(name)
+            end
+          end
+        end
+      end
+
+      # Write resources for output data.
+      def write_output_resources
+        @outputs.flatten.compact.each do |output|
+          path = File.join(@working_directory, output.name)
+          FileCache.put(path, output.uri)
+        end
+      end
+
+      # Writes action environment information file.
+      def write_env_info
+        path = File.join(@working_directory, ".pione-env")
+        File.open(path, "w+") do |out|
+          @variable_table.variables.each do |var|
+            val = @variable_table.get(var)
+            out.puts "%s: %s" % [var.name, val.textize]
+          end
+        end
+      end
+
+      # Writes resources for other intermediate files.
+      def write_other_resources
+        Dir.new(@working_directory).each do |name|
+          path = File.join(@working_directory, name)
+          if File.ftype(path) == "file"
+            uri = make_resource_uri(name, @domain)
+            Resource[uri].link_from(path)
+          end
+        end
+      end
+
+      # Writes output tuples into the tuple space server.
+      def write_output_tuples
+        @outputs.flatten.compact.each {|output| write(output)}
+      end
+    end
+  end
+end

data/lib/pione/rule-handler/basic-handler.rb

@@ -0,0 +1,217 @@
+module Pione
+  # RuleHandler is a handler for rule execution.
+  module RuleHandler
+    # Exception class for rule execution failure.
+    class RuleExecutionError < StandardError
+      def initialize(handler)
+        @handler = handler
+      end
+
+      def message
+        "Execution error when handling the rule '%s': inputs=%s, output=%s, params=%s" % [
+          @handler.rule.rule_path,
+          @handler.inputs,
+          @handler.outputs,
+          @handler.params.inspect
+        ]
+      end
+    end
+
+    class UnknownRule < StandardError; end
+
+    # BasicHandler is a base class for rule handlers.
+    class BasicHandler
+      include TupleSpaceServerInterface
+
+      attr_reader :rule
+      attr_reader :inputs
+      attr_reader :outputs
+      attr_reader :params
+      attr_reader :base_uri
+      attr_reader :task_id
+      attr_reader :domain
+      attr_reader :variable_table
+      attr_reader :call_stack
+
+      # Create a new handler for rule.
+      # @param [TupleSpaceServer] ts_server
+      #   tuple space server
+      # @param [Rule] rule
+      #   rule instance
+      # @param [Array<Data,Array<Data>>] inputs
+      #   input tuples
+      # @param [Hash] opts
+      #   optionals
+      def initialize(ts_server, rule, inputs, params, call_stack, opts={})
+        # check arguments
+        raise ArgumentError.new(inputs) unless inputs.kind_of?(Array)
+        raise ArgumentError.new(inputs) unless inputs.size == rule.inputs.size
+        raise ArgumentError.new(params) unless params.kind_of?(Parameters)
+
+        # set tuple space server
+        set_tuple_space_server(ts_server)
+
+        # set informations
+        @rule = rule
+        @inputs = inputs
+        @outputs = []
+        @params = @rule.params.merge(params)
+        @original_params = params
+        @content = rule.body
+        @domain = get_handling_domain(opts)
+        @variable_table = VariableTable.new(@params.data)
+        @base_uri = read(Tuple[:base_uri].any).uri
+        @dry_run = begin read0(Tuple[:dry_run].any).availability rescue false end
+        @task_id = ID.task_id(@inputs, @params)
+        @call_stack = call_stack
+
+        setup_variable_table
+      end
+
+      # Puts environment variable into pione variable table.
+      # @param [Hash{String => String}] env
+      #   environment table
+      # @return [void]
+      def setenv(env)
+        env.each do |key, value|
+          @variable_table.set(Variable.new("ENV_" + key), PioneString.new(value))
+        end
+      end
+
+      # Handles the rule and returns the outputs.
+      # @return [Array<Data,Array<Data>>]
+      #   outputs
+      def handle
+        name = self.class.message_name
+
+        # show begin message
+        user_message_begin("Start %s Rule: %s" % [name, handler_digest])
+
+        # call stack
+        debug_message("call stack:")
+        @call_stack.each_with_index do |domain, i|
+          debug_message("%s:%s" % [i, domain], 1)
+        end
+
+        # execute the rule
+        outputs = execute
+
+        # show output list
+        debug_message("%s Rule %s Result:" % [name, handler_digest])
+
+        @outputs.compact.each_with_index do |output, i|
+          if output.kind_of?(Array)
+            output.each_with_index do |o, ii|
+              debug_message("%s,%s:%s" % [i, ii, o.name], 1)
+            end
+          else
+            debug_message("%s:%s" % [i, output.name], 1)
+          end
+        end
+
+        # show end message
+        user_message_end "End %s Rule: %s" % [name, handler_digest]
+
+        return outputs.compact
+      end
+
+      # Returns true if it is root rule handler.
+      # @return [Boolean]
+      #   true if it is root rule handler
+      def root?
+        self.kind_of?(RootHandler)
+      end
+
+      # @api private
+      def ==(other)
+        return false unless @rule == other.rule
+        return false unless @inputs == other.inputs
+        return false unless @outputs == other.outputs
+        return false unless @params == other.params
+        return true
+      end
+
+      # @api private
+      alias :eql? :==
+
+      # @api private
+      def hash
+        @rule.hash + @inputs.hash + @outputs.hash + @params.hash
+      end
+
+      private
+
+      # Executes the rule.
+      # @return [Array<Data,Array<Data>>]
+      #   outputs
+      # @api private
+      def execute
+        raise NotImplementError
+      end
+
+      # Return the domain.
+      def get_handling_domain(opts)
+        opts[:domain] || ID.domain_id(
+          @rule.expr.package.name,
+          @rule.expr.name,
+          @inputs,
+          @original_params
+        )
+      end
+
+      # Makes resource uri.
+      def make_resource_uri(name, domain)
+        if domain == "root" || domain.nil?
+          return URI.parse(@base_uri) + "./%s" % name
+        else
+          # make relative path
+          rule_name = domain.split("_")[0..-2].join("_")
+          digest = domain.split("_").last
+          path = "./.%s/%s/%s" % [rule_name, digest, name]
+
+          # make uri
+          return URI.parse(@base_uri) + path
+        end
+      end
+
+      # Makes output resource uri.
+      def make_output_resource_uri(name)
+        # get parent domain or root domain
+        make_resource_uri(name, @call_stack.last)
+      end
+
+      # Make output tuple by name.
+      def make_output_tuple(name)
+        uri = make_output_resource_uri(name).to_s
+        Tuple[:data].new(name: name, domain: @domain, uri: uri, time: nil)
+      end
+
+      # Setup variable table. The following variables are introduced in variable
+      # table:
+      # - input auto variables
+      # - output auto variables
+      def setup_variable_table
+        @variable_table.make_input_auto_variables(@rule.inputs, @inputs)
+        outputs = @rule.outputs.map {|expr| expr.eval(@variable_table) }
+        output_tuples = outputs.map {|expr| make_output_tuple(expr.name) }
+        @variable_table.make_output_auto_variables(outputs, output_tuples)
+      end
+
+      # Returns digest string of this handler.
+      def handler_digest
+        params = @params.data.select{|k,_|
+          not(k.toplevel?)
+        }.map{|k,v| "%s:%s" % [k.name, v.textize]}.join(",")
+        "%s([%s],{%s})" % [
+          @rule.rule_path,
+          @inputs.map{|i|
+            i.kind_of?(Array) ? "[%s, ...]" % i[0].name : i.name
+          }.join(","),
+          params
+        ]
+      end
+
+    end
+  end
+end
+