pione 0.1.2 → 0.1.3

data/lib/pione/parser/expr-parser.rb

@@ -107,13 +107,18 @@ module Pione
         integer |
         string |
         ticket |
-        data_name |
+        data_expr |
         rule_expr |
         feature_expr |
         parameters |
         variable
       }
 
+      # +data_expr+ matches data expressions.
+      rule(:data_expr) {
+        data_name | keyword_null.as(:data_null)
+      }
+
       # @!attribute [r] rule_expr
       #   +rule_expr+ matches rule expressions.
       #   @return [Parslet::Atoms::Entity] +rule_expr+ atom

data/lib/pione/patch/em-ftpd-patch.rb

@@ -0,0 +1,21 @@
+module EM::FTPD
+  class Server
+    COMMANDS << "mdtm"
+
+    # get the file's mtime
+    def cmd_mdtm(param)
+      send_unauthorised and return unless logged_in?
+      send_param_required and return if param.nil?
+
+      path = build_path(param)
+
+      @driver.mtime(path) do |result|
+        if result.kind_of?(Time)
+          send_response result.strftime("213 %Y%m%d%H%M%S%L")
+        else
+          send_response "550 file not available"
+        end
+      end
+    end
+  end
+end

data/lib/pione/patch/rinda-patch.rb

@@ -127,17 +127,14 @@ module Rinda
         @bin = {}
       end
 
-      # Adds the tuple.
+      # Add the tuple into the tuple space.
+      #
       # @param [Array] tuple
       #   the tuple
       # @return [void]
       def add(tuple)
         if dom = domain(tuple)
-          unless @bin[dom]
-            @bin[dom] = tuple
-          else
-            raise RedundantTupleError.new(tuple.value)
-          end
+          @bin[dom] = tuple
         else
           raise RuntimeError
         end
@@ -527,7 +524,7 @@ module Rinda
         :finished => TupleBag::DomainTupleBin,
         :working => TupleBag::DomainTupleBin,
         :data => TupleBag::DataTupleBin,
-        :shift => TupleBag::HashTupleBin
+        :lift => TupleBag::HashTupleBin
       )
       @mutex = Mutex.new
     end
@@ -542,12 +539,12 @@ module Rinda
     end
 
     def read(tuple, sec=nil)
-      shift_tuple(real_read(tuple, sec))
+      lift_tuple(real_read(tuple, sec))
     end
 
     def read_all(tuple)
       real_read_all(tuple).map do |res|
-        shift_tuple(res)
+        lift_tuple(res)
       end
     end
 
@@ -730,28 +727,39 @@ module Rinda
       end
     end
 
-    def shift_tuple(tuple)
+    # Lift the location of the tuple.
+    #
+    # @param tuple [BasicTuple]
+    #   tuple
+    # @return [BasicTuple]
+    #   lifted tuple
+    def lift_tuple(tuple)
       if Pione::Tuple[tuple.first]
-        if pos = Pione::Tuple[tuple.first].uri_position
-          if new_uri = shift_uri(tuple[pos])
+        if pos = Pione::Tuple[tuple.first].location_position
+          if new_location = lift_location(tuple[pos])
             tuple = tuple.clone
-            tuple[pos] = new_uri
+            tuple[pos] = new_location
           end
         end
       end
       return tuple
     end
 
-    def shift_uri(uri, old=[])
-      return nil if old.include?(uri)
-
-      template = TemplateEntry.new([:shift, uri, nil])
-      if shift_tuple = @bag.find(template)
-        next_uri = shift_tuple[2]
-        if last_uri = shift_uri(next_uri, old + [uri])
-          next_uri = last_uri
-        end
-        return next_uri
+    # Lift the location.
+    #
+    # @param location [Location::BasicLocation]
+    #   location that lift from
+    # @param history [Array<Location::BasicLocation>]
+    #   history of lifted location
+    # @return [Location::BasicLocation or nil]
+    #   new location
+    def lift_location(location, history=[])
+      return nil if history.include?(location)
+
+      template = TemplateEntry.new([:lift, location, nil])
+      if lift_tuple = @bag.find(template)
+        new_location = lift_tuple[2]
+        return lift_location(new_location, history << location) || new_location
       end
       return nil
     end

data/lib/pione/rule-handler/action-handler.rb

@@ -22,8 +22,8 @@ module Pione
         write_shell_script {|path| call_shell_script(path) }
         # collect outputs
         collect_outputs
-        # write resouces
-        write_output_resources
+        # write output data
+        write_output_data
         # write tuples
         write_output_tuples
         # write environment info
@@ -54,9 +54,12 @@ module Pione
       end
 
       # Make a working directory.
+      #
+      # @return [Pathname]
+      #   path of working directory
       def make_working_directory
         # build directory path
-        task_dirname = ID.domain_id(
+        dirname = ID.domain_id(
           @rule.expr.package.name,
           @rule.expr.name,
           @inputs,
@@ -64,20 +67,20 @@ module Pione
         )
 
         # create a directory
-        path = Global.working_directory + task_dirname
-        FileUtils.makedirs(path)
-
-        return path
+        return (Global.working_directory + dirname).tap{|path| path.mkpath}
       end
 
       # Synchronize input data into working directory.
       def setup_working_directory
         @inputs.flatten.each do |input|
           # get file path in working directory
-          path = File.join(@working_directory, input.name)
+          wd_location = @working_directory + input.name
           # create a link to cache
-          cache_path = FileCache.get(input.uri)
-          FileUtils.symlink(cache_path, path, {:force => true})
+          cache_location = FileCache.get(input.location)
+          wd_location.make_symlink(cache_location.path)
+          unless wd_location.exist?
+            raise RuleExecutionError.new(self)
+          end
         end
       end
 
@@ -122,12 +125,14 @@ module Pione
 
       # Make output tuple by name.
       def make_output_tuple_with_time(name)
-        time = File.mtime(File.join(@working_directory, name))
-        uri = make_output_resource_uri(name).to_s
-        Tuple[:data].new(name: name, domain: @domain, uri: uri, time: time)
+        time = (@working_directory + name).mtime
+        location = make_output_location(name)
+        Tuple[:data].new(name: name, domain: @domain, location: location, time: time)
       end
 
       # Collect output data by names from working directory.
+      #
+      # @return [void]
       def collect_outputs
         list = Dir.entries(@working_directory)
         @rule.outputs.each_with_index do |output, i|
@@ -145,18 +150,22 @@ module Pione
         end
       end
 
-      # Write resources for output data.
-      def write_output_resources
+      # Write output data with caching.
+      #
+      # @return [void]
+      def write_output_data
         @outputs.flatten.compact.each do |output|
-          path = File.join(@working_directory, output.name)
-          FileCache.put(path, output.uri)
+          src = Location[@working_directory + output.name]
+          dest = output.location
+          FileCache.put(src, dest)
         end
       end
 
-      # Writes action environment information file.
+      # Write action environment information file.
+      #
+      # @return [void]
       def write_env_info
-        path = File.join(@working_directory, ".pione-env")
-        File.open(path, "w+") do |out|
+        (@working_directory + ".pione-env").open("w+") do |out|
           @variable_table.variables.each do |var|
             val = @variable_table.get(var)
             out.puts "%s: %s" % [var.name, val.textize]
@@ -164,13 +173,14 @@ module Pione
         end
       end
 
-      # Writes resources for other intermediate files.
+      # Write resources for other intermediate files.
+      #
+      # @return [void]
       def write_other_resources
-        Dir.new(@working_directory).each do |name|
-          path = File.join(@working_directory, name)
+        @working_directory.entries.each do |name|
+          path = @working_directory + name
           if File.ftype(path) == "file"
-            uri = make_resource_uri(name, @domain)
-            Resource[uri].link_from(path)
+            Location[path].move(make_location(name, @domain))
           end
         end
       end

data/lib/pione/rule-handler/basic-handler.rb

@@ -1,5 +1,5 @@
 module Pione
-  # RuleHandler is a handler for rule execution.
+  # RuleHandler is a handler for rule application.
   module RuleHandler
     # Exception class for rule execution failure.
     class RuleExecutionError < StandardError
@@ -27,13 +27,19 @@ module Pione
       attr_reader :inputs
       attr_reader :outputs
       attr_reader :params
-      attr_reader :base_uri
+      attr_reader :original_params
+      attr_reader :base_location
+      attr_reader :dry_run
       attr_reader :task_id
       attr_reader :domain
       attr_reader :variable_table
       attr_reader :call_stack
+      attr_reader :domain_location
+      attr_reader :rule_process_record
+      attr_reader :task_process_record
 
       # Create a new handler for rule.
+      #
       # @param [TupleSpaceServer] ts_server
       #   tuple space server
       # @param [Rule] rule
@@ -60,15 +66,42 @@ module Pione
         @content = rule.body
         @domain = get_handling_domain(opts)
         @variable_table = VariableTable.new(@params.data)
-        @base_uri = read(Tuple[:base_uri].any).uri
-        @dry_run = begin read0(Tuple[:dry_run].any).availability rescue false end
+        @base_location = read!(Tuple[:base_location].any).location
+        @dry_run = begin read!(Tuple[:dry_run].any).availability rescue false end
         @task_id = ID.task_id(@inputs, @params)
         @call_stack = call_stack
+        @domain_location = make_location("", @domain)
+
+        caller = @call_stack[-1]
+
+        # build rule process record
+        @rule_process_record = Log::RuleProcessRecord.new.tap do |record|
+          record.name = "&%s:%s" % [@rule.expr.package.name, @rule.expr.name]
+          record.rule_type = @rule.rule_type
+          record.caller = caller.split("_").first.tap do |dname|
+            if dname.include?("-")
+              package, name = dname.split("-")
+              break "&%s:%s" % [package, name]
+            else
+              break "&root:Root"
+            end
+          end if caller
+        end
+
+        # build task process record
+        @task_process_record = Log::TaskProcessRecord.new.tap do |record|
+          record.name = handler_digest
+          record.rule_name = "&%s:%s" % [@rule.expr.package.name, @rule.expr.name]
+          record.rule_type = @rule.rule_type
+          record.inputs = @inputs.flatten.map{|input| input.name}.join(",")
+          record.parameters = @params.textize
+        end
 
         setup_variable_table
       end
 
-      # Puts environment variable into pione variable table.
+      # Put environment variable into pione variable table.
+      #
       # @param [Hash{String => String}] env
       #   environment table
       # @return [void]
@@ -78,10 +111,15 @@ module Pione
         end
       end
 
-      # Handles the rule and returns the outputs.
+      # Handle the rule and returns the outputs.
+      #
       # @return [Array<Data,Array<Data>>]
       #   outputs
       def handle
+        # put rule and task process log
+        process_log(@task_process_record.merge(transition: "start"))
+        process_log(@rule_process_record.merge(transition: "start"))
+
         name = self.class.message_name
 
         # show begin message
@@ -93,6 +131,9 @@ module Pione
           debug_message("%s:%s" % [i, domain], 1)
         end
 
+        # save rule condition informations
+        save_rule_condition_infos
+
         # execute the rule
         outputs = execute
 
@@ -112,6 +153,10 @@ module Pione
         # show end message
         user_message_end "End %s Rule: %s" % [name, handler_digest]
 
+        # put rule and task process log
+        process_log(@rule_process_record.merge(transition: "complete"))
+        process_log(@task_process_record.merge(transition: "complete"))
+
         return outputs.compact
       end
 
@@ -141,7 +186,15 @@ module Pione
 
       private
 
+      # Save domain information file.
+      #
+      # @return [void]
+      def save_rule_condition_infos
+        Log::DomainInfo.new(self).save
+      end
+
       # Executes the rule.
+      #
       # @return [Array<Data,Array<Data>>]
       #   outputs
       # @api private
@@ -159,31 +212,48 @@ module Pione
         )
       end
 
-      # Makes resource uri.
-      def make_resource_uri(name, domain)
+      # Make location by data name and the domain.
+      #
+      # @param name [String]
+      #   data name
+      # @param domain [String]
+      #   domain of the data
+      # @return [BasicLocation]
+      #   the location
+      def make_location(name, domain)
         if domain == "root" || domain.nil?
-          return URI.parse(@base_uri) + "./%s" % name
+          return @base_location + "./%s" % name
         else
           # make relative path
           rule_name = domain.split("_")[0..-2].join("_")
           digest = domain.split("_").last
           path = "./.%s/%s/%s" % [rule_name, digest, name]
 
-          # make uri
-          return URI.parse(@base_uri) + path
+          # make location
+          return @base_location + path
         end
       end
 
-      # Makes output resource uri.
-      def make_output_resource_uri(name)
+      # Make output data location by the name.
+      #
+      # @param name [String]
+      #   data name
+      # @return [BasicLocation]
+      #   output data location
+      def make_output_location(name)
         # get parent domain or root domain
-        make_resource_uri(name, @call_stack.last)
+        make_location(name, @call_stack.last)
       end
 
-      # Make output tuple by name.
+      # Make output tuple by the name.
+      #
+      # @param name [String]
+      #   data name
+      # @return [Tuple::DataTuple]
+      #   data tuple
       def make_output_tuple(name)
-        uri = make_output_resource_uri(name).to_s
-        Tuple[:data].new(name: name, domain: @domain, uri: uri, time: nil)
+        location = make_output_location(name)
+        Tuple[:data].new(name: name, domain: @domain, location: location, time: nil)
       end
 
       # Setup variable table. The following variables are introduced in variable
@@ -205,7 +275,11 @@ module Pione
         "%s([%s],{%s})" % [
           @rule.rule_path,
           @inputs.map{|i|
-            i.kind_of?(Array) ? "[%s, ...]" % i[0].name : i.name
+            if i.kind_of?(Array)
+              i.empty? ? "[]" : "[%s, ...]" % i[0].name
+            else
+              i.name
+            end
           }.join(","),
           params
         ]