pione 0.1.3 → 0.1.4

data/lib/pione/rule-handler/flow-handler.rb

@@ -80,7 +80,7 @@ module Pione
           list + callee.eval(@variable_table).expr.to_set.to_a.map{|expr| CallRule.new(expr)}
         end
 
-        # ticket check
+        # check input tickets
         callees = callees.inject([]) do |list, callee|
           target = nil
           # check if tickets exist in the domain
@@ -99,28 +99,37 @@ module Pione
           # find callee rule
           rule = find_callee_rule_tuple(callee).content
 
-          # update callee parameters
+          # callee parameters inherit caller parameters
           @variable_table.variables.each do |var|
             val = @variable_table.get(var)
-            unless val == UndefinedValue.new
+            if val.kind_of?(Callable) or val.kind_of?(Variable) or val.void?
               if rule.params.keys.include?(var)
                 callee.expr.params.set_safety!(var, val)
               end
             end
           end
 
-          # eval callee rule by the context
-          vtable = callee.expr.params.eval(@variable_table).as_variable_table
-          rule = rule.eval(vtable)
-
-          # check rule status and find combinations
-          @data_finder.find(:input, rule.inputs, vtable).each do |res|
-            combinations << [
-              callee,
-              rule,
-              res.combination,
-              res.variable_table
-            ]
+          # build callee parameter from rule definition
+          callee_params = rule.params.merge(callee.expr.params)
+
+          # expand parameters
+          callee_params.eval(@variable_table).each do |atomic_params|
+
+            # eval callee rule by the context
+            vtable = atomic_params.eval(@variable_table).as_variable_table
+            rule = rule.eval(vtable)
+
+            # check rule status and find combinations
+            @data_finder.find(:input, rule.inputs, vtable).each do |res|
+              combinations << [
+                callee,
+                atomic_params,
+                rule,
+                res.combination,
+                res.variable_table,
+                ID.domain_id3(rule, res.combination, atomic_params)
+              ] if rule.constraints.satisfy?(res.variable_table)
+            end
           end
 
           # find next
@@ -145,13 +154,7 @@ module Pione
 
       # Find inputs and variables for flow element rules.
       def select_updatables(combinations)
-        combinations.select do |callee, rule, inputs, vtable|
-          # task domain
-          task_domain = ID.domain_id3(rule, inputs, callee)
-
-          # import finished tuples's data
-          import_finished_outputs(task_domain)
-
+        combinations.map do |callee, params, rule, inputs, vtable, task_domain|
           # find outputs combination
           outputs_combination = @data_finder.find(
             :output,
@@ -163,10 +166,14 @@ module Pione
           outputs_combination = [[]] if outputs_combination.empty?
 
           # check update criterias
-          outputs_combination.any?{|outputs|
-            UpdateCriteria.satisfy?(rule, inputs, outputs, vtable)
+          orders = outputs_combination.map {|outputs|
+            UpdateCriteria.order(rule, inputs, outputs, vtable)
           }
-        end
+          order = nil
+          order = :weak if orders.include?(:weak)
+          order = :force if orders.include?(:force)
+          [callee, params, rule, inputs, vtable, task_domain, order]
+        end.select {|_, _, _, _, _, _, order| not(order.nil?)}
       end
 
       # Distribute tasks.
@@ -181,58 +188,51 @@ module Pione
         process_log(@task_process_record.merge(transition: "suspend"))
         process_log(@rule_process_record.merge(transition: "suspend"))
 
-        applications.uniq.each do |callee, rule, inputs, vtable|
-          # task domain
-          task_domain = ID.domain_id3(rule, inputs, callee)
-
+        applications.uniq.each do |callee, params, rule, inputs, vtable, task_domain, order|
           # make a task tuple
           task = Tuple[:task].new(
             rule.rule_path,
             inputs,
-            callee.expr.params,
+            params,
             rule.features,
             task_domain,
             @call_stack + [@domain] # current call stack + caller
           )
 
-          # check if same task exists
-          begin
-            if need_to_process_task?(task)
-              # copy input data from the handler domain to task domain
-              copy_data_into_domain(inputs, task_domain)
-
-              # write the task
-              write(task)
-
-              # put task schedule process log
-              task_process_record = Log::TaskProcessRecord.new.tap do |record|
-                record.name = task.digest
-                record.rule_name = rule.rule_path
-                record.rule_type = rule.rule_type
-                record.inputs = inputs.flatten.map{|input| input.name}.join(",")
-                record.parameters = callee.expr.params.textize
-                record.transition = "schedule"
-              end
-              process_log(task_process_record)
-
-              msg = "distributed task %s on %s" % [task.digest, handler_digest]
-              user_message(msg, 1)
-
-              next
+          # check if the same task exists or finished already
+          if need_to_process_task?(task, order)
+            # clear finished tuple
+            remove_finished_tuple(task.domain)
+
+            # copy input data from this domain to the task domain
+            inputs.flatten.each {|input| copy_data_into_domain(input, task_domain)}
+
+            # write the task
+            write(task)
+
+            # put task schedule process log
+            task_process_record = Log::TaskProcessRecord.new.tap do |record|
+              record.name = task.digest
+              record.rule_name = rule.rule_path
+              record.rule_type = rule.rule_type
+              record.inputs = inputs.flatten.map{|input| input.name}.join(",")
+              record.parameters = params.textize
+              record.transition = "schedule"
             end
-          rescue Rinda::RedundantTupleError
-            # ignore
-          end
+            process_log(task_process_record)
 
-          show "cancel task %s on %s" % [task.digest, handler_digest]
-          canceled << task_domain
+            # message
+            msg = "distributed task %s on %s" % [task.digest, handler_digest]
+            user_message(msg, 1)
+          else
+            # cancel the task
+            show "cancel task %s on %s" % [task.digest, handler_digest]
+            canceled << task_domain
+          end
         end
 
         # wait to finish threads
-        applications.uniq.each do |callee, rule, inputs, vtable|
-          # task domain
-          task_domain = ID.domain_id3(rule, inputs, callee)
-
+        applications.uniq.each do |callee, params, rule, inputs, vtable, task_domain, order|
           # wait to finish the work
           template = Tuple[:finished].new(
             domain: task_domain,
@@ -240,16 +240,16 @@ module Pione
           )
           finished = read(template)
 
+          # show message about canceled tasks
           unless canceled.include?(task_domain)
             msg = "finished task %s on %s" % [finished.domain, handler_digest]
             user_message(msg, 1)
           end
 
-          # copy data from task domain to this domain
-          @finished << finished
-          copy_data_into_domain(finished.outputs, @domain)
+          # copy write operation data tuple from the task domain to this domain
+          update_by_finished_tuple(rule, finished, vtable)
 
-          # output ticket
+          # publish tickets into the domain
           callee.expr.output_ticket_expr.names.each do |name|
             write(Tuple[:ticket].new(@domain, name))
           end
@@ -262,39 +262,21 @@ module Pione
 
       # Return true if we need to write the task into the tuple space.
       #
-      # @param [TaskTuple] task
-      #   task tuple
-      # @return [Boolean]
-      #   true if we need to write the task into the tuple space
-      def need_to_process_task?(task)
-        not(read!(task) or working?(task))
-      end
-
-      # Return true if any task worker is working on the task.
-      #
       # @param task [TaskTuple]
       #   task tuple
+      # @param order [Symbol]
+      #   update order type
       # @return [Boolean]
-      #   true if any task worker is working on the task
-      def working?(task)
-        read!(Tuple[:working].new(domain: task.domain))
-      end
-
-      # Find outputs from the domain.
-      #
-      # @return [void]
-      def find_outputs
-        tuples = read_all(Tuple[:data].new(domain: @domain))
-        @rule.outputs.each_with_index do |output, i|
-          output = output.eval(@variable_table)
-          case output.modifier
-          when :all
-            @outputs[i] = tuples.find_all {|data| output.match(data.name)}
-          when :each
-            # FIXME
-            @outputs[i] = tuples.find {|data| output.match(data.name)}
+      #   true if we need to write the task into the tuple space
+      def need_to_process_task?(task, order)
+        # reuse task finished result if order is weak update
+        if order == :weak
+          if read!(Tuple[:finished].new(domain: task.domain, status: :succeeded))
+            return false
           end
         end
+        # check task status
+        not(read!(task) or read!(Tuple[:working].new(domain: task.domain)))
       end
 
       # Lift output data from child domains to this domain.
@@ -339,36 +321,71 @@ module Pione
         end
       end
 
+      # Remove finished tuple.
+      #
+      # @param domain [String]
+      #   domain of the finished tuple
+      # @return [void]
+      def remove_finished_tuple(domain)
+        take!(Tuple[:finished].new(domain: domain))
+      end
+
       # Import finished tuple's outputs from the domain.
       #
       # @param [String] task_domain
       #   target task domain
       # @return [void]
-      def import_finished_outputs(task_domain)
-        return if @finished.any?{|t| t.domain == task_domain}
-        if task_domain != @domain
-          template = Tuple[:finished].new(
-            domain: task_domain,
-            status: :succeeded
-          )
-          if finished = read!(template)
-            copy_data_into_domain(finished.outputs, @domain)
+      def update_by_finished_tuple(rule, finished, vtable)
+        finished.outputs.each_with_index do |output, i|
+          data_expr = rule.outputs[i].eval(vtable)
+          case data_expr.operation
+          when :write
+            if output.kind_of?(Array)
+              output.each {|o| copy_data_into_domain(o, @domain)}
+            else
+              copy_data_into_domain(output, @domain)
+            end
+          when :remove
+            if output.kind_of?(Array)
+              output.each {|o| remove_data_from_domain(o, @domain)}
+            else
+              remove_data_from_domain(output, @domain)
+            end
+          when :touch
+            if output.kind_of?(Array)
+              output.each {|o| touch_data_in_domain(o, @domain)}
+            else
+              touch_data_in_domain(output, @domain)
+            end
           end
         end
       end
 
-      # Copy data into specified domain and return the tuple list
-      def copy_data_into_domain(src_data, dest_domain)
-        src_data.flatten.compact.map do |d|
-          new_data = d.clone
-          new_data.domain = dest_domain
-          begin
-            write(new_data)
-          rescue Rinda::RedundantTupleError
-            # ignore
-          end
-          new_data
+      # Copy the data tuple with the specified domain and return the tuple list.
+      #
+      # @param data [DataTuple]
+      #   target data tuple
+      # @param domain [String]
+      #   new domain of the copied data tuple
+      # @return [DataTuple]
+      #   new data tuple with the domain
+      def copy_data_into_domain(data, domain)
+        new_data = data.clone.tap {|x| x.domain = domain}
+        write(new_data)
+        return new_data
+      end
+
+      # Remove the data from the domain.
+      def remove_data_from_domain(data, domain)
+        take!(Tuple[:data].new(name: data.name, domain: domain))
+      end
+
+      def touch_data_in_domain(data, domain)
+        if target = read!(Tuple[:data].new(name: data.name, domain: domain))
+          data = target
         end
+        new_data = data.clone.tap {|x| x.domain = domain; x.time = Time.now}
+        write(new_data)
       end
     end
   end

data/lib/pione/rule-handler/root-handler.rb

@@ -9,11 +9,15 @@ module Pione
       # @api private
       def execute
         # import initial input tuples from input domain
-        copy_data_into_domain(@inputs.flatten, @domain)
+        @inputs.flatten.each do |input|
+          copy_data_into_domain(input, @domain)
+        end
         # execute the rule
         result = super
         # export outputs to output domain
-        copy_data_into_domain(@outputs.flatten, '/output')
+        @outputs.flatten.each do |output|
+          copy_data_into_domain(output, '/output')
+        end
         # substantiate symbolic links
         # substantiate_date
 

data/lib/pione/rule-handler/update-criteria.rb

@@ -0,0 +1,152 @@
+# UpdateCriteria repesents rule application criteria.
+module Pione
+  module RuleHandler
+    module UpdateCriteria
+      class << self
+        # Return true if the rule has no output conditions.
+        #
+        # @param rule [Rule]
+        #   rule
+        # @param inputs [Tuple::Data]
+        #   input data tuples
+        # @param outputs [Tuple::Data]
+        #   output data tuples
+        # @param vtable [VariableTable]
+        #   variable table
+        # @return [Boolean]
+        #   true if the rule has no output conditions
+        def no_output_conditions?(rule, inputs, outputs, vtable)
+          rule.outputs.empty?
+        end
+
+        # Return true if data tuples don't exist against output conditions with
+        # write operation.
+        #
+        # @param rule [Rule]
+        #   rule
+        # @param inputs [Tuple::Data]
+        #   input data tuples
+        # @param outputs [Tuple::Data]
+        #   output data tuples
+        # @param vtable [VariableTable]
+        #   variable table
+        # @return [Boolean]
+        #   true if data tuples don't exist against output conditions with write
+        #   operation
+        def not_exist_output_data?(rule, inputs, outputs, vtable)
+          result = false
+          rule.outputs.each_with_index do |data_expr, i|
+            data_expr = data_expr.eval(vtable)
+            if data_expr.write? or data_expr.touch?
+              case data_expr.distribution
+              when :all
+                if outputs[i].nil? or outputs[i].select{|data| data_expr.match(data.name)}.empty?
+                  result = true
+                end
+              when :each
+                if outputs[i].nil? or not(data_expr.match(outputs[i].name))
+                  result = true
+                end
+              end
+            end
+            break if result
+          end
+          return result
+        end
+
+
+        # Return true if data tuples exist against output conditions with remove
+        # or touch operation.
+        #
+        # @param rule [Rule]
+        #   rule
+        # @param inputs [Tuple::Data]
+        #   input data tuples
+        # @param outputs [Tuple::Data]
+        #   output data tuples
+        # @param vtable [VariableTable]
+        #   variable table
+        # @return [Boolean]
+        #   if data tuples exist against output conditions with remove or touch
+        #   operation
+        def exist_output_data?(rule, inputs, outputs, vtable)
+          result = false
+          rule.outputs.each_with_index do |data_expr, i|
+            data_expr = data_expr.eval(vtable)
+            if data_expr.remove?
+              case data_expr.distribution
+              when :all
+                if not(outputs[i].nil? or outputs[i].select{|data| data_expr.match(data.name)}.empty?)
+                  result = true
+                end
+              when :each
+                if not(outputs[i].nil?) and data_expr.match(outputs[i].name)
+                  result = true
+                end
+              end
+            end
+            break if result
+          end
+          return result
+        end
+
+        # Return true if newer input data exist.
+        #
+        # @param rule [Rule]
+        #   rule
+        # @param inputs [Tuple::Data]
+        #   input data tuples
+        # @param outputs [Tuple::Data]
+        #   output data tuples
+        # @param vtable [VariableTable]
+        #   variable table
+        # @return [Boolean]
+        #   true if newer input data exist
+        def exist_newer_input_data?(rule, inputs, outputs, vtable)
+          # get output oldest time
+          outputs = outputs.select.with_index{|output, i| rule.outputs[i].eval(vtable).care?}
+          output_oldest_time = outputs.flatten.map{|output| output.time}.sort.first
+
+          # get input last time
+          inputs = inputs.select.with_index{|input, i| rule.inputs[i].eval(vtable).care?}
+          input_last_time = inputs.flatten.map{|input| input.time}.sort.last
+
+          #p output_oldest_time
+          #p input_last_time
+
+          # criteria
+          return false unless output_oldest_time
+          return false unless input_last_time
+          return output_oldest_time < input_last_time
+        end
+
+        # Return update order name if we need to update.
+        #
+        # @param [Rule] rule
+        #   rule
+        # @param [Tuple::DataTuple] inputs
+        #   input tuples
+        # @param [Tuple::DataTuple] outputs
+        #   output tuples
+        # @param [VariableTable] vtable
+        #   variable table
+        # @return [Symbol,nil]
+        #   update order or nil
+        def order(rule, inputs, outputs, vtable)
+          if FORCE_UPDATE.any? {|name| self.send(name, rule, inputs, outputs, vtable)}
+            return :force
+          end
+          if WEAK_UPDATE.any? {|name| self.send(name, rule, inputs, outputs, vtable)}
+            return :weak
+          end
+        end
+
+        # force update criteria
+        FORCE_UPDATE = [:no_output_conditions?, :exist_newer_input_data?]
+
+        # update criteria
+        WEAK_UPDATE = [:not_exist_output_data?, :exist_output_data?]
+      end
+    end
+  end
+end

data/lib/pione/rule-handler.rb

@@ -0,0 +1,14 @@
+module Pione
+  # RuleHandler is a name space for rule handling classes.
+  module RuleHandler
+  end
+end
+
+require 'pione/rule-handler/basic-handler'
+require 'pione/rule-handler/update-criteria'
+require 'pione/rule-handler/flow-handler'
+require 'pione/rule-handler/action-handler'
+require 'pione/rule-handler/root-handler'
+require 'pione/rule-handler/system-handler'
+require 'pione/rule-handler/empty-handler'
+

data/lib/pione/system/identifier.rb

@@ -37,19 +37,19 @@ module Pione
       end
       module_function :domain_id
 
-      # Makes target domain name by module name, inputs, and outputs.
-      # @param [String] rule
+      # Make a domain name by rule path, inputs, and parameters.
+      #
+      # @param rule [String]
       #   package name
-      # @param [Array<Pione::Tuple::Data>] inputs
+      # @param inputs [Array<Pione::Tuple::Data>]
       #   input data tuples
-      # @param [Pione::Model::] callee
-      #   parameters object
+      # @param params [Model::Parameters]
+      #   parameters
       # @return [String]
       #   domain string
-      def domain_id3(rule, inputs, callee)
-        package_name = rule.expr.package.name
-        rule_name = rule.expr.name
-        params = callee.expr.params
+      def domain_id3(rule, inputs, params)
+        package_name = rule.rule_expr.package.name
+        rule_name = rule.rule_expr.name
         return domain_id(package_name, rule_name, inputs, params)
       end
       module_function :domain_id3

data/lib/pione/transformer/block-transformer.rb

@@ -19,6 +19,10 @@ module Pione
           x.set_line_and_column(keyword_Action.line_and_column)
         end
       }
+
+      rule(:empty_block => simple(:any)) {
+        EmptyBlock.instance
+      }
     end
   end
 end

data/lib/pione/transformer/expr-transformer.rb

@@ -67,7 +67,7 @@ module Pione
 
       # data null
       rule(:data_null => simple(:obj)) {
-        Model::DataExprNull.instance
+        Model::DataExprNull.instance.to_seq
       }
 
       # Extract the content of +:rule_expr+.

data/lib/pione/transformer/flow-element-transformer.rb

@@ -11,7 +11,6 @@ module Pione
 
       # Transform +:call_rule: as Model::CallRule.
       rule(:call_rule => subtree(:rule_expr)) {
-        TypeRuleExpr.check(rule_expr)
         CallRule.new(rule_expr)
       }
 
@@ -21,7 +20,10 @@ module Pione
              :if_true_elements => sequence(:if_true),
              :if_else_block => simple(:if_false)
            }) {
-        block = { Model::PioneBoolean.true => Model::FlowBlock.new(*if_true) }
+        block = {
+          Model::BooleanSequence.new([Model::PioneBoolean.true]) =>
+          Model::FlowBlock.new(*if_true)
+        }
         block[:else] = if_false if if_false
         Model::ConditionalBlock.new(condition, block)
       }

data/lib/pione/transformer/literal-transformer.rb

@@ -9,6 +9,7 @@ module Pione
         val = (s == "true")
         Model::PioneBoolean.new(val).tap do |x|
           x.set_line_and_column(s.line_and_column)
+          break Model::BooleanSequence.new([x])
         end
       end
 
@@ -18,13 +19,21 @@ module Pione
         val = s.str.gsub(/\\(.)/){$1}
         Model::PioneString.new(val).tap do |x|
           x.set_line_and_column(s.line_and_column)
+          break Model::StringSequence.new([x])
         end
       end
 
+      # NOTE: how do we get the position of empty string?
+      rule(:string => sequence(:empty)) do
+        # convert backslash notations
+        Model::StringSequence.new([Model::PioneString.new("")])
+      end
+
       # Transform +:integer+ as Model::PioneInteger.
       rule(:integer => simple(:i)) do
         Model::PioneInteger.new(i.to_i).tap do |x|
           x.set_line_and_column(i.line_and_column)
+          break Model::IntegerSequence.new([x])
         end
       end
 
@@ -32,6 +41,7 @@ module Pione
       rule(:float => simple(:f)) do
         Model::PioneFloat.new(f.to_f).tap do |x|
           x.set_line_and_column(f.line_and_column)
+          break Model::FloatSequence.new([x])
         end
       end
 
@@ -48,6 +58,7 @@ module Pione
         val = name.str.gsub(/\\(.)/){$1}
         Model::DataExpr.new(val).tap do |x|
           x.set_line_and_column(name.line_and_column)
+          break Model::DataExprSequence.new([x])
         end
       end
 
@@ -63,7 +74,7 @@ module Pione
       rule(:rule_name => simple(:name)) do
         package = Package.new(Thread.current[:current_package_name])
         val = name.str
-        RuleExpr.new(package, val, Parameters.empty, TicketExpr.empty, TicketExpr.empty).tap do |x|
+        RuleExpr.new(package, val).tap do |x|
           x.set_line_and_column(name.line_and_column)
         end
       end
@@ -71,9 +82,9 @@ module Pione
       # ticket
       rule(:ticket => simple(:name)) do
         line_and_column = name.line_and_column
-        TicketExpr.new([name.to_s]) do
+        TicketExpr.new(name.to_s) do
           set_line_and_column(line_and_column)
-        end
+        end.to_seq
       end
     end
   end