pione 0.1.2 → 0.1.3

data/lib/pione/rule-handler/flow-handler.rb

@@ -1,30 +1,29 @@
 module Pione
   module RuleHandler
-    # FlowHandler represents a handler for flow actions.
+    # FlowHandler is a rule handler for flow elements.
     class FlowHandler < BasicHandler
       def self.message_name
         "Flow"
       end
 
-      # @api private
       def initialize(*args)
         super
         @data_finder = DataFinder.new(tuple_space_server, @domain)
         @finished = []
       end
 
-      # Starts to process flow elements. This processes rule application, output
-      # search, resource shift, and output validation.
+      # Start to process flow elements.
+      #
       # @return [void]
       def execute
-        # restore data tuples from resource
-        restore_data_tuples_from_resource
+        # restore data tuples from domain_location
+        restore_data_tuples_from_domain_location
         # rule application
         apply_rules(@rule.body.eval(@variable_table).elements)
         # find outputs
         find_outputs
-        # shift resource
-        shift_output_resources
+        # lift output data from child domains to this domain
+        lift_output_data
         # check output validation
         validate_outputs
 
@@ -33,45 +32,51 @@ module Pione
 
       private
 
-      # Restores data tuples from the domain resource.
+      # Restore data tuples from the domain location. This reads files in the
+      # location and write it as data tuples.
+      #
       # @return [void]
-      def restore_data_tuples_from_resource
-        uri = root? ? @base_uri : @base_uri + (".%s/%s" % @domain.split("_"))
-        if Resource[uri].exist?
-          Resource[uri].entries.each do |res|
-            write(Tuple[:data].new(@domain, res.basename, res.uri.to_s, res.mtime))
+      def restore_data_tuples_from_domain_location
+        dir = root? ? @base_location : @base_location + (".%s/%s" % @domain.split("_"))
+        if dir.exist?
+          dir.entries.each do |location|
+            unless location.basename[0] == "."
+              write(Tuple[:data].new(@domain, location.basename, location, location.mtime))
+            end
           end
         end
       end
 
-      # Applies target input data to rules.
-      # @param [Array<CallRule>] callees
+      # Apply input data to rules.
+      #
+      # @param callees [Array<CallRule>]
       #   elements of call rule lines
       # @return [void]
       def apply_rules(callees)
         user_message_begin("Start Rule Application: %s" % handler_digest)
 
-        # apply flow-element rules
         while true do
           # find updatable rule applications
           applications = select_updatables(find_applicable_rules(callees))
 
-          unless applications.empty?
-            # push task tuples into tuple space
-            distribute_tasks(applications)
-          else
-            break # finish applications
-          end
+          # check wheather applications completed
+          break if applications.empty?
+
+          # push tasks into tuple space
+          distribute_tasks(applications)
         end
 
         user_message_end("End Rule Application: %s" % handler_digest)
       end
 
-      # Finds applicable flow-element rules with inputs and variables.
+      # Find applicable rules with inputs and variables.
+      #
+      # @param callees [Array<CallRule>]
+      #   callee rules
       def find_applicable_rules(callees)
         callees = callees.inject([]) do |list, callee|
-          # evaluate callee expr by handling rule context
-          # and expand compositional rule expressions as simple rule expressions
+          # evaluate callee by handling rule context and expand compositional
+          # rule expressions as simple rule expressions
           list + callee.eval(@variable_table).expr.to_set.to_a.map{|expr| CallRule.new(expr)}
         end
 
@@ -81,14 +86,7 @@ module Pione
           # check if tickets exist in the domain
           names = callee.expr.input_ticket_expr.names
           if not(names.empty?)
-            if names.all? do |name|
-              begin
-                read0(Tuple[:ticket].new(@domain, name))
-                true
-              rescue Rinda::RequestExpiredError
-                false # when the ticket doesn't exist
-              end
-            end
+            if names.all? {|name| read!(Tuple[:ticket].new(@domain, name))}
               target = callee
             end
           else
@@ -99,7 +97,7 @@ module Pione
 
         callees.inject([]) do |combinations, callee|
           # find callee rule
-          rule = find_callee_rule(callee)
+          rule = find_callee_rule_tuple(callee).content
 
           # update callee parameters
           @variable_table.variables.each do |var|
@@ -130,23 +128,22 @@ module Pione
         end
       end
 
-      # Finds the rule of the callee.
-      def find_callee_rule(callee)
-        return read(Tuple[:rule].new(rule_path: callee.rule_path), 0).content
-      rescue Rinda::RequestExpiredError
-        debug_message "Request loading a rule #{callee.rule_path}"
-        write(Tuple[:request_rule].new(callee.rule_path))
-        tuple = read(Tuple[:rule].new(rule_path: callee.rule_path))
-
-        # check whether known or unknown
-        if tuple.status == :known
-          return tuple.content
+      # Find the rule tuple of the callee.
+      #
+      # @param callee [CallRule]
+      #   callee rule
+      # @return [RuleTuple]
+      #   rule tuple
+      def find_callee_rule_tuple(callee)
+        if rule = read!(Tuple[:rule].new(rule_path: callee.rule_path))
+          return rule
         else
-          raise UnknownRule.new(callee.rule_path)
+          write(Tuple[:request_rule].new(callee.rule_path))
+          return read(Tuple[:rule].new(rule_path: callee.rule_path))
         end
       end
 
-      # Finds inputs and variables for flow element rules.
+      # Find inputs and variables for flow element rules.
       def select_updatables(combinations)
         combinations.select do |callee, rule, inputs, vtable|
           # task domain
@@ -172,7 +169,8 @@ module Pione
         end
       end
 
-      # Distributes tasks.
+      # Distribute tasks.
+      #
       # @param [Array] applications
       #   application informations
       # @return [void]
@@ -180,6 +178,9 @@ module Pione
         user_message_begin("Start Task Distribution: %s" % handler_digest)
         canceled = []
 
+        process_log(@task_process_record.merge(transition: "suspend"))
+        process_log(@rule_process_record.merge(transition: "suspend"))
+
         applications.uniq.each do |callee, rule, inputs, vtable|
           # task domain
           task_domain = ID.domain_id3(rule, inputs, callee)
@@ -196,13 +197,24 @@ module Pione
 
           # check if same task exists
           begin
-            if need_task?(task)
+            if need_to_process_task?(task)
               # copy input data from the handler domain to task domain
               copy_data_into_domain(inputs, task_domain)
 
               # write the task
               write(task)
 
+              # put task schedule process log
+              task_process_record = Log::TaskProcessRecord.new.tap do |record|
+                record.name = task.digest
+                record.rule_name = rule.rule_path
+                record.rule_type = rule.rule_type
+                record.inputs = inputs.flatten.map{|input| input.name}.join(",")
+                record.parameters = callee.expr.params.textize
+                record.transition = "schedule"
+              end
+              process_log(task_process_record)
+
               msg = "distributed task %s on %s" % [task.digest, handler_digest]
               user_message(msg, 1)
 
@@ -243,97 +255,92 @@ module Pione
           end
         end
 
+        process_log(@rule_process_record.merge(transition: "resume"))
+        process_log(@task_process_record.merge(transition: "resume"))
         user_message_end("End Task Distribution: %s" % handler_digest)
       end
 
-      # Returns true if we need to write the task into the tuple space.
-      # @param [Task] task
+      # Return true if we need to write the task into the tuple space.
+      #
+      # @param [TaskTuple] task
       #   task tuple
       # @return [Boolean]
       #   true if we need to write the task into the tuple space
-      def need_task?(task)
-        not(exist_task?(task) or working?(task))
+      def need_to_process_task?(task)
+        not(read!(task) or working?(task))
       end
 
-      # Returns true if the task exists in the tuple space already.
-      # @param [Task] task
-      #   task tuple
-      # @return [Boolean]
-      #   true if the task exists in the tuple space already
-      def exist_task?(task)
-        read0(task)
-        return true
-      rescue Rinda::RequestExpiredError
-        return false
-      end
-
-      # Returns true if any task worker is working on the task.
-      # @param [Task] task
+      # Return true if any task worker is working on the task.
+      #
+      # @param task [TaskTuple]
       #   task tuple
       # @return [Boolean]
       #   true if any task worker is working on the task
       def working?(task)
-        read0(Tuple[:working].new(domain: task.domain))
-        return true
-      rescue Rinda::RequestExpiredError
-        return false
+        read!(Tuple[:working].new(domain: task.domain))
       end
 
       # Find outputs from the domain.
+      #
       # @return [void]
       def find_outputs
+        tuples = read_all(Tuple[:data].new(domain: @domain))
         @rule.outputs.each_with_index do |output, i|
           output = output.eval(@variable_table)
-          tuples = read_all(Tuple[:data].new(domain: @domain))
           case output.modifier
           when :all
             @outputs[i] = tuples.find_all {|data| output.match(data.name)}
           when :each
+            # FIXME
             @outputs[i] = tuples.find {|data| output.match(data.name)}
           end
         end
       end
 
-      # Shifts output resource locations.
+      # Lift output data from child domains to this domain.
+      #
       # @return [void]
-      def shift_output_resources
-        # cheker for double shift
-        shifted = []
-        # shift output resources
-        @outputs.flatten.compact.each do |output|
-          old_uri = URI.parse(output.uri)
-          new_uri = URI.parse(make_output_resource_uri(output.name).to_s)
-          unless new_uri.path == old_uri.path or shifted.include?(old_uri.path)
-            # shift resource
-            Resource[new_uri].shift_from(Resource[old_uri])
-            # shift cache if the old is cached in this machine
-            FileCache.shift(old_uri, new_uri)
-            # write shift tuple
-            write(Tuple[:shift].new(old_uri.to_s, new_uri.to_s))
-            # push moved path
-            shifted << old_uri.path
+      def lift_output_data
+        @outputs.flatten.compact.inject([]) do |lifted, output|
+          old_location = output.location
+          new_location = make_output_location(output.name)
+          unless new_location == old_location or lifted.include?(old_location)
+            # move data from old to new
+            old_location.move(new_location)
+            # sync cache if the old is cached in this machine
+            FileCache.sync(old_location, new_location)
+            # write lift tuple
+            write(Tuple[:lift].new(old_location, new_location))
+            # push history
+            lifted << old_location
           end
+          lifted
         end
       end
 
-      # Validates outputs size.
+      # Validate outputs size.
       def validate_outputs
         # size check
         if @rule.outputs.size > 0 and not(@rule.outputs.size == @outputs.size)
           raise RuleExecutionError.new(self)
         end
+
         # nil check
         if @outputs.any?{|tuple| tuple.nil?}
           raise RuleExecutionError.new(self)
         end
 
         # empty list check
-        if @outputs.any?{|tuple| tuple.kind_of?(Array) && tuple.empty?}
-          raise RuleExecutionError.new(self)
+        @outputs.each_with_index do |tuple, i|
+          output = @rule.outputs[i].eval(@variable_table)
+          if tuple.kind_of?(Array) && tuple.empty? && not(output.accept_nonexistence?)
+            raise RuleExecutionError.new(@outputs.inspect)
+          end
         end
       end
 
-      # Imports finished tuple's outputs from the domain.
+      # Import finished tuple's outputs from the domain.
+      #
       # @param [String] task_domain
       #   target task domain
       # @return [void]
@@ -344,11 +351,10 @@ module Pione
             domain: task_domain,
             status: :succeeded
           )
-          finished = read0(template)
-          copy_data_into_domain(finished.outputs, @domain)
+          if finished = read!(template)
+            copy_data_into_domain(finished.outputs, @domain)
+          end
         end
-      rescue Rinda::RequestExpiredError
-        # ignore
       end
 
       # Copy data into specified domain and return the tuple list

data/lib/pione/rule-handler/root-handler.rb

@@ -14,9 +14,20 @@ module Pione
         result = super
         # export outputs to output domain
         copy_data_into_domain(@outputs.flatten, '/output')
+        # substantiate symbolic links
+        # substantiate_date
 
         return result
       end
+
+      # Substantiate symbolic links to files.
+      def substantiate_date
+        @outputs.flatten.compact.each do |output|
+          if output.location.cached? and output.link?
+            FileCache.get(output.location).turn(output.location)
+          end
+        end
+      end
     end
   end
 end

data/lib/pione/system/common.rb

@@ -18,5 +18,15 @@ module Pione
       Signal.trap(:INT, finalizer)
     end
     module_function :set_signal_trap
+
+    # Return temporary path of the file.
+    #
+    # @param filename [String]
+    #   the file's name
+    # @return [Pathname]
+    #  temporary path of the file
+    def temporary_path(filename)
+      Global.temporary_directory + filename
+    end
   end
 end

data/lib/pione/system/file-cache.rb

@@ -2,76 +2,124 @@ module Pione
   module System
     # FileCache is a caching system for task workers.
     module FileCache
-      # Returns file cache class.
+      # Return file cache class.
+      #
       # @return [Class]
       #   CacheMethod class
       def self.cache_method
         @klass || SimpleCacheMethod
       end
 
-      # Sets a file cache class.
-      # @param [Class] klass
+      # Set a file cache class.
+      #
+      # @param klass [Class]
       #   CacheMethod class
       # @return [void]
       def self.set_cache_method(klass)
         @klass = klass
       end
 
-      # Returns the singleton.
+      # Return the singleton.
+      #
       # @return [CacheMethod]
       #   cache method instance
       def self.instance
         @instance ||= cache_method.new
       end
 
-      # Gets cached data path from the uri resource.
-      # @param [String] uri
-      #   uri to get
-      # @return [String]
-      #   cached path string
-      def self.get(uri)
-        instance.get(uri)
+      # Get cached data path from the uri resource.
+      #
+      # @param location [BasicLocation]
+      #   data location
+      # @return [Pathname]
+      #   cached path
+      def self.get(location)
+        instance.get(location)
       end
 
-      # Puts the data to uri resource and caches it.
-      def self.put(src, uri)
-        instance.put(src, uri)
+      # Put the data to the location and caches it.
+      #
+      # @param src [String]
+      #   source path
+      # @param location [BasicLocation]
+      #   destination location
+      # @return [void]
+      def self.put(src, location)
+        instance.put(src, location)
+      end
+
+      # Synchronize cache of old location with new location.
+      #
+      # @param old_location [BasicLocation]
+      #   old data location
+      # @param new_location [BasicLocation]
+      #   new data location
+      # @return [void]
+      def self.sync(old_location, new_location)
+        instance.sync(old_location, new_location)
+      end
+
+      # Return true if the location is cached.
+      #
+      # @return [Boolean]
+      #   true if the location is cached
+      def self.cached?(location)
+        instance.cached?(location)
       end
 
-      # Shifts the resource from old uri to new uri.
-      # @param [String] old_uri
-      #   old resource uri
-      # @param [String] new_uri
-      #   new resource uri
+      # Clear cache.
+      #
       # @return [void]
-      def self.shift(old_uri, new_uri)
-        instance.shift(old_uri, new_uri)
+      def self.clear
+        instance.clear
       end
 
       # FileCache is an interface class of cache methods.
       class FileCacheMethod
-        # Gets the cache location path of the URI.
-        # @param [String] uri
-        #   resource uri
-        # @return [String]
+        # Get the cache path of the location.
+        #
+        # @param location [BasicLocation]
+        #   data location
+        # @return [BasicLocalLocation]
         #   cached path
-        def get(uri)
+        def get(location)
           raise NotImplementedError
         end
 
-        # Puts the file into cache with the URI.
+        # Put and cache the source data at the location.
+        #
+        # @param src [BasicLocation]
+        #   source path
+        # @param location [BasicLocation]
+        #   destination
         # @return [void]
-        def put(src, uri)
+        def put(src, location)
+          raise NotImplementedError
+        end
+
+        # Synchronize cache of old location with new location.
+        #
+        # @param old_location [BasicLocation]
+        #   old data location
+        # @param new_location [BasicLocation]
+        #   new data location
+        # @return [void]
+        def sync(old_location, new_location)
+          raise NotImplementedError
+        end
+
+        # Return true if the location is cached.
+        #
+        # @return [Boolean]
+        #   true if the location is cached
+        def cached?(location)
           raise NotImplementedError
         end
 
-        # Shitfs the URI.
-        # @param [String] old_uri
-        #   old resource uri
-        # @param [String] new_uri
-        #   new resource uri
+        # Clear cache.
+        #
         # @return [void]
-        def shift(old_uri, new_uri)
+        def clear
           raise NotImplementedError
         end
       end
@@ -84,65 +132,36 @@ module Pione
           @tmpdir = Global.file_cache_directory
         end
 
-        # Gets cached data path from the uri resource.
-        # @param [String] uri
-        #   resource uri
-        # @return [String]
-        #   cached path
-        def get(uri)
-          # check cached or not
-          unless @table.has_key?(uri)
-            # prepare cache path
-            path = prepare_cache_path
-
-            # link the resource file to cache path
-            Resource[uri].link_to(path)
-            @table[uri.to_s] = path
+        def get(location)
+          # cache if record doesn't exist
+          unless @table.has_key?(location)
+            cache_location = Location[Temppath.create(nil, @tmpdir)]
+            location.turn(cache_location)
+            @table[location] = cache_location
           end
-
-          return @table[uri.to_s]
+          unless @table[location].exist?
+            location.turn(@table[location])
+          end
+          return @table[location]
         end
 
-        # Puts the data to uri resource and caches it in local.
-        # @return [void]
-        def put(src, uri)
-          # prepare cache path
-          path = prepare_cache_path
-
-          # move the file from the working directory to cache
-          FileUtils.mv(src, path)
-
-          # make a symbolic link from original location to the cache
-          FileUtils.symlink(path, src)
-
-          # copy from cache to the resource file
-          @table[uri.to_s] = path
-          Resource[uri].link_from(path)
+        def put(src, dest)
+          # make cache from source and link between cache and destination
+          @table[dest] = get(src).tap{|cache_location| cache_location.turn(dest)}
         end
 
-        # @param [String] old_uri
-        #   old resource uri
-        # @param [String] new_uri
-        #   new resource uri
-        # @return [void]
-        def shift(old_uri, new_uri)
-          if path = @table[old_uri.to_s]
-            if new_uri.scheme == "local"
-              FileUtils.symlink(new_uri.path, path, :force => true)
-            end
-            @table[new_uri.to_s] = path
+        def sync(old_location, new_location)
+          if cache_location = @table[old_location]
+            @table[new_location] = cache_location
           end
         end
 
-        private
+        def cached?(location)
+          @table.has_key?(location)
+        end
 
-        # Makes new cache path.
-        # @api private
-        def prepare_cache_path
-          cache = Tempfile.new("", Global.file_cache_directory)
-          path = cache.path
-          cache.close(true)
-          return path
+        def clear
+          @table.clear
         end
       end
     end