pione 0.1.4 → 0.2.0

data/lib/pione/location/basic-location.rb

@@ -51,9 +51,17 @@ module Pione
           @scheme = name
           SCHEMES[name] = self
         end
+
+        def set_real_appendable(b)
+          @appendable = b
+        end
+
+        def real_appendable?
+          @appendable
+        end
       end
 
-      forward :class, :scheme
+      forward! :class, :scheme, :real_appendable?
       forward :@uri, :host
 
       # @return [URI]
@@ -81,7 +89,7 @@ module Pione
       # @return [BasicLocation]
       #   new location
       def +(name)
-        self.class.new(@uri.as_directory + name)
+        self.class.new(@uri.as_directory + name.to_s)
       end
 
       # Create new location that has URI as a directory.
@@ -125,11 +133,25 @@ module Pione
         System::FileCache.cached?(self)
       end
 
-      # Creates a location.
+      # Write a data into the location.
+      #
+      # @param data [String]
+      #   data content
+      # @return [void]
+      def write(data)
+        if exist?
+          update(data)
+        else
+          create(data)
+        end
+      end
+
+      # Creates a file at the location. If a file exists at the location aleady,
+      # it raises an exception.
       #
       # @param data [String]
       #   data content
-      # @return[void]
+      # @return [void]
       def create(data)
         raise NotImplementedError
       end
@@ -183,14 +205,32 @@ module Pione
         raise NotImplementedError
       end
 
-      # Return entries of the resource path.
+      # Return entries of the location.
       #
       # @return [Array<Location>]
-      #   entries of the location path
+      #   entries of the location
       def entries
         raise NotImplementedError
       end
 
+      # Return file entries of the location.
+      #
+      # @return [Array<Location>]
+      #    file entries of the location
+      def file_entries
+        entries.select{|entry| entry.file?}
+      end
+
+      # Return directory entries of the location.
+      #
+      # @return [Array<Location>]
+      #    directory entries of the location
+      def directory_entries
+        entries.select do |entry|
+          entry.directory? and not(entry.path.basename == "." or entry.path.basename == "..")
+        end
+      end
+
       # Return true if there is data in the location.
       #
       # @return [Boolean]
@@ -233,7 +273,9 @@ module Pione
         raise NotImplementedError
       end
 
-      # Link to the destination.
+      # Link to the destination. If the location scheme is same to destination,
+      # create link by a symbolic link or lightweight copy method. If not, copy
+      # it simply.
       #
       # @param dest [BasicLocation]
       #   destination

data/lib/pione/location/ftp-location.rb

@@ -3,6 +3,15 @@ module Pione
     # FTPLocation represents locations on FTP server.
     class FTPLocation < BasicLocation
       set_scheme "ftp"
+      set_real_appendable false
+
+      # for myftp scheme
+      SCHEMES["myftp"] = self
+
+      def initialize(uri)
+        uri = uri.to_ftp_scheme if uri.scheme == "myftp"
+        super(uri)
+      end
 
       def rebuild(path)
         scheme = @uri.scheme
@@ -19,15 +28,19 @@ module Pione
         else
           connect do |ftp|
             makedirs(ftp, @path.dirname)
-            file = Temppath.create
-            file.open("w") {|f| f.write(data)}
-            ftp.put(file.to_s, @path.to_s)
+            path = Temppath.create
+            Location[path].create(data)
+            ftp.put(path, @path.to_s)
           end
         end
       end
 
       def append(data)
-        exist? ? update(read + data) : create(data)
+        if exist?
+          update(read + data)
+        else
+          create(data)
+        end
       end
 
       def read
@@ -70,7 +83,7 @@ module Pione
         connect do |ftp|
           ftp.nlst(@path.to_s).map do |entry|
             rebuild(@path + entry)
-          end.select {|entry| entry.file?}
+          end
         end
       end
 
@@ -107,7 +120,7 @@ module Pione
 
       def move(dest)
         if dest.scheme == scheme and dest.host == host
-          ftp.rename(@path.to_s, dest.path.to_s)
+          connect{|ftp| ftp.rename(@path.to_s, dest.path.to_s)}
         else
           copy(dest)
           delete

data/lib/pione/location/local-location.rb

@@ -3,6 +3,7 @@ module Pione
     # LocalLocation represents local disk locations.
     class LocalLocation < BasicLocation
       set_scheme "local"
+      set_real_appendable true
 
       def initialize(uri)
         super(uri.absolute)
@@ -19,13 +20,13 @@ module Pione
           raise ExistAlready.new(self)
         else
           @path.dirname.mkpath unless @path.dirname.exist?
-          @path.open("w+"){|f| f.write(data)}
+          @path.open("w"){|f| f.write(data)}
         end
       end
 
       def append(data)
         if exist?
-          update(read + data)
+          @path.open("a"){|f| f.write(data)}
         else
           create(data)
         end
@@ -37,14 +38,20 @@ module Pione
 
       def update(data)
         if @path.exist?
-          @path.open("w+"){|file| file.write(data)}
+          @path.open("w"){|file| file.write(data)}
         else
           raise NotFound.new(@uri)
         end
       end
 
       def delete
-        @path.delete if @path.exist?
+        if @path.exist?
+          if @path.file?
+            @path.delete
+          else
+            FileUtils.remove_entry_secure(@path)
+          end
+        end
       end
 
       def mtime
@@ -56,7 +63,9 @@ module Pione
       end
 
       def entries
-        @path.entries.select{|entry| (@path + entry).file?}.map do |entry|
+        @path.entries.select do |entry|
+          not(entry.to_s == "." or entry.to_s == "..")
+        end.map do |entry|
           Location["local:%s" % (@path + entry).expand_path]
         end
       rescue Errno::ENOENT
@@ -79,7 +88,7 @@ module Pione
         raise NotFound.new(self) unless exist?
         if dest.kind_of?(LocalLocation)
           dest.path.dirname.mkpath unless dest.path.dirname.exist?
-          FileUtils.mv(@path, dest.path)
+          FileUtils.mv(@path, dest.path, force: true)
         else
           copy(dest)
           delete

data/lib/pione/log.rb

@@ -4,7 +4,10 @@ module Pione
   end
 end
 
+require 'pione/log/message-log'
 require 'pione/log/process-record'
 require 'pione/log/process-log'
 require 'pione/log/xes-log'
-require 'pione/log/domain-info'
+require 'pione/log/domain-log'
+require 'pione/log/system-log'
+

data/lib/pione/log/{domain-info.rb → domain-log.rb}

@@ -1,7 +1,13 @@
 module Pione
   module Log
-    # DomainInfo is a domain log that records domain inputs and parameters.
-    class DomainInfo
+    # DomainLog is a domain log that records domain inputs and parameters.
+    class DomainLog
+      include Sys # for Uname
+
+      FILENAME = ".domain.log"
+
+      forward_as_key! :@record, :system_name, :system_nodename, :system_machine, :system_version, :system_release
+
       # @return [Location::BasicLocation]
       #   domain's location
       attr_reader :domain_location
@@ -15,15 +21,21 @@ module Pione
       def initialize(handler)
         @domain_location = handler.domain_location
         @record = {
-          "uname" => `uname -a`.chomp,
-          "params" => handler.params.textize,
-          "original_params" => handler.original_params.textize,
-          "inputs" => inputs_string(handler.inputs),
-          "domain" => handler.domain,
-          "domain_location" => @domain_location.uri.to_s,
-          "task_id" => handler.task_id.to_s,
-          "dry_run" => handler.dry_run.to_s
+          :system_name => Uname.sysname,
+          :system_nodename => Uname.nodename,
+          :system_machine => Uname.machine,
+          :system_version => Uname.version,
+          :system_release => Uname.release,
+          :params => handler.params.textize,
+          :original_params => handler.original_params.textize,
+          :inputs => inputs_string(handler.inputs),
+          :domain => handler.domain,
+          :domain_location => @domain_location.uri.to_s,
+          :dry_run => handler.dry_run.to_s
         }
+        if handler.respond_to?(:working_directory)
+          @record[:working_directory] = handler.working_directory.uri.to_s
+        end
       end
 
       # Save domain information file.
@@ -33,7 +45,7 @@ module Pione
         text = "== %s\n\n" % Time.now
         text << @record.map{|key, val| "- %s: %s" % [key,val]}.join("\n")
         text << "\n\n"
-        (@domain_location + ".domain_info").append(text)
+        (@domain_location + FILENAME).append(text)
       end
 
       private

data/lib/pione/{util/console-message.rb → log/message-log.rb}

@@ -1,11 +1,11 @@
 module Pione
-  module Util
+  module Log
     # ConsoleMessage is a set of utility methods for sending messages to user.
-    module ConsoleMessage
+    module MessageLog
       # @api private
       MESSAGE_QUEUE = Queue.new
 
-      # message queue thread
+      # Message queue thread
       Thread.new {
         while msg = MESSAGE_QUEUE.pop
           puts msg
@@ -93,12 +93,11 @@ module Pione
       # @param type [String]
       #   message heading type
       # @return [void]
-      def debug_message(msg, level=0, type="debug")
+      def debug_message(msg, level=0, head="debug")
         if debug_mode? and not(quiet_mode?)
-          message(type, :magenta, "  "*level + msg)
+          message(:debug, head, :magenta, "  "*level + msg)
         end
       end
-      module_function :debug_message
 
       # Send the debug message to notify that something begins.
       #
@@ -127,9 +126,9 @@ module Pione
       # @param type [String]
       #   message heading type
       # @return [void]
-      def user_message(msg, level=0, type="info")
+      def user_message(msg, level=0, head="info", color=:green)
         if not(quiet_mode?)
-          message(type, :green, "  "*level + msg)
+          message(:info, head, color, msg, level)
         end
       end
 
@@ -138,8 +137,8 @@ module Pione
       # @param msg [String]
       #   user message
       # @return [void]
-      def user_message_begin(msg)
-        user_message(msg, 0, ">>>")
+      def user_message_begin(msg, level=0)
+        user_message(msg, level, "-->")
       end
 
       # Send the debug message to notify that something ends.
@@ -147,8 +146,8 @@ module Pione
       # @param [String] msg
       #   debug message
       # @return [void]
-      def user_message_end(msg)
-        user_message(msg, 0, "<<<")
+      def user_message_end(msg, level=0)
+        user_message(msg, level, "<--")
       end
 
       # Show the message.
@@ -161,7 +160,7 @@ module Pione
       #
       # @api private
       def show(msg)
-        message("show", :red, msg)
+        message(:debug, "show", :red, msg)
       end
 
       # Print the message with the color.
@@ -174,8 +173,10 @@ module Pione
       #   message content
       #
       # @api private
-      def message(type, color, msg)
-        MESSAGE_QUEUE.push "%s %s" % [Terminal.color(color, "%5s" % type), msg]
+      def message(type, head, color, msg, level=0)
+        write(Tuple[:message].new(type: type, head: head, color: color, contents: msg, level: level))
+      rescue NoMethodError
+        MESSAGE_QUEUE.push "%s%s %s" % ["  "*level, ("%5s" % head).color(color), msg]
       end
       module_function :message
     end

data/lib/pione/log/process-log.rb

@@ -1,6 +1,6 @@
 module Pione
   module Log
-    # ProcessLog represents process log records of PIONE.
+    # ProcessLog represents process log file.
     class ProcessLog
       # formatter table
       @format = {}
@@ -29,13 +29,25 @@ module Pione
         #
         # @param location [Location]
         #   path of process log file
-        # @return [ProcessLogFile]
-        #   log object
+        # @return [Hash{String => ProcessLog}]
+        #   pairs of log id and the log
         def read(location)
-          if location.file?
-            read_file(location)
-          else
-            read_directory(location)
+          cache = location
+          unless location.scheme == "local"
+            cache = Location[Temppath.create]
+            cache.create(location.read)
+          end
+          records = cache.path.each_line.map do |line|
+            JSON.parse(line).inject({}) do |data, pair|
+              data[pair[0].to_sym] = pair[1]
+              data
+            end.tap do |table|
+              break ProcessRecord.build(table)
+            end
+          end
+          group_by(:log_id, records).inject({}) do |table, pair|
+            key, _records = pair
+            table[key] = new(_records)
           end
         end
 
@@ -48,38 +60,14 @@ module Pione
 
         attr_reader :filter_block
 
-        private
-
-        # Read records from the log file at the location.
+        # Return the record table grouped by the key.
         #
-        # @param location [Location::BasicLocation]
-        #   log location
-        def read_file(location)
-          cache = Location[Temppath.create]
-          cache.create(location.read)
-          cache.path.each_line.map do |line|
-            JSON.parse(line).inject({}) do |data, pair|
-              data[pair[0].to_sym] = pair[1]
-              data
-            end.tap do |table|
-              break ProcessRecord.build(table)
-            end
-          end.tap do |records|
-            break new([ProcessLogBundle.new(records)])
-          end
-        end
-
-        # Read records from log files in directory at the location.
-        #
-        # @param location [Location::BasicLocation]
-        #   log directory location
-        def read_directory(location)
-          location.entries.inject(new([])) do |formatter, entry|
-            if /pione_\d{14}\.log/.match(entry.path.basename.to_s)
-              new(formatter.bundles + read_file(entry).bundles)
-            else
-              formatter
-            end
+        # @return [Hash{String => Array<ProcessRecord>}]
+        #   grouping records table
+        def group_by(key, records)
+          records.inject({}) do |table, record|
+            table[record.send(key)] ||= []
+            table.tap {|x| x[record.send(key)] << record}
           end
         end
       end
@@ -93,7 +81,13 @@ module Pione
       # @param records [Array<ProcessRecord>]
       #   log records
       def initialize(records)
-        @records = records.select {|record| self.class.filter_block.call(record)}
+        @records = records.select do |record|
+          if block = self.class.filter_block
+            block.call(record)
+          else
+            true
+          end
+        end
       end
 
       # Return the record table grouped by the key.
@@ -101,26 +95,10 @@ module Pione
       # @return [Hash{String => Array<ProcessRecord>}]
       #   grouping records table
       def group_by(key)
-        @records.inject({}) do |table, record|
-          table[record.send(key)] ||= []
-          table.tap {|x| x[record.send(key)] << record}
-        end
-      end
-    end
-
-    # ProcessLogFormatter is a basic class for fomatting process logs.
-    class ProcessLogFormatter < ProcessLog
-      # @return [Array<ProcessBunle>]
-      #   target logs that we format
-      attr_reader :bundles
-
-      # @param bundles [Array<ProcessLogBundle>]
-      #   log bundles
-      def initialize(bundles)
-        @bundles = bundles
+        self.class.group_by(key, @records)
       end
 
-      # Format bundle logs.
+      # Format records.
       #
       # @return [String]
       #   result string
@@ -129,35 +107,6 @@ module Pione
       end
     end
 
-    # ProcessLogFormatError is raised when formatter cannot format some objects.
-    class ProcessLogFormatError < StandardError
-      # @param object [Object]
-      #   the object that we cannnot format
-      def initialize(object)
-        @object = object
-      end
-
-      # @api private
-      def message
-        "not formattable: %s" % @object.inspect
-      end
-    end
-
-    # ProcessLogBundle is a bundle of raw logs.
-    class ProcessLogBundle
-      attr_reader :agent_activity_log
-      attr_reader :rule_process_log
-      attr_reader :task_process_log
-
-      # @param records [Array<ProcessRecord>]
-      #   log records
-      def initialize(records)
-        @agent_activity_log = AgentActivityLog.new(records)
-        @rule_process_log = RuleProcessLog.new(records)
-        @task_process_log = TaskProcessLog.new(records)
-      end
-    end
-
     # AgentActivityLog is a set of records written about agent activities.
     class AgentActivityLog < ProcessLog
       set_filter {|record| record.type == :agent_activity}