pione 0.1.2 → 0.1.3

data/lib/pione/location/local-location.rb

@@ -0,0 +1,116 @@
+module Pione
+  module Location
+    # LocalLocation represents local disk locations.
+    class LocalLocation < BasicLocation
+      set_scheme "local"
+
+      def initialize(uri)
+        super(uri.absolute)
+      end
+
+      def rebuild(path)
+        scheme = @uri.scheme
+        path = path.expand_path.to_s
+        Location["%s:%s" % [scheme, path]]
+      end
+
+      def create(data)
+        if @path.exist?
+          raise ExistAlready.new(self)
+        else
+          @path.dirname.mkpath unless @path.dirname.exist?
+          @path.open("w+"){|f| f.write(data)}
+        end
+      end
+
+      def append(data)
+        if exist?
+          update(read + data)
+        else
+          create(data)
+        end
+      end
+
+      def read
+        @path.exist? ? @path.read : (raise NotFound.new(self))
+      end
+
+      def update(data)
+        if @path.exist?
+          @path.open("w+"){|file| file.write(data)}
+        else
+          raise NotFound.new(@uri)
+        end
+      end
+
+      def delete
+        @path.delete if @path.exist?
+      end
+
+      def mtime
+        @path.exist? ? @path.mtime : (raise NotFound.new(self))
+      end
+
+      def size
+        @path.exist? ? @path.size : (raise NotFound.new(self))
+      end
+
+      def entries
+        @path.entries.select{|entry| (@path + entry).file?}.map do |entry|
+          Location["local:%s" % (@path + entry).expand_path]
+        end
+      rescue Errno::ENOENT
+        raise NotFound.new(self)
+      end
+
+      def file?
+        @path.file?
+      end
+
+      def directory?
+        @path.directory?
+      end
+
+      def exist?
+        @path.exist?
+      end
+
+      def move(dest)
+        raise NotFound.new(self) unless exist?
+        if dest.kind_of?(LocalLocation)
+          dest.path.dirname.mkpath unless dest.path.dirname.exist?
+          FileUtils.mv(@path, dest.path)
+        else
+          copy(dest)
+          delete
+        end
+      end
+
+      def copy(dest)
+        if dest.kind_of?(LocalLocation)
+          dest.path.dirname.mkpath unless dest.path.dirname.exist?
+          IO.copy_stream(@path.open, dest.path)
+        else
+          dest.exist? ? dest.update(read) : dest.create(read)
+        end
+      end
+
+      def link(orig)
+        if orig.kind_of?(LocalLocation)
+          @path.make_symlink(orig.path)
+        else
+          orig.copy(self)
+        end
+      end
+
+      def turn(dest)
+        if dest.kind_of?(LocalLocation)
+          move(dest)
+          link(dest)
+        else
+          copy(dest)
+        end
+      end
+    end
+  end
+end

data/lib/pione/log.rb

@@ -0,0 +1,10 @@
+module Pione
+  # Log is a name space for log models.
+  module Log
+  end
+end
+
+require 'pione/log/process-record'
+require 'pione/log/process-log'
+require 'pione/log/xes-log'
+require 'pione/log/domain-info'

data/lib/pione/log/domain-info.rb

@@ -0,0 +1,72 @@
+module Pione
+  module Log
+    # DomainInfo is a domain log that records domain inputs and parameters.
+    class DomainInfo
+      # @return [Location::BasicLocation]
+      #   domain's location
+      attr_reader :domain_location
+
+      # @return [Hash{String => String}]
+      #   record table
+      attr_reader :record
+
+      # @param handler [RuleHandler::BasicHandler]
+      #   rule handler
+      def initialize(handler)
+        @domain_location = handler.domain_location
+        @record = {
+          "uname" => `uname -a`.chomp,
+          "params" => handler.params.textize,
+          "original_params" => handler.original_params.textize,
+          "inputs" => inputs_string(handler.inputs),
+          "domain" => handler.domain,
+          "domain_location" => @domain_location.uri.to_s,
+          "task_id" => handler.task_id.to_s,
+          "dry_run" => handler.dry_run.to_s
+        }
+      end
+
+      # Save domain information file.
+      #
+      # @return [void]
+      def save
+        text = "== %s\n\n" % Time.now
+        text << @record.map{|key, val| "- %s: %s" % [key,val]}.join("\n")
+        text << "\n\n"
+        (@domain_location + ".domain_info").append(text)
+      end
+
+      private
+
+      # Build a record string for inputs.
+      #
+      # @param inputs [Array<Tuple::DataTuple>]
+      #   input tuples
+      # @return [String]
+      #   a record string
+      def inputs_string(inputs)
+        "\n%s" % inputs.map.with_index(1) do |input, i|
+          if input.kind_of?(Array)
+            input.map{|_input| input_data_string(_input, i)}.join("\n")
+          else
+            input_data_string(input, i)
+          end
+        end.join("\n")
+      end
+
+      # Build a data tuple string.
+      #
+      # @param input [Tuple::DataTuple]
+      #   data tuple
+      # @param i [Integer]
+      #   index number
+      def input_data_string(input, i)
+        i = i.to_s
+        sp1 = " "*4
+        sp2 = " "*(6+i.size)
+        args = [sp1, i, input.name, sp2, input.location.uri, sp2, input.timestamp]
+        "%s%s. %s,\n%s%s,\n%s%s" % args
+      end
+    end
+  end
+end

data/lib/pione/log/process-log.rb

@@ -0,0 +1,176 @@
+module Pione
+  module Log
+    # ProcessLog represents process log records of PIONE.
+    class ProcessLog
+      # formatter table
+      @format = {}
+
+      class << self
+        # Set formatter name of this class.
+        #
+        # @param name [Symbol]
+        #   formatter name
+        # @return [void]
+        def set_format_name(name)
+          ProcessLog.instance_variable_get(:@format)[name] = self
+        end
+
+        # Return the named formatter class.
+        #
+        # @param name [Symbol]
+        #   formatter name
+        # @return [Class]
+        #   formatter class
+        def [](name)
+          ProcessLog.instance_variable_get(:@format)[name]
+        end
+
+        # Read log files and return the process log.
+        #
+        # @param location [Location]
+        #   path of process log file
+        # @return [ProcessLogFile]
+        #   log object
+        def read(location)
+          if location.file?
+            read_file(location)
+          else
+            read_directory(location)
+          end
+        end
+
+        # Set record filter.
+        #
+        # @yield block
+        def set_filter(&block)
+          @filter_block = block
+        end
+
+        attr_reader :filter_block
+
+        private
+
+        # Read records from the log file at the location.
+        #
+        # @param location [Location::BasicLocation]
+        #   log location
+        def read_file(location)
+          cache = Location[Temppath.create]
+          cache.create(location.read)
+          cache.path.each_line.map do |line|
+            JSON.parse(line).inject({}) do |data, pair|
+              data[pair[0].to_sym] = pair[1]
+              data
+            end.tap do |table|
+              break ProcessRecord.build(table)
+            end
+          end.tap do |records|
+            break new([ProcessLogBundle.new(records)])
+          end
+        end
+
+        # Read records from log files in directory at the location.
+        #
+        # @param location [Location::BasicLocation]
+        #   log directory location
+        def read_directory(location)
+          location.entries.inject(new([])) do |formatter, entry|
+            if /pione_\d{14}\.log/.match(entry.path.basename.to_s)
+              new(formatter.bundles + read_file(entry).bundles)
+            else
+              formatter
+            end
+          end
+        end
+      end
+
+      forward :@records, :map
+
+      # @return [Array<ProcessRcord>]
+      #   records of the log
+      attr_reader :records
+
+      # @param records [Array<ProcessRecord>]
+      #   log records
+      def initialize(records)
+        @records = records.select {|record| self.class.filter_block.call(record)}
+      end
+
+      # Return the record table grouped by the key.
+      #
+      # @return [Hash{String => Array<ProcessRecord>}]
+      #   grouping records table
+      def group_by(key)
+        @records.inject({}) do |table, record|
+          table[record.send(key)] ||= []
+          table.tap {|x| x[record.send(key)] << record}
+        end
+      end
+    end
+
+    # ProcessLogFormatter is a basic class for fomatting process logs.
+    class ProcessLogFormatter < ProcessLog
+      # @return [Array<ProcessBunle>]
+      #   target logs that we format
+      attr_reader :bundles
+
+      # @param bundles [Array<ProcessLogBundle>]
+      #   log bundles
+      def initialize(bundles)
+        @bundles = bundles
+      end
+
+      # Format bundle logs.
+      #
+      # @return [String]
+      #   result string
+      def format(trace_filters=[])
+        raise NotImplementedError
+      end
+    end
+
+    # ProcessLogFormatError is raised when formatter cannot format some objects.
+    class ProcessLogFormatError < StandardError
+      # @param object [Object]
+      #   the object that we cannnot format
+      def initialize(object)
+        @object = object
+      end
+
+      # @api private
+      def message
+        "not formattable: %s" % @object.inspect
+      end
+    end
+
+    # ProcessLogBundle is a bundle of raw logs.
+    class ProcessLogBundle
+      attr_reader :agent_activity_log
+      attr_reader :rule_process_log
+      attr_reader :task_process_log
+
+      # @param records [Array<ProcessRecord>]
+      #   log records
+      def initialize(records)
+        @agent_activity_log = AgentActivityLog.new(records)
+        @rule_process_log = RuleProcessLog.new(records)
+        @task_process_log = TaskProcessLog.new(records)
+      end
+    end
+
+    # AgentActivityLog is a set of records written about agent activities.
+    class AgentActivityLog < ProcessLog
+      set_filter {|record| record.type == :agent_activity}
+    end
+
+    # RuleProcessLog is a set of records written about details of rule process.
+    class RuleProcessLog < ProcessLog
+      set_filter {|record| record.type == :rule_process}
+    end
+
+    # TaskProcessLog is a set of records written about details of task process.
+    class TaskProcessLog < ProcessLog
+      set_filter {|record| record.type == :task_process}
+    end
+  end
+end

data/lib/pione/log/process-record.rb

@@ -0,0 +1,189 @@
+module Pione
+  module Log
+    # UnknownProcessRecordType is raised when we find unknown process types.
+    class UnknownProcessRecordType < StandardError
+      # @param type [Symbol]
+      #   type name
+      def initialize(type)
+        @type = type
+      end
+
+      # @api private
+      def message
+        'Unknown process type "%s"' % @type
+      end
+    end
+
+    # ProcessRecord is a class that represents records of process log. Records
+    # are in tuple spaces and handled by PIONE's logger agent.
+    class ProcessRecord
+      # known process record types and classes
+      TYPE_TABLE = {}
+
+      class << self
+        # @return [String]
+        #   record type
+        attr_reader :type
+
+        # @return [Array<Symbol>]
+        #   field names
+        attr_reader :fields
+
+        # Build a new record from hash table.
+        #
+        # @param hash [Hash{Symbol=>String}]
+        def build(hash)
+          if klass = TYPE_TABLE[hash[:type].to_sym]
+            klass.new(hash)
+          else
+            raise UnknownProcessRecordType.new(hash[:type])
+          end
+        end
+
+        private
+
+        # Set type of the record class.
+        def set_type(name)
+          @type = name
+          ProcessRecord::TYPE_TABLE[name] = self
+        end
+
+        # Declare to append the named field into records.
+        #
+        # @param name [Symbol]
+        #   field name of the record
+        # @return [void]
+        def field(name)
+          unless (@fields ||= []).include?(name)
+            @fields << name
+
+            define_method(name) do
+              instance_variable_get("@%s" % name)
+            end
+
+            define_method("%s=" % name) do |val|
+              val = Time.parse(val) if name == :timestamp and val.kind_of?(String)
+              instance_variable_set("@%s" % name, val)
+            end
+          end
+        end
+
+        # @api private
+        def inherited(klass)
+          klass.instance_variable_set(:@fields, @fields.clone)
+        end
+      end
+
+      # @!attribute [rw]
+      # @return [Time]
+      #   record timestamp
+      field :timestamp
+
+      # @!attribute [rw]
+      # @return [String]
+      #   transition name
+      field :transition
+
+      forward! :class, :type, :fields
+
+      # Create a new process log record.
+      #
+      # @param data [Hash{String => Object}]
+      #   log content
+      def initialize(data={})
+        data.each {|key, val| send("%s=" % key, val) unless key == :type}
+      end
+
+      # Create a copy of the record and merge the data into it.
+      #
+      # @return [ProcessRecord]
+      #   new merged record
+      def merge(data)
+        ProcessRecord.build(to_hash).tap do |record|
+          data.each do |key, val|
+            record.send("%s=" % key, val)
+          end
+        end
+      end
+
+      # Format as JSON string.
+      #
+      # @return [String]
+      #   JSON string
+      def format
+        JSON.dump(to_hash)
+      end
+
+      # Convert record into a hash table.
+      #
+      # @return [Hash]
+      #   hash table representation of the record
+      def to_hash
+        fields.inject({type: type}) do |table, name|
+          table.tap do
+            if val = send(name)
+              val = val.iso8601(3) if name == :timestamp
+              table[name] = val
+            end
+          end
+        end
+      end
+
+      # @api private
+      def to_json(*args)
+        to_hash.to_json(*args)
+      end
+    end
+
+    # CreateChildTaskWorkerProcessRecord represents the event that a task worker
+    # creates the child task worker.
+    class CreateChildTaskWorkerProcessRecord < ProcessRecord
+      set_type :create_child_task_worker
+      field :parent
+      field :child
+    end
+
+    # PutDataProcessRecord represents data uploading to some location.
+    class PutDataProcessRecord < ProcessRecord
+      set_type :put_data
+      field :agent_type
+      field :agent_uuid
+      field :location
+      field :size
+    end
+
+    # AgentActivityProcessRecord represents agent acitivity about state transition.
+    class AgentActivityProcessRecord < ProcessRecord
+      set_type :agent_activity
+      field :agent_type
+      field :agent_uuid
+      field :state
+    end
+
+    # AgentConnectionProcessRecord represents hello and bye message from agents.
+    class AgentConnectionProcessRecord < ProcessRecord
+      set_type :agent_connection
+      field :agent_type
+      field :agent_uuid
+      field :message
+    end
+
+    # RuleProcessRecord represents lifecyles of rule process.
+    class RuleProcessRecord < ProcessRecord
+      set_type :rule_process
+      field :name
+      field :rule_type
+      field :caller
+    end
+
+    # TaskProcessRecord represents lifecyle of task process.
+    class TaskProcessRecord < ProcessRecord
+      set_type :task_process
+      field :name
+      field :rule_name
+      field :rule_type
+      field :inputs
+      field :parameters
+    end
+  end
+end