roby 0.8.0 → 3.0.0

data/lib/roby/droby/event_logging.rb

@@ -0,0 +1,57 @@
+module Roby
+    module DRoby
+        # Mixin to add event-logging related functionality to a class
+        #
+        # The class must provide a #event_logger object. It must be non-nil, and
+        # can be initialized with {NullEventLogger} for a no-op logger
+        module EventLogging
+            # Log an event on the underlying logger
+            def log(m, *args)
+                event_logger.dump(m, Time.now, args)
+            end
+
+            # Log a timepoint on the underlying logger
+            def log_timepoint(name)
+                current_thread = Thread.current
+                event_logger.dump_timepoint(:timepoint, Time.now, [current_thread.droby_id, current_thread.name, name])
+            end
+
+            # Run a block within a timepoint group
+            def log_timepoint_group(name)
+                log_timepoint_group_start(name)
+                yield
+            ensure
+                log_timepoint_group_end(name)
+            end
+
+            # Log a timepoint on the underlying logger
+            #
+            # The logger will NOT do any validation of the group start/end
+            # pairing at logging time. This is done at replay time
+            def log_timepoint_group_start(name)
+                current_thread = Thread.current
+                event_logger.dump_timepoint(:timepoint_group_start, Time.now, [current_thread.droby_id, current_thread.name, name])
+            end
+
+            # End a timepoint group
+            #
+            # The logger will NOT do any validation of the group start/end
+            # pairing at logging time. This is done at replay time
+            def log_timepoint_group_end(name)
+                current_thread = Thread.current
+                event_logger.dump_timepoint(:timepoint_group_end, Time.now, [current_thread.droby_id, current_thread.name, name])
+            end
+
+            # The amount of cycles pending in the {#event_logger}'s dump queue
+            def log_queue_size
+                event_logger.log_queue_size
+            end
+
+            def log_flush_cycle(m, *args)
+                event_logger.flush_cycle(m, Time.now, args)
+            end
+        end
+    end
+end
+
+

data/lib/roby/droby/exceptions.rb

@@ -0,0 +1,14 @@
+module Roby
+    module DRoby
+        # Exception raised when trying to resolve a sibling for which we
+        # don't have any information
+        class UnknownSibling < RuntimeError
+        end
+
+        # Exception raised when an attempt to resolve a marshalled object into a
+        # local object failed
+        class NoLocalObject < RuntimeError
+        end
+    end
+end
+

data/lib/roby/droby/identifiable.rb

@@ -0,0 +1,22 @@
+module Roby
+    module DRoby
+        module Identifiable
+            # The DRobyID for this object
+            def droby_id
+                if @__droby_remote_id__
+                    @__droby_remote_id__
+                elsif !frozen?
+                    @__droby_remote_id__ = DRobyID.allocate
+                end
+            end
+
+            def initialize_copy(old) # :nodoc:
+                super
+                @__droby_remote_id__ = nil
+            end
+        end
+    end
+end
+
+
+

data/lib/roby/droby/logfile.rb

@@ -0,0 +1,141 @@
+module Roby
+    module DRoby
+        # Module containing all the logfile-related code
+        #
+        # Note that unlike other Roby facilities, this file is not meant to be
+        # required directly. Require either logfile/reader or logfile/writer
+        # depending on what you need
+        module Logfile
+            extend Logger::Hierarchy
+
+            MAGIC_CODE = "ROBYLOG"
+            PROLOGUE_SIZE = MAGIC_CODE.size + 4
+            FORMAT_VERSION = 5
+
+            class IndexInvalid < RuntimeError; end
+            class IndexMissing < RuntimeError; end
+
+            # Exception raised when attempting to guess the format version of a
+            # log file, and the guess fails
+            class UnknownFormatVersion < RuntimeError; end
+
+            # Execption raised when loading a good-looking prologue of a log
+            # file, but the contained log version does not match our
+            # expectations
+            class InvalidFormatVersion < RuntimeError; end
+
+            # Exception raised when attempting to load a file that does not look
+            # like a Roby log file
+            class InvalidFileError < RuntimeError; end
+
+            class TruncatedFileError < InvalidFileError; end
+
+            def self.write_prologue(io)
+                io.write(MAGIC_CODE)
+                io.write([FORMAT_VERSION].pack("L<"))
+            end
+
+            # Write a log file header
+            #
+            # The created log file will always have {FORMAT_VERSION} as its
+            # version field
+            def self.write_header(io, options = Hash.new)
+                write_prologue(io)
+                options = ::Marshal.dump(options)
+                io.write [options.size].pack("L<")
+                io.write options
+            end
+
+            # Guess the log file format version for the given IO
+            def self.guess_version(input)
+                input.rewind
+
+                magic = input.read(Logfile::MAGIC_CODE.size)
+                if magic == Logfile::MAGIC_CODE
+                    return input.read(4).unpack("L<").first
+                else
+                    input.rewind
+                    header =
+                        begin ::Marshal.load(input)
+                        rescue TypeError
+                            return
+                        end
+
+                    case header
+                    when Hash then return header[:log_format]
+                    when Symbol
+                        first_chunk =
+                            begin ::Marshal.load(input)
+                            rescue TypeError
+                                return
+                            end
+
+                        if first_chunk.kind_of?(Array)
+                            return 0
+                        end
+                    when Array
+                        if header[-2] == :cycle_end
+                            return 1 
+                        end
+                    end
+                end
+
+            ensure
+                input.rewind
+            end
+
+            # Read a file prologue, validating that it is of the latest file
+            # format
+            #
+            # @raise [InvalidFileError,InvalidFormatVersion,UnknownFormatVersion]
+            def self.read_prologue(io)
+                magic = io.read(MAGIC_CODE.size)
+                if magic != MAGIC_CODE
+                    raise InvalidFileError, "no magic code at beginning of file"
+                end
+
+                log_format = io.read(4).unpack('I').first
+                validate_format(log_format)
+
+            rescue InvalidFormatVersion
+                raise
+            rescue Exception => e
+                if format = guess_version(io)
+                    validate_format(format)
+                else
+                    raise
+                end
+            end
+
+            # @api private
+            #
+            # Validate the given format version
+            #
+            # @raise [InvalidFormatVersion]
+            def self.validate_format(format)
+                if format < FORMAT_VERSION
+                    raise InvalidFormatVersion, "this is an outdated format (#{format}, current is #{FORMAT_VERSION}). Please run roby-log upgrade-format"
+                elsif format > FORMAT_VERSION
+                    raise InvalidFormatVersion, "this is an unknown format version #{format}: expected #{FORMAT_VERSION}. This file can be read only by newest version of Roby"
+                end
+            end
+
+            # Load a chunk of data from an event file. +buffer+, if given, must be
+            # a String object that will be used as intermediate buffer in the
+            # process
+            def self.read_one_chunk(io)
+                data_size = io.read(4)
+                return if !data_size
+
+                data_size = data_size.unpack("L<").first
+                buffer = io.read(data_size) || ''
+                if buffer.size < data_size
+                    raise TruncatedFileError, "expected a chunk of size #{data_size} at #{io.tell}, but got only #{buffer ? buffer.size : '0'} bytes"
+                end
+
+                buffer
+            end
+        end
+    end
+end
+

data/lib/roby/droby/logfile/client.rb

@@ -0,0 +1,176 @@
+require 'roby/droby/logfile/reader'
+
+module Roby
+    module DRoby
+        module Logfile
+            # The client part of the event log distribution service
+            class Client
+                include Hooks
+                include Hooks::InstanceHooks
+
+                # @!group Hooks
+
+                # @!method on_init_progress()
+                #   @yieldparam [Integer] rx the amount of bytes processed so far
+                #   @yieldparam [Integer] init_size the amount of bytes expected to
+                #     be received for the init phase
+                #   @return [void]
+                define_hooks :on_init_progress
+
+                # @!method on_init_done()
+                #   Hooks called when we finished processing the initial set of data
+                #   @return [void]
+                define_hooks :on_init_done
+
+                # @!method on_data
+                #   Hooks called with one cycle worth of data
+                #
+                #   @yieldparam [Array] data the data as logged, unmarshalled (with
+                #     Marshal.load) but not unmarshalled by Roby. It is a flat array
+                #     of 4-elements tuples of the form (event_name, sec, usec,
+                #     args). See {lib/roby/droby/logfile/file_format.md} for more
+                #     details.
+                #   @return [void]
+                define_hooks :on_data
+
+                # @!endgroup
+
+                # The socket through which we are connected to the remote host
+                attr_reader :socket
+                # The host we are contacting
+                attr_reader :host
+                # The port on which a connection is created
+                attr_reader :port
+                # Data that is not a full cycle worth of data (i.e. buffer needed
+                # for packet reassembly)
+                attr_reader :buffer
+                # The amount of bytes received so far
+                attr_reader :rx
+
+                def initialize(host, port = Server::DEFAULT_PORT)
+                    @host = host
+                    @port = port
+                    @buffer = ""
+
+                    @rx = 0
+                    @socket =
+                        begin TCPSocket.new(host, port)
+                        rescue Errno::ECONNREFUSED => e
+                            raise Interface::ConnectionError, "cannot contact Roby log server at '#{host}:#{port}': #{e.message}"
+                        end
+                    socket.fcntl(Fcntl::FD_CLOEXEC, 1)
+                    socket.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
+                rescue Exception
+                    socket.close if socket
+                    raise
+                end
+
+                def disconnect
+                    @socket.close
+                end
+            
+                def close
+                    @socket.close
+                end
+
+                def closed?
+                    @socket.closed?
+                end
+
+                def add_listener(&block)
+                    on_data(&block)
+                end
+
+                def alive?
+                    @alive
+                end
+
+                # Read and process data
+                #
+                # @param [Numeric] max max time we can spend processing. The method
+                #   will at least process one cycle worth of data regardless of this
+                #   parameter
+                # @return [Boolean] true if the last call processed something and
+                #   false otherwise. It is an indicator of whether there could be
+                #   still some data pending
+                def read_and_process_pending(max: 0)
+                    current_time = start = Time.now
+                    while (processed_one_cycle = read_and_process_one_pending_cycle) && (current_time - start) <= max
+                        current_time = Time.now
+                    end
+                    processed_one_cycle
+                end
+
+                # The number of bytes that have to be transferred to finish
+                # initializing the connection
+                attr_reader :init_size
+
+                def init_done?
+                    @init_done
+                end
+
+                # @api private
+                #
+                # Read data from the underlying socket
+                #
+                # @return [Boolean] true if some data was read, false otherwise
+                def read_from_socket(size = Server::DATA_CHUNK_SIZE)
+                    @buffer.concat(socket.read_nonblock(size))
+                    true
+                rescue EOFError, Errno::ECONNRESET, Errno::EPIPE => e
+                    raise Interface::ComError, e.message, e.backtrace
+                rescue Errno::EAGAIN
+                    false
+                end
+
+                # @api private
+                #
+                # Reads the socket and processes at most one cycle of data
+                #
+                # @return [Boolean] whether there might be one more cyc
+                def read_and_process_one_pending_cycle
+                    Logfile.debug "#{buffer.size} bytes of data in buffer"
+
+                    while true
+                        if buffer.size > 4
+                            data_size = buffer.unpack('L<').first
+                            Logfile.debug "expecting data block of #{data_size} bytes"
+                            if buffer.size >= data_size + 4
+                                break
+                            end
+                            return if !read_from_socket([Server::DATA_CHUNK_SIZE, buffer.size - data_size].max)
+                        else
+                            return if !read_from_socket
+                        end
+
+                    end
+
+                    if data_size && (buffer.size >= data_size + 4)
+                        cycle_data = buffer[4, data_size]
+                        @buffer = buffer[(data_size + 4)..-1]
+                        data = ::Marshal.load_with_missing_constants(cycle_data)
+                        if data.kind_of?(Hash)
+                            Reader.process_options_hash(data)
+                        elsif data == Server::CONNECTION_INIT_DONE
+                            @init_done = true
+                            run_hook :on_init_done
+                        elsif data[0] == Server::CONNECTION_INIT
+                            @init_size = data[1]
+                        else
+                            @rx += (data_size + 4)
+                            if !init_done?
+                                run_hook :on_init_progress, rx, init_size
+                            end
+                            run_hook :on_data, data
+                        end
+                        Logfile.debug "processed #{data_size} bytes of data, #{@buffer.size} remaining in buffer"
+                        true
+                    end
+
+                rescue Errno::EAGAIN
+                end
+            end
+        end
+    end
+end
+

data/lib/roby/droby/logfile/file_format.md

@@ -0,0 +1,97 @@
+The Roby log file format uses Ruby's Marshal and Roby's own DRoby mechanisms to
+save data to disk.
+
+A log file starts with 
+
+  ROBYLOG#{FORMAT_VERSION}#{OPTIONS}
+
+FORMAT_VERSION is an integer encoded in little-endian 32-bit. This version is
+incremented each time the format is changed in a backward-incompatible way.
+When the log file format changes, newer ruby versions will require one to run
+the roby-log upgrade command on older log files before they can be interpreted.
+
+OPTIONS is a Hash marshalled using Ruby's Marshal facility and prefixed by the
+size of the marshalled hash, in bytes, encoded as a little-endian 32-bit. This
+hash contains the following keys:
+
+  plugins: [String] the list of plugins loaded in the Roby instance
+
+The rest of the file is a list of blocks. Each block is prefixed by its size
+represented with a 32-bit unsigned integer. The rest of the block is a Ruby
+array marshalled using Marshal.dump. The array contains a flat list of log
+messages for one execution cycle. All cycles end with the same cycle_end message.
+
+Each message is represented by 4 elements in the array
+
+   mesage_name, time_seconds, time_microseconds, args
+
+where message_name is an event name (listed below), Time is the time where the
+message was queued for logging and 'args' are a list of message-specific
+parameters
+
+= Messages in Roby log format version 5
+
+Only executable plans log operations that are performed on them. Transactions
+and template plans are currently not logged.
+
+A new ExecutablePlan registers itself on the logger with
+
+    register_executable_plan plan_id
+
+Where plan_id is ExecutablePlan#remote_id
+
+Every plan object must be registered in an executable plan before it can be
+referred to into other messages or objects. A plan object is registered through
+the merged_plan hook, where the 'plan' parameter is a full dump of the task and
+events that it contains.
+
+After a merge_plan message, all further messages will refer to a plan object
+through its #remote_id attribute.
+
+    merged_plan(plan, merged_plan)
+
+Plan modification hooks
+
+    added_edge(parent, child, relations, info)
+    updated_edge_info(parent, child, relation, info)
+    removed_edge(parent, child, relations)
+    notify_plan_status_change(task, status)
+    garbage(object)
+    finalized_task(plan, task)
+    finalized_event(plan, event)
+    task_arguments_updated(task, key, value)
+
+Event propagation hooks:
+
+    task_failed_to_start(task, reason)
+    generator_fired(event)
+    generator_emit_failed(generator, error)
+    generator_propagate_event(is_forwarding, events, generator)
+    generator_unreachable(generator, reason)
+    exception_notification(plan_id, mode, error, involved_objects)
+      mode == EXCEPTION_FATAL -> involved_objects Roby::Task
+      mode == EXCEPTION_NONFATAL -> involved_objects Roby::Task
+      mode == EXCEPTION_HANDLED -> involved_objects Roby::Task or Roby::Plan
+
+Scheduler state:
+
+    scheduler_report_pending_non_executable_task(msg, task, *args)
+    scheduler_report_trigger(generator)
+    scheduler_report_holdoff(msg, task, *args)
+    scheduler_report_action(msg, task, *args)
+
+Cycle information. This message always ends one cycle of data, e.g. each entry
+in a log file will end with this message
+
+    cycle_end(info)
+
+The info parameter is a Hash containing a list of statistics and general
+information from the Roby ExecutionEngine. In addition, it contains the
+following two logger-specific informations:
+
+    pos: the position of this cycle in the file
+    event_count: the number of log messages received during this cycle
+
+The log file generated along a log file is simply a list of such cycle_end
+messages.
+