roby 0.8.0 → 3.0.0

data/lib/roby/droby/logfile/index.rb

@@ -0,0 +1,117 @@
+module Roby
+    module DRoby
+        module Logfile
+            class Index
+                # Creates an index file for +event_log+ in +index_log+
+                def self.rebuild(event_io, index_io)
+                    stat = File.stat(event_io.path)
+                    event_log = Reader.new(event_io)
+
+                    index_io.write [stat.size, stat.mtime.tv_sec, stat.mtime.tv_nsec].pack("Q<L<L<")
+                    dump_io     = StringIO.new("", 'w')
+                    while !event_log.eof?
+                        current_pos = event_log.tell
+                        cycle = event_log.load_one_cycle
+                        info  = cycle.last.last
+                        event_count = 0
+                        cycle.each_slice(4) do |m, *|
+                            if m.to_s !~ /^timepoint/
+                                event_count += 1
+                            end
+                        end
+                        info[:event_count] = event_count
+                        info[:pos] = current_pos
+
+                        if block_given?
+                            yield(Float(event_io.tell) / end_pos)
+                        end
+
+                        info = ::Marshal.dump(info)
+                        index_io.write [info.size].pack("L<")
+                        index_io.write info
+                    end
+
+                rescue EOFError
+                ensure index_io.flush if index_io
+                end
+
+                # The size in bytes of the file that has been indexed
+                attr_reader :file_size
+                # The modification time of the file that has been indexed
+                attr_reader :file_time
+                # The index data
+                #
+                # @return [Array<Hash>]
+                attr_reader :data
+
+                def initialize(file_size, file_time, data)
+                    @file_size = file_size
+                    @file_time = file_time
+                    @data = data
+                end
+
+                def size
+                    data.size
+                end
+
+                def [](*args)
+                    data[*args]
+                end
+
+                def each(&block)
+                    data.each(&block)
+                end
+
+                include Enumerable
+
+                # Tests whether this index is valid for a given file
+                #
+                # @param [String] path the log file path
+                # @return [Boolean]
+                def valid_for?(path)
+                    stat = File.stat(path)
+                    stat.size == file_size && stat.mtime == file_time
+                end
+
+                # Returns the number of cycles in this index
+                def cycle_count
+                    data.size
+                end
+
+                # Tests whether this index contains cycles
+                def empty?
+                    data.empty?
+                end
+
+                # The time range
+                #
+                # @return [nil,(Time,Time)]
+                def range
+                    if !data.empty?
+                        [Time.at(*data.first[:start]), 
+                         Time.at(*data.last[:start]) + data.last[:end]]
+                    end
+                end
+
+                # Read an index file
+                #
+                # @param [String] filename the index file path
+                def self.read(filename)
+                    io = File.open(filename)
+                    file_info = io.read(16)
+                    size, tv_sec, tv_nsec = file_info.unpack("Q<L<L<")
+                    data = Array.new
+                    begin
+                        while !io.eof?
+                            data << ::Marshal.load(Logfile.read_one_chunk(io))
+                        end
+                    rescue EOFError
+                    end
+
+                    new(size, Time.at(tv_sec, Rational(tv_nsec, 1000)), data)
+                end
+            end
+        end
+    end
+end
+

data/lib/roby/droby/logfile/reader.rb

@@ -0,0 +1,139 @@
+require 'roby/droby/logfile'
+require 'roby/droby/logfile/index'
+
+module Roby
+    module DRoby
+        module Logfile
+            # A class that reads log files generated by {Writer}
+            class Reader
+                # The current log format version
+                FORMAT_VERSION = 5
+
+                attr_reader :event_io
+
+                def initialize(event_io)
+                    @event_io = event_io
+                    event_io.rewind
+                    options_hash = read_header
+                    self.class.process_options_hash(options_hash)
+                end
+
+                def read_header
+                    Logfile.read_prologue(event_io)
+                    if chunk = Logfile.read_one_chunk(event_io)
+                        ::Marshal.load(chunk)
+                    else
+                        raise InvalidFileError, "expected the prologue to be followed by one chunk, but got nothing"
+                    end
+                end
+
+                def dup
+                    Reader.new(event_io.dup)
+                end
+
+                def tell
+                    event_io.tell
+                end
+
+                def close
+                    event_io.close
+                end
+
+                def eof?
+                    event_io.eof?
+                end
+
+                def closed?
+                    event_io.closed?
+                end
+
+                def seek(pos)
+                    event_io.seek(pos)
+                end
+
+                def load_one_cycle
+                    if chunk = Logfile.read_one_chunk(event_io)
+                        begin ::Marshal.load_with_missing_constants(chunk)
+                        rescue ArgumentError => e
+                            if e.message == "marshal data too short"
+                                raise TruncatedFileError, "marshal data invalid"
+                            else raise
+                            end
+                        end
+                    end
+                rescue Exception => e
+                    raise e, "#{e.message}, running roby-log repair might repair the file", e.backtrace
+                end
+
+                def self.process_options_hash(options_hash)
+                    if options_hash[:plugins]
+                        options_hash[:plugins].each do |plugin_name|
+                            begin
+                                Roby.app.using plugin_name
+                            rescue ArgumentError => e
+                                Roby.warn "the log file mentions the #{plugin_name} plugin, but it is not available on this system. Some information might not be displayed"
+                            end
+                        end
+                    end
+                end
+
+                # The standard Roby index path, inferred from the log file's own
+                # path
+                #
+                # @return [String]
+                def index_path
+                    event_io.path.gsub(/\.log$/, '') + ".idx"
+                end
+
+                def rebuild_index(path = index_path)
+                    Logfile.warn "rebuilding index file for #{event_io.path}"
+                    File.open(path, 'w') do |index_io|
+                        event_io = self.event_io.dup
+                        Index.rebuild(File.open(event_io.path), index_io)
+                    end
+                    @index = nil
+                end
+
+                # Returns the index object for this event log
+                def index(path = index_path, rebuild: true)
+                    if @index
+                        return @index
+                    elsif !File.file?(path)
+                        if !rebuild
+                            raise IndexMissing, "there's no file #{path}"
+                        end
+                        rebuild_index(path)
+                    end
+
+                    index =
+                        begin Index.read(path)
+                        rescue Exception => e
+                            raise e, "while reading index file #{path}: #{e.message}", e.backtrace
+                        end
+                    if index.valid_for?(event_io.path)
+                        @index = index
+                    elsif !rebuild
+                        raise IndexInvalid, "#{path} is not a valid index for #{self}"
+                    else
+                        rebuild_index(path)
+                        @index = Index.read(path)
+                    end
+                end
+                
+                def self.open(path)
+                    io = new(File.open(path))
+                    if block_given?
+                        begin
+                            yield(io)
+                        ensure
+                            io.close unless io.closed?
+                        end
+                    else
+                        io
+                    end
+                end
+            end
+        end
+    end
+end
+

data/lib/roby/droby/logfile/server.rb

@@ -0,0 +1,199 @@
+require 'socket'
+require 'fcntl'
+require 'stringio'
+require 'roby/interface/exceptions'
+require 'roby/droby/logfile'
+
+module Roby
+    module DRoby
+        module Logfile
+            # This is the server part of the log distribution mechanism
+            #
+            # It is basically a file distribution mechanism: it "listens" to the
+            # event log file and sends new data to the clients that are connected to
+            # it.
+            #
+            # When a client connects, it will send the complete file
+            class Server
+                extend Logger::Hierarchy
+                make_own_logger("Log Server", Logger::WARN)
+
+                DEFAULT_PORT  = 20200
+                DEFAULT_SAMPLING_PERIOD = 0.05
+                DATA_CHUNK_SIZE = 512 * 1024
+
+                # The IO we are listening on
+                attr_reader :server_io
+                # The sampling period (in seconds)
+                attr_reader :sampling_period
+                # The path to the event file this server is listening to
+                attr_reader :event_file_path
+                # The IO object that we use to read the event file
+                attr_reader :event_file
+                # A mapping from socket to data chunks representing the data that
+                # should be sent to a particular client
+                attr_reader :pending_data
+                # The server socket
+                attr_reader :server
+
+                def initialize(event_file_path, sampling_period = DEFAULT_SAMPLING_PERIOD, io)
+                    @server = io
+                    @pending_data = Hash.new
+                    @sampling_period = sampling_period
+                    @event_file_path = event_file_path
+                    @event_file = File.open(event_file_path, 'r:BINARY')
+                end
+
+                def found_header?
+                    @found_header
+                end
+
+                def exec
+                    while true
+                        sockets_with_pending_data = pending_data.find_all do |socket, chunks|
+                            !chunks.empty?
+                        end.map(&:first)
+                        if !sockets_with_pending_data.empty?
+                            Server.debug "#{sockets_with_pending_data.size} sockets have pending data"
+                        end
+
+                        readable_sockets, _ =
+                            select([server], sockets_with_pending_data, nil, sampling_period)
+
+                        # Incoming connections
+                        if readable_sockets && !readable_sockets.empty?
+                            socket = server.accept
+                            socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true)
+                            socket.fcntl(Fcntl::FD_CLOEXEC, 1)
+
+                            Server.debug "new connection: #{socket}"
+                            if found_header?
+                                all_data = File.binread(event_file_path, 
+                                                        event_file.tell - Logfile::PROLOGUE_SIZE,
+                                                        Logfile::PROLOGUE_SIZE)
+
+                                Server.debug "  queueing #{all_data.size} bytes of data"
+                                chunks = split_in_chunks(all_data)
+                            else
+                                Server.debug "  log file is empty, not queueing any data"
+                                chunks = Array.new
+                            end
+                            connection_init      = ::Marshal.dump([CONNECTION_INIT, chunks.inject(0) { |s, c| s + c.size }])
+                            connection_init_done = ::Marshal.dump(CONNECTION_INIT_DONE)
+                            chunks.unshift([connection_init.size].pack('L<') + connection_init)
+                            chunks << [connection_init_done.size].pack('L<') + connection_init_done
+                            @pending_data[socket] = chunks
+                        end
+
+                        # Read new data
+                        read_new_data
+                        # Send data to our peers
+                        send_pending_data
+                    end
+                rescue Exception
+                    pending_data.each_key(&:close)
+                    raise
+                end
+
+                # Splits the data block in +data+ in blocks of size DATA_CHUNK_SIZE
+                def split_in_chunks(data)
+                    result = []
+
+                    index = 0
+                    while index != data.size
+                        remaining = (data.size - index)
+                        if remaining > DATA_CHUNK_SIZE
+                            result << data[index, DATA_CHUNK_SIZE]
+                            index += DATA_CHUNK_SIZE
+                        else
+                            result << data[index, remaining]
+                            index = data.size
+                        end
+                    end
+                    result
+                end
+
+                # Reads new data from the underlying file and queues it to dispatch
+                # for our clients
+                def read_new_data
+                    new_data = event_file.read
+                    return if new_data.empty?
+
+                    if !found_header?
+                        if new_data.size >= Logfile::PROLOGUE_SIZE
+                            # This will read and validate the prologue
+                            Logfile.read_prologue(StringIO.new(new_data))
+                            new_data = new_data[Logfile::PROLOGUE_SIZE..-1]
+                            @found_header = true
+                        else
+                            # Go back to the beginning of the file so that, next
+                            # time, we read the complete prologue again
+                            event_file.rewind
+                            return
+                        end
+                    end
+
+                    # Split the data in chunks of DATA_CHUNK_SIZE, and add the
+                    # chunks in the pending_data hash
+                    new_chunks = split_in_chunks(new_data)
+                    pending_data.each_value do |chunks|
+                        chunks.concat(new_chunks)
+                    end
+                end
+
+                CONNECTION_INIT = :log_server_connection_init
+                CONNECTION_INIT_DONE = :log_server_connection_init_done
+
+                # Tries to send all pending data to the connected clients
+                def send_pending_data
+                    needs_looping = true
+                    while needs_looping
+                        needs_looping = false
+                        pending_data.delete_if do |socket, chunks|
+                            if chunks.empty?
+                                # nothing left to send for this socket
+                                next
+                            end
+
+                            buffer = chunks.shift
+                            while !chunks.empty? && (buffer.size + chunks[0].size < DATA_CHUNK_SIZE)
+                                buffer.concat(chunks.shift)
+                            end
+                            Server.debug "sending #{buffer.size} bytes to #{socket}"
+
+
+                            begin
+                                written = socket.write_nonblock(buffer)
+                            rescue Errno::EAGAIN
+                                Server.debug "cannot send: send buffer full"
+                                chunks.unshift(buffer)
+                                next
+                            rescue Exception => e
+                                Server.warn "disconnecting from #{socket}: #{e.message}"
+                                e.backtrace.each do |line|
+                                    Server.warn "  #{line}"
+                                end
+                                socket.close
+                                next(true)
+                            end
+
+                            remaining = buffer.size - written
+                            if remaining == 0
+                                Server.debug "wrote complete chunk of #{written} bytes to #{socket}"
+                                # Loop if we wrote the complete chunk and there
+                                # is still stuff to write for this socket
+                                needs_looping = !chunks.empty?
+                            else
+                                Server.debug "wrote partial chunk #{written} bytes instead of #{buffer.size} bytes to #{socket}"
+                                chunks.unshift(buffer[written, remaining])
+                            end
+                            false
+                        end
+                    end
+                end
+            end
+
+        end
+    end
+end
+