roby 0.7

data/lib/roby/log/dot.rb

@@ -0,0 +1,279 @@
+require 'roby/distributed/protocol'
+require 'roby/log'
+require 'tempfile'
+require 'fileutils'
+
+module Roby
+    module LoggedPlan
+	attr_accessor :layout_level
+	def all_events(display)
+	    known_tasks.inject(free_events.dup) do |events, task|
+		if display.displayed?(task)
+		    events.merge(task.events.values.to_value_set)
+		else
+		    events
+		end
+	    end
+	end
+
+	attr_reader :dot_id
+	def to_dot(display, io, level)
+	    @layout_level = level
+	    id = io.layout_id(self)
+	    @dot_id = "plan_#{id}"
+	    io << "subgraph cluster_#{dot_id} {\n"
+	    (known_tasks | finalized_tasks | free_events | finalized_events).
+		each do |obj|
+		    obj.to_dot(display, io) if display.displayed?(obj)
+		end
+
+	    io << "};\n"
+
+	    transactions.each do |trsc|
+		trsc.to_dot(display, io, level + 1)
+	    end
+
+	    relations_to_dot(display, io, TaskStructure, known_tasks)
+	end
+
+	def each_displayed_relation(display, space, objects)
+	    space.relations.each do |rel|
+		next unless display.relation_enabled?(rel)
+
+		objects.each do |from|
+		    next unless display.displayed?(from)
+		    unless display[from]
+			Roby::Log.warn "no display item for #{from} in #each_displayed_relation"
+			next
+		    end
+
+		    from.each_child_object(rel) do |to|
+			next unless display.displayed?(to)
+			unless display[to]
+			    Roby::Log.warn "no display item for child in #{from} <#{rel}> #{to} in #each_displayed_relation"
+			    next
+			end
+
+			yield(rel, from, to)
+		    end
+		end
+	    end
+	end
+
+	def relations_to_dot(display, io, space, objects)
+	    each_displayed_relation(display, space, objects) do |rel, from, to|
+		from_id, to_id = from.dot_id, to.dot_id
+		if from_id && to_id
+		    io << "  #{from_id} -> #{to_id}\n"
+		else
+		    Roby::Log.warn "ignoring #{from}(#{from.object_id} #{from_id}) -> #{to}(#{to.object_id} #{to_id}) in #{rel} in #{caller(1).join("\n  ")}"
+		end
+	    end
+	end
+
+	def layout_relations(positions, display, space, objects)
+	    each_displayed_relation(display, space, objects) do |rel, from, to|
+		display.task_relation(from, to, rel, from[to, rel])
+	    end
+	end
+
+	# The distance from the root plan
+	attr_reader :depth
+
+	# Computes the plan depths and max_depth for this plan and all its
+	# children. +depth+ is this plan depth
+	#
+	# Returns max_depth
+	def compute_depth(depth)
+	    @depth = depth
+	    child_depth = transactions.
+		map { |trsc| trsc.compute_depth(depth + 1) }.
+		max
+	    child_depth || depth
+	end
+	
+	def apply_layout(bounding_rects, positions, display, max_depth = nil)
+	    max_depth ||= compute_depth(0)
+
+	    if rect = bounding_rects[dot_id]
+		item = display[self]
+                rect[2] *= 1.2
+                rect[3] *= 1.2
+		item.z_value = Log::PLAN_LAYER + depth - max_depth
+		item.set_rect *rect
+	    else
+		Roby::Log.warn "no bounding rectangle for #{self} (#{dot_id})"
+	    end
+
+
+	    (known_tasks | finalized_tasks | free_events | finalized_events).
+		each do |obj|
+		    obj.apply_layout(positions, display)
+		end
+
+	    transactions.each do |trsc|
+		trsc.apply_layout(bounding_rects, positions, display, max_depth)
+	    end
+	    layout_relations(positions, display, TaskStructure, known_tasks)
+	end
+    end
+
+    module LoggedPlanObject
+	attr_reader :dot_id
+
+	def dot_label(display); display_name(display) end
+
+	# Adds the dot definition for this object in +io+
+	def to_dot(display, io)
+	    return unless display.displayed?(self)
+	    @dot_id ||= "plan_object_#{io.layout_id(self)}"
+	    io << "  #{dot_id}[label=\"#{dot_label(display).split("\n").join('\n')}\"];\n"
+	end
+
+	# Applys the layout in +positions+ to this particular object
+	def apply_layout(positions, display)
+	    return unless display.displayed?(self)
+	    if p = positions[dot_id]
+		raise "no graphics for #{self}" unless graphics_item = display[self]
+		graphics_item.pos = p
+	    else
+		STDERR.puts "WARN: #{self} has not been layouted"
+	    end
+	end
+    end
+
+    class PlanObject::DRoby
+	include LoggedPlanObject
+    end
+
+    class TaskEventGenerator::DRoby
+	def dot_label(display); symbol.to_s end
+	def dot_id; task.dot_id end
+    end
+
+    module LoggedTask
+	include LoggedPlanObject
+	def dot_label(display)
+	    event_names = events.values.find_all { |ev| display.displayed?(ev) }.
+		map { |ev| ev.dot_label(display) }.
+		join(" ")
+
+	    own = super
+	    if own.size > event_names.size then own
+	    else event_names
+	    end
+	end
+    end
+
+    module Log
+	class Layout
+	    @@bkpindex = 0
+
+	    def layout_id(object)
+		id = Object.address_from_id(object.object_id).to_s
+		object_ids[id] = object
+		id
+	    end
+
+	    attribute(:object_ids) { Hash.new }
+	    attr_reader :dot_input
+
+	    def <<(string); dot_input << string end
+	    def layout(display, plan)
+		@@index ||= 0
+		@@index += 1
+
+		# Dot input file
+		@dot_input  = Tempfile.new("roby_dot")
+		# Dot output file
+		dot_output = Tempfile.new("roby_layout")
+
+		dot_input << "digraph relations {\n"
+		display.layout_options.each do |k, v|
+		    dot_input << "  #{k}=#{v};\n"
+		end
+		plan.to_dot(display, self, 0)
+
+		# Take the signalling into account for the layout
+		display.propagated_events.each do |_, sources, to, _|
+		    sources.each do |from|
+			from_id, to_id = from.dot_id, to.dot_id
+			if from_id && to_id
+			    dot_input << "  #{from.dot_id} -> #{to.dot_id}\n"
+			end
+		    end
+		end
+
+		dot_input << "\n};"
+		dot_input.flush
+
+		# Make sure the GUI keeps being updated while dot is processing
+		FileUtils.cp dot_input.path, "/tmp/dot-input-#{@@index}.dot"
+		system("#{display.layout_method} #{dot_input.path} > #{dot_output.path}")
+		#pid = fork do
+		#    exec("#{display.layout_method} #{dot_input.path} > #{dot_output.path}")
+		#end
+		#while !Process.waitpid(pid, Process::WNOHANG)
+		#    if Qt::Application.has_pending_events
+		#	Qt::Application.process_events
+		#    else
+		#	sleep(0.05)
+		#    end
+		#end
+		FileUtils.cp dot_output.path, "/tmp/dot-output-#{@@index}.dot"
+
+		# Load only task bounding boxes from dot, update arrows later
+		current_graph_id = nil
+		bounding_rects = Hash.new
+		object_pos     = Hash.new
+		lines = File.open(dot_output.path) { |io| io.readlines  }
+		full_line = ""
+		lines.each do |line|
+		    line.chomp!
+		    full_line << line
+		    if line[-1] == ?\\
+			full_line.chop!
+			next
+		    end
+
+		    case full_line
+		    when /((?:\w+_)+\d+) \[.*pos="(\d+),(\d+)"/
+			object_pos[$1] = Qt::PointF.new(Integer($2), Integer($3))
+		    when /subgraph cluster_(plan_\d+)/
+			current_graph_id = $1
+		    when /graph \[bb="(\d+),(\d+),(\d+),(\d+)"\]/
+			bb = [$1, $2, $3, $4].map do |c|
+			    c = Integer(c)
+			end
+			bounding_rects[current_graph_id] = [bb[0], bb[1], bb[2] - bb[0], bb[3] - bb[1]]
+		    end
+		    full_line = ""
+		end
+
+		graph_bb = bounding_rects.delete(nil)
+		bounding_rects.each_value do |coords|
+		    coords[0] -= graph_bb[0]
+		    coords[1] = graph_bb[1] - coords[1] - coords[3]
+		end
+		object_pos.each do |id, pos|
+		    pos.x -= graph_bb[0]
+		    pos.y = graph_bb[1] - pos.y
+		end
+
+		@display         = display
+		@plan            = plan
+		@object_pos      = object_pos
+		@bounding_rects  = bounding_rects
+
+	    ensure
+		dot_input.close!  if dot_input
+		dot_output.close! if dot_output
+	    end
+
+	    attr_reader :bounding_rects, :object_pos, :display, :plan
+	    def apply
+		plan.apply_layout(bounding_rects, object_pos, display)
+	    end
+	end
+    end
+end

data/lib/roby/log/event_stream.rb

@@ -0,0 +1,151 @@
+require 'roby/log/data_stream'
+
+module Roby
+    module Log
+	# This class is a logger-compatible interface which read event and index logs,
+	# and may rebuild the task and event graphs from the marshalled events
+	# that are saved using for instance FileLogger
+	class EventStream < DataStream
+	    def splat?; true end
+
+	    # The event log
+	    attr_reader :logfile
+
+	    # The index of the currently displayed cycle in +index_data+
+	    attr_reader :current_cycle
+            # The index of the first non-empty cycle
+            attr_reader :start_cycle
+	    # A [min, max] array of the minimum and maximum times for this
+	    # stream
+	    def range; [start_time, logfile.range.last] end
+
+	    def initialize(basename, file = nil)
+		super(basename, "roby-events")
+		if file
+		    @logfile = file
+		    reinit!
+		end
+	    end
+	    def open
+		@logfile = Roby::Log.open(name)
+		reinit!
+		self
+	    end
+	    def close; @logfile.close end
+
+	    def index_data; logfile.index_data end
+
+	    # True if the stream has been reinitialized
+	    def reinit?
+		@reinit ||= (!index_data.empty? && logfile.stat.size < index_data.last[:pos])
+	    end
+
+	    # Reinitializes the stream
+	    def reinit!
+		prepare_seek(nil)
+
+		super
+
+                start_cycle = 0
+                while start_cycle < index_data.size && index_data[start_cycle][:event_count] == 4
+                    start_cycle += 1
+                end
+                @start_cycle   = start_cycle
+		@current_cycle = start_cycle
+            end
+
+	    # True if there is at least one sample available
+	    def has_sample?
+		logfile.update_index
+		!index_data.empty? && (index_data.last[:pos] > logfile.tell)
+	    end
+
+	    # Seek the data stream to the specified time.
+	    def prepare_seek(time)
+		if !time || !current_time || time < current_time
+		    clear
+
+		    @current_time  = nil
+		    @current_cycle = start_cycle
+		    logfile.rewind
+		end
+	    end
+
+	    def start_time
+		return if start_cycle == index_data.size
+		Time.at(*index_data[start_cycle][:start])
+	    end
+	    
+	    # The current time
+	    def current_time
+		return if index_data.empty?
+		time = Time.at(*index_data[current_cycle][:start])
+		if index_data.size == current_cycle + 1
+		    time += index_data[current_cycle][:end]
+		end
+		time
+	    end
+
+	    # The time we will reach when the next sample is processed
+	    def next_time
+		return if index_data.empty?
+		if index_data.size > current_cycle + 1
+		    Time.at(*index_data[current_cycle + 1][:start])
+		end
+	    end
+
+	    # Reads a sample of data and returns it. It will be fed to
+	    # decoders' #decode method.
+	    #
+	    # In this stream, this is the chunk of the marshalled file which
+	    # corresponds to a cycle
+	    def read
+		if reinit?
+		    reinit!
+		end
+
+		start_pos = index_data[current_cycle][:pos]
+		end_pos   = if index_data.size > current_cycle + 1
+				index_data[current_cycle + 1][:pos]
+			    else
+				logfile.stat.size
+			    end
+
+		logfile.seek(start_pos)
+		logfile.read(end_pos - start_pos)
+
+	    ensure
+		@current_cycle += 1
+	    end
+
+	    # Unmarshalls a set of data returned by #read_all and yield
+	    # each sample that should be fed to the decoders
+	    def self.init(data)
+		io = StringIO.new(data)
+		while !io.eof?
+		    yield(Marshal.load(io))
+		end
+	    rescue EOFError
+	    end
+
+	    # Unmarshalls one cycle of data returned by #read and feeds
+	    # it to the decoders
+	    def self.decode(data)
+		Marshal.load(data)
+	    end
+
+	    # Read all data read so far in a format suitable to feed to
+	    # #init_stream on the decoding side
+	    def read_all
+		end_pos   = if index_data.size > current_cycle + 1
+				index_data[current_cycle + 1][:pos]
+			    else
+				logfile.stat.size
+			    end
+		logfile.rewind
+		logfile.read(end_pos)
+	    end
+	end
+    end
+end
+

data/lib/roby/log/file.rb

@@ -0,0 +1,340 @@
+require 'roby/log/logger'
+require 'roby/distributed'
+require 'tempfile'
+require 'fileutils'
+
+module Roby::Log
+    class Logfile < DelegateClass(File)
+	attr_reader :event_io
+	attr_reader :index_io
+	attr_reader :index_data
+	attr_reader :basename
+	def range
+	    [Time.at(*index_data.first[:start]), 
+		Time.at(*index_data.last[:start]) + index_data.last[:end]]
+	end
+
+	def initialize(file, allow_old_format = false, force_rebuild_index = false)
+	    @event_io = if file.respond_to?(:to_str)
+			    @basename = if file =~ /-events\.log$/ then $`
+					else file
+					end
+
+			    File.open("#{basename}-events.log")
+			else
+			    @basename = file.path.gsub(/-events\.log$/, '')
+			    file
+			end
+
+	    if !allow_old_format
+		FileLogger.check_format(@event_io)
+	    end
+
+	    index_path = "#{basename}-index.log"
+	    if force_rebuild_index || !File.file?(index_path)
+		rebuild_index
+	    else
+		@index_io = File.open(index_path)
+		if @index_io.stat.size == 0
+		    rebuild_index
+		end
+	    end
+
+	    super(@event_io)
+
+	    @index_data = Array.new
+	    update_index
+	    rewind
+	end
+
+	# Reads as much index data as possible
+	def update_index
+	    begin
+		pos = nil
+		loop do
+		    pos = index_io.tell
+		    length = index_io.read(4)
+		    raise EOFError unless length
+		    length = length.unpack("N").first
+		    index_data << Marshal.load(index_io.read(length))
+		end
+	    rescue EOFError
+		index_io.seek(pos, IO::SEEK_SET)
+	    end
+
+	    return if index_data.empty?
+	end
+
+	def rewind
+	    @event_io.rewind
+	    Marshal.load(@event_io)
+	    @index_io.rewind
+	    @index_data.clear
+	    update_index
+	end
+
+	def rebuild_index
+	    STDOUT.puts "rebuilding index file for #{basename}"
+	    @index_io.close if @index_io
+	    @index_io = File.open("#{basename}-index.log", 'w+')
+	    FileLogger.rebuild_index(@event_io, @index_io)
+	end
+
+	def self.open(path)
+	    io = new(path)
+	    if block_given?
+		begin
+		    yield(io)
+		ensure
+		    io.close unless io.closed?
+		end
+	    else
+		io
+	    end
+	end
+    end
+
+    # A logger object which marshals all available events in two files. The
+    # event log is the full log, the index log contains only the timings given
+    # to Control#cycle_end, along with the corresponding position in the event
+    # log file.
+    #
+    # You can use FileLogger.replay(io) to send the events back into the
+    # logging system (using Log.log), for instance to feed an offline display
+    class FileLogger
+	# The current log format version
+	FORMAT_VERSION = 3
+
+	@dumped = Hash.new
+	class << self
+	    attr_reader :dumped
+	end
+
+	# The IO object for the event log
+	attr_reader :event_log
+	# The IO object for the index log
+	attr_reader :index_log
+	# The set of events for the current cycle. This is dumped only
+	# when the +cycle_end+ event is received
+	attr_reader :current_cycle
+	# StringIO object on which we dump the data
+	attr_reader :dump_io
+
+	def initialize(basename)
+	    @current_pos   = 0
+	    @dump_io	   = StringIO.new('', 'w')
+	    @current_cycle = Array.new
+	    @event_log = File.open("#{basename}-events.log", 'w')
+	    event_log.sync = true
+	    FileLogger.write_header(@event_log)
+	    @index_log = File.open("#{basename}-index.log", 'w')
+	    index_log.sync = true
+	end
+
+	attr_accessor :stats_mode
+	def splat?; false end
+
+	def dump_method(m, time, args)
+	    if m == :cycle_end || !stats_mode
+		current_cycle << m << time.tv_sec << time.tv_usec << args
+	    end
+	    if m == :cycle_end
+		info = args.first
+		info[:pos] = event_log.tell
+		info[:event_count] = current_cycle.size
+		Marshal.dump(current_cycle, event_log)
+
+		dump_io.truncate(0)
+		dump_io.seek(0)
+		Marshal.dump(info, dump_io)
+		index_log.write [dump_io.size].pack("N")
+		index_log.write dump_io.string
+		current_cycle.clear
+	    end
+
+	rescue 
+	    puts "failed to dump #{m}#{args}: #{$!.full_message}"
+	    args.each do |obj|
+		unless (Marshal.dump(obj) rescue nil)
+		    puts "there is a problem with"
+		    pp obj
+		end
+	    end
+	end
+
+	Roby::Log.each_hook do |klass, m|
+	    define_method(m) { |time, args| dump_method(m, time, args) }
+	end
+	
+	# Creates an index file for +event_log+ in +index_log+
+	def self.rebuild_index(event_log, index_log)
+	    event_log.rewind
+	    # Skip the file header
+	    Marshal.load(event_log)
+
+	    current_pos = event_log.tell
+	    dump_io	   = StringIO.new("", 'w')
+
+	    loop do
+		cycle = Marshal.load(event_log)
+		info               = cycle.last.last
+		info[:pos]         = current_pos
+		info[:event_count] = cycle.size
+
+		dump_io.truncate(0)
+		dump_io.seek(0)
+		Marshal.dump(info, dump_io)
+		index_log.write [dump_io.size].pack("N")
+		index_log.write dump_io.string
+
+		current_pos = event_log.tell
+	    end
+
+	rescue EOFError
+	ensure
+	    event_log.rewind
+	    index_log.rewind
+	end
+
+
+	def self.log_format(input)
+	    input.rewind
+	    format = begin
+			 header = Marshal.load(input)
+			 case header
+			 when Hash: header[:log_format]
+			 when Symbol
+			     if Marshal.load(input).kind_of?(Array)
+				 0
+			     end
+			 when Array
+			     1 if header[-2] == :cycle_end
+			 end
+		     rescue
+		     end
+
+	    unless format
+		raise "#{input.path} does not look like a Roby event log file"
+	    end
+	    format
+
+	ensure
+	    input.rewind
+	end
+
+	def self.check_format(input)
+	    format = log_format(input)
+	    if format < FORMAT_VERSION
+		raise "this is an outdated format. Please run roby-log upgrade-format"
+	    elsif format > FORMAT_VERSION
+		raise "this is an unknown format version #{format}: expected #{FORMAT_VERSION}. This file can be read only by newest version of Roby"
+	    end
+	end
+
+	def self.write_header(io)
+	    header = { :log_format => FORMAT_VERSION }
+	    Marshal.dump(header, io)
+	end
+
+	def self.from_format_0(input, output)
+	    current_cycle = []
+	    while !input.eof?
+		m    = Marshal.load(input)
+		args = Marshal.load(input)
+
+		current_cycle << m << args
+		if m == :cycle_end
+		    Marshal.dump(current_cycle, output)
+		    current_cycle.clear
+		end
+	    end
+
+	    unless current_cycle.empty?
+		Marshal.dump(current_cycle, output)
+	    end
+	end
+
+	def self.from_format_1(input, output)
+	    # The only difference between v1 and v2 is the header. Just copy
+	    # data from input to output
+	    output.write(input.read)
+	end
+
+	def self.from_format_2(input, output)
+	    # In format 3, we did two things:
+	    #   * changed the way exceptions were dumped: instead of relying on
+	    #     DRbRemoteError, we are now using DRobyModel
+	    #   * stopped dumping Time objects and instead marshalled tv_sec
+	    #     and tv_usec directly
+	    Exception::DRoby.class_eval do
+		def self._load(str)
+		    Marshal.load(str)
+		end
+	    end
+
+	    new_cycle = []
+	    input_stream = EventStream.new("input", Logfile.new(input, true))
+	    while !input.eof?
+		new_cycle.clear
+		begin
+		    m_data = input_stream.read
+		    cycle_data = Marshal.load(m_data)
+		    cycle_data.each_slice(2) do |m, args|
+			time = args.shift
+			new_cycle << m << time.tv_sec << time.tv_usec << args
+		    end
+
+		    stats   = new_cycle.last.first
+		    reftime = stats[:start]
+		    stats[:start] = [reftime.tv_sec, reftime.tv_usec]
+		    new_cycle.last[0] = stats.inject(Hash.new) do |new_stats, (name, value)|
+			new_stats[name] = if value.kind_of?(Time) then value - reftime
+					  else value
+					  end
+			new_stats
+		    end
+
+		    Marshal.dump(new_cycle, output)
+		rescue Exception => e
+		    STDERR.puts "dropped cycle because of the following error:"
+		    STDERR.puts "  #{e.full_message}"
+		end
+	    end
+	end
+
+	def self.to_new_format(file, into = file)
+	    input = File.open(file)
+	    log_format = self.log_format(input)
+
+	    if log_format == FORMAT_VERSION
+		STDERR.puts "#{file} is already at format #{log_format}"
+	    else
+		if into =~ /-events\.log$/
+		    into = $`
+		end
+		STDERR.puts "upgrading #{file} from format #{log_format} into #{into}"
+
+		input.rewind
+		Tempfile.open('roby_to_new_format') do |output|
+		    write_header(output)
+		    send("from_format_#{log_format}", input, output)
+		    output.flush
+
+		    input.close
+		    FileUtils.cp output.path, "#{into}-events.log"
+		end
+
+		File.open("#{into}-events.log") do |event_log|
+		    File.open("#{into}-index.log", 'w') do |index_log|
+			puts "rebuilding index file for #{into}"
+			rebuild_index(event_log, index_log)
+		    end
+		end
+	    end
+
+	ensure
+	    input.close if input && !input.closed?
+	end
+    end
+end
+