tracksperanto 2.1.1 → 2.2.0

data/lib/import/maya_live.rb

@@ -15,13 +15,13 @@ class Tracksperanto::Import::MayaLive < Tracksperanto::Import::Base
   
   COMMENT = /^# /
   
-  def stream_parse(original_io)
-    io = Tracksperanto::ExtIO.new(original_io)
+  def each
+    io = Tracksperanto::ExtIO.new(@io)
     extract_width_height_and_aspect(io.gets_non_empty)
     
     while line = io.gets_and_strip
       if line =~ COMMENT
-        send_tracker(@last_tracker) if @last_tracker
+        yield(@last_tracker) if @last_tracker
         @last_tracker = Tracksperanto::Tracker.new(:name => line.gsub(/#{COMMENT} Name(\s+)/, ''))
         next
       end
@@ -32,7 +32,7 @@ class Tracksperanto::Import::MayaLive < Tracksperanto::Import::Base
       @last_tracker.keyframe! :frame => frame, :abs_x => abs_x, :abs_y => abs_y,  :residual => set_residual(residual)
     end
     
-    send_tracker(@last_tracker) if @last_tracker
+    yield(@last_tracker) if @last_tracker
   end
   
   private

data/lib/import/nuke_grammar/utils.rb

@@ -0,0 +1,32 @@
+class Tracksperanto::NukeGrammarUtils
+  SECTION_START = /^x(\d+)$/
+  KEYFRAME = /^([-\d\.]+)$/
+  
+  # Scan a TCL curve expression into a number of tuples of [frame, value]
+  def parse_curve(curve_text)
+    # Replace the closing curly brace with a curly brace with space so that it gets caught by split
+    atoms, tuples = curve_text.gsub(/\}/m, ' }').split, []
+    # Nuke saves curves very efficiently. x(keyframe_number) means that an uninterrupted sequence of values will start,
+    # after which values follow. When the curve is interrupted in some way a new x(keyframe_number) will signifu that we
+    # skip to that specified keyframe and the curve continues from there, in gap size defined by the last fragment.
+    # That is, x1 1 x3 2 3 4 will place 2, 3 and 4 at 2-frame increments
+    
+    last_processed_keyframe = 1
+    intraframe_gap_size = 1
+    while atom = atoms.shift
+      if atom =~ SECTION_START
+        last_processed_keyframe = $1.to_i
+        if tuples.any?
+          last_captured_frame = tuples[-1][0]
+          intraframe_gap_size = last_processed_keyframe - last_captured_frame
+        end
+      elsif  atom =~ KEYFRAME
+        tuples << [last_processed_keyframe, $1.to_f]
+        last_processed_keyframe += intraframe_gap_size
+      elsif atom == '}'
+        return tuples
+      end
+    end
+    tuples
+  end
+end

data/lib/import/nuke_script.rb

@@ -1,4 +1,5 @@
 require 'delegate'
+require File.expand_path(File.dirname(__FILE__)) + "/nuke_grammar/utils"
 
 class Tracksperanto::Import::NukeScript < Tracksperanto::Import::Base
   
@@ -10,14 +11,13 @@ class Tracksperanto::Import::NukeScript < Tracksperanto::Import::Base
     ".nk"
   end
   
-  def stream_parse(in_io)
-    io = Tracksperanto::ExtIO.new(in_io)
+  def each
+    io = Tracksperanto::ExtIO.new(@io)
     while line = io.gets_and_strip
       if line =~ TRACKER_3_PATTERN
-        scan_tracker_node(io).each { |t| send_tracker(t) }
+        scan_tracker_node(io).each { |t| yield(t) }
       end
     end
-    
   end
   
   private
@@ -45,43 +45,13 @@ class Tracksperanto::Import::NukeScript < Tracksperanto::Import::Base
     end
     
     def scan_track(line_with_curve)
-      x_curve, y_curve = line_with_curve.split(/\}/).map{ | one_curve| parse_curve(one_curve) }
+      x_curve, y_curve = line_with_curve.split(/\}/).map do | one_curve| 
+        Tracksperanto::NukeGrammarUtils.new.parse_curve(one_curve)
+      end
       return nil unless (x_curve && y_curve)
       zip_curve_tuples(x_curve, y_curve)
     end
     
-    SECTION_START = /^x(\d+)$/
-    KEYFRAME = /^([-\d\.]+)$/
-    
-    # Scan a curve to a number of tuples of [frame, value]
-    def parse_curve(curve_text)
-      # Replace the closing curly brace with a curly brace with space so that it gets caught by split
-      atoms, tuples = curve_text.gsub(/\}/m, ' }').split, []
-      # Nuke saves curves very efficiently. x(keyframe_number) means that an uninterrupted sequence of values will start,
-      # after which values follow. When the curve is interrupted in some way a new x(keyframe_number) will signifu that we
-      # skip to that specified keyframe and the curve continues from there, in gap size defined by the last fragment.
-      # That is, x1 1 x3 2 3 4 will place 2, 3 and 4 at 2-frame increments
-      
-      last_processed_keyframe = 1
-      intraframe_gap_size = 1
-      while atom = atoms.shift
-        if atom =~ SECTION_START
-          last_processed_keyframe = $1.to_i
-          if tuples.any?
-            last_captured_frame = tuples[-1][0]
-            intraframe_gap_size = last_processed_keyframe - last_captured_frame
-          end
-        elsif  atom =~ KEYFRAME
-          report_progress("Reading curve at frame #{last_processed_keyframe}")
-          tuples << [last_processed_keyframe, $1.to_f]
-          last_processed_keyframe += intraframe_gap_size
-        elsif atom == '}'
-          return tuples
-        end
-      end
-      tuples
-    end
-    
     def extract_tracker(line)
       tuples = scan_track(line)
       return nil unless (tuples && tuples.any?)

data/lib/import/pftrack.rb

@@ -10,16 +10,16 @@ class Tracksperanto::Import::PFTrack < Tracksperanto::Import::Base
   CHARACTERS_OR_QUOTES = /[AZaz"]/
   INTS = /^\d+$/
   
-  def stream_parse(io)
-    until io.eof?
-      line = io.gets
+  def each
+    until @io.eof?
+      line = @io.gets
       next if (!line || line =~ /^#/)
       
       if line =~ CHARACTERS_OR_QUOTES # Tracker with a name
         t = Tracksperanto::Tracker.new(:name => unquote(line.strip))
         report_progress("Reading tracker #{t.name}")
-        parse_tracker(t, io)
-        send_tracker(t)
+        parse_tracker(t, @io)
+        yield(t)
       end
     end
   end

data/lib/import/shake_grammar/lexer.rb

@@ -112,6 +112,9 @@ module Tracksperanto::ShakeGrammar
       string.strip.gsub(/^\"/, '').gsub(/\"$/, '').gsub(/\\\"/, '"')
     end
     
+    # In the default impl. this just puts things on the stack. However,
+    # if you want to unwrap structures as they come along (whych you do for big files)
+    # you have to override this
     def push(atom_array)
       @stack << atom_array
     end

data/lib/import/shake_script.rb

@@ -1,5 +1,6 @@
-require File.dirname(__FILE__) + "/shake_grammar/lexer"
-require File.dirname(__FILE__) + "/shake_grammar/catcher"
+require File.expand_path(File.dirname(__FILE__)) + "/shake_grammar/lexer"
+require File.expand_path(File.dirname(__FILE__)) + "/shake_grammar/catcher"
+
 class Tracksperanto::Import::ShakeScript < Tracksperanto::Import::Base
   
   def self.human_name
@@ -10,9 +11,9 @@ class Tracksperanto::Import::ShakeScript < Tracksperanto::Import::Base
     ".shk"
   end
   
-  def stream_parse(script_io)
+  def each
     progress_proc = lambda{|msg| report_progress(msg) }
-    Traxtractor.new(script_io, [method(:send_tracker), progress_proc])
+    Traxtractor.new(@io, [Proc.new, progress_proc])
   end
   
   private
@@ -216,6 +217,8 @@ class Tracksperanto::Import::ShakeScript < Tracksperanto::Import::Base
     def collect_stabilizer_tracker(name, x_curve, y_curve)
       return unless valid_curves?(x_curve, y_curve)
       
+      report_progress("Scavenging tracker #{name}")
+      
       keyframes = zip_curve_tuples(x_curve, y_curve).map do | (frame, x, y) |
         Tracksperanto::Keyframe.new(:frame => frame - 1, :abs_x => x, :abs_y => y)
       end

data/lib/import/shake_text.rb

@@ -5,10 +5,10 @@ class Tracksperanto::Import::ShakeText < Tracksperanto::Import::Base
     "Shake .txt tracker file"
   end
   
-  def stream_parse(io)
-    io.each do | line |
+  def each
+    @io.each do | line |
       if line =~ /TrackName (.+)/
-        send_tracker(@last_tracker) if @last_tracker && @last_tracker.any?
+        yield(@last_tracker) if @last_tracker && @last_tracker.any?
         @last_tracker = Tracksperanto::Tracker.new(:name => $1)
         # Toss the next following string - header
         io.gets
@@ -25,7 +25,7 @@ class Tracksperanto::Import::ShakeText < Tracksperanto::Import::Base
       end
     end
     
-    send_tracker(@last_tracker) if @last_tracker && @last_tracker.any?
+    yield(@last_tracker) if @last_tracker && @last_tracker.any?
   end
   
 end

data/lib/import/syntheyes.rb

@@ -5,13 +5,13 @@ class Tracksperanto::Import::Syntheyes < Tracksperanto::Import::Base
     "Syntheyes 2D tracker paths file"
   end
   
-  def stream_parse(io)
-    io.each_line do | line |
+  def each
+    @io.each_line do | line |
       name, frame, x, y, frame_status = line.split
       
       # Do we already have this tracker?
       unless @last_tracker && @last_tracker.name == name
-        send_tracker(@last_tracker) if @last_tracker
+        yield(@last_tracker) if @last_tracker
         report_progress("Allocating tracker #{name}")
         @last_tracker = Tracksperanto::Tracker.new{|t| t.name = name }
       end
@@ -26,7 +26,7 @@ class Tracksperanto::Import::Syntheyes < Tracksperanto::Import::Base
       @last_tracker.push(k)
       report_progress("Adding keyframe #{frame} to #{name}")
     end
-    send_tracker(@last_tracker) if @last_tracker && @last_tracker.any?
+    yield(@last_tracker) if @last_tracker && @last_tracker.any?
     
   end
 end

data/lib/pipeline/base.rb

@@ -116,24 +116,30 @@ class Tracksperanto::Pipeline::Base
     end
     @ios << io_with_progress
     
-    accumulator = Tracksperanto::Accumulator.new
-    importer.receiver = accumulator
-    
-    importer.stream_parse(io_with_progress)
+    @accumulator = Tracksperanto::Accumulator.new
+    importer.io = io_with_progress
+    importer.each {|t| @accumulator.push(t) }
+    
+    # OBSOLETE - for this version we are going to permit it
+    if importer.respond_to?(:stream_parse)
+      importer.receiver = @accumulator
+      importer.stream_parse(io_with_progress)
+    end
     
-    report_progress(percent_complete = 50.0, "Validating #{accumulator.length} imported trackers")
-    if accumulator.num_objects.zero?
+    report_progress(percent_complete = 50.0, "Validating #{@accumulator.size} imported trackers")
+    if @accumulator.size.zero?
       raise "Could not recover any non-empty trackers from this file. Wrong import format maybe?"
     end
     
     report_progress(percent_complete, "Starting export")
     
-    percent_per_tracker = (100.0 - percent_complete) / accumulator.num_objects
+    percent_per_tracker = (100.0 - percent_complete) / @accumulator.size
     
     # Use the width and height provided by the parser itself
     exporter.start_export(importer.width, importer.height)
-    accumulator.each_object_with_index do | t, tracker_idx |
-      raise "Not a Tracker" unless t.is_a?(Tracksperanto::Tracker)
+    
+    # Now send each tracker through the middleware chain
+    @accumulator.each_with_index do | t, tracker_idx |
       
       kf_weight = percent_per_tracker / t.keyframes.length
       points += 1
@@ -143,7 +149,7 @@ class Tracksperanto::Pipeline::Base
         exporter.export_point(kf.frame, kf.abs_x, kf.abs_y, kf.residual)
         report_progress(
             percent_complete += kf_weight,
-            "Writing keyframe #{idx+1} of #{t.name.inspect}, #{accumulator.num_objects - tracker_idx} trackers to go"
+            "Writing keyframe #{idx+1} of #{t.name.inspect}, #{@accumulator.size - tracker_idx} trackers to go"
         )
       end
       exporter.end_tracker_segment
@@ -154,8 +160,10 @@ class Tracksperanto::Pipeline::Base
     
     [points, keyframes]
   ensure
+    @accumulator.clear
     @ios.map!{|e| e.close! rescue e.close }
     @ios = []
+    @accumulator = nil
   end
   
   # Setup output files and return a single output
@@ -176,4 +184,4 @@ class Tracksperanto::Pipeline::Base
     @ios ||= []
     @ios.push(File.open(path_to_file, "wb"))[-1]
   end
-end
+end

data/lib/tracksperanto.rb

@@ -5,7 +5,7 @@ require 'tempfile'
 
 module Tracksperanto
   PATH = File.expand_path(File.dirname(__FILE__))
-  VERSION = '2.1.1'
+  VERSION = '2.2.0'
   
   module Import; end
   module Export; end

data/lib/tracksperanto/accumulator.rb

@@ -1,42 +1,68 @@
 # An accumulator buffer for Ruby objects. Use it to sequentially store a shitload
-# of objects on disk and then retreive them one by one. Make sure to call #close! when done with it to
-# discard the stored blob. This object is intended to be used as a Tracksperanto::Import::Base#receiver
+# of objects on disk and then retreive them one by one. Make sure to call clear when done
+# with it to discard the stored blob.
+#
+# This object is intended to be used as a Tracksperanto::Import::Base#receiver, but can be used
+# in general like a disk-based object buffer.
+#
+#  a = Tracksperanto::Accumulator.new
+#  parse_big_file do | one_node |
+#    a.push(one_node)
+#  end
+#
+#  a.size #=> 30932
+#  a.each do | node_read_from_disk |
+#     # do something with node
+#  end
+#
+#  a.clear # ensure that the file is deleted
 class Tracksperanto::Accumulator
+  include Enumerable
   
-  # Stores the number of objects stored so far
-  attr_reader :num_objects
-  alias_method :length, :num_objects
+  # Returns the number of objects stored so far
+  attr_reader :size
   
   def initialize
     @store = Tracksperanto::BufferIO.new
-    @num_objects = 0
+    @size = 0
+    @byte_size = 0
+    
     super
   end
   
   # Store an object
   def push(object_to_store)
-    @num_objects += 1
-    d = Marshal.dump(object_to_store)
-    [d.size, "\t", d, "\n"].map(&@store.method(:write))
+    @store.seek(@byte_size)
+    blob = marshal_object(object_to_store)
+    @store.write(blob)
+    @size += 1
+    @byte_size = @byte_size + blob.size
+    object_to_store
+  end
+  
+  # Retreive each stored object in succession. All other Enumerable
+  # methods are also available (but be careful with Enumerable#map)
+  def each
+    @store.rewind
+    @size.times { yield(recover_object) }
   end
   
-  # Retreive each stored object in succession and unlink the buffer
-  def each_object_with_index
-    begin
-      @store.rewind
-      @num_objects.times { |i| yield(recover_object, i - 1) }
-    ensure
-      @store.close!
-    end
+  # Calls close! on the datastore and deletes the objects in it
+  def clear
+    @store.close!
+    @size = 0
   end
-
   
   private
   
+  def marshal_object(object_to_store)
+    d = Marshal.dump(object_to_store)
+    blob = [d.size, "\t", d, "\n"].join
+  end
+  
   def recover_object
     # Up to the tab is the amount of bytes to read
     demarshal_bytes = @store.gets("\t").strip.to_i
-    # Then read the bytes and unmarshal it
     Marshal.load(@store.read(demarshal_bytes))
   end
 end

data/lib/tracksperanto/buffer_io.rb

@@ -1,7 +1,8 @@
 require "tempfile"
 
-# BufferIO is used for writing big segments of text. When the segment is bigger than a certain number of bytes,
-# the underlying memory buffer will be swapped with a tempfile
+# BufferIO is used for writing big segments of text. It works like a StringIO, but when the size
+# of the underlying string buffer exceeds MAX_IN_MEM_BYTES the string will be flushed to disk
+# and it automagically becomes a Tempfile
 class Tracksperanto::BufferIO < DelegateClass(IO)
   include Tracksperanto::Returning
   

data/lib/tracksperanto/const_name.rb

@@ -1,3 +1,5 @@
+# Provides const_name that returns the name of the class or module (or the name of the class
+# an instance belongs to)  without it's parent namespace. Useful for building module tables
 module Tracksperanto::ConstName
   module C
     def const_name

data/lib/tracksperanto/progressive_io.rb

@@ -1,5 +1,6 @@
 # Used for IO objects that need to report the current offset at each operation that changes the said offset
-# (useful for building progress bars that report on a file read operation)
+# (useful for building progress bars that report on a file read operation). Pass the actual IO and a block to
+# report progress to the constructor.
 class Tracksperanto::ProgressiveIO < DelegateClass(IO)
   include Tracksperanto::Returning
   
@@ -14,6 +15,7 @@ class Tracksperanto::ProgressiveIO < DelegateClass(IO)
     @progress_block = blk.to_proc if blk
   end
   
+  # Report offset at each line
   def each(sep_string = $/, &blk)
     # Report offset at each call of the iterator
     result = super(sep_string) do | line |

data/lib/tracksperanto/tracker.rb

@@ -1,4 +1,5 @@
-# Internal representation of a tracker point with keyframes. A Tracker is an array of Keyframe objects and should act and work like one
+# Internal representation of a tracker point with keyframes. A Tracker is an array of Keyframe objects
+# with a few methods added for convenience
 class Tracksperanto::Tracker < DelegateClass(Array)
   include Tracksperanto::Casts
   include Tracksperanto::BlockInit