tracksperanto 2.2.4 → 2.3.0

data/History.txt

@@ -1,3 +1,9 @@
+=== 2.3.0 / 2011-17-04
+
+* Fixed issue#1 (Last tracker in some Flame exports receives an empty shift channel)
+* Nicer progress reports
+* Modified the Accumulator to support nested iterations and random access
+
 === 2.2.4 / 2011-04-04
 
 * Properly deal with negative frames happening in Shake scripts sometimes. We will now recognize a script starting at negative

data/MIT_LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2004-2010 Julik Tarkhanov
+Copyright (c) 2004-2011 Julik Tarkhanov
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/Manifest.txt

@@ -56,6 +56,7 @@ lib/tracksperanto/ext_io.rb
 lib/tracksperanto/flame_builder.rb
 lib/tracksperanto/format_detector.rb
 lib/tracksperanto/keyframe.rb
+lib/tracksperanto/pbar.rb
 lib/tracksperanto/progressive_io.rb
 lib/tracksperanto/returning.rb
 lib/tracksperanto/safety.rb

data/bin/tracksperanto

@@ -17,6 +17,7 @@ require 'progressbar'
 require "net/http"
 require "open-uri"
 require "timeout"
+require File.dirname(__FILE__) + '/../lib/tracksperanto/pbar'
 
 def version_check(version_present)
   begin
@@ -104,12 +105,11 @@ unless File.exist?(input_file)
   exit(-1)
 end
 
-pbar = ProgressBar.new("Converting", 100, $stderr)
-pipe = Tracksperanto::Pipeline::Base.new(:progress_block => lambda{|p,m| pbar.set(p) }, :middleware_tuples => $middlewares)
+pbar = Tracksperanto::PBar.new("Converting", 100, $stderr)
+pipe = Tracksperanto::Pipeline::Base.new(:progress_block => lambda{|p,m| pbar.set_with_message(p, m) }, :middleware_tuples => $middlewares)
 pipe.exporters = [Tracksperanto.get_exporter(writer_klass_name)] if writer_klass_name
 pipe.run(input_file, options)
 pbar.finish
 
-puts "Converted #{pipe.converted_points} trackers with #{pipe.converted_keyframes} keys."
 puts disclaimer
 version_check(Tracksperanto::VERSION)

data/lib/export/flame_stabilizer.rb

@@ -44,6 +44,9 @@ class Tracksperanto::Export::FlameStabilizer < Tracksperanto::Export::Base
         t.tolerance 100
       end
     end
+    
+    # Write the finalizing "End"
+    @writer.write_loose!("end")
   end
   
   def start_tracker_segment(tracker_name)
@@ -66,6 +69,11 @@ class Tracksperanto::Export::FlameStabilizer < Tracksperanto::Export::Base
     # how many keyframes they should contain
     write_shift_channel("shift/x", @x_shift_values)
     write_shift_channel("shift/y", @y_shift_values)
+    
+    # And finish with the offset channels. The order of channels is important!
+    # (otherwise the last tracker's shift animation is not imported by Flame)
+    # https://github.com/guerilla-di/tracksperanto/issues/1
+    write_offset_channels
   end
   
   private
@@ -140,7 +148,6 @@ class Tracksperanto::Export::FlameStabilizer < Tracksperanto::Export::Base
     write_ref_width_and_height
     write_ref_channels(x, y)
     write_deltax_and_deltay_channels
-    write_offset_channels
   end
   
   def write_track_channels

data/lib/import/flame_stabilizer.rb

@@ -75,13 +75,15 @@ class Tracksperanto::Import::FlameStabilizer < Tracksperanto::Import::Base
     report_progress("Extracting setup size")
     self.width, self.height = extract_width_and_height_from_stream(@io)
     report_progress("Extracting all animation channels")
-    channels = extract_channels_from_stream(@io)
+    channels, names = extract_channels_from_stream(@io)
     
     raise "The setup contained no channels that we could process" if channels.empty?
     raise "A channel was nil" if channels.find{|e| e.nil? }
     
     report_progress("Assembling tracker curves from channels")
-    scavenge_trackers_from_channels(channels) {|t| yield(t) }
+    scavenge_trackers_from_channels(channels, names) {|t| yield(t) }
+  ensure
+    channels.clear
   end
   
   private
@@ -103,54 +105,20 @@ class Tracksperanto::Import::FlameStabilizer < Tracksperanto::Import::Base
       end
       
     end
-=begin
-Here's how a Flame channel looks like
-The Size will not be present if there are no keyframes
-  
-Channel tracker1/ref/x
-	Extrapolation constant
-	Value 770.41
-	Size 4
-	KeyVersion 1
-	Key 0
-		Frame 1
-		Value 770.41
-		Interpolation constant
-		End
-	Key 1
-		Frame 44
-		Value 858.177
-		Interpolation constant
-		RightSlope 2.31503
-		LeftSlope 2.31503
-		End
-	Key 2
-		Frame 74
-		Value 939.407
-		Interpolation constant
-		RightSlope 2.24201
-		LeftSlope 2.24201
-		End
-	Key 3
-		Frame 115
-		Value 1017.36
-		Interpolation constant
-		End
-	Colour 50 50 50 
-	End
-=end
 
     def extract_channels_from_stream(io)
-      channels = []
+      channels = Tracksperanto::Accumulator.new
+      names = []
       channel_matcher = /Channel (.+)\n/
       until io.eof?
         line = io.gets
         if line =~ channel_matcher && channel_is_useful?(line)
           report_progress("Extracting channel #{$1}")
           channels << ChannelBlock.new(io, $1)
+          names << $1
         end
       end
-      channels
+      [channels, names]
     end
     
     USEFUL_CHANNELS = %w( /shift/x /shift/y /ref/x /ref/y ).map(&Regexp.method(:new))
@@ -166,21 +134,32 @@ Channel tracker1/ref/x
     
     REF_CHANNEL = "ref" # or "track" - sometimes works sometimes don't
     
-    def scavenge_trackers_from_channels(channels)
-      channels.select{|e| e.name =~ /\/#{REF_CHANNEL}\/x/}.each do | track_x |
-        report_progress("Detected reference channel #{track_x}")
-        yield grab_tracker(channels, track_x)
+    def scavenge_trackers_from_channels(channels, names)
+      
+      
+      channels.each do |c|
+        next unless c.name =~ /\/#{REF_CHANNEL}\/x/
+        
+        report_progress("Detected reference channel #{c.name}")
+        
+        yield grab_tracker(channels, c, names)
       end
     end
     
-    def grab_tracker(channels, track_x)
+    def grab_tracker(channels, track_x, names)
       t = Tracksperanto::Tracker.new(:name => track_x.name.split('/').shift)
       
       report_progress("Extracting tracker #{t.name}")
       
-      track_y = channels.find{|e| e.name == "#{t.name}/#{REF_CHANNEL}/y" }
-      shift_x = channels.find{|e| e.name == "#{t.name}/shift/x" }
-      shift_y = channels.find{|e| e.name == "#{t.name}/shift/y" }
+      # This takes a LONG time when we have alot of channels, we need a precache of
+      # some sort to do this
+      ref_idx = names.index("#{t.name}/#{REF_CHANNEL}/y")
+      shift_x_idx = names.index("#{t.name}/shift/x")
+      shift_y_idx = names.index("#{t.name}/shift/y")
+      
+      track_y = channels[ref_idx]
+      shift_x = channels[shift_x_idx]
+      shift_y = channels[shift_y_idx]
       
       shift_tuples = zip_curve_tuples(shift_x, shift_y)
       track_tuples = zip_curve_tuples(track_x, track_y)
@@ -207,4 +186,41 @@ Channel tracker1/ref/x
       end || track_tuples[0]
       base_track_tuple[1..2]
     end
-end
+end
+
+__END__
+
+Here's how a Flame channel looks like
+The Size will not be present if there are no keyframes
+  
+Channel tracker1/ref/x
+	Extrapolation constant
+	Value 770.41
+	Size 4
+	KeyVersion 1
+	Key 0
+		Frame 1
+		Value 770.41
+		Interpolation constant
+		End
+	Key 1
+		Frame 44
+		Value 858.177
+		Interpolation constant
+		RightSlope 2.31503
+		LeftSlope 2.31503
+		End
+	Key 2
+		Frame 74
+		Value 939.407
+		Interpolation constant
+		RightSlope 2.24201
+		LeftSlope 2.24201
+		End
+	Key 3
+		Frame 115
+		Value 1017.36
+		Interpolation constant
+		End
+	Colour 50 50 50 
+	End

data/lib/import/shake_grammar/lexer.rb

@@ -78,7 +78,7 @@ module Tracksperanto::ShakeGrammar
       end
     end
     
-    INT_ATOM = /^(-?\d+)$/
+    INT_ATOM = /^(\d+)$/
     FLOAT_ATOM = /^([\-\d\.]+)$/
     STR_ATOM = /^\"/
     AT_FRAME = /^@(-?\d+)/

data/lib/import/shake_script.rb

@@ -243,7 +243,7 @@ class Tracksperanto::Import::ShakeScript < Tracksperanto::Import::Base
     def collect_stabilizer_tracker(name, x_curve, y_curve)
       return unless valid_curves?(x_curve, y_curve)
       
-      report_progress("Scavenging tracker #{name}")
+      report_progress("Assembling Stabilizer node tracker #{name}")
       
       keyframes = zip_curve_tuples(x_curve, y_curve).map do | (frame, x, y) |
         Tracksperanto::Keyframe.new(:frame => frame - 1, :abs_x => x, :abs_y => y)

data/lib/import/syntheyes.rb

@@ -26,6 +26,7 @@ class Tracksperanto::Import::Syntheyes < Tracksperanto::Import::Base
       @last_tracker.push(k)
       report_progress("Adding keyframe #{frame} to #{name}")
     end
+    
     yield(@last_tracker) if @last_tracker && @last_tracker.any?
     
   end

data/lib/pipeline/base.rb

@@ -140,52 +140,54 @@ module Tracksperanto::Pipeline
     @ios.push(io_with_progress)
     
     @accumulator = Tracksperanto::Accumulator.new
+    begin
+    
+      # OBSOLETE - for this version we are going to permit it.
+      if importer.respond_to?(:stream_parse)
+        STDERR.puts "Import::Base#stream_parse(io) is obsolete, please rewrite your importer to use each instead"
+        importer.receiver = @accumulator
+        importer.stream_parse(io_with_progress)
+      else
+        importer.io = io_with_progress
+        importer.each {|t| @accumulator.push(t) unless t.empty? }
+      end
     
-    # OBSOLETE - for this version we are going to permit it.
-    if importer.respond_to?(:stream_parse)
-      STDERR.puts "Import::Base#stream_parse(io) is obsolete, please rewrite your importer to use each instead"
-      importer.receiver = @accumulator
-      importer.stream_parse(io_with_progress)
-    else
-      importer.io = io_with_progress
-      importer.each {|t| @accumulator.push(t) unless t.empty? }
-    end
-    
-    report_progress(percent_complete = 50.0, "Validating #{@accumulator.size} imported trackers")
-    raise NoTrackersRecoveredError if @accumulator.size.zero?
+      report_progress(percent_complete = 50.0, "Validating #{@accumulator.size} imported trackers")
+      raise NoTrackersRecoveredError if @accumulator.size.zero?
     
-    report_progress(percent_complete, "Starting export")
+      report_progress(percent_complete, "Starting export")
     
-    percent_per_tracker = (100.0 - percent_complete) / @accumulator.size
+      percent_per_tracker = (100.0 - percent_complete) / @accumulator.size
     
-    # Use the width and height provided by the parser itself
-    exporter.start_export(importer.width, importer.height)
+      # Use the width and height provided by the parser itself
+      exporter.start_export(importer.width, importer.height)
     
-    # Now send each tracker through the middleware chain
-    @accumulator.each_with_index do | t, tracker_idx |
+      # Now send each tracker through the middleware chain
+      @accumulator.each_with_index do | t, tracker_idx |
       
-      kf_weight = percent_per_tracker / t.keyframes.length
-      points += 1
-      exporter.start_tracker_segment(t.name)
-      t.each_with_index do | kf, idx |
-        keyframes += 1
-        exporter.export_point(kf.frame, kf.abs_x, kf.abs_y, kf.residual)
-        report_progress(
-            percent_complete += kf_weight,
-            "Writing keyframe #{idx+1} of #{t.name.inspect}, #{@accumulator.size - tracker_idx} trackers to go"
-        )
+        kf_weight = percent_per_tracker / t.keyframes.length
+        points += 1
+        exporter.start_tracker_segment(t.name)
+        t.each_with_index do | kf, idx |
+          keyframes += 1
+          exporter.export_point(kf.frame, kf.abs_x, kf.abs_y, kf.residual)
+          report_progress(
+              percent_complete += kf_weight,
+              "Writing keyframe #{idx+1} of #{t.name.inspect}, #{@accumulator.size - tracker_idx} trackers to go"
+          )
+        end
+        exporter.end_tracker_segment
       end
-      exporter.end_tracker_segment
-    end
-    exporter.end_export
+      exporter.end_export
     
-    report_progress(100.0, "Wrote #{points} points and #{keyframes} keyframes")
+      report_progress(100.0, "Wrote #{points} points and #{keyframes} keyframes")
     
-    [points, keyframes]
-  ensure
-    @accumulator.clear
-    @ios.map!{|e| e.close! rescue e.close }
-    @ios.clear
+      [points, keyframes]
+    ensure
+      @accumulator.clear
+      @ios.map!{|e| e.close! rescue e.close }
+      @ios.clear
+    end
   end
   
   # Setup output files and return a single output

data/lib/tracksperanto.rb

@@ -4,7 +4,7 @@ require 'tempfile'
 
 module Tracksperanto
   PATH = File.expand_path(File.dirname(__FILE__))
-  VERSION = '2.2.4'
+  VERSION = '2.3.0'
   
   module Import; end
   module Export; end

data/lib/tracksperanto/accumulator.rb

@@ -17,26 +17,27 @@
 class Tracksperanto::Accumulator
   include Enumerable
   
+  DELIM = "\n"
+  
   # Returns the number of objects stored so far
   attr_reader :size
   
   def initialize
     @store = Tracksperanto::BufferIO.new
-    
     @size = 0
-    @byte_size = 0
     
     super
   end
   
+  def empty?
+    @size.zero?
+  end
+  
   # Store an object
   def push(object_to_store)
-    @store.seek(@byte_size)
-    
     blob = marshal_object(object_to_store)
     @store.write(blob)
     @size += 1
-    @byte_size = @byte_size + blob.size
     
     object_to_store
   end
@@ -46,27 +47,57 @@ class Tracksperanto::Accumulator
   # Retreive each stored object in succession. All other Enumerable
   # methods are also available (but be careful with Enumerable#map and to_a)
   def each
-    @store.rewind
-    @size.times { yield(recover_object) }
+    with_separate_read_io do | iterable |
+      @size.times { yield(recover_object_from(iterable)) }
+    end
   end
   
   # Calls close! on the datastore and deletes the objects in it
   def clear
     @store.close!
     @size = 0
-    @offsets = []
+  end
+  
+  # Retreive a concrete object at index
+  def [](idx)
+    idx.respond_to?(:each) ? idx.map{|i| recover_at(i) } : recover_at(idx)
   end
   
   private
   
+  def recover_at(idx)
+    with_separate_read_io do | iterable |
+      iterable.seek(0)
+      
+      # Do not unmarshal anything but wind the IO in fixed offsets
+      idx.times do
+        skip_bytes = iterable.gets("\t").to_i
+        iterable.seek(iterable.pos + skip_bytes)
+      end
+      
+      recover_object_from(iterable)
+    end
+  end
+  
+  # We first ensure that we have a disk-backed file, then reopen it as read-only
+  # and iterate through that (we will have one IO handle per loop nest)
+  def with_separate_read_io
+    iterable = File.open(@store.to_file.path, "r")
+    yield(iterable)
+  ensure
+    iterable.close
+  end
+  
   def marshal_object(object_to_store)
     d = Marshal.dump(object_to_store)
-    blob = [d.size, "\t", d, "\n"].join
+    blob = [d.size, "\t", d, DELIM].join
   end
   
-  def recover_object
+  def recover_object_from(io)
     # Up to the tab is the amount of bytes to read
-    demarshal_bytes = @store.gets("\t").strip.to_i
-    Marshal.load(@store.read(demarshal_bytes))
+    demarshal_bytes = io.gets("\t").to_i
+    blob = io.read(demarshal_bytes)
+    
+    Marshal.load(blob)
   end
 end

data/lib/tracksperanto/buffer_io.rb

@@ -30,18 +30,34 @@ class Tracksperanto::BufferIO < DelegateClass(IO)
     __setobj__(nil)
   end
   
+  # Sometimes you just need to upgrade to a File forcibly (for example if you want)
+  # to have an object with many iterators sitting on it. We also flush here.
+  def to_file
+    replace_with_tempfile unless @tempfile_in
+    flush
+    self
+  end
+  
+  # Tells whether this one is on disk
+  def file_backed?
+    @tempfile_in
+  end
+  
   private
   
+  def replace_with_tempfile
+    sio = __getobj__
+    tf = Tempfile.new("tracksperanto-xbuf")
+    tf.write(sio.string)
+    tf.flush # Needed of we will reopen this file soon from another thread/loop
+    sio.string = ""
+    GC.start
+    __setobj__(tf)
+    
+    @tempfile_in = true
+  end
+  
   def replace_with_tempfile_if_needed
-    return if @tempfile_in
-    io = __getobj__
-    if io.pos > MAX_IN_MEM_BYTES
-      tf = Tempfile.new("tracksperanto-xbuf")
-      tf.write(io.string)
-      io.string = ""
-      GC.start
-      __setobj__(tf)
-      @tempfile_in = true
-    end
+    replace_with_tempfile if !@tempfile_in && pos > MAX_IN_MEM_BYTES
   end
 end

data/lib/tracksperanto/pbar.rb

@@ -0,0 +1,13 @@
+# We make use of the implementation details of the progress bar to also show our current status
+class Tracksperanto::PBar < ::ProgressBar
+  
+  def fmt_title
+    "%20s" % @title
+  end
+  
+  def set_with_message(pcnt, message)
+    @title = message
+    set(pcnt)
+  end
+  
+end

data/test/export/samples/ref_flame.stabilizer

@@ -122,14 +122,6 @@ Channel tracker1/ref/dy
 		Interpolation constant
 		End
 	End
-Channel tracker1/offset/x
-	Extrapolation constant
-	Value 0
-	End
-Channel tracker1/offset/y
-	Extrapolation constant
-	Value 0
-	End
 Channel tracker1/shift/x
 	Extrapolation constant
 	Value 0
@@ -408,6 +400,14 @@ Channel tracker1/shift/y
 		RightSlope 2.400
 		End
 	End
+Channel tracker1/offset/x
+	Extrapolation constant
+	Value 0
+	End
+Channel tracker1/offset/y
+	Extrapolation constant
+	Value 0
+	End
 Channel tracker2/track/x
 	Extrapolation constant
 	Value 960
@@ -508,14 +508,6 @@ Channel tracker2/ref/dy
 		Interpolation constant
 		End
 	End
-Channel tracker2/offset/x
-	Extrapolation constant
-	Value 0
-	End
-Channel tracker2/offset/y
-	Extrapolation constant
-	Value 0
-	End
 Channel tracker2/shift/x
 	Extrapolation constant
 	Value 0
@@ -794,6 +786,14 @@ Channel tracker2/shift/y
 		RightSlope 2.400
 		End
 	End
+Channel tracker2/offset/x
+	Extrapolation constant
+	Value 0
+	End
+Channel tracker2/offset/y
+	Extrapolation constant
+	Value 0
+	End
 Channel tracker3/track/x
 	Extrapolation constant
 	Value 960
@@ -894,14 +894,6 @@ Channel tracker3/ref/dy
 		Interpolation constant
 		End
 	End
-Channel tracker3/offset/x
-	Extrapolation constant
-	Value 0
-	End
-Channel tracker3/offset/y
-	Extrapolation constant
-	Value 0
-	End
 Channel tracker3/shift/x
 	Extrapolation constant
 	Value 0
@@ -928,6 +920,14 @@ Channel tracker3/shift/y
 		RightSlope 2.400
 		End
 	End
+Channel tracker3/offset/x
+	Extrapolation constant
+	Value 0
+	End
+Channel tracker3/offset/y
+	Extrapolation constant
+	Value 0
+	End
 ChannelEnd
 Tracker 0
 	Active yes
@@ -959,3 +959,4 @@ Tracker 2
 	FixedX no
 	FixedY no
 	Tolerance 100
+End

data/test/import/test_flame_import.rb

@@ -151,8 +151,7 @@ class FlameImportTest < Test::Unit::TestCase
   
   def test_from_tracksperanto
     fixture = File.open(File.dirname(__FILE__) + '/samples/flame_stabilizer/fromTracksperanto.stabilizer')
-    parser = Tracksperanto::Import::FlameStabilizer.new(:io => fixture)
-    trackers = parser.to_a
+    trackers = Tracksperanto::Import::FlameStabilizer.new(:io => fixture).to_a
     
     assert_equal 3, trackers.length
     assert_equal 3, trackers[0].length

data/test/test_accumulator.rb

@@ -17,6 +17,49 @@ class TestAccumulator < Test::Unit::TestCase
     assert_equal 50_000, a.size
   end
   
+  def test_accumulator_saves_shitload_of_objs
+    a = Tracksperanto::Accumulator.new
+    4.times { a.push("A \tstring") }
+    a.each {|e| assert_equal "A \tstring", e }
+  end
+  
+  def test_accumulator_empty
+    a = Tracksperanto::Accumulator.new
+    assert a.empty?
+    a.push(1)
+    assert !a.empty?
+  end
+  
+  def test_accumulator_supports_nested_iteration
+    a = Tracksperanto::Accumulator.new
+    ("A".."Z").each{|e| a << e}
+    
+    accumulated = []
+    seen_g = false
+    a.each do | first_level |
+      if first_level == "G"
+        seen_g = true
+        # Force a nested iteration and break it midway
+        a.each do | second_level |
+          accumulated.push(second_level)
+          break if second_level == "E"
+        end
+      elsif seen_g
+        assert_equal "H", first_level
+        return
+      end
+    end
+  end
+  
+  def test_random_access
+    a = Tracksperanto::Accumulator.new
+    letters = ("A".."Z").map{|e| "#{e}\r\nWow!"}.to_a
+    letters.map(&a.method(:push))
+    
+    assert_equal "B\r\nWow!", a[1]
+    assert_equal "E\r\nWow!", a[4]
+  end
+  
   def test_clear_calls_close_on_buffer
     io = Tracksperanto::BufferIO.new
     flexmock(io).should_receive(:close!)

data/test/test_buffer_io.rb

@@ -19,4 +19,15 @@ class TestBufferIO < Test::Unit::TestCase
     flexmock(f).should_receive(:close!).once
     io.close!
   end
+  
+  def test_to_file_forces_immediate_promotion_to_file
+    io = Tracksperanto::BufferIO.new
+    io.write("a" * 3000)
+    assert_equal 3000, io.pos
+    assert !io.file_backed?
+    
+    f = io.to_file
+    assert_equal 3000, f.pos
+    assert f.file_backed?
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: tracksperanto
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 3
   prerelease: 
   segments: 
   - 2
-  - 2
-  - 4
-  version: 2.2.4
+  - 3
+  - 0
+  version: 2.3.0
 platform: ruby
 authors: 
 - Julik Tarkhanov
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-06 00:00:00 +02:00
+date: 2011-04-17 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -134,6 +134,7 @@ files:
 - lib/tracksperanto/flame_builder.rb
 - lib/tracksperanto/format_detector.rb
 - lib/tracksperanto/keyframe.rb
+- lib/tracksperanto/pbar.rb
 - lib/tracksperanto/progressive_io.rb
 - lib/tracksperanto/returning.rb
 - lib/tracksperanto/safety.rb