tracksperanto 2.6.3 → 2.7.0

data/History.txt

@@ -1,3 +1,12 @@
+=== 2.7.0 / 2011-13-10
+
+* Speedup Shake script parsing by using a buffering reader
+
+=== 2.6.4 / 2011-29-08
+
+* Use various external dependencies that we extracted
+* Note that Nuke now supports import and export of camera tracker features!
+
 === 2.6.3 / 2011-31-07
 
 * Report progress from the parser for Flame, like it has been before

data/Manifest.txt

@@ -51,16 +51,15 @@ lib/middleware/slipper.rb
 lib/middleware/start_trim.rb
 lib/pipeline/base.rb
 lib/tracksperanto.rb
-lib/tracksperanto/accumulator.rb
 lib/tracksperanto/block_init.rb
 lib/tracksperanto/buffer_io.rb
+lib/tracksperanto/buffering_reader.rb
 lib/tracksperanto/casts.rb
 lib/tracksperanto/const_name.rb
 lib/tracksperanto/ext_io.rb
 lib/tracksperanto/flame_builder.rb
 lib/tracksperanto/format_detector.rb
 lib/tracksperanto/keyframe.rb
-lib/tracksperanto/progressive_io.rb
 lib/tracksperanto/returning.rb
 lib/tracksperanto/safety.rb
 lib/tracksperanto/simple_export.rb
@@ -94,7 +93,6 @@ test/export/test_pftrack5_export.rb
 test/export/test_pftrack_export.rb
 test/export/test_shake_export.rb
 test/export/test_syntheyes_export.rb
-test/fixtures/processing_log.txt
 test/helper.rb
 test/import/samples/3de_v3/3de_export_v3.txt
 test/import/samples/3de_v4/3de_export_cube.txt
@@ -135,6 +133,7 @@ test/import/samples/shake_script/stabilize_nodes_with_hermite.shk
 test/import/samples/shake_script/three_tracks_in_one_stabilizer.shk
 test/import/samples/shake_script/track.shk
 test/import/samples/shake_script/two_tracks_in_one_tracker.shk
+test/import/samples/shake_text/nuke_camtracker.txt
 test/import/samples/shake_text/one_shake_tracker.txt
 test/import/samples/shake_text/one_shake_tracker_from_first.txt
 test/import/samples/shake_text/two_shake_trackers.txt
@@ -168,8 +167,8 @@ test/middleware/test_scaler_middleware.rb
 test/middleware/test_shift_middleware.rb
 test/middleware/test_slip_middleware.rb
 test/middleware/test_start_trim_middleware.rb
-test/test_accumulator.rb
 test/test_buffer_io.rb
+test/test_bufferingreader.rb
 test/test_cli.rb
 test/test_const_name.rb
 test/test_extio.rb
@@ -177,7 +176,6 @@ test/test_flame_builder.rb
 test/test_format_detector.rb
 test/test_keyframe.rb
 test/test_pipeline.rb
-test/test_progressive_io.rb
 test/test_simple_export.rb
 test/test_tracker.rb
 test/test_tracksperanto.rb

data/Rakefile

@@ -11,6 +11,8 @@ begin
     p.version = Tracksperanto::VERSION
     
     p.extra_deps = {
+      "obuf" => "~> 1.0",
+      "progressive_io" => "~> 1.0",
       "flame_channel_parser" => "2.2.1",
       "progressbar" => "~> 0.9",
       "update_hints" => "~> 1.0"

data/lib/export/shake_text.rb

@@ -9,7 +9,7 @@ class Tracksperanto::Export::ShakeText < Tracksperanto::Export::Base
   end
   
   def self.human_name
-    "Shake trackers in a .txt file"
+    "Shake trackers in a .txt file (also usable with Nuke's CameraTracker)"
   end
   
   def start_tracker_segment(tracker_name)

data/lib/import/shake_grammar/lexer.rb

@@ -20,8 +20,13 @@ module Tracksperanto::ShakeGrammar
     # to the downstream lexers instantiated for nested data structures.
     # You can use the sentinel to collect data from child nodes for example.
     def initialize(with_io, sentinel = nil, limit_to_one_stmt = false, stack_depth = 0)
+      # We parse byte by byte, but reading byte by byte is very slow. We therefore use a buffering reader
+      # that will cache in chunks, and then read from there byte by byte. This yields a substantial speedup (4.9 seconds for the test
+      # as opposed to 7.9 without this). We do check for the proper class only once so that when we use nested lexers
+      # we only wrap the passed IO once, and only if necessary.
+      with_io = Tracksperanto::BufferingReader.new(with_io) unless with_io.respond_to?(:read_one_byte)
       @io, @stack, @buf, @sentinel, @limit_to_one_stmt, @stack_depth  = with_io, [], '', sentinel, limit_to_one_stmt, stack_depth
-      catch(STOP_TOKEN) { parse until @io.eof? }
+      catch(STOP_TOKEN) { parse until @io.data_exhausted? }
       @in_comment ? consume_comment! : consume_atom!
     end
     
@@ -38,7 +43,7 @@ module Tracksperanto::ShakeGrammar
     
     def parse
       
-      c = @io.read(1)
+      c = @io.read_one_byte
       
       if @buf.length > MAX_BUFFER_SIZE # Wrong format and the buffer is filled up, bail
         raise WrongInputError, "Atom buffer overflow at #{MAX_BUFFER_SIZE} bytes, this is definitely not a Shake script"

data/lib/import/shake_text.rb

@@ -2,7 +2,7 @@
 class Tracksperanto::Import::ShakeText < Tracksperanto::Import::Base
   
   def self.human_name
-    "Shake .txt tracker file"
+    "Shake .txt tracker file and Nuke CameraTracker auto tracks export"
   end
   
   def each

data/lib/pipeline/base.rb

@@ -20,7 +20,8 @@ module Tracksperanto::Pipeline
     end
   end
   
-  # The base pipeline is the whole process of track conversion from start to finish. The pipeline object organizes the import formats, scans them,
+  # The base pipeline is the whole process of track conversion from start to finish. 
+  # The pipeline object organizes the import formats, scans them,
   # applies the middlewares. Here's how a calling sequence for a pipeline looks like:
   #
   #   pipe = Tracksperanto::Pipeline::Base.new
@@ -157,18 +158,17 @@ module Tracksperanto::Pipeline
     
     @ios.push(io_with_progress)
     
-    @accumulator = Tracksperanto::Accumulator.new
+    @accumulator = Obuf.new
     
     begin
     
-      # OBSOLETE - for this version we are going to permit it.
-      if importer.respond_to?(:stream_parse)
+      if importer.respond_to?(:each)
+        importer.io = io_with_progress
+        importer.each {|t| @accumulator.push(t) unless t.empty? }
+      else # OBSOLETE - for this version we are going to permit it.
         STDERR.puts "Import::Base#stream_parse(io) is obsolete, please rewrite your importer to use each instead"
         importer.receiver = @accumulator
         importer.stream_parse(io_with_progress)
-      else
-        importer.io = io_with_progress
-        importer.each {|t| @accumulator.push(t) unless t.empty? }
       end
     
       report_progress(percent_complete = 50.0, "Validating #{@accumulator.size} imported trackers")

data/lib/tracksperanto/buffering_reader.rb

@@ -0,0 +1,40 @@
+# Shake uses this reader to parse byte by byte without having to read byte by byte.
+# Reading byte by byte is very inefficient, but we want to parse byte by byte since
+# this makes parser construction much easier. So what we do is cache some chunk of the
+# passed buffer and read from that. Once exhausted there will be some caching again,
+# and ad infinitum until the passed buffer is exhausted
+class Tracksperanto::BufferingReader
+  
+  # By default will read in chunks of 4K
+  DEFAULT_BUFFER_SIZE = 10240
+  
+  def initialize(with_io, buffer_size = DEFAULT_BUFFER_SIZE)
+    @io = with_io
+    @bufsize = buffer_size
+    @buf = StringIO.new
+  end
+  
+  # Will transparently read one byte off the contained IO, maintaining the internal cache.
+  # If the cache has been depleted it will read a big chunk from the IO and cache it and then
+  # return the byte
+  def read_one_byte
+    cache if @buf.pos == @buf.size
+    
+    return nil if @buf.size.zero?
+    return @buf.read(1)
+  end
+  
+  # Tells whether all the data has been both read from the passed buffer
+  # and from the internal cache buffer (checks whether there is anything that
+  # can be retreived using read_one_byte)
+  def data_exhausted?
+    @buf.eof? && @io.eof?
+  end
+  
+  private
+  
+  def cache
+    data = @io.read(@bufsize)
+    @buf = StringIO.new(data.to_s) # Make nil become ""
+  end
+end

data/lib/tracksperanto.rb

@@ -2,10 +2,12 @@ require 'stringio'
 require 'delegate'
 require 'tempfile'
 require 'flame_channel_parser'
+require "obuf"
+require "progressive_io"
 
 module Tracksperanto
   PATH = File.expand_path(File.dirname(__FILE__))
-  VERSION = '2.6.3'
+  VERSION = '2.7.0'
   
   module Import; end
   module Export; end
@@ -70,6 +72,15 @@ module Tracksperanto
     
     raise UnknownImporterError, "Unknown importer #{name.inspect}"
   end
+  
+  # DEPRECATED, will be removed in Tracksperanto 3
+  class Accumulator < Obuf
+  end
+  
+  # DEPRECATED, will be removed in Tracksperanto 3
+  class Tracksperanto::ProgressiveIO < ProgressiveIO
+  end
+  
 end
 
 %w(
@@ -83,12 +94,11 @@ end
   tracker
   format_detector
   ext_io
-  progressive_io
   buffer_io
   simple_export
   uv_coordinates
   flame_builder
-  accumulator
+  buffering_reader
 ).each do | submodule |
   require File.join(Tracksperanto::PATH, "tracksperanto", submodule)
 end

data/test/export/test_shake_export.rb

@@ -10,6 +10,6 @@ class ShakeTextExportTest < Test::Unit::TestCase
   
   def test_exporter_meta
     assert_equal "shake_trackers.txt", Tracksperanto::Export::ShakeText.desc_and_extension
-    assert_equal "Shake trackers in a .txt file", Tracksperanto::Export::ShakeText.human_name
+    assert_equal "Shake trackers in a .txt file (also usable with Nuke's CameraTracker)", Tracksperanto::Export::ShakeText.human_name
   end
 end