ffi-libav 0.1.0 → 0.2.0

data/.gitignore

@@ -25,3 +25,6 @@ lib/ffi/*.xml
 # OSX artifacts
 .AppleDouble
 .DS_Store
+
+# We don't want to include the test video
+spec/data/big_buck_bunny_480p_surround-fix.avi

data/.rspec

@@ -0,0 +1,2 @@
+--format doc
+--color

data/Gemfile

@@ -7,4 +7,5 @@ group :development do
   gem 'pry'
   gem 'ffi-swig-generator',
     :git => 'https://github.com/ffi/ffi-swig-generator.git'
+  gem 'rspec'
 end

data/README.md

@@ -16,13 +16,13 @@ Or install it yourself as:
 
     $ gem install ffi-libav
 
-## Usage
+## Basic Usage
 
 ```ruby
-require 'ffi-ffmpeg'
+require 'ffi-libav'
 
 # Allocate a reader for our video file
-reader = FFmpeg::Reader.new(ARGV[0])
+reader = Libav::Reader.new(ARGV[0])
 
 # Dump the format information for the file
 reader.dump_format
@@ -37,6 +37,117 @@ stream.each_frame do |frame|
 end
 ```
 
+## Advanced Usage
+In addition to the basic decoding of video frames, `ffi-libav` supports
+buffered decoding, for multi-threaded frame processing, stream rewind support,
+and accurate frame seeking.
+
+### Buffered Read
+Buffered decoding for Stream::Video#each_frame allows frame decoding
+to happen independently of frame processing by providing a series of
+frames for the stream instance to use while decoding.  As long as
+frames are available to the stream, it will continue decoding.  The
+caller must call Stream::Video#release_frame or Frame#release to
+notify the Stream that yielded frames are no longer being used in
+another thread.
+
+```ruby
+# Set up a work queue
+frames = Queue.new
+
+# Create 10 worker threads to perform some computationally expensive
+# task on each frame, while not holding the GIL, think ffi opencv
+running = true
+threads = 10.times.map do
+  while running
+
+    # Grab a frame of the queue
+    frame = frames.pop
+
+    # Run something for a long time
+    process_for_a_really_long_time(frame)
+
+    # Release the frame so the stream can reuse it
+    frame.release
+  end
+end
+
+# In the main thread, let's read frames and throw them on the work
+# queue.  #each_frame will block once it has decoded 20 frames, until one of
+# the worker threads calls frame.release.
+stream.each_frame(:buffer => 20) do |frame|
+  frames.push frame
+end
+```
+
+### Rewind
+A simplified rewind capability has been implemented in conjunction with the
+buffered decoding.  Libav::Reader will keep track of all frames released by the
+stream, and if the caller calls Libav::Reader#rewind or Libav::Stream.rewind,
+the Reader will rewind as many frame as possible, up to the number requested by
+the caller.
+
+NOTE: You can only rewind as many frames as have been buffered.  So if you have
+no buffer set up, you can only call `rewind(1)` to get #next_frame to emit the
+current frame again.
+
+```ruby
+# Decode frames, and buffer about 5 seconds worth (at 24 fps)
+stream.each_frame(24 * 5) do |frame|
+
+  # Check the frame to see if it matches something
+  if frame_has_some_data?(frame)
+    puts "match at %d, rewound %d frames" % [frame.number, rewind(24 * 5)]
+    frame.release
+    break
+  end
+
+  # Release the frame for reuse.  Note that releasing a frame DOES NOT mean
+  # that it will be immediately overwritten.
+  frame.release
+end
+
+# Now grab the frame from 5 seconds before we matched
+frame = stream.next_frame
+```
+
+### Accurate Frame Seek (AFS)
+Accurate Frame Seek (AFS) attempts to provide a reliable mechanism for seeking
+accurately using libav.  It works by reading through the file once and
+generating an index of frame numbers, pts, and file offset for each key frame.
+It then uses these locations when seeking accurately.
+
+
+```ruby
+# open our video file
+reader = Libav::Reader.new(ARGV[0], :afs => true)
+
+# read all the frames; we do nothing with the data here, but this could be a
+# first processing pass.
+reader.default_stream.each_frame { }
+
+# Save off our AFS data
+afs = reader.afs
+
+# Open the video file again, but supply the afs data this time
+reader = Libav::Reader.new(ARGV[0], :afs => afs)
+
+# Seek to a specific frame (by number)
+begin
+  reader.default_stream.seek { :frame => 100 }
+rescue Libav::Stream::FrameNotFound => e
+  # This can happen for the first few frames if the first frame is not a key
+  # frame (MPEG-TS).
+  puts "Unable to access frame 100"
+end
+
+# If the exception was not raised, the following call to #next_frame should
+# give you the exact same data as you got for frame 100 on the first read.
+frame = reader.default_stream.next_frame
+```
+
+A more indepth example can be found in `examples/afs_test.rb`.
+
 ## Contributing
 
 1. Fork it ( http://github.com/<my-github-username>/ffi-libav/fork )

data/Rakefile

@@ -7,3 +7,30 @@ end
 rule ".xml" => ".i" do |t|
   sh "swig -I/usr/include -xml -o #{t.source.sub(/\.i$/, ".xml")} #{t.source}"
 end
+
+task(:test) do |t|
+  unless File.exists? "spec/data/big_buck_bunny_480p_surround-fix.avi"
+    puts <<EOM
+
+=================================== NOTICE ====================================
+Some of these tests require a video file to function.  For testing purposes we
+use the freely available, creative commons movie Big Buck Bunny.  To get the
+video file used for testing visit: http://bbb3d.renderfarming.net/download.html
+Under the "Standard 2D" section, you can download the "480p HD (854x480)"
+video; it should be named "big_buck_bunny_480p_surround-fix.avi".
+
+Place this file, or a link to it at:
+  spec/data/big_buck_bunny_480p_surround-fix.avi
+
+Once that file is in place, you will no longer see this notice when starting
+the tests.
+
+                WITHOUT THIS VIDEO, SOME OF THE TESTS WILL FAIL.
+===============================================================================
+
+[ Press Enter to continue, ^C to abort ]
+EOM
+    STDIN.getc
+  end
+  sh "rspec"
+end

data/examples/afs_test.rb

@@ -0,0 +1,92 @@
+#!/usr/bin/env ruby
+#
+# Simple program that reads the first 1000 frames of a video file, then uses
+# AFS to seek to each frame and verify that the seeked frame matches our
+# original frame.  Frame comparison done using crc.
+
+$: << File.join(File.dirname(__FILE__), "..", "lib")
+require 'ffi-libav'
+require 'zlib'
+require 'pp'
+
+# We use this to verify that we got back to the same frame
+class Libav::Frame::Video
+  def crc
+    height.times.inject(0) do |crc,row|
+      Zlib::crc32(data[0].get_bytes(linesize[0] * row, width), crc)
+    end
+  end
+end
+
+def format_seconds(s)
+  s = s.to_i
+  h = s/3600
+  s -= h * 3600
+  m = s/60
+  s -= m * 60
+  "%02d:%02d:%02d" % [ h, m, s ]
+end
+
+GLYPHS = %w{ | / - \\ }
+@first = nil
+@last  = nil
+@calls = 0
+def spin(frame)
+  now = Time.now()
+  @first ||= now
+  @last  ||= now
+
+  @calls += 1
+
+  if (@last + 0.25 < now)
+    fps = @calls.to_f / (now - @first)
+    pos = frame.number / frame.stream.fps.to_f
+    printf("\r[%s] %0.03f fps, frame: %d, pts %d, pos: %s",
+            format_seconds(now - @first), fps, frame.number, frame.pts, 
+            format_seconds(pos))
+    @last = now
+  end
+end
+
+# open our video file
+reader = Libav::Reader.new(ARGV[0], :afs => true)
+reader.dump_format
+
+# Read through the default stream until we get say 100 key frames in our
+# accurate frame seek data.
+stream = reader.default_stream
+crc = []
+stream.each_frame do |frame|
+  spin(frame)
+  crc[frame.number] = frame.crc
+  break if crc.size > 999
+end
+puts ""
+
+# Save off the AFS data, and create a new reader with it
+afs = reader.afs
+reader = Libav::Reader.new(ARGV[0], :afs => afs)
+stream = reader.default_stream
+
+# Shuffle our frame numbers, and seek to each one.
+crc.size.times.to_a.shuffle.each_with_index do |number, attempt|
+  STDOUT.write "%3d: frame %3d, " % [attempt, number]
+  STDOUT.flush
+  begin
+    # Perform the seek
+    stream.seek(:frame => number)
+  rescue Libav::Stream::FrameNotFound => e
+    # This can happen for some codeds
+    puts "warn -- inaccessable frame; #{e}"
+    next
+  end
+
+  # Verify the frame data matches our original pass
+  frame = stream.next_frame
+  frame_crc = frame.crc
+  if (crc[number] != frame_crc)
+    puts "FAILED -- got %08x, expected %08x" % [frame_crc, crc[number]]
+  else
+    puts "pass"
+  end
+end

data/lib/ffi-libav.rb

@@ -1,3 +1,4 @@
+require 'forwardable'
 require 'ffi/libav'
 require 'libav/version'
 require 'libav/frame'

data/lib/ffi/libav.i

@@ -16,6 +16,7 @@ module FFI::Libav
 %include "libavutil/rational.h"
 %include "libavutil/mem.h"
 %include "libavutil/attributes.h"
+%include "libavutil/mathematics.h"
 
 %{
 

data/lib/ffi/libav.rb

@@ -251,6 +251,11 @@ module FFI::Libav
       self[:num].to_f / self[:den]
     end
   end
+
+  AV_TIME_BASE_Q = AVRational.new
+  AV_TIME_BASE_Q[:num] = 1
+  AV_TIME_BASE_Q[:den] = AV_TIME_BASE
+
   # inline function av_cmp_q
   # inline function av_q2d
   attach_function :av_reduce, :av_reduce, [ :pointer, :pointer, :int64, :int64, :int64 ], :int
@@ -267,6 +272,35 @@ module FFI::Libav
   attach_function :av_mallocz, :av_mallocz, [ :uint ], :pointer
   attach_function :av_strdup, :av_strdup, [ :string ], :string
   attach_function :av_freep, :av_freep, [ :pointer ], :void
+  M_E = 2.7182818284590452354
+  M_LN2 = 0.69314718055994530942
+  M_LN10 = 2.30258509299404568402
+  M_LOG2_10 = 3.32192809488736234787
+  M_PHI = 1.61803398874989484820
+  M_PI = 3.14159265358979323846
+  M_SQRT1_2 = 0.70710678118654752440
+  M_SQRT2 = 1.41421356237309504880
+  NAN = (0.0/0.0)
+  INFINITY = (1.0/0.0)
+  AV_ROUND_ZERO = 0
+  AV_ROUND_INF = 1
+  AV_ROUND_DOWN = 2
+  AV_ROUND_UP = 3
+  AV_ROUND_NEAR_INF = 5
+  AVRounding = enum :AVRounding, [
+    :zero, 0,
+    :inf, 1,
+    :down, 2,
+    :up, 3,
+    :near_inf, 5,
+  ]
+
+  attach_function :av_gcd, :av_gcd, [ :int64, :int64 ], :int64
+  attach_function :av_rescale, :av_rescale, [ :int64, :int64, :int64 ], :int64
+  attach_function :av_rescale_rnd, :av_rescale_rnd, [ :int64, :int64, :int64, AVRounding ], :int64
+  attach_function :av_rescale_q, :av_rescale_q, [ :int64, AVRational.by_value, AVRational.by_value ], :int64
+  attach_function :av_compare_ts, :av_compare_ts, [ :int64, AVRational.by_value, :int64, AVRational.by_value ], :int
+  attach_function :av_compare_mod, :av_compare_mod, [ :uint64, :uint64, :uint64 ], :int64
 
 
   ffi_lib [ "libavcodec.so.53", "libavcodec.53.dylib" ]
@@ -2292,7 +2326,6 @@ module FFI::Libav
   end
   class AVIOContext < FFI::Struct
     layout(
-           :av_class, :pointer,
            :buffer, :pointer,
            :buffer_size, :int,
            :buf_ptr, :pointer,

data/lib/libav/frame.rb

@@ -3,10 +3,12 @@ require 'ffi/libav'
 module Libav::Frame; end
 
 class Libav::Frame::Video
+  extend Forwardable
   include FFI::Libav
 
   attr_reader :av_frame, :stream
-  attr_accessor :number
+  attr_accessor :number, :pos
+  def_delegator :@av_frame, :[], :[]
 
   # Initialize a new frame, and optionally allocate memory for the frame data.
   #
@@ -38,23 +40,28 @@ class Libav::Frame::Video
     end
   end
 
-  # Throw together a bunch of helper methods for accessing the AVFrame
-  # attributes.
-  AVFrame.members.each do |member|
-    define_method(member) { @av_frame[member] }
-    define_method(member.to_s + "=") { |v| @av_frame[member] = v }
+  # define a few reader methods to read the underlying AVFrame
+  %w{ width height data linesize }.each do |field|
+    define_method(field) { @av_frame[field.to_sym] }
+  end
+
+  # define a few accessor methods to read/write fields in the AVFrame
+  %w{ pts key_frame }.each do |field|
+    define_method(field) { @av_frame[field.to_sym] }
+    define_method("#{field}=") { |v| @av_frame[field.to_sym] = v }
   end
 
   def key_frame?
-    key_frame == 1
+    @av_frame[:key_frame] != 0
   end
 
   def pixel_format
-    format.is_a?(Fixnum) ? PixelFormat[format] : format
+    @av_frame[:format]
   end
 
-  def pixel_format=(v)
-    send("format=", (v.is_a?(Fixnum) ? v : PixelFormat[v]))
+  # Get the presentation timestamp for this frame in fractional seconds
+  def timestamp
+    pts * @stream[:time_base].to_f
   end
 
   # Scale the frame
@@ -94,14 +101,19 @@ class Libav::Frame::Video
     sws_freeContext(ctx) unless p[:scale_ctx]
 
     # Let's copy a handful of attributes to the scaled frame
-    %w{ pts number key_frame }.each do |k|
-      out.send("#{k}=", send(k))
+    %w{ pts number pos key_frame }.each do |field|
+      out.send("#{field}=", send(field))
     end
 
     # Return our scaled frame
     out
   end
 
+  # Release frame back to buffered stream for re-use
+  def release
+    stream.release_frame(self)
+  end
+
   private
 
   # Returns Proc responsible for cleaning up the picture memory when it gets

data/lib/libav/reader.rb

@@ -2,15 +2,37 @@ require 'ffi/libav'
 
 # Libav file reader
 class Libav::Reader
+  extend Forwardable
   include FFI::Libav
 
-  attr_reader :filename, :streams, :av_format_ctx
+  attr_reader :filename, :streams, :av_format_ctx, :afs
+  def_delegator :@av_format_ctx, :[], :[]
 
   # Initialize a Reader for a specific file
   #
   # ==== Attributes
   # * +filename+ - file to read
   #
+  # ==== Options
+  # * +:afs+ - Enable and supply fast-frame-seek data (default: false)
+  #
+  # ==== Usage
+  #   # open a file named 'video.ts' for reading
+  #   r = Libav::Reader.new("video.ts")
+  #
+  #   # open the same video and enable AFS
+  #   r = Libav::Reader.new("video.ts", :afs => true)
+  #   r.each_frame { |f| do_something(f) }
+  #
+  #   # After reading any portion of the file, save the AFS data
+  #   File.open("video_afs.yml", "w") do |file|
+  #     file.write(r.afs.to_yaml)
+  #   end
+  #
+  #   # open a video file and use AFS data from a previous run
+  #   afs = Yaml.load_file("video_afs.yml")
+  #   r = Libav::Reader.new("video.ts", :afs => afs)
+  #
   def initialize(filename, p={})
     @filename = filename or raise ArgumentError, "No filename"
 
@@ -24,6 +46,11 @@ class Libav::Reader
     rc = avformat_find_stream_info(@av_format_ctx, nil)
     raise RuntimeError, "av_find_stream_info() failed, rc=#{rc}" if rc < 0
 
+    # Fast frame seeking data; initialize it if @afs is enabled, but no data
+    # has been provided.
+    @afs = p[:afs]
+    @afs = Array.new(@av_format_ctx[:nb_streams]) {[]} if @afs == true
+
     # Open all of our streams
     initialize_streams(p)
 
@@ -34,6 +61,13 @@ class Libav::Reader
     # Our packet for reading
     @packet = AVPacket.new
     av_init_packet(@packet)
+
+    # output frame buffer; used for #rewind
+    @output_frames = []
+
+    # This is our rewind queue, frames from @output_frame get stuck on here
+    # by #rewind, and shifted off by #each_frame
+    @rewound = []
   end
 
   # Call +av_dump_format+ to print out the format info for the video
@@ -51,6 +85,10 @@ class Libav::Reader
   # ==== Argument
   # * +block+ - block of code to call with the frame
   #
+  # ==== Options
+  # * +:stream+ stream index or indexes to get frames for
+  # * +:buffer+ number of frames to buffer in each stream
+  #
   # ==== Usage
   #   # Read each frame
   #   reader.each_frame do |frame|
@@ -59,14 +97,41 @@ class Libav::Reader
   #     my_show_frame(frame)
   #   end
   #
-  def each_frame(&block)
+  def each_frame(p={}, &block)
     raise ArgumentError, "No block provided" unless block_given?
 
+    # Patch up the :stream argument
+    p[:stream] ||= @streams.map { |s| s[:index] }
+    p[:stream] = [ p[:stream] ] unless p[:stream].is_a? Array
+
+    # Notify each stream of the requested buffer size
+    p[:stream].each { |i| @streams[i].buffer = p[:buffer] if p[:buffer]}
+
+    # If we have any frames on our @rewound list
+    while frame = @rewound.shift
+      @output_frames.push frame
+      next if p[:stream].include? frame.stream
+      break if yield(frame) == false
+    end
+
+    # Let's read frames
     while av_read_frame(@av_format_ctx, @packet) >= 0
-      frame = @streams[@packet[:stream_index]].decode_frame(@packet)
+
+      # Only call the decoder if the packet is from a stream we're interested
+      # in.
+      frame = nil
+      frame = @streams[@packet[:stream_index]].decode_frame(@packet) if
+        p[:stream].include? @packet[:stream_index]
+
+      # release our packet memory
       av_free_packet(@packet)
-      rc = frame ? yield(frame) : true
-      break if rc == false
+
+      next unless frame
+
+      # Before yielding the frame, add it to our output list for rewind
+      @output_frames.push frame
+
+      yield frame
     end
   end
 
@@ -80,6 +145,68 @@ class Libav::Reader
     default_stream.seek(p)
   end
 
+  # Rewind the reader
+  #
+  # This method will rewind the reader at most +count+ frames for the whole
+  # file, or for the +:stream+ provided.  If not enough frames are available,
+  # rewind() will rewind as many as possible.
+  #
+  # After calling #rewind, #each_frame will yield frames
+  #
+  # Arguments:
+  #   +count+ Number of frames to rewind
+  #
+  # Options:
+  #   +:stream+ [optional] stream +count+ applies to
+  #
+  # Return:
+  #   Number of frames for stream that were rewound
+  def rewind(count=nil, p={})
+
+    # Reduce our output frames based on any :stream provided
+    frames = @output_frames.select do |frame|
+      p[:stream].nil? or frame.stream == p[:stream]
+    end
+
+    # Cap the count at the number of frames we can rewind
+    count = frames.size if count.nil? or count > frames.size
+
+    # find the count-th frame from the end of our reduced frame array.
+    frame = frames[-1 * count.to_i]
+
+    # Find the index of that frame in the real output frames array
+    index = @output_frames.find_index(frame) or return 0
+
+    # Split the output frames into two arrays, those that have been yielded
+    # (output_frames), and those that have been rewound (rewound)
+    @rewound = (@output_frames.slice!(index, @output_frames.size) || []) +
+      @rewound
+
+    # Flush the buffer of every stream we rewound.  We need to do this because
+    # other threads may have references to some of the frames we rewound.  If
+    # it were possible to reverse Libav::Stream#release_frame, there would
+    # still be a problem if another thread released one of our rewound frames
+    # before it was yielded by #each_frame.
+    #
+    # The solution to this problem is to have each stream clear all of their
+    # frame buffers.  The next time the stream decodes a frame, it will have to
+    # allocate a new buffer.  Expensive, but this shouldn't happen very often.
+    @rewound.map { |f| f.stream }.uniq.each { |s| s.release_all_frames }
+  
+    # Return the number of frames rewound for the requested stream.  If no
+    # stream were requested, this would be the total number of frames rewound.
+    frames.size - frames.find_index(frame)
+  end
+
+  # This method is used to notify the Reader that the frame is about to be
+  # modified.  This call is used to update the output buffer that is used
+  # by #rewind.  The supplied frame, and all preceding frames are dropped from
+  # the output buffer.  This reduces how far we can rewind.
+  def frame_dirty(frame)
+    index = @output_frames.index(frame) or return
+    @output_frames.shift(index + 1)
+  end
+
   private
 
   # Generate the Proc that is responsible for releasing our avcodec, avformat
@@ -93,6 +220,7 @@ class Libav::Reader
 
   # Lookup and initialize the streams
   def initialize_streams(p={})
+
     @streams = @av_format_ctx[:nb_streams].times.map do |i|
       av_stream = AVStream.new \
             @av_format_ctx[:streams].get_pointer(i * FFI::Pointer::SIZE)
@@ -101,10 +229,11 @@ class Libav::Reader
       case av_codec_ctx[:codec_type]
       when :video
         Libav::Stream::Video.new(:reader => self,
-                                  :av_stream => av_stream,
-                                  :pixel_format => p[:pixel_format],
-                                  :width => p[:width],
-                                  :height => p[:height])
+                                 :av_stream => av_stream,
+                                 :pixel_format => p[:pixel_format],
+                                 :width => p[:width],
+                                 :height => p[:height],
+                                 :afs => @afs && @afs[av_stream[:index]])
       else
         Libav::Stream::Unsupported.new(:reader => self,
                                         :av_stream => av_stream)