flvedit 0.6.1

data/lib/flv/edit/processor/base.rb

@@ -0,0 +1,209 @@
+module FLV
+  module Edit  
+    module Processor
+      # Processors are used to process FLV files. Processors form are chain and the data (header and tags) will 'flow'
+      # through the chain in sequence. Each processor can inspect the data and change the flow,
+      # either by modifying the data, inserting new data in the flow or stopping the propagation of data.
+      #
+      # A FLV file can be seen as a sequence of chunks, the first one being a Header and the following ones
+      # a series of Tags with different types of bodies: Audio, Video or Event. Events store all meta data
+      # related information: onMetaData, onCuePoint, ...
+      #
+      # To tap into the flow of chunks, a processor can define any of the following methods:
+      #   on_chunk
+      #     on_header
+      #     on_tag
+      #       on_audio
+      #       on_video
+      #         on_keyframe
+      #         on_interframe
+      #         on_disposable_interframe
+      #       on_event
+      #         on_meta_data
+      #         on_cue_point
+      #         on_other_event
+      #
+      # All of these methods will have one argument: the current chunk being processed.
+      # For example, if the current chunk is an 'onMetaData' event, then
+      # the following will be called (from the most specialized to the least).
+      #   processor.on_meta_data(chunk)
+      #   processor.on_event(chunk)
+      #   processor.on_chunk(chunk)
+      #   # later on, the next processor will handle it:
+      #   next_processor.on_meta_data(chunk)
+      #   #...
+      #
+      # The methods need not return anything. It is assumed that the chunk will continue to flow through the
+      # processing chain. When the chunk should not continue down the chain, call +absorb+.
+      # To insert other tags in the flow, call +dispatch_instead+.
+      # Finally, it's possible to +stop+ the processing of the file completely.
+      #
+      # It is possible to look back at already processed chunks (up to a certain limit) with +look_back+
+      # or even in the future with +look_ahead+
+      #
+      class Base
+        attr_reader :options
+
+        def initialize(source=nil, options={})
+          @options = options.freeze
+          @source = source
+          on_calls = self.class.instance_methods(false).select{|m| m.to_s.start_with?("on_")}.map(&:to_sym) #Note: to_s needed for ruby 1.9, to_sym for ruby 1.8
+          unless (potential_errors = on_calls - ALL_EVENTS).empty?
+            warn "The following are not events: #{potential_errors.join(',')} (class #{self.class})"
+          end
+        end
+
+        def has_next_file?
+          @source.has_next_file?
+        end
+
+        def rewind
+          @source.rewind
+        end
+
+        def absorb(*) # Note: the (*) is so that we can alias events like on_meta_data
+          throw :absorb
+        end
+      
+        def stop
+          throw :stop
+        end
+      
+        def process_all
+          each{} while has_next_file?
+        end
+
+        def each(&block)
+          return to_enum(:each_chunk) unless block_given?
+          @block = block
+          catch :stop do
+            process_next_file
+          end
+        end
+      
+        def process_next_file
+          dispatch_chunks(@source)
+        end
+        
+        def dispatch_chunks(enum)
+          enum.each do |chunk|
+            dispatch_chunk(chunk)
+          end
+        end
+
+        def dispatch_chunk(chunk)
+          evt = chunk.main_event
+          catch :absorb do
+            EVENT_TRIGGER_LIST[evt].each do |event|
+              send(event, chunk) if respond_to?(event)
+            end
+            @block.call chunk
+          end
+        end
+      
+        def dispatch_instead(*chunks)
+          chunks.each do |chunk|
+            @block.call chunk
+          end
+          absorb
+        end
+
+        def stdout
+          options[:out] || STDOUT
+        end
+
+        EVENT_TRIGGER = {
+          :on_header      => :on_chunk,
+          :on_tag         => :on_chunk,
+          :on_audio       => :on_tag,
+          :on_video       => :on_tag,
+          :on_event       => :on_tag,
+          :on_meta_data   => :on_event,
+          :on_cue_point   => :on_event,
+          :on_last_second => :on_event,
+          :on_other_event => :on_event,
+          :on_keyframe    => :on_video,
+          :on_interframe  => :on_video,
+          :on_disposable_interframe => :on_video
+        }.freeze
+      
+        ALL_EVENTS = (EVENT_TRIGGER.keys | EVENT_TRIGGER.values).freeze
+      
+        MAIN_EVENTS = ALL_EVENTS.reject{ |k| EVENT_TRIGGER.has_value?(k)}.freeze
+      
+        EVENT_TRIGGER_LIST = Hash.new{|h, k| h[k] = [k] + h[EVENT_TRIGGER[k]]}.tap{|h| h[nil] = []; h.values_at(*MAIN_EVENTS)}.freeze
+      
+      protected
+      
+        def self.desc(text, options = {})
+          registry << [self, text, options]
+        end
+    
+        def self.registry # :nodoc:
+          @@registry ||= []
+        end
+    
+        def self.absorb(*events)
+          events.each{|evt| alias_method evt, :absorb}
+        end
+        
+      end #class Base
+
+      def self.chain(chain_classes, options = {})
+        next_chain_class = chain_classes.pop
+        next_chain_class.new(chain(chain_classes, options), options) if next_chain_class
+      end
+
+      # Let's extend the different kinds of chunks so that any_chunk.main_event returns
+      # the desired trigger.
+
+      module MainEvent # :nodoc:
+        module Header
+          def main_event
+            :on_header
+          end
+        end
+
+        module Video
+          MAPPING = {
+            :keyframe => :on_keyframe,
+            :interframe => :on_interframe,
+            :disposable_interframe => :on_disposable_interframe
+          }.freeze
+
+          def main_event
+            MAPPING[frame_type]
+          end
+        end
+
+        module Audio
+          def main_event
+            :on_audio
+          end
+        end
+
+        module Event
+          MAPPING = Hash.new(:on_other_event).merge!(
+            :onMetaData => :on_meta_data,
+            :onCuePoint => :on_cue_point,
+            :onLastSecond => :on_last_second
+          ).freeze
+          def main_event
+            MAPPING[event]
+          end
+        end
+        
+        module Tag
+          def main_event
+            body.main_event
+          end
+        end
+      end #module MainEvent
+    end #module Processor
+  end #module Edit
+  
+  [Header, Tag, Audio, Video, Event].each do |klass|
+    klass.class_eval{ include Edit::Processor::MainEvent.const_get(klass.to_s.sub('FLV::', '')) }
+  end
+  
+end #module FLV

data/lib/flv/edit/processor/command_line.rb

@@ -0,0 +1,23 @@
+module FLV
+  module Edit
+    module Processor
+      class CommandLine < Base
+        def on_header(h)
+          @last = h.path
+        end
+      
+        def process_all
+          ok, errors = [], []
+          begin
+            each{}
+            ok << @last
+          rescue Exception => e
+            errors << [@last, e]
+          end while has_next_file?
+          puts (["Processed successfully:"] + ok).join("\n") unless ok.empty?
+          puts (["**** Processed with errors: ****"] + errors.map{|path, err| "#{path}: #{err}"}).join("\n") unless errors.empty?
+        end
+      end
+    end
+  end
+end

data/lib/flv/edit/processor/cut.rb

@@ -0,0 +1,27 @@
+module FLV
+  module Edit  
+    module Processor
+      class Cut < Base
+        desc ["Cuts file using the given RANGE"],
+              :param => {:class => TimestampRange, :name => "RANGE"}, :shortcut => "x"
+
+        def on_header(*)
+          @from, @to = options[:cut].begin, options[:cut].end
+          @wait_for_keyframe = options[:keyframe_mode]
+          @first_timestamp = nil
+        end
+
+        def on_tag(tag)
+          if tag.timestamp > @to
+            stop
+          elsif (tag.timestamp < @from) || (@wait_for_keyframe &&= !tag.body.is?(:keyframe))
+            absorb
+          else
+            @first_timestamp ||= tag.timestamp
+            tag.timestamp -= @first_timestamp
+          end
+        end
+      end
+    end
+  end
+end

data/lib/flv/edit/processor/debug.rb

@@ -0,0 +1,30 @@
+require_relative "printer"
+
+module FLV
+  module Edit  
+    module Processor
+      class Debug < Base
+        desc ["Prints out the details of all tags. Information that stays the same",
+              "from one tag type to the next will not be repeated.",
+              "A RANGE argument will limit the output to tags within that range;",
+              "similarily, a given TIMESTAMP will limit the output to tags",
+              "within 0.1s of this timestamp."],
+            :param => {:class => TimestampOrTimestampRange, :name => "[RANGE/TS]"}
+        def on_header(tag)
+          @range = self.options[:debug] || TimestampRange.new(0, INFINITY)
+          @range = @range.widen(0.1) unless @range.is_a? Range
+          @last = {}
+          @printer = Printer.new(stdout)
+          tag.debug(@printer) if @range.include? 0
+        end
+
+        def on_tag(tag)
+          return unless @range.include? tag.timestamp
+          tag.debug(@printer, @last[tag.body.class])
+          @last[tag.body.class] = tag
+        end
+
+      end
+    end
+  end
+end

data/lib/flv/edit/processor/head.rb

@@ -0,0 +1,16 @@
+module FLV
+  module Edit
+    module Processor
+      class Head < Base
+        desc "Processes only the first NB tags.", :param => {:class => Integer, :name => "NB"}, :shortcut => "n"
+        def on_header(header)
+          @count = self.options[:head]
+        end
+
+        def on_tag(tag)
+          throw :stop if (@count -= 1) < 0
+        end
+      end
+    end
+  end
+end

data/lib/flv/edit/processor/join.rb

@@ -0,0 +1,52 @@
+module FLV
+  module Edit  
+    module Processor
+
+      class Join < Base
+        desc "Join the FLV files"
+    
+        def process_next_file
+          dispatch_chunks(@source) while @source.has_next_file?
+        end
+
+        def on_tag(tag)
+          if @wait_for_keyframe
+            absorb
+          else
+            tag.timestamp += @delta
+          end
+          @last_timestamp = tag.timestamp
+        end
+        
+        def on_video(tag)
+          @next_to_last_video_timestamp, @last_video_timestamp = @last_video_timestamp, tag.timestamp
+        end
+        
+        def on_keyframe(tag)
+          if @wait_for_keyframe
+            @wait_for_keyframe = false
+            @delta -= tag.timestamp
+          end
+        end
+
+        def on_header(h)
+          if is_first_header = !@delta
+            @delta = 0
+            @wait_for_keyframe = false
+            @last_video_timestamp = 0
+          else
+            if @last_video_timestamp
+              last_interval = @last_video_timestamp - @next_to_last_video_timestamp
+              @delta += [@last_video_timestamp + last_interval, @last_timestamp].max
+              @wait_for_keyframe = true
+            else
+              @delta = @last_timestamp
+            end
+            dispatch_instead(Tag.new(@last_timestamp, evt = Event.new(:onNextSegment, :file => ::File.basename(h.path))))
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/flv/edit/processor/meta_data_maker.rb

@@ -0,0 +1,127 @@
+module FLV
+  module Edit
+    module Processor
+      class MetaDataMaker < Base        
+        CHUNK_LENGTH_SIZE = 4 # todo: calc instead?
+        TAG_HEADER_SIZE = 11 # todo: calc instead?
+        TOTAL_EXTRA_SIZE_PER_TAG = CHUNK_LENGTH_SIZE + TAG_HEADER_SIZE
+        Info = Struct.new(:bytes, :tag_count, :first, :last)
+      
+        def total_size
+          @info.values.sum{|h| h.bytes + h.tag_count * TOTAL_EXTRA_SIZE_PER_TAG}
+        end
+      
+        def on_header(header)
+          @cue_points = []
+          @key_frames = []
+          @video_interval_stats = Hash.new(0)
+          @info = Hash.new{|h, key| h[key] = Info.new(0,0,nil,nil)}
+          @info[Header].bytes = header.size
+          # Header is not a tag, so leave tag_count at 0
+        end
+  
+        def on_tag(tag)
+          h = @info[tag.body.class]
+          h.first ||= tag
+          h.last = tag
+          h.bytes += tag.body.size
+          h.tag_count += 1
+        end
+      
+        def on_meta_data(t)
+          @previous_meta = t
+          absorb
+        end
+      
+        def on_video(t)
+          @video_interval_stats[t.timestamp.in_milliseconds - @info[Video].last.timestamp.in_milliseconds] += 1   if @info[Video].last
+        end
+        
+        def on_keyframe(t)
+          @key_frames << [t.timestamp.in_seconds, self.total_size]
+        end
+
+        def on_cue_point(t)
+          @cue_points << t
+        end
+
+        def time_positions_to_hash(time_position_pairs)
+          times, filepositions = time_position_pairs.transpose
+          {:times => times || [], :filepositions => filepositions || []}
+        end
+      
+        def meta_data
+          frame_sequence_in_ms = @video_interval_stats.index(@video_interval_stats.values.max)
+          last_ts = @info.values.map{|h| h.last}.compact.map(&:timestamp).map(&:in_seconds).max
+          duration = last_ts + frame_sequence_in_ms/1000.0
+
+          meta = @info[Video].last.body.dimensions if @info[Video].last
+          meta ||= {:width => @previous_meta[:width], :height => @previous_meta[:height]} if @previous_meta
+          meta ||= {}
+          [Video, Audio].each do |k|
+            h = @info[k]
+            type = k.name.gsub!("FLV::","").downcase
+
+            meta[:"#{type}size"] = h.bytes + h.tag_count * TAG_HEADER_SIZE
+            meta[:"#{type}datarate"] = h.bytes / duration * 8 / 1000 #kbits/s
+            if meta[:"has#{type.capitalize}"] = h.first != nil
+              meta[:"#{type}codecid"] = h.first.codec_id
+            end
+          end
+                
+          meta.merge!(
+            :filesize               => total_size,
+            :datasize               => total_size - @info[Header].bytes - CHUNK_LENGTH_SIZE*(1 + @info.values.sum(&:tag_count)),
+            :lasttimestamp          => last_ts,
+            :framerate              => frame_sequence_in_ms ? 1000.0/frame_sequence_in_ms : 0,
+            :duration               => duration,
+
+            :hasCuePoints           => !@cue_points.empty?,
+            :cuePoints              => @cue_points.map(&:body),
+            :hasKeyframes           => !@key_frames.empty?,
+            :keyframes              => time_positions_to_hash(@key_frames),
+
+            :hasMetadata            => true, #duh! Left for compatibility with FLVTool2, but I can't believe anyone would actually check on this...
+            :metadatadate           => Time.now,
+            :metadatacreator        => 'flvedit........................................................'
+          )
+
+          meta.merge!(
+            :audiodelay             => @info[Video].first.timestamp.in_seconds,
+            :canSeekToEnd           => @info[Video].last.frame_type == :keyframe,
+            :lastkeyframetimestamp  => @key_frames.last.first || 0
+          ) if meta[:hasVideo]
+        
+          meta.merge!(
+            :stereo                 => @info[Audio].first.channel == :stereo,
+            :audiosamplerate        => @info[Audio].first.rate,
+            :audiosamplesize        => @info[Audio].first.sample_size
+          ) if meta[:hasAudio]
+        
+          # Adjust all sizes for this new meta tag
+          meta_size = Tag.new(0, meta).size
+          # p "Size of meta: #{meta_size}"
+          # p "Info: #{@info.values.map(&:bytes).inspect} #{@info.values.map(&:tag_count).inspect}"
+          meta[:filesize] += meta_size
+          meta[:datasize] += meta_size
+          meta[:keyframes][:filepositions].map! {|ts| ts + meta_size} if meta[:hasKeyframes]
+          meta[:cuePoints][:filepositions].map! {|ts| ts + meta_size} if meta[:hasCuePoints]
+        
+          meta
+        end
+    
+          # def add_meta_data_tag(stream, options)
+          #   # add onLastSecond tag
+          #   onlastsecond = FLV::FLVMetaTag.new
+          #   onlastsecond.timestamp = ((stream.duration - 1) * 1000).to_int
+          #   stream.add_tags(onlastsecond, false) if onlastsecond.timestamp >= 0
+          # 
+          #   stream.add_meta_tag({ 'metadatacreator' => options[:metadatacreator], 'metadatadate' => Time.now }.merge(options[:metadata]))
+          #   unless options[:compatibility_mode]
+          #     stream.on_meta_data_tag.meta_data['duration'] += (stream.frame_sequence_in_ms || 0) / 1000.0
+          #   end
+          # end
+      end
+    end
+  end
+end

data/lib/flv/edit/processor/print.rb

@@ -0,0 +1,13 @@
+require_relative "printer"
+module FLV
+  module Edit
+    module Processor
+      class Print < Base
+        desc "Prints out meta data to stdout"
+        def on_meta_data(tag)
+          tag.debug(Printer.new(stdout))
+        end
+      end
+    end
+  end
+end

data/lib/flv/edit/processor/printer.rb

@@ -0,0 +1,27 @@
+module FLV
+  module Edit  
+    module Processor
+      
+      class Printer
+        def initialize(io, options={})
+          @io = io
+          @options = {:width => 50, :column_width => 15, :separator => "| "}.merge(options)
+          @margin_left = ""
+        end
+
+        def header(left, right)
+          @io.puts left + @options[:separator] + right
+          @margin_left = " "*left.length + @options[:separator]
+        end
+
+        # Prints out a hash (or any list of key-value pairs) in two columns
+        def values(hash)
+          hash.map{|k,v| [k.to_s, v] }.sort.each do |key, value|
+            @io.puts "#{@margin_left}#{key.to_s.ljust(@options[:column_width])}: #{value.inspect.delete(':"')}"
+          end
+        end
+      end
+    
+    end
+  end
+end

data/lib/flv/edit/processor/reader.rb

@@ -0,0 +1,30 @@
+module FLV
+  module Edit  
+    module Processor
+      class Reader < Base
+        def initialize(*)
+          super
+          rewind
+          raise "Oups, Filenames were #{@options[:files].inspect}" if @options[:files].include? nil
+          raise "Please specify at least one filename" if @options[:files].empty?
+        end
+
+        def has_next_file?
+          @to_process > 0
+        end
+  
+        def rewind
+          @to_process = @options[:files].length
+        end
+
+        def process_next_file
+          raise IndexError, "No more filenames to process" unless has_next_file?
+          @to_process -= 1
+          FLV::File.open(@options[:files][-1- @to_process]) do |f|
+            dispatch_chunks(f)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/flv/edit/processor/save.rb

@@ -0,0 +1,28 @@
+module FLV
+  module Edit
+    module Processor
+      class Save < Base
+        desc "Saves the result to PATH", :param => {:class => String, :name => "PATH"}
+        def process_next_file
+          super
+        ensure
+          if @out
+            @out.close
+            finalpath = @out.path.sub(/\.temp$/, '')
+            FileUtils.mv(@out.path, finalpath) unless finalpath == @out.path
+          end
+        end
+
+        def on_header(h)
+          @out = FLV::File::open(options[:save] || (h.path+".temp"), "w+b")
+          @out << h
+        end
+        
+        def on_tag(t)
+          @out << t
+        end
+      end
+    
+    end
+  end
+end

data/lib/flv/edit/processor/update.rb

@@ -0,0 +1,27 @@
+require_relative "meta_data_maker"
+
+module FLV
+  module Edit
+    module Processor
+      class Update < Base
+        desc "Updates FLV with an onMetaTag event"
+        def initialize(source=nil, options={})
+          super
+          @meta_data_maker = MetaDataMaker.new(source.dup, options)
+        end
+      
+        def each(&block)
+          @meta_data_maker.each {}
+        ensure  # even if each throws, we better call super otherwise we won't be synchronized anymore!
+          super
+        end
+      
+        absorb :on_meta_data, :on_last_second
+        
+        def on_header(header)
+          dispatch_instead header, Tag.new(0, @meta_data_maker.meta_data)
+        end
+      end
+    end
+  end
+end

data/lib/flv/edit/processor.rb

@@ -0,0 +1,3 @@
+%w(base add command_line cut debug head join print reader save update).each do |proc|
+  require_relative "processor/#{proc}"
+end

data/lib/flv/edit/runner.rb

@@ -0,0 +1,23 @@
+require_relative 'options'
+require_relative 'processor'
+
+module FLV
+  module Edit
+    class Runner
+      attr_reader :options, :commands
+      
+      def initialize(*arg)
+        @commands, @options = (arg.length == 1 ? Options.new(arg.first).to_a : arg)
+      end
+      
+      def run
+        commands = [*@commands].map{|c| c.is_a?(Class) ? c : Processor.const_get(c.to_s.camelize)}
+        commands.unshift Processor::Reader
+        commands << Processor::CommandLine unless options[:dont_catch_errors]
+        Processor.chain(commands, @options).process_all
+      end
+      
+      alias_method :run!,  :run
+    end
+  end
+end

data/lib/flv/edit/version.rb

@@ -0,0 +1,15 @@
+module FLV
+  module Edit
+    FILE = ::File.dirname(__FILE__) + '/../../../VERSION.yml'
+    
+    class Version < Struct.new(:major, :minor, :patch)
+      def to_s
+        "#{major}.#{minor}.#{patch}"
+      end
+    end
+    
+    def self.version
+      Version.new(*YAML.load_file(FILE).values_at('major','minor','patch'))
+    end
+  end
+end

data/lib/flv/edit.rb

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'backports'
+require_relative '../flv'
+require_relative 'edit/version'
+require_relative 'edit/options'
+require_relative 'edit/processor'
+require_relative 'edit/runner'
+
+#todo Change bin/flvedit
+#todo Auto write to files
+#todo Command Save no|false
+#todo add & nearest keyframe & overwrite
+#todo fix timestamps
+#todo cut & nearest keyframe?
+#todo print & xml/yaml & multiple files
+#todo onLastSecond
+#todo in|out pipe
+#todo recursive?
+#todo fix offset, both 24bit issue & apparent skips
+#todo bug join & time for audio vs video