media_processing_tool 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 297b7fe61e129519bdfe33fb90bf29d7b199ab64
+  data.tar.gz: 6127946e2a8d8865ca4846d183376e8d3fbecf68
+SHA512:
+  metadata.gz: 0f8d9ce57c58f76c19f2336866da0be2f20ba887bc9b1edb1811857fa6aa223169cc5457eb7b5bb30c6f8bfaaaf67cf365299d4e05d2b95d67e4a9dda6c0420b
+  data.tar.gz: 96a77a2e6729efec6cd3dfafaa24b0e3a9313543c78ab967fbfa45bd1bf870a3274392f20d8017376bfc198896322bdb9b40f4496b7ce0f79205707bdaa4558e

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.bundle
+.config
+bin/dev/
+bin/*_option
+coverage
+dev
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/README.md

@@ -0,0 +1,2 @@
+media_processing_tool
+=====================

data/bin/catalog

@@ -0,0 +1,181 @@
+#!/usr/bin/env ruby
+# A utility to create a CSV of the file properties.
+#
+# If process-xml-files is true only the xml files then the file paths inside of supported xml files will be parsed and the files will be
+# analyzed and added to the output
+#
+lib_path = File.expand_path('../../lib', __FILE__)
+$:.unshift(lib_path) if !$:.include?(lib_path) and File.exists?(lib_path)
+
+require 'cgi'
+require 'csv'
+require 'json'
+require 'logger'
+require 'open3'
+require 'optparse'
+require 'shellwords'
+require 'time'
+require 'uri'
+
+require 'mig/modules/ffmpeg'
+
+DEFAULT_MEDIA_FILE_PATH_FIELD_NAME = 'Media File Path'
+
+module Find
+
+  #
+  # Calls the associated block with the name of every file and directory listed
+  # as arguments, then recursively on their subdirectories, and so on.
+  #
+  # See the +Find+ module documentation for an example.
+  #
+  def find(*paths) # :yield: path
+    block_given? or return enum_for(__method__, *paths)
+
+    paths.collect!{|d| raise Errno::ENOENT unless File.exist?(d); d.dup}
+    while file = paths.shift
+      catch(:prune) do
+        begin
+          s = File.lstat(file)
+        rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG
+          next
+        end
+        if s.directory? then
+          begin
+            fs = Dir.entries(file)
+          rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG
+            next
+          end
+          fs.sort!
+          fs.reverse_each {|f|
+            next if f == "." or f == ".."
+            f = File.join(file, f)
+            paths.unshift f.untaint
+          }
+        else
+          yield file.dup.taint, s
+        end
+      end
+    end
+  end
+
+  #
+  # Skips the current file or directory, restarting the loop with the next
+  # entry. If the current file is a directory, that directory will not be
+  # recursively entered. Meaningful only within the block associated with
+  # Find::find.
+  #
+  # See the +Find+ module documentation for an example.
+  #
+  def prune
+    throw :prune
+  end
+
+  module_function :find, :prune
+end
+
+File::Stat.class_eval do
+
+  def to_hash
+    (self.methods - Object.methods - [__callee__]).each_with_object({}) { |meth, acc| acc[meth] = self.send(meth) if self.method(meth).arity == 0 }
+  end
+
+end
+
+options = { }
+options_file_name = nil
+op = OptionParser.new
+op.on('--path PATH', 'The path to catalog') { |v| options[:path_to_catalog] = v }
+op.on('--[no-]recursive', 'Determines if The directory tree will be recursive or not. If not then path can be a glob pattern.') { |v| options[:recursive] = v }
+op.on('--[no-]process-xml-files', 'Media File Paths will be pulled from FCP xml files found using the path.') { |v| options[:process_xml_files] = v }
+op.on('--destination-path PATH', 'The path and file name of the csv to create.') { |v| options[:destination_path] = v }
+op.on('--help', 'Display this message.') { puts op; exit }
+op.parse!(command_line_options = ARGV.dup)
+op.parse!(ARGV.dup) if op.load(options_file_name)
+
+@logger = Logger.new(STDOUT)
+def logger; @logger end
+
+@ffmpeg = FFMPEG.new
+path_to_catalog = options[:path_to_catalog]
+recursive = options[:recursive]
+process_xml_files = options[:process_xml_files]
+destination_path = options[:destination_path]
+media_file_path_field_name = options[:media_file_path_field_name] || DEFAULT_MEDIA_FILE_PATH_FIELD_NAME
+
+path_to_catalog  ||= '/assets/**/*.mov'
+destination_path ||= '/tmp/assets.csv'
+
+
+def unescape_fcp_file_path_url(file_path_url)
+  URI(file_path_url).path.gsub(/(%(?:[2-9]|[A-F])(?:\d|[A-F]))/) { |v| CGI.unescape(v) }
+end
+
+def process_media_file_paths(media_file_paths, options = { })
+  xml_file_path = options[:xml_file_path]
+  xml_file_path_field_name = options[:xml_file_path_field_name] || 'XML File Path'
+
+  media_file_paths = media_file_paths.uniq
+  media_file_path_data = { }
+  media_file_path_count = media_file_paths.length
+  media_file_path_counter = 0
+  media_file_paths.each do |mfp|
+    media_file_path_counter += 1
+    logger.debug { "Processing Media File Path #{media_file_path_counter} of #{media_file_path_count}. '#{mfp}'" }
+    next unless File.exists?(mfp)
+    md = @ffmpeg.run(mfp)
+    md[xml_file_path_field_name] = xml_file_path if xml_file_path
+    md = md.delete_if { |k, v| %w(output command).include? k }
+    md.merge!(File.stat(mfp).to_hash)
+    media_file_path_data[mfp] = md
+  end
+  media_file_path_data
+end
+
+def get_media_file_paths_from_xml(xml_file_path)
+  _media_file_paths = [ ]
+  File.open(xml_file_path).read().scan(/<pathurl>(.*)<\/pathurl>/) { |file_path_url_ary| _media_file_paths << unescape_fcp_file_path_url(file_path_url_ary.first) }
+  _media_file_paths
+end
+
+def get_media_file_paths_from_xml_files(path_to_catalog)
+  media_file_paths = [ ]
+  media_file_paths_by_xml_file = { }
+  xml_file_paths = Dir.glob(path_to_catalog)
+  xml_file_path_count = xml_file_paths.length
+  xml_file_path_counter = 0
+  xml_file_paths.each do |xml_file_path|
+    xml_file_path_counter += 1
+    logger.debug { "Processing XML File #{xml_file_path_counter} of #{xml_file_path_count}. #{xml_file_path}" }
+    media_file_paths_from_xml = get_media_file_paths_from_xml(xml_file_path)
+    media_file_paths_by_xml_file[xml_file_path] = media_file_paths_from_xml
+    media_file_paths = media_file_paths + media_file_paths_from_xml
+  end
+  media_file_paths
+end
+
+def data_to_table(media_file_paths_with_data)
+
+end
+
+if process_xml_files
+  media_file_paths = get_media_file_paths_from_xml_files(path_to_catalog)
+else
+  media_file_paths = recursive ? Find.find(path_to_catalog) : Dir.glob(path_to_catalog)
+end
+
+media_file_paths_with_data = process_media_file_paths(media_file_paths)
+
+
+fields = { media_file_path_field_name => '' }
+media_file_paths_with_data.each { |_, data| fields.merge!(data) }
+empty_record = Hash[fields.map { |k,_| [k, ''] }]
+
+CSV.open(destination_path, 'w') do |csv|
+  csv << fields.keys
+  media_file_paths_with_data.each do |media_file_path, data|
+    data[media_file_path_field_name] = media_file_path
+    csv << empty_record.merge(data).values
+  end
+end
+puts "Output written to: #{destination_path}"

data/bin/catalog_assets

@@ -0,0 +1,187 @@
+#!/usr/bin/env ruby
+# A utility to create a CSV of the file properties.
+#
+# If process-xml-files is true only the xml files then the file paths inside of supported xml files will be parsed and the files will be
+# analyzed and added to the output
+#
+lib_path = File.expand_path('../../lib', __FILE__)
+$:.unshift(lib_path) if !$:.include?(lib_path) and File.exists?(lib_path)
+require 'rubygems'
+require 'cgi'
+require 'csv'
+require 'find'
+require 'json'
+require 'logger'
+require 'mig'
+require 'open3'
+require 'open3_backport' if RUBY_VERSION.start_with?('1.8.')
+require 'optparse'
+require 'pp'
+require 'shellwords'
+require 'time'
+require 'uri'
+
+#require 'mig/modules/ffmpeg'
+module Find
+
+  #
+  # Calls the associated block with the name of every file and directory listed
+  # as arguments, then recursively on their subdirectories, and so on.
+  #
+  # See the +Find+ module documentation for an example.
+  #
+  def find(*paths) # :yield: path
+    block_given? or return enum_for(__method__, *paths)
+
+    paths.collect!{|d| raise Errno::ENOENT unless File.exist?(d); d.dup}
+    while file = paths.shift
+      catch(:prune) do
+
+        begin
+          s = File.lstat(file)
+        rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG
+          next
+        end
+
+        yield file.dup.taint, s
+
+        if s.directory? then
+          begin
+            fs = Dir.entries(file)
+          rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG
+            next
+          end
+          fs.sort!
+          fs.reverse_each {|f|
+            next if f == "." or f == ".."
+            f = File.join(file, f)
+            paths.unshift f.untaint
+          }
+        end
+      end
+    end
+  end
+
+  #
+  # Skips the current file or directory, restarting the loop with the next
+  # entry. If the current file is a directory, that directory will not be
+  # recursively entered. Meaningful only within the block associated with
+  # Find::find.
+  #
+  # See the +Find+ module documentation for an example.
+  #
+  def prune
+    throw :prune
+  end
+
+  module_function :find, :prune
+end
+
+DEFAULT_MEDIA_FILE_PATH_FIELD_NAME = 'Media File Path'
+
+options = { }
+options_file_name = nil
+op = OptionParser.new
+op.on('--path PATH', 'The path to catalog') { |v| options[:path_to_catalog] = v }
+op.on('--[no-]recursive', 'Determines if The directory tree will be recursive or not. If not then path can be a glob pattern.') { |v| options[:recursive] = v }
+op.on('--destination-path PATH', 'The path and file name of the csv to create.') { |v| options[:destination_path] = v }
+op.on('--help', 'Display this message.') { puts op; exit }
+op.parse!(ARGV.dup)
+op.parse!(ARGV.dup) if op.load(options_file_name)
+
+@logger = Logger.new(STDOUT)
+def logger; @logger end
+
+@mig = MediaInformationGatherer.new(options)
+def mig; @mig end
+
+@path_to_catalog = options[:path_to_catalog]
+def path_to_catalog; @path_to_catalog end
+
+@recursive = options[:recursive]
+def recursive; @recursive end
+
+@destination_path = options[:destination_path]
+def destination_path; @destination_path end
+
+@media_file_path_field_name = options[:media_file_path_field_name] || DEFAULT_MEDIA_FILE_PATH_FIELD_NAME
+def media_file_path_field_name; @media_file_path_field_name end
+
+# @path_to_catalog  ||= '/assets/**/*.mov'
+# @destination_path ||= '/tmp/assets.csv'
+
+abort("Path is a required argument.\n#{op}") unless path_to_catalog
+abort("Destination Path is a required argument.\n#{op}") unless destination_path
+
+def unescape_fcp_file_path_url(file_path_url)
+  URI(file_path_url).path.gsub(/(%(?:[2-9]|[A-F])(?:\d|[A-F]))/) { |v| CGI.unescape(v) }
+end
+
+def process_media_file_path(media_file_path)
+  mds = mig.run(media_file_path)
+  md = mds[:common]
+  md
+end
+
+def process_media_file_paths(media_file_paths, options = { })
+  xml_file_path = options[:xml_file_path]
+  xml_file_path_field_name = options[:xml_file_path_field_name] || 'XML File Path'
+
+  media_file_paths = media_file_paths.uniq
+  media_file_path_data = { }
+  media_file_path_count = media_file_paths.length
+  media_file_path_counter = 0
+  attempt_counter = 0
+  media_file_paths.each do |mfp|
+    attempt_counter += 1
+    media_file_path_counter += 1
+    logger.debug { "Processing Media File Path #{media_file_path_counter} of #{media_file_path_count}. '#{mfp}'" }
+    #next unless File.exists?(mfp)
+    #md = @ffmpeg.run(mfp)
+    md = process_media_file_path(mfp)
+    unless (mfp == md[:file_path])
+      warn("File Path Mismatch A1. '#{mfp}' != '#{md[:file_path]}'")
+      media_file_path_counter -= 1
+      attempt_counter < 4 ? redo : warn("Failing After #{attempt_counter}")
+    end
+    md[xml_file_path_field_name] = xml_file_path if xml_file_path
+    media_file_path_data[mfp] = md
+    # unless (mfp == md[:file_path])
+    #   warn("File Path Mismatch A2. '#{mfp}' != '#{md[:file_path]}'")
+    #   media_file_path_counter -= 1
+    #   redo
+    # end
+    attempt_counter = 0
+  end
+
+  media_file_path_data
+end
+
+def data_to_table(media_file_paths_with_data)
+
+end
+
+if recursive then
+  media_file_paths = [ ]
+  Find.find(path_to_catalog) { |fp, stat| media_file_paths << fp if stat.file?; puts fp }
+else
+ media_file_paths =  Dir.glob(path_to_catalog)
+end
+# require 'pp'
+# abort("MFP Size: #{media_file_paths.length} #{PP.pp(media_file_paths, '')}")
+media_file_paths_with_data = process_media_file_paths(media_file_paths)
+
+
+fields = { media_file_path_field_name => '' }
+media_file_paths_with_data.each { |_, data| fields.merge!(data) }
+empty_record = Hash[fields.map { |k,_| [k, ''] }]
+
+CSV.open(destination_path, 'w') do |csv|
+  csv << fields.keys
+  media_file_paths_with_data.each do |media_file_path, data|
+    data[media_file_path_field_name] = media_file_path
+    csv << empty_record.merge(data).values
+  end
+end
+
+puts "Output written to: #{destination_path}"

data/bin/fcp_xml_parser

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '../lib')))
+require 'json'
+require 'optparse'
+require 'final_cut_pro/xml_parser'
+
+options = { }
+
+op = OptionParser.new
+op.banner = "Usage: #{File.basename(__FILE__)} [options] filepath"
+options[:pretty_print] = true
+op.on('--[no-]pretty-print', '') { |v| options[:pretty_print] = v }
+options[:output_files] = true
+op.on('--[no-]files', '') { |v| options[:output_files] = v }
+options[:output_sequences] = true
+op.on('--[no-]sequences', '') { |v| options[:output_sequences] = v }
+op.parse!
+
+file_path = ARGV.shift
+#file_path ||= '/Users/admin/Dropbox/work/bbc/examples/final_cut_pro/SX100117B-Football Samoa RE-EDIT.xml'
+file_path ||= '/Users/admin/Dropbox/work/bbc/examples/final_cut_pro/multipleEdits-SameSource-NNAT069S Nigeria master EDIT.xml'
+# file_path ||= '/Users/admin/Dropbox/work/bbc/examples/final_cut_pro/SX071209A_Kings of Congo.xml'
+unless file_path
+  puts op
+  exit
+end
+
+# { files: { }, sequences: { } }
+doc = FinalCutPro::XMLParser.parse(file_path)#.to_hash
+
+output = { }
+output[:files] = doc.files if options[:output_files]
+output[:sequences] = doc.sequences if options[:output_sequences]
+
+json_generate_options = { :max_nesting => 250 }
+if options[:pretty_print]
+  puts JSON.pretty_generate(output, json_generate_options)
+else
+  puts JSON.fast_generate(output, json_generate_options)
+end
+

data/bin/mig

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+#lib_path = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+lib_path = File.expand_path('../../lib', __FILE__)
+$:.unshift(lib_path) if !$:.include?(lib_path) and File.exists?(lib_path)
+
+require 'rubygems'
+require 'json'
+require 'optparse'
+
+require 'mig'
+
+options = { }
+
+op = OptionParser.new
+#op.banner = "Usage: #{File.basename(__FILE__)} [options] filepath"
+op.on('--ffmpeg-bin-path PATH', '') { |v| options[:ffmpeg_cmd_path] = v }
+op.on('--mediainfo-bin-path PATH', '') { |v| options[:mediainfo_cmd_path] = v }
+op.on('--exiftool-bin-path PATH', '') { |v| options[:exiftool_cmd_path] = v }
+op.on('--[no-]pretty-print', '') { |v| options[:pretty_print] = v }
+op.on('--help', 'Displays this message.') { puts op; exit }
+op.load
+op.parse!
+
+logger = Logger.new(STDERR)
+logger.level = Logger::ERROR
+options[:logger] = logger
+
+file_path = ARGV.shift
+unless file_path
+  puts op
+  exit
+end
+
+mig = MediaInformationGatherer.new(options)
+abort("File Not Found. #{file_path}") unless File.exist?(file_path)
+output = mig.run(file_path)
+
+json_options = { :allow_nan => true }
+
+if options[:pretty_print]
+  puts JSON.pretty_generate(output, json_options)
+else
+  puts JSON.fast_generate(output, json_options)
+end