media_processing_tool 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +21 -0
- data/README.md +2 -0
- data/bin/catalog +181 -0
- data/bin/catalog_assets +187 -0
- data/bin/fcp_xml_parser +41 -0
- data/bin/mig +44 -0
- data/bin/mig_http +52 -0
- data/bin/xml_processor +51 -0
- data/config/default/xml_processor_config +49 -0
- data/lib/axml.rb +59 -0
- data/lib/cli.rb +88 -0
- data/lib/final_cut_pro.rb +31 -0
- data/lib/final_cut_pro/sequence_processor.rb +135 -0
- data/lib/final_cut_pro/xml_parser.rb +15 -0
- data/lib/final_cut_pro/xml_parser/common.rb +121 -0
- data/lib/final_cut_pro/xml_parser/document.rb +18 -0
- data/lib/final_cut_pro/xml_parser/fcpxml/version_1.rb +28 -0
- data/lib/final_cut_pro/xml_parser/xmeml/version_5.rb +234 -0
- data/lib/itunes/xml_parser.rb +51 -0
- data/lib/media_processing_tool/publisher.rb +52 -0
- data/lib/media_processing_tool/xml_parser.rb +30 -0
- data/lib/media_processing_tool/xml_parser/document.rb +38 -0
- data/lib/media_processing_tool/xml_parser/identifier.rb +43 -0
- data/lib/media_processing_tool/xml_processor.rb +132 -0
- data/lib/mig.rb +158 -0
- data/lib/mig/http.rb +54 -0
- data/lib/mig/modules/common.rb +333 -0
- data/lib/mig/modules/exiftool.rb +26 -0
- data/lib/mig/modules/ffmpeg.rb +225 -0
- data/lib/mig/modules/media_type.rb +23 -0
- data/lib/mig/modules/mediainfo.rb +91 -0
- data/lib/timecode_methods.rb +108 -0
- data/lib/udam_utils/publish_map_processor.rb +710 -0
- metadata +111 -0
@@ -0,0 +1,52 @@
|
|
1
|
+
# THIS CLASS IS CURRENTLY JUST A PASSTHROUGH TO THE GENERIC PUBLISH MAP PROCESSOR
|
2
|
+
|
3
|
+
require 'logger'
|
4
|
+
require 'udam_utils/publish_map_processor'
|
5
|
+
module MediaProcessingTool
|
6
|
+
|
7
|
+
class Publisher
|
8
|
+
|
9
|
+
attr_accessor :logger, :object, :publisher
|
10
|
+
|
11
|
+
# @param [Hash] params
|
12
|
+
# @option params [Object|nil] :logger
|
13
|
+
# @Option params [String] :config_file_path
|
14
|
+
def initialize(params = {})
|
15
|
+
@logger = params[:logger] ||= Logger.new(STDOUT)
|
16
|
+
|
17
|
+
@interrupted = false
|
18
|
+
Signal.trap 'INT' do stop end
|
19
|
+
Signal.trap 'TERM' do stop end
|
20
|
+
Signal.trap 'SIGINT' do stop end
|
21
|
+
|
22
|
+
#@config_file_path = params[:config_file_path]
|
23
|
+
#raise ArgumentError, 'Missing required parameter :config_file_path' unless @config_file_path
|
24
|
+
#load_configuration_from_file(@config_file_path)
|
25
|
+
|
26
|
+
|
27
|
+
publisher_options = params
|
28
|
+
@publisher = UDAMUtils::GenericPublishMapProcessor.new(publisher_options)
|
29
|
+
|
30
|
+
end # initialize
|
31
|
+
|
32
|
+
def stop
|
33
|
+
@interrupted = true
|
34
|
+
puts 'Quitting on interrupt signal.'
|
35
|
+
while true
|
36
|
+
puts 'Exiting...'
|
37
|
+
exit
|
38
|
+
end
|
39
|
+
end # stop
|
40
|
+
|
41
|
+
def process(object, params = { })
|
42
|
+
publisher.process_object(params.merge(:object => object))
|
43
|
+
end
|
44
|
+
|
45
|
+
def publish(object, params = {})
|
46
|
+
#@object = object
|
47
|
+
#publisher.process_object(object)
|
48
|
+
end # publish
|
49
|
+
|
50
|
+
end # Publisher
|
51
|
+
|
52
|
+
end # MediaProcessingTool
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'media_processing_tool/xml_parser/identifier'
|
2
|
+
require 'final_cut_pro/xml_parser'
|
3
|
+
require 'itunes/xml_parser'
|
4
|
+
module MediaProcessingTool
|
5
|
+
|
6
|
+
class XMLParser
|
7
|
+
|
8
|
+
# Gives access to the last document returned by the Identifier
|
9
|
+
# This gives access to the identifiers instance parameters (such determined type) for use later
|
10
|
+
def self.identifier_document
|
11
|
+
@identifier_document
|
12
|
+
end # self.identifier_document
|
13
|
+
|
14
|
+
def self.parse(xml, args = { })
|
15
|
+
@identifier_document = Identifier.load(xml, args)
|
16
|
+
|
17
|
+
case @identifier_document.type
|
18
|
+
when :final_cut_pro
|
19
|
+
doc = FinalCutPro::XMLParser.parse(@identifier_document.xml_document, args)
|
20
|
+
when :itunes
|
21
|
+
doc = ITunes::XMLParser.parse(xml, args)
|
22
|
+
else
|
23
|
+
doc = @identifier_document
|
24
|
+
end
|
25
|
+
doc
|
26
|
+
end # self.parse
|
27
|
+
|
28
|
+
end # XMLParser
|
29
|
+
|
30
|
+
end # MediaProcessingTool
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'axml'
|
2
|
+
|
3
|
+
module MediaProcessingTool
|
4
|
+
|
5
|
+
class XMLParser
|
6
|
+
|
7
|
+
class Document
|
8
|
+
|
9
|
+
def self.xml_as_document(xml, params = {})
|
10
|
+
AXML.xml_as_document(xml)
|
11
|
+
end # self.xml_as_document
|
12
|
+
|
13
|
+
def self.load(xml, params = { })
|
14
|
+
new(xml, params)
|
15
|
+
end # self.load
|
16
|
+
|
17
|
+
def initialize(xml, params = { })
|
18
|
+
@xml_document = self.class.xml_as_document(xml, params)
|
19
|
+
end # initialize
|
20
|
+
|
21
|
+
def xml_document
|
22
|
+
@xml_document
|
23
|
+
end # xml_document
|
24
|
+
|
25
|
+
def root
|
26
|
+
xml_document.root
|
27
|
+
end # root
|
28
|
+
|
29
|
+
# Gets the
|
30
|
+
def root_type
|
31
|
+
@root_type ||= root.name
|
32
|
+
end # type
|
33
|
+
|
34
|
+
end # Document
|
35
|
+
|
36
|
+
end # XMLParser
|
37
|
+
|
38
|
+
end # MediaProcessingTool
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'media_processing_tool/xml_parser/document'
|
2
|
+
module MediaProcessingTool
|
3
|
+
|
4
|
+
class XMLParser
|
5
|
+
|
6
|
+
class Identifier < Document
|
7
|
+
|
8
|
+
def initialize(xml, params = { })
|
9
|
+
super(xml, params)
|
10
|
+
end # initialize
|
11
|
+
|
12
|
+
def is_fcpxml?
|
13
|
+
root_type == 'fcpxml'
|
14
|
+
end # is_fcpxml?
|
15
|
+
|
16
|
+
def is_xmeml?
|
17
|
+
root_type == 'xmeml'
|
18
|
+
end # is_xmeml?
|
19
|
+
|
20
|
+
def is_plist?
|
21
|
+
root_type == 'plist'
|
22
|
+
end # is_plist?
|
23
|
+
|
24
|
+
def is_final_cut_pro?
|
25
|
+
is_xmeml? || is_fcpxml?
|
26
|
+
end # is_final_cut_pro?
|
27
|
+
|
28
|
+
def is_itunes?
|
29
|
+
is_plist? and !xml_document.find('/plist/dict/key[text()="Tracks"]').empty?
|
30
|
+
end # is_itunes?
|
31
|
+
|
32
|
+
def type
|
33
|
+
return :final_cut_pro if is_final_cut_pro?
|
34
|
+
return :itunes if is_itunes?
|
35
|
+
return :plist if is_plist?
|
36
|
+
return :unknown
|
37
|
+
end # source_application
|
38
|
+
|
39
|
+
end # Identifier
|
40
|
+
|
41
|
+
end # XMLParser
|
42
|
+
|
43
|
+
end # MediaProcessingTool
|
@@ -0,0 +1,132 @@
|
|
1
|
+
require 'media_processing_tool/xml_parser'
|
2
|
+
require 'media_processing_tool/publisher'
|
3
|
+
require 'mig'
|
4
|
+
module MediaProcessingTool
|
5
|
+
|
6
|
+
class XMLProcessor
|
7
|
+
|
8
|
+
def self.process(xml, params = { })
|
9
|
+
|
10
|
+
end # self.process
|
11
|
+
|
12
|
+
DEFAULT_FILE_PATH_FIELD_NAME = :path_on_file_system
|
13
|
+
|
14
|
+
attr_accessor :logger
|
15
|
+
|
16
|
+
# @return [Boolean] determines if the files parsed from the XML should be sent to a publisher
|
17
|
+
attr_accessor :publish
|
18
|
+
|
19
|
+
def initialize(params = { })
|
20
|
+
initialize_logger(params)
|
21
|
+
|
22
|
+
@publish = params.fetch(:publish, true)
|
23
|
+
@default_file_path_field_name = params[:@default_file_path_field_name] || DEFAULT_FILE_PATH_FIELD_NAME
|
24
|
+
|
25
|
+
initialize_mig(params.dup)
|
26
|
+
initialize_default_publisher(params.dup)
|
27
|
+
end # initialize
|
28
|
+
|
29
|
+
def initialize_logger(params = { })
|
30
|
+
@logger = params[:logger] ||= Logger.new(params[:log_to] || STDOUT)
|
31
|
+
logger.level = params[:log_level] if params[:log_level]
|
32
|
+
params[:logger] = logger unless params[:logger]
|
33
|
+
end # initialize_logger
|
34
|
+
|
35
|
+
def initialize_mig(params = {})
|
36
|
+
logger.debug { "Initializing Media Processing Tool. #{params}" }
|
37
|
+
@mig = MediaInformationGatherer.new(params)
|
38
|
+
end # initialize_mig
|
39
|
+
|
40
|
+
def initialize_default_publisher(params = {})
|
41
|
+
logger.debug { "Initializing Default Publisher. #{params}" }
|
42
|
+
params[:file_path_field_name] = @default_file_path_field_name
|
43
|
+
@default_publisher = MediaProcessingTool::Publisher.new(params)
|
44
|
+
end # initialize_publisher
|
45
|
+
|
46
|
+
def document
|
47
|
+
@document
|
48
|
+
end # document
|
49
|
+
alias :doc :document
|
50
|
+
|
51
|
+
def document_type
|
52
|
+
@identifier_document.type
|
53
|
+
end # document_type
|
54
|
+
|
55
|
+
def publisher(params = {})
|
56
|
+
@publisher
|
57
|
+
end # publisher
|
58
|
+
|
59
|
+
def process(xml, params = {})
|
60
|
+
@xml_file_path = File.exists?(xml) ? xml : nil
|
61
|
+
|
62
|
+
@document = XMLParser.parse(xml)
|
63
|
+
@identifier_document = XMLParser.identifier_document
|
64
|
+
@params = params
|
65
|
+
|
66
|
+
@files = document.respond_to?(:files) ? document.files : [ ]
|
67
|
+
@results = { }
|
68
|
+
|
69
|
+
#force_default_publisher = params[:force_default_publisher]
|
70
|
+
force_default_publisher = params.fetch(:force_default_publisher, true)
|
71
|
+
|
72
|
+
if force_default_publisher
|
73
|
+
@publisher = @default_publisher.dup
|
74
|
+
@results[:files] = process_document_files(@files, :publisher => @publisher) if @files
|
75
|
+
else
|
76
|
+
# TODO PUT IN DYNAMIC PUBLISHER HANDLING
|
77
|
+
doc_type = document_type
|
78
|
+
end
|
79
|
+
|
80
|
+
#{ :files => files, :sequences => sequences }
|
81
|
+
|
82
|
+
@results
|
83
|
+
end # process
|
84
|
+
|
85
|
+
def process_document_files(_files, params = {})
|
86
|
+
publisher = params[:publisher]
|
87
|
+
|
88
|
+
run_mig = params.fetch(:run_mig, true)
|
89
|
+
|
90
|
+
_results = [ ]
|
91
|
+
total_files = _files.length
|
92
|
+
current_file_counter = 0
|
93
|
+
_files.each do |file|
|
94
|
+
current_file_counter += 1
|
95
|
+
full_file_path = file[@default_file_path_field_name]
|
96
|
+
|
97
|
+
logger.debug { "Processing Document File #{current_file_counter} of #{total_files}. File Path: #{full_file_path}" }
|
98
|
+
if run_mig
|
99
|
+
_mig = run_mig_on_file(full_file_path)
|
100
|
+
file[:metadata_sources] = _mig ? _mig.metadata_sources : { }
|
101
|
+
else
|
102
|
+
logger.debug { 'Media Information Gathering SKIPPED. run_mig set to false.' }
|
103
|
+
end
|
104
|
+
file[:xml_file_path] = @xml_file_path
|
105
|
+
file_result = { file: file }
|
106
|
+
file_result[:publish_result] = publisher.process(file) if publish and publisher
|
107
|
+
|
108
|
+
_results << file_result
|
109
|
+
end
|
110
|
+
_results
|
111
|
+
end # process_files
|
112
|
+
|
113
|
+
def process_document_sequences(params = {})
|
114
|
+
sequences = doc.respond_to?(:sequences) ? doc.sequences : [ ]
|
115
|
+
end # process_sequences
|
116
|
+
|
117
|
+
def process_document_tracks(params = {})
|
118
|
+
tracks = doc.respond_to?(:tracks) ? doc.tracks : [ ]
|
119
|
+
end # process_tracks
|
120
|
+
|
121
|
+
def run_mig_on_file(full_file_path, params = {})
|
122
|
+
if File.exists?(full_file_path)
|
123
|
+
@mig.run(full_file_path)
|
124
|
+
return @mig
|
125
|
+
else
|
126
|
+
logger.debug { "Media Information Gathering SKIPPED. File Not Found. #{full_file_path}" }
|
127
|
+
return false
|
128
|
+
end
|
129
|
+
end # run_mig_on_file
|
130
|
+
|
131
|
+
end # XMLProcessor
|
132
|
+
end # MediaProcessingTool
|
data/lib/mig.rb
ADDED
@@ -0,0 +1,158 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'mig/modules/exiftool'
|
3
|
+
require 'mig/modules/ffmpeg'
|
4
|
+
require 'mig/modules/mediainfo'
|
5
|
+
require 'mig/modules/media_type'
|
6
|
+
require 'mig/modules/common'
|
7
|
+
|
8
|
+
class MediaInformationGatherer
|
9
|
+
|
10
|
+
File::Stat.class_eval do
|
11
|
+
|
12
|
+
TO_HASH_METHODS = [
|
13
|
+
:atime, :blksize, :blockdev?, :blocks, :chardev?, :ctime, :dev, :dev_major, :dev_minor, :directory?,
|
14
|
+
:executable?, :executable_real?, :file?, :ftype, :gid, :grpowned?, :ino, :mode, :mtime, :nlink, :owned?,
|
15
|
+
:pipe?, :rdev, :rdev_major, :rdev_minor, :readable?, :readable_real?,
|
16
|
+
:setgid?, :setuid?, :size, :size?, :socket?, :sticky?, :symlink?, :uid,
|
17
|
+
:world_readable?, :world_writable?, :writable?, :writable_real?, :zero?
|
18
|
+
]
|
19
|
+
|
20
|
+
def to_hash
|
21
|
+
if defined?(__callee__)
|
22
|
+
return (self.methods - Object.methods - [__callee__]).each_with_object({}) { |meth, acc| acc[meth] = self.send(meth) if self.method(meth).arity == 0 }
|
23
|
+
else
|
24
|
+
#(self.methods - Object.methods).each({}) { |meth, acc| acc[meth] = self.send(meth) if self.method(meth).arity == 0 }
|
25
|
+
hash_out = { }
|
26
|
+
TO_HASH_METHODS.each do |meth|
|
27
|
+
hash_out[meth] = self.send(meth) if self.methods.include?(meth.to_s) and self.method(meth).arity == 0
|
28
|
+
end
|
29
|
+
return hash_out
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
attr_reader :log, :options
|
36
|
+
|
37
|
+
# @param [Hash] _options
|
38
|
+
# @option options [String] :exiftool_cmd_path
|
39
|
+
# @option options [String] :ffmpeg_cmd_path
|
40
|
+
# @option options [String] :mediainfo_cmd_path
|
41
|
+
def initialize(_options = { })
|
42
|
+
@options = { }
|
43
|
+
@options.merge!(_options)
|
44
|
+
|
45
|
+
@log = options[:logger] || $log || Logger.new(STDOUT)
|
46
|
+
log.debug { "#{self.class.name} - Options loaded. #{options}" }
|
47
|
+
|
48
|
+
options[:logger] ||= log
|
49
|
+
|
50
|
+
params = options.dup
|
51
|
+
|
52
|
+
@exiftool = ExifTool.new( params )
|
53
|
+
@ffmpeg = FFMPEG.new( params )
|
54
|
+
@mediainfo = Mediainfo.new( params )
|
55
|
+
|
56
|
+
@media_typer = MediaType.new
|
57
|
+
|
58
|
+
@default_run_file_stat = options.fetch(:run_file_stat, true)
|
59
|
+
@default_run_file_magic = options.fetch(:run_filemagic, true)
|
60
|
+
@default_run_mediainfo = options.fetch(:run_mediainfo, true)
|
61
|
+
@default_run_ffmpeg = options.fetch(:run_ffmpeg, true)
|
62
|
+
@default_run_exiftool = options.fetch(:run_exiftool, true)
|
63
|
+
@default_run_common = options.fetch(:run_common, true)
|
64
|
+
end # initialize
|
65
|
+
|
66
|
+
def media_type; @media_type ||= { } end
|
67
|
+
def metadata_sources; @metadata_sources ||= { } end
|
68
|
+
|
69
|
+
# @param [String] file_path The path to the file to gather information about
|
70
|
+
def run(file_path, options = { })
|
71
|
+
@media_type = { }
|
72
|
+
@metadata_sources = { }
|
73
|
+
|
74
|
+
raise Errno::ENOENT, "File Not Found. File Path: '#{file_path}'" unless File.exist?(file_path)
|
75
|
+
|
76
|
+
|
77
|
+
gathering_start = Time.now
|
78
|
+
log.debug { "Gathering metadata for file: #{file_path}"}
|
79
|
+
@metadata_sources = run_modules(file_path, options)
|
80
|
+
log.debug { "Metadata gathering completed. Took: #{Time.now - gathering_start} seconds" }
|
81
|
+
|
82
|
+
metadata_sources
|
83
|
+
end # run
|
84
|
+
|
85
|
+
# @param [String] file_path The path of the file to gather information about
|
86
|
+
# @return [Hash]
|
87
|
+
def run_modules(file_path, options = { })
|
88
|
+
run_file_stat = options.fetch(:run_file_stat, @default_run_file_stat)
|
89
|
+
run_filemagic = options.fetch(:run_filemagic, @default_run_filemagic)
|
90
|
+
run_mediainfo = options.fetch(:run_mediainfo, @default_run_mediainfo)
|
91
|
+
run_ffmpeg = options.fetch(:run_ffmpeg, @default_run_ffmpeg)
|
92
|
+
run_exiftool = options.fetch(:run_exiftool, @default_run_exiftool)
|
93
|
+
run_common = options.fetch(:run_common, @default_run_common)
|
94
|
+
|
95
|
+
if run_file_stat
|
96
|
+
log.debug { 'Running File Stat.' }
|
97
|
+
start = Time.now and metadata_sources[:stat] = File.stat(file_path).to_hash rescue { :error => { :message => $!.message, :backtrace => $!.backtrace } }
|
98
|
+
log.debug { "File Stat took #{Time.now - start}" }
|
99
|
+
end
|
100
|
+
|
101
|
+
if run_filemagic
|
102
|
+
log.debug { 'Running Filemagic.' }
|
103
|
+
start = Time.now and metadata_sources[:filemagic] = @media_typer.run(file_path, options) rescue { :error => { :message => $!.message, :backtrace => $!.backtrace } }
|
104
|
+
log.debug { "Filemagic took #{Time.now - start}" }
|
105
|
+
end
|
106
|
+
|
107
|
+
if run_mediainfo
|
108
|
+
log.debug { 'Running MediaInfo.' }
|
109
|
+
start = Time.now and metadata_sources[:mediainfo] = @mediainfo.run(file_path, options) rescue { :error => { :message => $!.message, :backtrace => $!.backtrace } }
|
110
|
+
log.debug { "MediaInfo took #{Time.now - start}" }
|
111
|
+
end
|
112
|
+
|
113
|
+
if run_ffmpeg
|
114
|
+
log.debug { 'Running FFMPEG.' }
|
115
|
+
start = Time.now and metadata_sources[:ffmpeg] = @ffmpeg.run(file_path, options) rescue { :error => { :message => $!.message, :backtrace => $!.backtrace } }
|
116
|
+
log.debug { "FFMpeg took #{Time.now - start}" }
|
117
|
+
end
|
118
|
+
|
119
|
+
if run_exiftool
|
120
|
+
log.debug { 'Running ExifTool.' }
|
121
|
+
start = Time.now and metadata_sources[:exiftool] = @exiftool.run(file_path) rescue { :error => { :message => $!.message, :backtrace => $!.backtrace } }
|
122
|
+
log.debug { "ExifTool took #{Time.now - start}" }
|
123
|
+
end
|
124
|
+
|
125
|
+
set_media_type
|
126
|
+
metadata_sources[:media_type] = media_type
|
127
|
+
|
128
|
+
metadata_sources[:common] = Common.common_variables(metadata_sources) if run_common
|
129
|
+
|
130
|
+
metadata_sources
|
131
|
+
end # run_modules
|
132
|
+
|
133
|
+
def get_media_type_using_exiftool
|
134
|
+
exiftool_md = metadata_sources[:exiftool]
|
135
|
+
return unless exiftool_md.is_a?(Hash)
|
136
|
+
|
137
|
+
mime_type = exiftool_md['MIMEType']
|
138
|
+
return unless mime_type.is_a?(String)
|
139
|
+
|
140
|
+
type, sub_type = mime_type.split('/')
|
141
|
+
return unless type
|
142
|
+
|
143
|
+
{ :type => type, :subtype => sub_type }
|
144
|
+
end
|
145
|
+
|
146
|
+
def get_media_type_using_filemagic
|
147
|
+
filemagic_md = metadata_sources[:filemagic]
|
148
|
+
return unless filemagic_md.is_a?(Hash)
|
149
|
+
return unless filemagic_md[:type]
|
150
|
+
|
151
|
+
filemagic_md
|
152
|
+
end
|
153
|
+
|
154
|
+
def set_media_type
|
155
|
+
@media_type = get_media_type_using_filemagic || get_media_type_using_exiftool
|
156
|
+
end
|
157
|
+
|
158
|
+
end # MediaInformationGatherer
|