active_encode 0.2 → 0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +9 -0
  3. data/README.md +1 -16
  4. data/active_encode.gemspec +4 -4
  5. data/app/jobs/active_encode/polling_job.rb +4 -4
  6. data/lib/active_encode/callbacks.rb +2 -21
  7. data/lib/active_encode/core.rb +1 -35
  8. data/lib/active_encode/engine_adapters.rb +1 -3
  9. data/lib/active_encode/engine_adapters/elastic_transcoder_adapter.rb +32 -28
  10. data/lib/active_encode/engine_adapters/ffmpeg_adapter.rb +243 -0
  11. data/lib/active_encode/engine_adapters/matterhorn_adapter.rb +64 -97
  12. data/lib/active_encode/engine_adapters/test_adapter.rb +0 -12
  13. data/lib/active_encode/engine_adapters/zencoder_adapter.rb +53 -44
  14. data/lib/active_encode/input.rb +6 -0
  15. data/lib/active_encode/output.rb +7 -0
  16. data/lib/active_encode/persistence.rb +1 -1
  17. data/lib/active_encode/polling.rb +2 -2
  18. data/lib/active_encode/status.rb +0 -3
  19. data/lib/active_encode/technical_metadata.rb +7 -0
  20. data/lib/active_encode/version.rb +1 -1
  21. data/spec/fixtures/ffmpeg/cancelled-id/error.log +0 -0
  22. data/spec/fixtures/ffmpeg/cancelled-id/input_metadata +90 -0
  23. data/spec/fixtures/ffmpeg/cancelled-id/pid +1 -0
  24. data/spec/fixtures/ffmpeg/cancelled-id/progress +11 -0
  25. data/spec/fixtures/ffmpeg/completed-id/error.log +0 -0
  26. data/spec/fixtures/ffmpeg/completed-id/input_metadata +102 -0
  27. data/spec/fixtures/ffmpeg/completed-id/output_metadata-high +90 -0
  28. data/spec/fixtures/ffmpeg/completed-id/output_metadata-low +90 -0
  29. data/spec/fixtures/ffmpeg/completed-id/pid +1 -0
  30. data/spec/fixtures/ffmpeg/completed-id/progress +11 -0
  31. data/spec/fixtures/ffmpeg/completed-id/video-high.mp4 +0 -0
  32. data/spec/fixtures/ffmpeg/completed-id/video-low.mp4 +0 -0
  33. data/spec/fixtures/ffmpeg/failed-id/error.log +1 -0
  34. data/spec/fixtures/ffmpeg/failed-id/input_metadata +90 -0
  35. data/spec/fixtures/ffmpeg/failed-id/pid +1 -0
  36. data/spec/fixtures/ffmpeg/failed-id/progress +11 -0
  37. data/spec/fixtures/ffmpeg/running-id/error.log +0 -0
  38. data/spec/fixtures/ffmpeg/running-id/input_metadata +90 -0
  39. data/spec/fixtures/ffmpeg/running-id/pid +1 -0
  40. data/spec/fixtures/ffmpeg/running-id/progress +11 -0
  41. data/spec/fixtures/fireworks.mp4 +0 -0
  42. data/spec/integration/elastic_transcoder_adapter_spec.rb +21 -12
  43. data/spec/integration/ffmpeg_adapter_spec.rb +120 -0
  44. data/spec/integration/matterhorn_adapter_spec.rb +30 -59
  45. data/spec/integration/zencoder_adapter_spec.rb +242 -22
  46. data/spec/shared_specs/engine_adapter_specs.rb +116 -16
  47. data/spec/units/core_spec.rb +31 -0
  48. data/spec/units/input_spec.rb +25 -0
  49. data/spec/units/output_spec.rb +28 -1
  50. data/spec/units/polling_job_spec.rb +10 -10
  51. metadata +51 -11
  52. data/lib/active_encode/engine_adapters/active_job_adapter.rb +0 -21
  53. data/lib/active_encode/engine_adapters/inline_adapter.rb +0 -47
  54. data/lib/active_encode/engine_adapters/shingoncoder_adapter.rb +0 -63
  55. data/spec/integration/shingoncoder_adapter_spec.rb +0 -170
@@ -7,11 +7,7 @@ module ActiveEncode
7
7
 
8
8
  def create(input_url, options = {})
9
9
  workflow_id = options[:preset] || "full"
10
- # workflow_om = if encode.input.is_a? Hash
11
- # create_multiple_files(encode.input, workflow_id)
12
- # else
13
- workflow_om = Rubyhorn.client.addMediaPackageWithUrl(DEFAULT_ARGS.merge('workflow' => workflow_id, 'url' => input_url, 'filename' => File.basename(input_url), 'title' => File.basename(input_url)))
14
- # end
10
+ workflow_om = Rubyhorn.client.addMediaPackageWithUrl(DEFAULT_ARGS.merge('workflow' => workflow_id, 'url' => input_url, 'filename' => File.basename(input_url), 'title' => File.basename(input_url)))
15
11
  build_encode(get_workflow(workflow_om))
16
12
  end
17
13
 
@@ -19,39 +15,11 @@ module ActiveEncode
19
15
  build_encode(fetch_workflow(id))
20
16
  end
21
17
 
22
- def list(*_filters)
23
- raise NotImplementedError # TODO: implement this
24
- end
25
-
26
18
  def cancel(id)
27
19
  workflow_om = Rubyhorn.client.stop(id)
28
20
  build_encode(get_workflow(workflow_om))
29
21
  end
30
22
 
31
- def purge(encode)
32
- workflow_om = begin
33
- Rubyhorn.client.stop(encode.id)
34
- rescue
35
- nil
36
- end
37
- workflow_om ||= begin
38
- Rubyhorn.client.get_stopped_workflow(encode.id)
39
- rescue
40
- nil
41
- end
42
- purged_workflow = purge_outputs(get_workflow(workflow_om))
43
- # Rubyhorn.client.delete_instance(encode.id) #Delete is not working so workflow instances can always be retrieved later!
44
- build_encode(purged_workflow)
45
- end
46
-
47
- def remove_output(encode, output_id)
48
- workflow = fetch_workflow(encode.id)
49
- output = encode.output.find { |o| o[:id] == output_id }
50
- return if output.nil?
51
- purge_output(workflow, output_id)
52
- output
53
- end
54
-
55
23
  private
56
24
 
57
25
  def fetch_workflow(id)
@@ -81,17 +49,28 @@ module ActiveEncode
81
49
 
82
50
  def build_encode(workflow)
83
51
  return nil if workflow.nil?
84
- encode = ActiveEncode::Base.new(convert_input(workflow), convert_options(workflow))
52
+ input_url = convert_input(workflow)
53
+ input_url = get_workflow_title(workflow) if input_url.blank?
54
+ encode = ActiveEncode::Base.new(input_url, convert_options(workflow))
85
55
  encode.id = convert_id(workflow)
86
56
  encode.state = convert_state(workflow)
87
57
  encode.current_operations = convert_current_operations(workflow)
88
58
  encode.percent_complete = calculate_percent_complete(workflow)
89
59
  encode.created_at = convert_created_at(workflow)
90
- encode.updated_at = convert_updated_at(workflow)
91
- encode.finished_at = convert_finished_at(workflow) unless encode.running?
60
+ encode.updated_at = convert_updated_at(workflow) || encode.created_at
92
61
  encode.output = convert_output(workflow, encode.options)
93
62
  encode.errors = convert_errors(workflow)
94
- encode.tech_metadata = convert_tech_metadata(workflow)
63
+
64
+ encode.input.id = "presenter/source"
65
+ encode.input.state = encode.state
66
+ encode.input.created_at = encode.created_at
67
+ encode.input.updated_at = encode.updated_at
68
+ tech_md = convert_tech_metadata(workflow)
69
+ [:width, :height, :duration, :frame_rate, :checksum, :audio_codec, :video_codec,
70
+ :audio_bitrate, :video_bitrate].each do |field|
71
+ encode.input.send("#{field}=", tech_md[field])
72
+ end
73
+
95
74
  encode
96
75
  end
97
76
 
@@ -99,8 +78,12 @@ module ActiveEncode
99
78
  workflow.attribute('id').to_s
100
79
  end
101
80
 
81
+ def get_workflow_state(workflow)
82
+ workflow.attribute('state').to_s
83
+ end
84
+
102
85
  def convert_state(workflow)
103
- case workflow.attribute('state').to_s
86
+ case get_workflow_state(workflow)
104
87
  when "INSTANTIATED", "RUNNING" # Should there be a queued state?
105
88
  :running
106
89
  when "STOPPED"
@@ -114,7 +97,11 @@ module ActiveEncode
114
97
 
115
98
  def convert_input(workflow)
116
99
  # Need to do anything else since this is a MH url? and this disappears when a workflow is cleaned up
117
- workflow.xpath('mediapackage/media/track[@type="presenter/source"]/url/text()').to_s
100
+ workflow.xpath('mediapackage/media/track[@type="presenter/source"]/url/text()').to_s.strip
101
+ end
102
+
103
+ def get_workflow_title(workflow)
104
+ workflow.xpath('mediapackage/title/text()').to_s.strip
118
105
  end
119
106
 
120
107
  def convert_tech_metadata(workflow)
@@ -122,17 +109,29 @@ module ActiveEncode
122
109
  end
123
110
 
124
111
  def convert_output(workflow, options)
125
- output = []
112
+ outputs = []
126
113
  workflow.xpath('//track[@type="presenter/delivery" and tags/tag[text()="streaming"]]').each do |track|
127
- label = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
128
- url = track.at("url/text()").to_s
129
- if url.start_with? "rtmp"
130
- url = File.join(options[:stream_base], MatterhornRtmpUrl.parse(url).to_path) if options[:stream_base]
114
+ output = ActiveEncode::Output.new
115
+ output.label = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
116
+ output.url = track.at("url/text()").to_s
117
+ if output.url.start_with? "rtmp"
118
+ output.url = File.join(options[:stream_base], MatterhornRtmpUrl.parse(output.url).to_path) if options[:stream_base]
119
+ end
120
+ output.id = track.at("@id").to_s
121
+
122
+ tech_md = convert_track_metadata(track)
123
+ [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
124
+ :audio_bitrate, :video_bitrate, :file_size].each do |field|
125
+ output.send("#{field}=", tech_md[field])
131
126
  end
132
- track_id = track.at("@id").to_s
133
- output << convert_track_metadata(track).merge(id: track_id, url: url, label: label)
127
+
128
+ output.state = :completed
129
+ output.created_at = convert_output_created_at(track, workflow)
130
+ output.updated_at = convert_output_updated_at(track, workflow)
131
+
132
+ outputs << output
134
133
  end
135
- output
134
+ outputs
136
135
  end
137
136
 
138
137
  def convert_current_operations(workflow)
@@ -154,9 +153,16 @@ module ActiveEncode
154
153
  updated_at.present? ? Time.strptime(updated_at, "%Q") : nil
155
154
  end
156
155
 
157
- def convert_finished_at(workflow)
158
- finished_at = workflow.xpath('//operation[@state!="INSTANTIATED"]/completed/text()').last.to_s
159
- finished_at.present? ? Time.strptime(finished_at, "%Q") : nil
156
+ def convert_output_created_at(track, workflow)
157
+ quality = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
158
+ created_at = workflow.xpath("//operation[@id=\"compose\"][configurations/configuration[@key=\"target-tags\" and contains(text(), \"#{quality}\")]]/started/text()").to_s
159
+ created_at.present? ? Time.at(created_at.to_i / 1000.0) : nil
160
+ end
161
+
162
+ def convert_output_updated_at(track, workflow)
163
+ quality = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
164
+ updated_at = workflow.xpath("//operation[@id=\"compose\"][configurations/configuration[@key=\"target-tags\" and contains(text(), \"#{quality}\")]]/completed/text()").to_s
165
+ updated_at.present? ? Time.at(updated_at.to_i / 1000.0) : nil
160
166
  end
161
167
 
162
168
  def convert_options(workflow)
@@ -169,20 +175,20 @@ module ActiveEncode
169
175
  def convert_track_metadata(track)
170
176
  return {} if track.nil?
171
177
  metadata = {}
172
- metadata[:mime_type] = track.at("mimetype/text()").to_s if track.at('mimetype')
173
- metadata[:checksum] = track.at("checksum/text()").to_s if track.at('checksum')
174
- metadata[:duration] = track.at("duration/text()").to_s if track.at('duration')
178
+ # metadata[:mime_type] = track.at("mimetype/text()").to_s if track.at('mimetype')
179
+ metadata[:checksum] = track.at("checksum/text()").to_s.strip if track.at('checksum')
180
+ metadata[:duration] = track.at("duration/text()").to_s.to_i if track.at('duration')
175
181
  if track.at('audio')
176
182
  metadata[:audio_codec] = track.at("audio/encoder/@type").to_s
177
183
  metadata[:audio_channels] = track.at("audio/channels/text()").to_s
178
- metadata[:audio_bitrate] = track.at("audio/bitrate/text()").to_s
184
+ metadata[:audio_bitrate] = track.at("audio/bitrate/text()").to_s.to_f
179
185
  end
180
186
  if track.at('video')
181
187
  metadata[:video_codec] = track.at("video/encoder/@type").to_s
182
- metadata[:video_bitrate] = track.at("video/bitrate/text()").to_s
183
- metadata[:video_framerate] = track.at("video/framerate/text()").to_s
184
- metadata[:width] = track.at("video/resolution/text()").to_s.split('x')[0]
185
- metadata[:height] = track.at("video/resolution/text()").to_s.split('x')[1]
188
+ metadata[:video_bitrate] = track.at("video/bitrate/text()").to_s.to_f
189
+ metadata[:frame_rate] = track.at("video/framerate/text()").to_s.to_f
190
+ metadata[:width] = track.at("video/resolution/text()").to_s.split('x')[0].to_i
191
+ metadata[:height] = track.at("video/resolution/text()").to_s.split('x')[1].to_i
186
192
  end
187
193
  metadata
188
194
  end
@@ -194,45 +200,6 @@ module ActiveEncode
194
200
  mp
195
201
  end
196
202
 
197
- def purge_outputs(workflow)
198
- # Delete hls tracks first since the next, more general xpath matches them as well
199
- workflow.xpath('//track[@type="presenter/delivery" and tags/tag[text()="streaming"] and tags/tag[text()="hls"]]/@id').map(&:to_s).each do |hls_track_id|
200
- begin
201
- purge_output(workflow, hls_track_id)
202
- rescue
203
- nil
204
- end
205
- end
206
- workflow.xpath('//track[@type="presenter/delivery" and tags/tag[text()="streaming"]]/@id').map(&:to_s).each do |track_id|
207
- begin
208
- purge_output(workflow, track_id)
209
- rescue
210
- nil
211
- end
212
- end
213
-
214
- workflow
215
- end
216
-
217
- def purge_output(workflow, track_id)
218
- media_package = get_media_package(workflow)
219
- hls = workflow.xpath("//track[@id='#{track_id}']/tags/tag[text()='hls']").present?
220
- job_url = if hls
221
- Rubyhorn.client.delete_hls_track(media_package, track_id)
222
- else
223
- Rubyhorn.client.delete_track(media_package, track_id)
224
- end
225
- sleep(0.1)
226
- job_status = Nokogiri::XML(Rubyhorn.client.get(URI(job_url).path)).root.attribute("status").value
227
- # FIXME: have this return a boolean based upon result of operation
228
- case job_status
229
- when "FINISHED"
230
- workflow.at_xpath("//track[@id=\"#{track_id}\"]").remove
231
- when "FAILED"
232
- workflow.at_xpath('//errors').add_child("<error>Output not purged: #{mp.at_xpath("//*[@id=\"#{track_id}\"]/tags/tag[starts-with(text(),\"quality\")]/text()")}</error>")
233
- end
234
- end
235
-
236
203
  def calculate_percent_complete(workflow)
237
204
  totals = {
238
205
  transcode: 70,
@@ -28,18 +28,6 @@ module ActiveEncode
28
28
  new_encode.updated_at = Time.now
29
29
  new_encode
30
30
  end
31
-
32
- def list(*_filters)
33
- raise NotImplementedError
34
- end
35
-
36
- def purge(encode)
37
- @encodes.delete(encode.id)
38
- end
39
-
40
- def remove_output(_encode, _output_id)
41
- raise NotImplementedError
42
- end
43
31
  end
44
32
  end
45
33
  end
@@ -11,23 +11,11 @@ module ActiveEncode
11
11
  build_encode(get_job_details(id))
12
12
  end
13
13
 
14
- def list(*_filters)
15
- raise NotImplementedError
16
- end
17
-
18
14
  def cancel(id)
19
15
  response = Zencoder::Job.cancel(id)
20
16
  build_encode(get_job_details(id)) if response.success?
21
17
  end
22
18
 
23
- def purge(_encode)
24
- raise NotImplementedError
25
- end
26
-
27
- def remove_output(_encode, _output_id)
28
- raise NotImplementedError
29
- end
30
-
31
19
  private
32
20
 
33
21
  def get_job_details(job_id)
@@ -42,32 +30,47 @@ module ActiveEncode
42
30
  return nil if job_details.nil?
43
31
  encode = ActiveEncode::Base.new(convert_input(job_details), convert_options(job_details))
44
32
  encode.id = job_details.body["job"]["id"].to_s
45
- encode.state = convert_state(job_details)
33
+ encode.state = convert_state(get_job_state(job_details))
46
34
  job_progress = get_job_progress(encode.id)
47
35
  encode.current_operations = convert_current_operations(job_progress)
48
36
  encode.percent_complete = convert_percent_complete(job_progress, job_details)
49
37
  encode.created_at = job_details.body["job"]["created_at"]
50
38
  encode.updated_at = job_details.body["job"]["updated_at"]
51
- encode.finished_at = job_details.body["job"]["finished_at"]
52
- encode.output = convert_output(job_details)
53
- encode.errors = convert_errors(job_details)
54
- encode.tech_metadata = convert_tech_metadata(job_details.body["job"]["input_media_file"])
39
+ encode.errors = []
40
+
41
+ encode.output = convert_output(job_details, job_progress)
42
+
43
+ encode.input.id = job_details.body["job"]["input_media_file"]["id"].to_s
44
+ encode.input.errors = convert_input_errors(job_details)
45
+ tech_md = convert_tech_metadata(job_details.body["job"]["input_media_file"])
46
+ [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
47
+ :audio_bitrate, :video_bitrate, :file_size].each do |field|
48
+ encode.input.send("#{field}=", tech_md[field])
49
+ end
50
+ encode.input.state = convert_state(job_details.body["job"]["input_media_file"]["state"])
51
+ encode.input.created_at = job_details.body["job"]["input_media_file"]["created_at"]
52
+ encode.input.updated_at = job_details.body["job"]["input_media_file"]["updated_at"]
53
+
55
54
  encode
56
55
  end
57
56
 
58
- def convert_state(job_details)
59
- case job_details.body["job"]["state"]
60
- when "pending", "waiting", "processing" # Should there be a queued state?
57
+ def convert_state(state)
58
+ case state
59
+ when "assigning", "pending", "waiting", "processing" # Should there be a queued state?
61
60
  :running
62
61
  when "cancelled"
63
62
  :cancelled
64
- when "failed"
63
+ when "failed", "no_input"
65
64
  :failed
66
65
  when "finished"
67
66
  :completed
68
67
  end
69
68
  end
70
69
 
70
+ def get_job_state(job_details)
71
+ job_details.body["job"]["state"]
72
+ end
73
+
71
74
  def convert_current_operations(job_progress)
72
75
  current_ops = []
73
76
  job_progress.body["outputs"].each { |output| current_ops << output["current_event"] unless output["current_event"].nil? }
@@ -76,7 +79,7 @@ module ActiveEncode
76
79
 
77
80
  def convert_percent_complete(job_progress, job_details)
78
81
  percent = job_progress.body["progress"]
79
- percent ||= 100 if convert_state(job_details) == :completed
82
+ percent ||= 100 if convert_state(get_job_state(job_details)) == :completed
80
83
  percent ||= 0
81
84
  percent
82
85
  end
@@ -89,23 +92,29 @@ module ActiveEncode
89
92
  {}
90
93
  end
91
94
 
92
- def convert_output(job_details)
93
- output = []
94
- job_details.body["job"]["output_media_files"].each do |o|
95
- track_id = o["id"].to_s
96
- label = o["label"]
97
- url = o["url"]
98
- output << convert_tech_metadata(o).merge(id: track_id, url: url, label: label)
95
+ def convert_output(job_details, job_progress)
96
+ job_details.body["job"]["output_media_files"].collect do |o|
97
+ output = ActiveEncode::Output.new
98
+ output.id = o["id"].to_s
99
+ output.label = o["label"]
100
+ output.url = o["url"]
101
+ output.errors = Array(o["error_message"])
102
+
103
+ tech_md = convert_tech_metadata(o)
104
+ [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
105
+ :audio_bitrate, :video_bitrate, :file_size].each do |field|
106
+ output.send("#{field}=", tech_md[field])
107
+ end
108
+ output_progress = job_progress.body["outputs"].find { |out_prog| out_prog["id"] = output.id }
109
+ output.state = convert_state(output_progress["state"])
110
+ output.created_at = o["created_at"]
111
+ output.updated_at = o["updated_at"]
112
+ output
99
113
  end
100
- output
101
114
  end
102
115
 
103
- def convert_errors(job_details)
104
- errors = []
105
- input_error = job_details.body["job"]["input_media_file"]["error_message"]
106
- errors << input_error if input_error.present?
107
- job_details.body["job"]["output_media_files"].each { |o| errors << o["error_message"] if o["error_message"].present? }
108
- errors
116
+ def convert_input_errors(job_details)
117
+ Array(job_details.body["job"]["input_media_file"]["error_message"])
109
118
  end
110
119
 
111
120
  def convert_tech_metadata(media_file)
@@ -120,23 +129,23 @@ module ActiveEncode
120
129
  when "format"
121
130
  metadata[:mime_type] = value
122
131
  when "duration_in_ms"
123
- metadata[:duration] = value.to_s
132
+ metadata[:duration] = value
124
133
  when "audio_codec"
125
- metadata[:audio_codec] = value.to_s
134
+ metadata[:audio_codec] = value
126
135
  when "channels"
127
- metadata[:audio_channels] = value.to_s
136
+ metadata[:audio_channels] = value
128
137
  when "audio_bitrate_in_kbps"
129
- metadata[:audio_bitrate] = value.to_s
138
+ metadata[:audio_bitrate] = value
130
139
  when "video_codec"
131
140
  metadata[:video_codec] = value
132
141
  when "frame_rate"
133
- metadata[:video_framerate] = value.to_s
142
+ metadata[:frame_rate] = value
134
143
  when "video_bitrate_in_kbps"
135
- metadata[:video_bitrate] = value.to_s
144
+ metadata[:video_bitrate] = value
136
145
  when "width"
137
- metadata[:width] = value.to_s
146
+ metadata[:width] = value
138
147
  when "height"
139
- metadata[:height] = value.to_s
148
+ metadata[:height] = value
140
149
  end
141
150
  end
142
151
  metadata
@@ -5,5 +5,11 @@ module ActiveEncode
5
5
 
6
6
  attr_accessor :id
7
7
  attr_accessor :url
8
+
9
+ def valid?
10
+ id.present? && url.present? &&
11
+ created_at.is_a?(Time) && updated_at.is_a?(Time) &&
12
+ updated_at >= created_at
13
+ end
8
14
  end
9
15
  end
@@ -5,5 +5,12 @@ module ActiveEncode
5
5
 
6
6
  attr_accessor :id
7
7
  attr_accessor :url
8
+ attr_accessor :label
9
+
10
+ def valid?
11
+ id.present? && url.present? && label.present? &&
12
+ created_at.is_a?(Time) && updated_at.is_a?(Time) &&
13
+ updated_at >= created_at
14
+ end
8
15
  end
9
16
  end