active_encode 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +72 -0
- data/.rubocop.yml +8 -70
- data/.rubocop_todo.yml +64 -0
- data/Gemfile +3 -3
- data/active_encode.gemspec +4 -1
- data/app/controllers/active_encode/encode_record_controller.rb +1 -0
- data/app/jobs/active_encode/polling_job.rb +1 -1
- data/app/models/active_encode/encode_record.rb +1 -0
- data/db/migrate/20180822021048_create_active_encode_encode_records.rb +1 -0
- data/db/migrate/20190702153755_add_create_options_to_active_encode_encode_records.rb +6 -0
- data/db/migrate/20190712174821_add_progress_to_active_encode_encode_records.rb +6 -0
- data/lib/active_encode.rb +1 -0
- data/lib/active_encode/base.rb +1 -0
- data/lib/active_encode/callbacks.rb +1 -0
- data/lib/active_encode/core.rb +4 -3
- data/lib/active_encode/engine.rb +1 -0
- data/lib/active_encode/engine_adapter.rb +1 -0
- data/lib/active_encode/engine_adapters.rb +2 -1
- data/lib/active_encode/engine_adapters/elastic_transcoder_adapter.rb +25 -24
- data/lib/active_encode/engine_adapters/ffmpeg_adapter.rb +43 -58
- data/lib/active_encode/engine_adapters/matterhorn_adapter.rb +5 -4
- data/lib/active_encode/engine_adapters/test_adapter.rb +5 -4
- data/lib/active_encode/engine_adapters/zencoder_adapter.rb +3 -2
- data/lib/active_encode/global_id.rb +2 -1
- data/lib/active_encode/input.rb +3 -2
- data/lib/active_encode/output.rb +3 -2
- data/lib/active_encode/persistence.rb +7 -3
- data/lib/active_encode/polling.rb +2 -1
- data/lib/active_encode/status.rb +1 -0
- data/lib/active_encode/technical_metadata.rb +3 -2
- data/lib/active_encode/version.rb +2 -1
- data/lib/file_locator.rb +7 -8
- data/spec/controllers/encode_record_controller_spec.rb +2 -1
- data/spec/integration/elastic_transcoder_adapter_spec.rb +26 -26
- data/spec/integration/ffmpeg_adapter_spec.rb +26 -22
- data/spec/integration/matterhorn_adapter_spec.rb +6 -5
- data/spec/integration/zencoder_adapter_spec.rb +15 -14
- data/spec/rails_helper.rb +1 -0
- data/spec/routing/encode_record_controller_routing_spec.rb +1 -0
- data/spec/shared_specs/engine_adapter_specs.rb +1 -1
- data/spec/spec_helper.rb +2 -1
- data/spec/test_app_templates/lib/generators/test_app_generator.rb +13 -12
- data/spec/units/callbacks_spec.rb +3 -2
- data/spec/units/core_spec.rb +9 -8
- data/spec/units/engine_adapter_spec.rb +1 -0
- data/spec/units/file_locator_spec.rb +19 -18
- data/spec/units/global_id_spec.rb +4 -3
- data/spec/units/input_spec.rb +8 -5
- data/spec/units/output_spec.rb +8 -5
- data/spec/units/persistence_spec.rb +8 -4
- data/spec/units/polling_job_spec.rb +7 -6
- data/spec/units/polling_spec.rb +1 -0
- data/spec/units/status_spec.rb +3 -3
- metadata +37 -7
- data/.travis.yml +0 -19
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'fileutils'
|
2
3
|
require 'nokogiri'
|
3
4
|
|
@@ -9,8 +10,8 @@ module ActiveEncode
|
|
9
10
|
def create(input_url, options = {})
|
10
11
|
new_encode = ActiveEncode::Base.new(input_url, options)
|
11
12
|
new_encode.id = SecureRandom.uuid
|
12
|
-
new_encode.created_at = Time.
|
13
|
-
new_encode.updated_at = Time.
|
13
|
+
new_encode.created_at = Time.now.utc
|
14
|
+
new_encode.updated_at = Time.now.utc
|
14
15
|
new_encode.current_operations = []
|
15
16
|
new_encode.output = []
|
16
17
|
|
@@ -26,11 +27,11 @@ module ActiveEncode
|
|
26
27
|
new_encode.state = :failed
|
27
28
|
new_encode.percent_complete = 1
|
28
29
|
|
29
|
-
if new_encode.input.file_size.blank?
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
30
|
+
new_encode.errors = if new_encode.input.file_size.blank?
|
31
|
+
["#{input_url} does not exist or is not accessible"]
|
32
|
+
else
|
33
|
+
["Error inspecting input: #{input_url}"]
|
34
|
+
end
|
34
35
|
|
35
36
|
write_errors new_encode
|
36
37
|
return new_encode
|
@@ -53,9 +54,10 @@ module ActiveEncode
|
|
53
54
|
end
|
54
55
|
|
55
56
|
# Return encode object from file system
|
56
|
-
def find(id, opts={})
|
57
|
+
def find(id, opts = {})
|
57
58
|
encode_class = opts[:cast]
|
58
|
-
|
59
|
+
encode_class ||= ActiveEncode::Base
|
60
|
+
encode = encode_class.new(nil, opts)
|
59
61
|
encode.id = id
|
60
62
|
encode.output = []
|
61
63
|
encode.created_at, encode.updated_at = get_times encode.id
|
@@ -84,9 +86,9 @@ module ActiveEncode
|
|
84
86
|
encode.state = :running
|
85
87
|
encode.current_operations = ["transcoding"]
|
86
88
|
elsif progress_ended?(encode.id) && encode.percent_complete == 100
|
87
|
-
|
89
|
+
encode.state = :completed
|
88
90
|
elsif encode.percent_complete < 100
|
89
|
-
|
91
|
+
encode.state = :cancelled
|
90
92
|
end
|
91
93
|
|
92
94
|
encode.output = build_outputs encode if encode.completed?
|
@@ -102,25 +104,25 @@ module ActiveEncode
|
|
102
104
|
find id
|
103
105
|
end
|
104
106
|
|
105
|
-
private
|
107
|
+
private
|
106
108
|
|
107
|
-
def get_times
|
109
|
+
def get_times(id)
|
108
110
|
updated_at = if File.file? working_path("progress", id)
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
111
|
+
File.mtime(working_path("progress", id))
|
112
|
+
elsif File.file? working_path("error.log", id)
|
113
|
+
File.mtime(working_path("error.log", id))
|
114
|
+
else
|
115
|
+
File.mtime(working_path("input_metadata", id))
|
116
|
+
end
|
115
117
|
|
116
|
-
|
118
|
+
[File.mtime(working_path("input_metadata", id)), updated_at]
|
117
119
|
end
|
118
120
|
|
119
|
-
def write_errors
|
121
|
+
def write_errors(encode)
|
120
122
|
File.write(working_path("error.log", encode.id), encode.errors.join("\n"))
|
121
123
|
end
|
122
124
|
|
123
|
-
def build_input
|
125
|
+
def build_input(encode)
|
124
126
|
input = ActiveEncode::Input.new
|
125
127
|
metadata = get_tech_metadata(working_path("input_metadata", encode.id))
|
126
128
|
input.url = metadata[:url]
|
@@ -132,7 +134,7 @@ private
|
|
132
134
|
input
|
133
135
|
end
|
134
136
|
|
135
|
-
def build_outputs
|
137
|
+
def build_outputs(encode)
|
136
138
|
id = encode.id
|
137
139
|
outputs = []
|
138
140
|
Dir["#{File.absolute_path(working_path('outputs', id))}/*"].each do |file_path|
|
@@ -162,15 +164,11 @@ private
|
|
162
164
|
" #{output[:ffmpeg_opt]} #{working_path(file_name, id)}"
|
163
165
|
end.join(" ")
|
164
166
|
|
165
|
-
"ffmpeg -y -loglevel error -progress #{working_path(
|
167
|
+
"ffmpeg -y -loglevel error -progress #{working_path('progress', id)} -i #{input_url} #{output_opt} > #{working_path('error.log', id)} 2>&1"
|
166
168
|
end
|
167
169
|
|
168
170
|
def get_pid(id)
|
169
|
-
if File.file? working_path("pid", id)
|
170
|
-
File.read(working_path("pid", id)).remove("\n")
|
171
|
-
else
|
172
|
-
nil
|
173
|
-
end
|
171
|
+
File.read(working_path("pid", id)).remove("\n") if File.file? working_path("pid", id)
|
174
172
|
end
|
175
173
|
|
176
174
|
def working_path(path, id)
|
@@ -178,15 +176,13 @@ private
|
|
178
176
|
end
|
179
177
|
|
180
178
|
def running?(pid)
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
false
|
186
|
-
end
|
179
|
+
Process.getpgid pid.to_i
|
180
|
+
true
|
181
|
+
rescue Errno::ESRCH
|
182
|
+
false
|
187
183
|
end
|
188
184
|
|
189
|
-
def calculate_percent_complete
|
185
|
+
def calculate_percent_complete(encode)
|
190
186
|
data = read_progress encode.id
|
191
187
|
if data.blank?
|
192
188
|
1
|
@@ -196,32 +192,25 @@ private
|
|
196
192
|
end
|
197
193
|
end
|
198
194
|
|
199
|
-
def read_progress
|
200
|
-
if File.file? working_path("progress", id)
|
201
|
-
File.read working_path("progress", id)
|
202
|
-
else
|
203
|
-
nil
|
204
|
-
end
|
195
|
+
def read_progress(id)
|
196
|
+
File.read working_path("progress", id) if File.file? working_path("progress", id)
|
205
197
|
end
|
206
198
|
|
207
|
-
def progress_ended?
|
199
|
+
def progress_ended?(id)
|
208
200
|
"end" == progress_value("progress=", read_progress(id))
|
209
201
|
end
|
210
202
|
|
211
|
-
def progress_value
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
else
|
216
|
-
nil
|
217
|
-
end
|
203
|
+
def progress_value(key, data)
|
204
|
+
return nil unless data.present? && key.present?
|
205
|
+
ri = data.rindex(key) + key.length
|
206
|
+
data[ri..data.index("\n", ri) - 1]
|
218
207
|
end
|
219
208
|
|
220
|
-
def get_tech_metadata
|
209
|
+
def get_tech_metadata(file_path)
|
221
210
|
doc = Nokogiri::XML File.read(file_path)
|
222
211
|
doc.remove_namespaces!
|
223
212
|
duration = get_xpath_text(doc, '//Duration/text()', :to_f)
|
224
|
-
duration
|
213
|
+
duration *= 1000 unless duration.nil? # Convert to milliseconds
|
225
214
|
{ url: get_xpath_text(doc, '//media/@ref', :to_s),
|
226
215
|
width: get_xpath_text(doc, '//Width/text()', :to_f),
|
227
216
|
height: get_xpath_text(doc, '//Height/text()', :to_f),
|
@@ -234,12 +223,8 @@ private
|
|
234
223
|
video_bitrate: get_xpath_text(doc, '//track[@type="Video"]/BitRate/text()', :to_i) }
|
235
224
|
end
|
236
225
|
|
237
|
-
def get_xpath_text
|
238
|
-
|
239
|
-
doc.xpath(xpath).first.text.send(cast_method)
|
240
|
-
else
|
241
|
-
nil
|
242
|
-
end
|
226
|
+
def get_xpath_text(doc, xpath, cast_method)
|
227
|
+
doc.xpath(xpath).first&.text&.send(cast_method)
|
243
228
|
end
|
244
229
|
end
|
245
230
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'rubyhorn'
|
2
3
|
|
3
4
|
module ActiveEncode
|
@@ -11,7 +12,7 @@ module ActiveEncode
|
|
11
12
|
build_encode(get_workflow(workflow_om))
|
12
13
|
end
|
13
14
|
|
14
|
-
def find(id,
|
15
|
+
def find(id, _opts = {})
|
15
16
|
build_encode(fetch_workflow(id))
|
16
17
|
end
|
17
18
|
|
@@ -145,7 +146,7 @@ module ActiveEncode
|
|
145
146
|
|
146
147
|
def convert_created_at(workflow)
|
147
148
|
created_at = workflow.xpath('mediapackage/@start').last.to_s
|
148
|
-
created_at.present? ? Time.parse(created_at) : nil
|
149
|
+
created_at.present? ? Time.parse(created_at).utc : nil
|
149
150
|
end
|
150
151
|
|
151
152
|
def convert_updated_at(workflow)
|
@@ -156,13 +157,13 @@ module ActiveEncode
|
|
156
157
|
def convert_output_created_at(track, workflow)
|
157
158
|
quality = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
|
158
159
|
created_at = workflow.xpath("//operation[@id=\"compose\"][configurations/configuration[@key=\"target-tags\" and contains(text(), \"#{quality}\")]]/started/text()").to_s
|
159
|
-
created_at.present? ? Time.at(created_at.to_i / 1000.0) : nil
|
160
|
+
created_at.present? ? Time.at(created_at.to_i / 1000.0).utc : nil
|
160
161
|
end
|
161
162
|
|
162
163
|
def convert_output_updated_at(track, workflow)
|
163
164
|
quality = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
|
164
165
|
updated_at = workflow.xpath("//operation[@id=\"compose\"][configurations/configuration[@key=\"target-tags\" and contains(text(), \"#{quality}\")]]/completed/text()").to_s
|
165
|
-
updated_at.present? ? Time.at(updated_at.to_i / 1000.0) : nil
|
166
|
+
updated_at.present? ? Time.at(updated_at.to_i / 1000.0).utc : nil
|
166
167
|
end
|
167
168
|
|
168
169
|
def convert_options(workflow)
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module ActiveEncode
|
2
3
|
module EngineAdapters
|
3
4
|
class TestAdapter
|
@@ -9,8 +10,8 @@ module ActiveEncode
|
|
9
10
|
new_encode = ActiveEncode::Base.new(input_url, options)
|
10
11
|
new_encode.id = SecureRandom.uuid
|
11
12
|
new_encode.state = :running
|
12
|
-
new_encode.created_at = Time.now
|
13
|
-
new_encode.updated_at = Time.now
|
13
|
+
new_encode.created_at = Time.now.utc
|
14
|
+
new_encode.updated_at = Time.now.utc
|
14
15
|
@encodes[new_encode.id] = new_encode
|
15
16
|
new_encode
|
16
17
|
end
|
@@ -18,14 +19,14 @@ module ActiveEncode
|
|
18
19
|
def find(id, _opts = {})
|
19
20
|
new_encode = @encodes[id]
|
20
21
|
# Update the updated_at time to simulate changes
|
21
|
-
new_encode.updated_at = Time.now
|
22
|
+
new_encode.updated_at = Time.now.utc
|
22
23
|
new_encode
|
23
24
|
end
|
24
25
|
|
25
26
|
def cancel(id)
|
26
27
|
new_encode = @encodes[id]
|
27
28
|
new_encode.state = :cancelled
|
28
|
-
new_encode.updated_at = Time.now
|
29
|
+
new_encode.updated_at = Time.now.utc
|
29
30
|
new_encode
|
30
31
|
end
|
31
32
|
end
|
@@ -1,13 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module ActiveEncode
|
2
3
|
module EngineAdapters
|
3
4
|
class ZencoderAdapter
|
4
5
|
# TODO: add a stub for an input helper (supplied by an initializer) that transforms encode.input.url into a zencoder accepted url
|
5
|
-
def create(input_url,
|
6
|
+
def create(input_url, _options = {})
|
6
7
|
response = Zencoder::Job.create(input: input_url.to_s)
|
7
8
|
build_encode(get_job_details(response.body["id"]))
|
8
9
|
end
|
9
10
|
|
10
|
-
def find(id,
|
11
|
+
def find(id, _opts = {})
|
11
12
|
build_encode(get_job_details(id))
|
12
13
|
end
|
13
14
|
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'globalid'
|
2
3
|
|
3
4
|
module ActiveEncode
|
@@ -9,7 +10,7 @@ module ActiveEncode
|
|
9
10
|
other.is_a?(ActiveEncode::Base) && to_global_id == other.to_global_id
|
10
11
|
end
|
11
12
|
|
12
|
-
def to_global_id(
|
13
|
+
def to_global_id(_options = {})
|
13
14
|
super(app: 'ActiveEncode')
|
14
15
|
end
|
15
16
|
end
|
data/lib/active_encode/input.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module ActiveEncode
|
2
3
|
class Input
|
3
4
|
include Status
|
@@ -8,8 +9,8 @@ module ActiveEncode
|
|
8
9
|
|
9
10
|
def valid?
|
10
11
|
id.present? && url.present? &&
|
11
|
-
|
12
|
-
|
12
|
+
created_at.is_a?(Time) && updated_at.is_a?(Time) &&
|
13
|
+
updated_at >= created_at
|
13
14
|
end
|
14
15
|
end
|
15
16
|
end
|
data/lib/active_encode/output.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module ActiveEncode
|
2
3
|
class Output
|
3
4
|
include Status
|
@@ -9,8 +10,8 @@ module ActiveEncode
|
|
9
10
|
|
10
11
|
def valid?
|
11
12
|
id.present? && url.present? && label.present? &&
|
12
|
-
|
13
|
-
|
13
|
+
created_at.is_a?(Time) && updated_at.is_a?(Time) &&
|
14
|
+
updated_at >= created_at
|
14
15
|
end
|
15
16
|
end
|
16
17
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'active_support'
|
2
3
|
|
3
4
|
module ActiveEncode
|
@@ -9,8 +10,10 @@ module ActiveEncode
|
|
9
10
|
persist(persistence_model_attributes(encode))
|
10
11
|
end
|
11
12
|
|
12
|
-
|
13
|
-
|
13
|
+
around_create do |encode, block|
|
14
|
+
create_options = encode.options
|
15
|
+
encode = block.call
|
16
|
+
persist(persistence_model_attributes(encode).merge(create_options: create_options.to_json))
|
14
17
|
end
|
15
18
|
|
16
19
|
after_cancel do |encode|
|
@@ -38,7 +41,8 @@ module ActiveEncode
|
|
38
41
|
# Need to ensure that these values come through or else validations will fail
|
39
42
|
created_at: encode.created_at,
|
40
43
|
updated_at: encode.updated_at,
|
41
|
-
raw_object: encode.to_json
|
44
|
+
raw_object: encode.to_json,
|
45
|
+
progress: encode.percent_complete
|
42
46
|
}
|
43
47
|
end
|
44
48
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'active_support'
|
2
3
|
require 'active_model/callbacks'
|
3
4
|
|
@@ -8,7 +9,7 @@ module ActiveEncode
|
|
8
9
|
POLLING_WAIT_TIME = 10.seconds.freeze
|
9
10
|
|
10
11
|
CALLBACKS = [
|
11
|
-
|
12
|
+
:after_status_update, :after_failed, :after_cancelled, :after_completed
|
12
13
|
].freeze
|
13
14
|
|
14
15
|
included do
|
data/lib/active_encode/status.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'active_support'
|
2
3
|
|
3
4
|
module ActiveEncode
|
@@ -23,10 +24,10 @@ module ActiveEncode
|
|
23
24
|
attr_accessor :video_bitrate
|
24
25
|
end
|
25
26
|
|
26
|
-
def assign_tech_metadata
|
27
|
+
def assign_tech_metadata(metadata)
|
27
28
|
[:width, :height, :frame_rate, :duration, :file_size, :checksum,
|
28
29
|
:audio_codec, :video_codec, :audio_bitrate, :video_bitrate].each do |field|
|
29
|
-
|
30
|
+
send("#{field}=", metadata[field]) if metadata.key?(field)
|
30
31
|
end
|
31
32
|
end
|
32
33
|
end
|
data/lib/file_locator.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'addressable/uri'
|
2
3
|
require 'aws-sdk'
|
3
4
|
|
@@ -10,7 +11,7 @@ class FileLocator
|
|
10
11
|
def initialize(uri)
|
11
12
|
uri = Addressable::URI.parse(uri)
|
12
13
|
@bucket = URI.decode(uri.host)
|
13
|
-
@key = URI.decode(uri.path).sub(%r
|
14
|
+
@key = URI.decode(uri.path).sub(%r{^/*(.+)/*$}, '\1')
|
14
15
|
end
|
15
16
|
|
16
17
|
def object
|
@@ -39,9 +40,7 @@ class FileLocator
|
|
39
40
|
end
|
40
41
|
end
|
41
42
|
|
42
|
-
if @uri.scheme.nil?
|
43
|
-
@uri = Addressable::URI.parse("file://#{URI.encode(File.expand_path(source))}")
|
44
|
-
end
|
43
|
+
@uri = Addressable::URI.parse("file://#{URI.encode(File.expand_path(source))}") if @uri.scheme.nil?
|
45
44
|
end
|
46
45
|
end
|
47
46
|
@uri
|
@@ -68,16 +67,16 @@ class FileLocator
|
|
68
67
|
false
|
69
68
|
end
|
70
69
|
end
|
71
|
-
|
70
|
+
alias exists? exist?
|
72
71
|
|
73
72
|
def reader
|
74
73
|
case uri.scheme
|
75
74
|
when 's3'
|
76
75
|
S3File.new(uri).object.get.body
|
77
76
|
when 'file'
|
78
|
-
File.open(location,'r')
|
77
|
+
File.open(location, 'r')
|
79
78
|
else
|
80
|
-
Kernel
|
79
|
+
Kernel.open(uri.to_s, 'r')
|
81
80
|
end
|
82
81
|
end
|
83
82
|
|
@@ -86,7 +85,7 @@ class FileLocator
|
|
86
85
|
when 's3'
|
87
86
|
uri
|
88
87
|
when 'file'
|
89
|
-
File.open(location,'r')
|
88
|
+
File.open(location, 'r')
|
90
89
|
else
|
91
90
|
location
|
92
91
|
end
|