active_encode 0.5.0 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +72 -0
- data/.rubocop.yml +8 -70
- data/.rubocop_todo.yml +64 -0
- data/Gemfile +3 -3
- data/active_encode.gemspec +4 -1
- data/app/controllers/active_encode/encode_record_controller.rb +1 -0
- data/app/jobs/active_encode/polling_job.rb +1 -1
- data/app/models/active_encode/encode_record.rb +1 -0
- data/db/migrate/20180822021048_create_active_encode_encode_records.rb +1 -0
- data/db/migrate/20190702153755_add_create_options_to_active_encode_encode_records.rb +6 -0
- data/db/migrate/20190712174821_add_progress_to_active_encode_encode_records.rb +6 -0
- data/lib/active_encode.rb +1 -0
- data/lib/active_encode/base.rb +1 -0
- data/lib/active_encode/callbacks.rb +1 -0
- data/lib/active_encode/core.rb +4 -3
- data/lib/active_encode/engine.rb +1 -0
- data/lib/active_encode/engine_adapter.rb +1 -0
- data/lib/active_encode/engine_adapters.rb +2 -1
- data/lib/active_encode/engine_adapters/elastic_transcoder_adapter.rb +25 -24
- data/lib/active_encode/engine_adapters/ffmpeg_adapter.rb +43 -58
- data/lib/active_encode/engine_adapters/matterhorn_adapter.rb +5 -4
- data/lib/active_encode/engine_adapters/test_adapter.rb +5 -4
- data/lib/active_encode/engine_adapters/zencoder_adapter.rb +3 -2
- data/lib/active_encode/global_id.rb +2 -1
- data/lib/active_encode/input.rb +3 -2
- data/lib/active_encode/output.rb +3 -2
- data/lib/active_encode/persistence.rb +7 -3
- data/lib/active_encode/polling.rb +2 -1
- data/lib/active_encode/status.rb +1 -0
- data/lib/active_encode/technical_metadata.rb +3 -2
- data/lib/active_encode/version.rb +2 -1
- data/lib/file_locator.rb +7 -8
- data/spec/controllers/encode_record_controller_spec.rb +2 -1
- data/spec/integration/elastic_transcoder_adapter_spec.rb +26 -26
- data/spec/integration/ffmpeg_adapter_spec.rb +26 -22
- data/spec/integration/matterhorn_adapter_spec.rb +6 -5
- data/spec/integration/zencoder_adapter_spec.rb +15 -14
- data/spec/rails_helper.rb +1 -0
- data/spec/routing/encode_record_controller_routing_spec.rb +1 -0
- data/spec/shared_specs/engine_adapter_specs.rb +1 -1
- data/spec/spec_helper.rb +2 -1
- data/spec/test_app_templates/lib/generators/test_app_generator.rb +13 -12
- data/spec/units/callbacks_spec.rb +3 -2
- data/spec/units/core_spec.rb +9 -8
- data/spec/units/engine_adapter_spec.rb +1 -0
- data/spec/units/file_locator_spec.rb +19 -18
- data/spec/units/global_id_spec.rb +4 -3
- data/spec/units/input_spec.rb +8 -5
- data/spec/units/output_spec.rb +8 -5
- data/spec/units/persistence_spec.rb +8 -4
- data/spec/units/polling_job_spec.rb +7 -6
- data/spec/units/polling_spec.rb +1 -0
- data/spec/units/status_spec.rb +3 -3
- metadata +37 -7
- data/.travis.yml +0 -19
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'fileutils'
|
2
3
|
require 'nokogiri'
|
3
4
|
|
@@ -9,8 +10,8 @@ module ActiveEncode
|
|
9
10
|
def create(input_url, options = {})
|
10
11
|
new_encode = ActiveEncode::Base.new(input_url, options)
|
11
12
|
new_encode.id = SecureRandom.uuid
|
12
|
-
new_encode.created_at = Time.
|
13
|
-
new_encode.updated_at = Time.
|
13
|
+
new_encode.created_at = Time.now.utc
|
14
|
+
new_encode.updated_at = Time.now.utc
|
14
15
|
new_encode.current_operations = []
|
15
16
|
new_encode.output = []
|
16
17
|
|
@@ -26,11 +27,11 @@ module ActiveEncode
|
|
26
27
|
new_encode.state = :failed
|
27
28
|
new_encode.percent_complete = 1
|
28
29
|
|
29
|
-
if new_encode.input.file_size.blank?
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
30
|
+
new_encode.errors = if new_encode.input.file_size.blank?
|
31
|
+
["#{input_url} does not exist or is not accessible"]
|
32
|
+
else
|
33
|
+
["Error inspecting input: #{input_url}"]
|
34
|
+
end
|
34
35
|
|
35
36
|
write_errors new_encode
|
36
37
|
return new_encode
|
@@ -53,9 +54,10 @@ module ActiveEncode
|
|
53
54
|
end
|
54
55
|
|
55
56
|
# Return encode object from file system
|
56
|
-
def find(id, opts={})
|
57
|
+
def find(id, opts = {})
|
57
58
|
encode_class = opts[:cast]
|
58
|
-
|
59
|
+
encode_class ||= ActiveEncode::Base
|
60
|
+
encode = encode_class.new(nil, opts)
|
59
61
|
encode.id = id
|
60
62
|
encode.output = []
|
61
63
|
encode.created_at, encode.updated_at = get_times encode.id
|
@@ -84,9 +86,9 @@ module ActiveEncode
|
|
84
86
|
encode.state = :running
|
85
87
|
encode.current_operations = ["transcoding"]
|
86
88
|
elsif progress_ended?(encode.id) && encode.percent_complete == 100
|
87
|
-
|
89
|
+
encode.state = :completed
|
88
90
|
elsif encode.percent_complete < 100
|
89
|
-
|
91
|
+
encode.state = :cancelled
|
90
92
|
end
|
91
93
|
|
92
94
|
encode.output = build_outputs encode if encode.completed?
|
@@ -102,25 +104,25 @@ module ActiveEncode
|
|
102
104
|
find id
|
103
105
|
end
|
104
106
|
|
105
|
-
private
|
107
|
+
private
|
106
108
|
|
107
|
-
def get_times
|
109
|
+
def get_times(id)
|
108
110
|
updated_at = if File.file? working_path("progress", id)
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
111
|
+
File.mtime(working_path("progress", id))
|
112
|
+
elsif File.file? working_path("error.log", id)
|
113
|
+
File.mtime(working_path("error.log", id))
|
114
|
+
else
|
115
|
+
File.mtime(working_path("input_metadata", id))
|
116
|
+
end
|
115
117
|
|
116
|
-
|
118
|
+
[File.mtime(working_path("input_metadata", id)), updated_at]
|
117
119
|
end
|
118
120
|
|
119
|
-
def write_errors
|
121
|
+
def write_errors(encode)
|
120
122
|
File.write(working_path("error.log", encode.id), encode.errors.join("\n"))
|
121
123
|
end
|
122
124
|
|
123
|
-
def build_input
|
125
|
+
def build_input(encode)
|
124
126
|
input = ActiveEncode::Input.new
|
125
127
|
metadata = get_tech_metadata(working_path("input_metadata", encode.id))
|
126
128
|
input.url = metadata[:url]
|
@@ -132,7 +134,7 @@ private
|
|
132
134
|
input
|
133
135
|
end
|
134
136
|
|
135
|
-
def build_outputs
|
137
|
+
def build_outputs(encode)
|
136
138
|
id = encode.id
|
137
139
|
outputs = []
|
138
140
|
Dir["#{File.absolute_path(working_path('outputs', id))}/*"].each do |file_path|
|
@@ -162,15 +164,11 @@ private
|
|
162
164
|
" #{output[:ffmpeg_opt]} #{working_path(file_name, id)}"
|
163
165
|
end.join(" ")
|
164
166
|
|
165
|
-
"ffmpeg -y -loglevel error -progress #{working_path(
|
167
|
+
"ffmpeg -y -loglevel error -progress #{working_path('progress', id)} -i #{input_url} #{output_opt} > #{working_path('error.log', id)} 2>&1"
|
166
168
|
end
|
167
169
|
|
168
170
|
def get_pid(id)
|
169
|
-
if File.file? working_path("pid", id)
|
170
|
-
File.read(working_path("pid", id)).remove("\n")
|
171
|
-
else
|
172
|
-
nil
|
173
|
-
end
|
171
|
+
File.read(working_path("pid", id)).remove("\n") if File.file? working_path("pid", id)
|
174
172
|
end
|
175
173
|
|
176
174
|
def working_path(path, id)
|
@@ -178,15 +176,13 @@ private
|
|
178
176
|
end
|
179
177
|
|
180
178
|
def running?(pid)
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
false
|
186
|
-
end
|
179
|
+
Process.getpgid pid.to_i
|
180
|
+
true
|
181
|
+
rescue Errno::ESRCH
|
182
|
+
false
|
187
183
|
end
|
188
184
|
|
189
|
-
def calculate_percent_complete
|
185
|
+
def calculate_percent_complete(encode)
|
190
186
|
data = read_progress encode.id
|
191
187
|
if data.blank?
|
192
188
|
1
|
@@ -196,32 +192,25 @@ private
|
|
196
192
|
end
|
197
193
|
end
|
198
194
|
|
199
|
-
def read_progress
|
200
|
-
if File.file? working_path("progress", id)
|
201
|
-
File.read working_path("progress", id)
|
202
|
-
else
|
203
|
-
nil
|
204
|
-
end
|
195
|
+
def read_progress(id)
|
196
|
+
File.read working_path("progress", id) if File.file? working_path("progress", id)
|
205
197
|
end
|
206
198
|
|
207
|
-
def progress_ended?
|
199
|
+
def progress_ended?(id)
|
208
200
|
"end" == progress_value("progress=", read_progress(id))
|
209
201
|
end
|
210
202
|
|
211
|
-
def progress_value
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
else
|
216
|
-
nil
|
217
|
-
end
|
203
|
+
def progress_value(key, data)
|
204
|
+
return nil unless data.present? && key.present?
|
205
|
+
ri = data.rindex(key) + key.length
|
206
|
+
data[ri..data.index("\n", ri) - 1]
|
218
207
|
end
|
219
208
|
|
220
|
-
def get_tech_metadata
|
209
|
+
def get_tech_metadata(file_path)
|
221
210
|
doc = Nokogiri::XML File.read(file_path)
|
222
211
|
doc.remove_namespaces!
|
223
212
|
duration = get_xpath_text(doc, '//Duration/text()', :to_f)
|
224
|
-
duration
|
213
|
+
duration *= 1000 unless duration.nil? # Convert to milliseconds
|
225
214
|
{ url: get_xpath_text(doc, '//media/@ref', :to_s),
|
226
215
|
width: get_xpath_text(doc, '//Width/text()', :to_f),
|
227
216
|
height: get_xpath_text(doc, '//Height/text()', :to_f),
|
@@ -234,12 +223,8 @@ private
|
|
234
223
|
video_bitrate: get_xpath_text(doc, '//track[@type="Video"]/BitRate/text()', :to_i) }
|
235
224
|
end
|
236
225
|
|
237
|
-
def get_xpath_text
|
238
|
-
|
239
|
-
doc.xpath(xpath).first.text.send(cast_method)
|
240
|
-
else
|
241
|
-
nil
|
242
|
-
end
|
226
|
+
def get_xpath_text(doc, xpath, cast_method)
|
227
|
+
doc.xpath(xpath).first&.text&.send(cast_method)
|
243
228
|
end
|
244
229
|
end
|
245
230
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'rubyhorn'
|
2
3
|
|
3
4
|
module ActiveEncode
|
@@ -11,7 +12,7 @@ module ActiveEncode
|
|
11
12
|
build_encode(get_workflow(workflow_om))
|
12
13
|
end
|
13
14
|
|
14
|
-
def find(id,
|
15
|
+
def find(id, _opts = {})
|
15
16
|
build_encode(fetch_workflow(id))
|
16
17
|
end
|
17
18
|
|
@@ -145,7 +146,7 @@ module ActiveEncode
|
|
145
146
|
|
146
147
|
def convert_created_at(workflow)
|
147
148
|
created_at = workflow.xpath('mediapackage/@start').last.to_s
|
148
|
-
created_at.present? ? Time.parse(created_at) : nil
|
149
|
+
created_at.present? ? Time.parse(created_at).utc : nil
|
149
150
|
end
|
150
151
|
|
151
152
|
def convert_updated_at(workflow)
|
@@ -156,13 +157,13 @@ module ActiveEncode
|
|
156
157
|
def convert_output_created_at(track, workflow)
|
157
158
|
quality = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
|
158
159
|
created_at = workflow.xpath("//operation[@id=\"compose\"][configurations/configuration[@key=\"target-tags\" and contains(text(), \"#{quality}\")]]/started/text()").to_s
|
159
|
-
created_at.present? ? Time.at(created_at.to_i / 1000.0) : nil
|
160
|
+
created_at.present? ? Time.at(created_at.to_i / 1000.0).utc : nil
|
160
161
|
end
|
161
162
|
|
162
163
|
def convert_output_updated_at(track, workflow)
|
163
164
|
quality = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
|
164
165
|
updated_at = workflow.xpath("//operation[@id=\"compose\"][configurations/configuration[@key=\"target-tags\" and contains(text(), \"#{quality}\")]]/completed/text()").to_s
|
165
|
-
updated_at.present? ? Time.at(updated_at.to_i / 1000.0) : nil
|
166
|
+
updated_at.present? ? Time.at(updated_at.to_i / 1000.0).utc : nil
|
166
167
|
end
|
167
168
|
|
168
169
|
def convert_options(workflow)
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module ActiveEncode
|
2
3
|
module EngineAdapters
|
3
4
|
class TestAdapter
|
@@ -9,8 +10,8 @@ module ActiveEncode
|
|
9
10
|
new_encode = ActiveEncode::Base.new(input_url, options)
|
10
11
|
new_encode.id = SecureRandom.uuid
|
11
12
|
new_encode.state = :running
|
12
|
-
new_encode.created_at = Time.now
|
13
|
-
new_encode.updated_at = Time.now
|
13
|
+
new_encode.created_at = Time.now.utc
|
14
|
+
new_encode.updated_at = Time.now.utc
|
14
15
|
@encodes[new_encode.id] = new_encode
|
15
16
|
new_encode
|
16
17
|
end
|
@@ -18,14 +19,14 @@ module ActiveEncode
|
|
18
19
|
def find(id, _opts = {})
|
19
20
|
new_encode = @encodes[id]
|
20
21
|
# Update the updated_at time to simulate changes
|
21
|
-
new_encode.updated_at = Time.now
|
22
|
+
new_encode.updated_at = Time.now.utc
|
22
23
|
new_encode
|
23
24
|
end
|
24
25
|
|
25
26
|
def cancel(id)
|
26
27
|
new_encode = @encodes[id]
|
27
28
|
new_encode.state = :cancelled
|
28
|
-
new_encode.updated_at = Time.now
|
29
|
+
new_encode.updated_at = Time.now.utc
|
29
30
|
new_encode
|
30
31
|
end
|
31
32
|
end
|
@@ -1,13 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module ActiveEncode
|
2
3
|
module EngineAdapters
|
3
4
|
class ZencoderAdapter
|
4
5
|
# TODO: add a stub for an input helper (supplied by an initializer) that transforms encode.input.url into a zencoder accepted url
|
5
|
-
def create(input_url,
|
6
|
+
def create(input_url, _options = {})
|
6
7
|
response = Zencoder::Job.create(input: input_url.to_s)
|
7
8
|
build_encode(get_job_details(response.body["id"]))
|
8
9
|
end
|
9
10
|
|
10
|
-
def find(id,
|
11
|
+
def find(id, _opts = {})
|
11
12
|
build_encode(get_job_details(id))
|
12
13
|
end
|
13
14
|
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'globalid'
|
2
3
|
|
3
4
|
module ActiveEncode
|
@@ -9,7 +10,7 @@ module ActiveEncode
|
|
9
10
|
other.is_a?(ActiveEncode::Base) && to_global_id == other.to_global_id
|
10
11
|
end
|
11
12
|
|
12
|
-
def to_global_id(
|
13
|
+
def to_global_id(_options = {})
|
13
14
|
super(app: 'ActiveEncode')
|
14
15
|
end
|
15
16
|
end
|
data/lib/active_encode/input.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module ActiveEncode
|
2
3
|
class Input
|
3
4
|
include Status
|
@@ -8,8 +9,8 @@ module ActiveEncode
|
|
8
9
|
|
9
10
|
def valid?
|
10
11
|
id.present? && url.present? &&
|
11
|
-
|
12
|
-
|
12
|
+
created_at.is_a?(Time) && updated_at.is_a?(Time) &&
|
13
|
+
updated_at >= created_at
|
13
14
|
end
|
14
15
|
end
|
15
16
|
end
|
data/lib/active_encode/output.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module ActiveEncode
|
2
3
|
class Output
|
3
4
|
include Status
|
@@ -9,8 +10,8 @@ module ActiveEncode
|
|
9
10
|
|
10
11
|
def valid?
|
11
12
|
id.present? && url.present? && label.present? &&
|
12
|
-
|
13
|
-
|
13
|
+
created_at.is_a?(Time) && updated_at.is_a?(Time) &&
|
14
|
+
updated_at >= created_at
|
14
15
|
end
|
15
16
|
end
|
16
17
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'active_support'
|
2
3
|
|
3
4
|
module ActiveEncode
|
@@ -9,8 +10,10 @@ module ActiveEncode
|
|
9
10
|
persist(persistence_model_attributes(encode))
|
10
11
|
end
|
11
12
|
|
12
|
-
|
13
|
-
|
13
|
+
around_create do |encode, block|
|
14
|
+
create_options = encode.options
|
15
|
+
encode = block.call
|
16
|
+
persist(persistence_model_attributes(encode).merge(create_options: create_options.to_json))
|
14
17
|
end
|
15
18
|
|
16
19
|
after_cancel do |encode|
|
@@ -38,7 +41,8 @@ module ActiveEncode
|
|
38
41
|
# Need to ensure that these values come through or else validations will fail
|
39
42
|
created_at: encode.created_at,
|
40
43
|
updated_at: encode.updated_at,
|
41
|
-
raw_object: encode.to_json
|
44
|
+
raw_object: encode.to_json,
|
45
|
+
progress: encode.percent_complete
|
42
46
|
}
|
43
47
|
end
|
44
48
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'active_support'
|
2
3
|
require 'active_model/callbacks'
|
3
4
|
|
@@ -8,7 +9,7 @@ module ActiveEncode
|
|
8
9
|
POLLING_WAIT_TIME = 10.seconds.freeze
|
9
10
|
|
10
11
|
CALLBACKS = [
|
11
|
-
|
12
|
+
:after_status_update, :after_failed, :after_cancelled, :after_completed
|
12
13
|
].freeze
|
13
14
|
|
14
15
|
included do
|
data/lib/active_encode/status.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'active_support'
|
2
3
|
|
3
4
|
module ActiveEncode
|
@@ -23,10 +24,10 @@ module ActiveEncode
|
|
23
24
|
attr_accessor :video_bitrate
|
24
25
|
end
|
25
26
|
|
26
|
-
def assign_tech_metadata
|
27
|
+
def assign_tech_metadata(metadata)
|
27
28
|
[:width, :height, :frame_rate, :duration, :file_size, :checksum,
|
28
29
|
:audio_codec, :video_codec, :audio_bitrate, :video_bitrate].each do |field|
|
29
|
-
|
30
|
+
send("#{field}=", metadata[field]) if metadata.key?(field)
|
30
31
|
end
|
31
32
|
end
|
32
33
|
end
|
data/lib/file_locator.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'addressable/uri'
|
2
3
|
require 'aws-sdk'
|
3
4
|
|
@@ -10,7 +11,7 @@ class FileLocator
|
|
10
11
|
def initialize(uri)
|
11
12
|
uri = Addressable::URI.parse(uri)
|
12
13
|
@bucket = URI.decode(uri.host)
|
13
|
-
@key = URI.decode(uri.path).sub(%r
|
14
|
+
@key = URI.decode(uri.path).sub(%r{^/*(.+)/*$}, '\1')
|
14
15
|
end
|
15
16
|
|
16
17
|
def object
|
@@ -39,9 +40,7 @@ class FileLocator
|
|
39
40
|
end
|
40
41
|
end
|
41
42
|
|
42
|
-
if @uri.scheme.nil?
|
43
|
-
@uri = Addressable::URI.parse("file://#{URI.encode(File.expand_path(source))}")
|
44
|
-
end
|
43
|
+
@uri = Addressable::URI.parse("file://#{URI.encode(File.expand_path(source))}") if @uri.scheme.nil?
|
45
44
|
end
|
46
45
|
end
|
47
46
|
@uri
|
@@ -68,16 +67,16 @@ class FileLocator
|
|
68
67
|
false
|
69
68
|
end
|
70
69
|
end
|
71
|
-
|
70
|
+
alias exists? exist?
|
72
71
|
|
73
72
|
def reader
|
74
73
|
case uri.scheme
|
75
74
|
when 's3'
|
76
75
|
S3File.new(uri).object.get.body
|
77
76
|
when 'file'
|
78
|
-
File.open(location,'r')
|
77
|
+
File.open(location, 'r')
|
79
78
|
else
|
80
|
-
Kernel
|
79
|
+
Kernel.open(uri.to_s, 'r')
|
81
80
|
end
|
82
81
|
end
|
83
82
|
|
@@ -86,7 +85,7 @@ class FileLocator
|
|
86
85
|
when 's3'
|
87
86
|
uri
|
88
87
|
when 'file'
|
89
|
-
File.open(location,'r')
|
88
|
+
File.open(location, 'r')
|
90
89
|
else
|
91
90
|
location
|
92
91
|
end
|