active_encode 0.5.0 → 0.8.1

data/lib/active_encode/engine_adapters/pass_through_adapter.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+require 'fileutils'
+require 'nokogiri'
+require 'shellwords'
+require 'file_locator'
+
+# PassThroughAdapter accepts an input file url and a number of derivative urls in the options
+# E.g. `create(input, outputs: [{ label: 'low',  url: 'file:///derivatives/low.mp4' }, { label: 'high', url: 'file:///derivatives/high.mp4' }])`
+# This adapter mirrors the ffmpeg adapter but differs in a few ways:
+#    1. It starts by copying the derivative files to the work directory
+#    2. It runs Mediainfo on the input and output files and skips ffmpeg
+#    3. All work is done in the create method so it's status is always completed or failed
+module ActiveEncode
+  module EngineAdapters
+    class PassThroughAdapter
+      WORK_DIR = ENV["ENCODE_WORK_DIR"] || "encodes" # Should read from config
+      MEDIAINFO_PATH = ENV["MEDIAINFO_PATH"] || "mediainfo"
+
+      def create(input_url, options = {})
+        # Decode file uris for ffmpeg (mediainfo works either way)
+        input_url = URI.decode(input_url) if input_url.starts_with? "file:///"
+
+        new_encode = ActiveEncode::Base.new(input_url, options)
+        new_encode.id = SecureRandom.uuid
+        new_encode.current_operations = []
+        new_encode.output = []
+
+        # Create a working directory that holds all output files related to the encode
+        FileUtils.mkdir_p working_path("", new_encode.id)
+        FileUtils.mkdir_p working_path("outputs", new_encode.id)
+
+        # Extract technical metadata from input file
+        `#{MEDIAINFO_PATH} --Output=XML --LogFile=#{working_path("input_metadata", new_encode.id)} #{input_url.shellescape}`
+        new_encode.input = build_input new_encode
+        new_encode.input.id = new_encode.id
+        new_encode.created_at, new_encode.updated_at = get_times new_encode.id
+
+        if new_encode.input.duration.blank?
+          new_encode.state = :failed
+          new_encode.percent_complete = 1
+
+          new_encode.errors = if new_encode.input.file_size.blank?
+                                ["#{input_url} does not exist or is not accessible"]
+                              else
+                                ["Error inspecting input: #{input_url}"]
+                              end
+
+          write_errors new_encode
+          return new_encode
+        end
+
+        # For saving filename to label map used to find the label when building outputs
+        filename_label_hash = {}
+
+        # Copy derivatives to work directory
+        options[:outputs].each do |opt|
+          url = opt[:url]
+          output_path = working_path("outputs/#{sanitize_base opt[:url]}#{File.extname opt[:url]}", new_encode.id)
+          FileUtils.cp FileLocator.new(url).location, output_path
+          filename_label_hash[output_path] = opt[:label]
+        end
+
+        # Write filename-to-label map so we can retrieve them on build_output
+        File.write working_path("filename_label.yml", new_encode.id), filename_label_hash.to_yaml
+
+        new_encode.percent_complete = 1
+        new_encode.state = :running
+        new_encode.errors = []
+
+        new_encode
+      rescue StandardError => e
+        new_encode.state = :failed
+        new_encode.percent_complete = 1
+        new_encode.errors = [e.full_message]
+        write_errors new_encode
+        return new_encode
+      end
+
+      # Return encode object from file system
+      def find(id, opts = {})
+        encode_class = opts[:cast]
+        encode_class ||= ActiveEncode::Base
+        encode = encode_class.new(nil, opts)
+        encode.id = id
+        encode.created_at, encode.updated_at = get_times encode.id
+        encode.input = build_input encode
+        encode.input.id = encode.id
+        encode.output = []
+        encode.current_operations = []
+
+        encode.errors = read_errors(id)
+        if encode.errors.present?
+          encode.state = :failed
+          encode.percent_complete = 1
+        elsif cancelled?(id)
+          encode.state = :cancelled
+          encode.percent_complete = 1
+        elsif completed?(id)
+          encode.state = :completed
+          encode.percent_complete = 100
+        else
+          encode.output = build_outputs encode
+          encode.state = :completed
+          encode.percent_complete = 100
+        end
+
+        encode
+      rescue StandardError => e
+        encode.state = :failed
+        encode.percent_complete = 1
+        encode.errors = [e.full_message]
+        write_errors encode
+        return encode
+      end
+
+      # Cancel ongoing encode using pid file
+      def cancel(id)
+        # Check for errors and if not then create cancelled file else raise CancelError?
+        if running?(id)
+          File.write(working_path("cancelled", id), "")
+          find id
+        else
+          raise CancelError
+        end
+      end
+
+    private
+
+      def running?(id)
+        !cancelled?(id) || !failed?(id) || !completed?(id)
+      end
+
+      def cancelled?(id)
+        File.exist? working_path("cancelled", id)
+      end
+
+      def failed?(id)
+        read_errors(id).present?
+      end
+
+      def completed?(id)
+        File.exist? working_path("completed", id)
+      end
+
+      def get_times(id)
+        updated_at = if File.file? working_path("completed", id)
+                       File.mtime(working_path("completed", id))
+                     elsif File.file? working_path("cancelled", id)
+                       File.mtime(working_path("cancelled", id))
+                     elsif File.file? working_path("error.log", id)
+                       File.mtime(working_path("error.log", id))
+                     else
+                       File.mtime(working_path("input_metadata", id))
+                     end
+
+        [File.mtime(working_path("input_metadata", id)), updated_at]
+      end
+
+      def write_errors(encode)
+        File.write(working_path("error.log", encode.id), encode.errors.join("\n"))
+      end
+
+      def read_errors(id)
+        err_path = working_path("error.log", id)
+        error = File.read(err_path) if File.file? err_path
+        if error.present?
+          [error]
+        else
+          []
+        end
+      end
+
+      def build_input(encode)
+        input = ActiveEncode::Input.new
+        metadata = get_tech_metadata(working_path("input_metadata", encode.id))
+        input.url = metadata[:url]
+        input.assign_tech_metadata(metadata)
+        created_at = File.mtime(working_path("input_metadata", encode.id))
+        input.created_at = created_at
+        input.updated_at = created_at
+
+        input
+      end
+
+      def build_outputs(encode)
+        id = encode.id
+        outputs = []
+        filename_label_hash = YAML.safe_load(File.read(working_path("filename_label.yml", id))) if File.exist?(working_path("filename_label.yml", id))
+        Dir["#{File.absolute_path(working_path('outputs', id))}/*"].each do |file_path|
+          output = ActiveEncode::Output.new
+          output.url = "file://#{file_path}"
+          output.label = filename_label_hash[file_path] if filename_label_hash
+          output.id = "#{encode.input.id}-#{output.label}"
+          output.created_at = encode.created_at
+          output.updated_at = File.mtime file_path
+
+          # Extract technical metadata from output file
+          metadata_path = working_path("output_metadata-#{output.label}", id)
+          `#{MEDIAINFO_PATH} --Output=XML --LogFile=#{metadata_path} #{output.url}` unless File.file? metadata_path
+          output.assign_tech_metadata(get_tech_metadata(metadata_path))
+
+          outputs << output
+        end
+        File.write(working_path("completed", id), "")
+
+        outputs
+      end
+
+      def sanitize_base(input_url)
+        File.basename(input_url, File.extname(input_url)).gsub(/[^0-9A-Za-z.\-]/, '_')
+      end
+
+      def working_path(path, id)
+        File.join(WORK_DIR, id, path)
+      end
+
+      def get_tech_metadata(file_path)
+        doc = Nokogiri::XML File.read(file_path)
+        doc.remove_namespaces!
+        duration = get_xpath_text(doc, '//Duration/text()', :to_f)
+        duration *= 1000 unless duration.nil? # Convert to milliseconds
+        { url: get_xpath_text(doc, '//media/@ref', :to_s),
+          width: get_xpath_text(doc, '//Width/text()', :to_f),
+          height: get_xpath_text(doc, '//Height/text()', :to_f),
+          frame_rate: get_xpath_text(doc, '//FrameRate/text()', :to_f),
+          duration: duration,
+          file_size: get_xpath_text(doc, '//FileSize/text()', :to_i),
+          audio_codec: get_xpath_text(doc, '//track[@type="Audio"]/CodecID/text()', :to_s),
+          audio_bitrate: get_xpath_text(doc, '//track[@type="Audio"]/BitRate/text()', :to_i),
+          video_codec: get_xpath_text(doc, '//track[@type="Video"]/CodecID/text()', :to_s),
+          video_bitrate: get_xpath_text(doc, '//track[@type="Video"]/BitRate/text()', :to_i) }
+      end
+
+      def get_xpath_text(doc, xpath, cast_method)
+        doc.xpath(xpath).first&.text&.send(cast_method)
+      end
+    end
+  end
+end

data/lib/active_encode/engine_adapters/test_adapter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module ActiveEncode
   module EngineAdapters
     class TestAdapter
@@ -9,8 +10,8 @@ module ActiveEncode
         new_encode = ActiveEncode::Base.new(input_url, options)
         new_encode.id = SecureRandom.uuid
         new_encode.state = :running
-        new_encode.created_at = Time.now
-        new_encode.updated_at = Time.now
+        new_encode.created_at = Time.now.utc
+        new_encode.updated_at = Time.now.utc
         @encodes[new_encode.id] = new_encode
         new_encode
       end
@@ -18,14 +19,14 @@ module ActiveEncode
       def find(id, _opts = {})
         new_encode = @encodes[id]
         # Update the updated_at time to simulate changes
-        new_encode.updated_at = Time.now
+        new_encode.updated_at = Time.now.utc
         new_encode
       end
 
       def cancel(id)
         new_encode = @encodes[id]
         new_encode.state = :cancelled
-        new_encode.updated_at = Time.now
+        new_encode.updated_at = Time.now.utc
         new_encode
       end
     end

data/lib/active_encode/engine_adapters/zencoder_adapter.rb

@@ -1,13 +1,14 @@
+# frozen_string_literal: true
 module ActiveEncode
   module EngineAdapters
     class ZencoderAdapter
       # TODO: add a stub for an input helper (supplied by an initializer) that transforms encode.input.url into a zencoder accepted url
-      def create(input_url, options = {})
+      def create(input_url, _options = {})
         response = Zencoder::Job.create(input: input_url.to_s)
         build_encode(get_job_details(response.body["id"]))
       end
 
-      def find(id, opts = {})
+      def find(id, _opts = {})
         build_encode(get_job_details(id))
       end
 

data/lib/active_encode/errors.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+module ActiveEncode #:nodoc:
+  class NotFound < RuntimeError; end
+  class NotRunningError < RuntimeError; end
+  class CancelError < RuntimeError; end
+end

data/lib/active_encode/global_id.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'globalid'
 
 module ActiveEncode
@@ -9,7 +10,7 @@ module ActiveEncode
       other.is_a?(ActiveEncode::Base) && to_global_id == other.to_global_id
     end
 
-    def to_global_id(options = {})
+    def to_global_id(_options = {})
       super(app: 'ActiveEncode')
     end
   end

data/lib/active_encode/input.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module ActiveEncode
   class Input
     include Status
@@ -8,8 +9,8 @@ module ActiveEncode
 
     def valid?
       id.present? && url.present? &&
-      created_at.is_a?(Time) && updated_at.is_a?(Time) &&
-      updated_at >= created_at
+        created_at.is_a?(Time) && updated_at.is_a?(Time) &&
+        updated_at >= created_at
     end
   end
 end

data/lib/active_encode/output.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module ActiveEncode
   class Output
     include Status
@@ -9,8 +10,8 @@ module ActiveEncode
 
     def valid?
       id.present? && url.present? && label.present? &&
-      created_at.is_a?(Time) && updated_at.is_a?(Time) &&
-      updated_at >= created_at
+        created_at.is_a?(Time) && updated_at.is_a?(Time) &&
+        updated_at >= created_at
     end
   end
 end

data/lib/active_encode/persistence.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'active_support'
 
 module ActiveEncode
@@ -9,8 +10,10 @@ module ActiveEncode
         persist(persistence_model_attributes(encode))
       end
 
-      after_create do |encode|
-        persist(persistence_model_attributes(encode))
+      around_create do |encode, block|
+        create_options = encode.options
+        encode = block.call
+        persist(persistence_model_attributes(encode, create_options))
       end
 
       after_cancel do |encode|
@@ -29,8 +32,8 @@ module ActiveEncode
         model.update(encode_attributes) # Don't fail if persisting doesn't succeed?
       end
 
-      def persistence_model_attributes(encode)
-        {
+      def persistence_model_attributes(encode, create_options = nil)
+        attributes = {
           global_id: encode.to_global_id.to_s,
           state: encode.state,
           adapter: encode.class.engine_adapter.class.name,
@@ -38,8 +41,11 @@ module ActiveEncode
           # Need to ensure that these values come through or else validations will fail
           created_at: encode.created_at,
           updated_at: encode.updated_at,
-          raw_object: encode.to_json
+          raw_object: encode.to_json,
+          progress: encode.percent_complete
         }
+        attributes[:create_options] = create_options.to_json if create_options.present?
+        attributes
       end
   end
 end

data/lib/active_encode/polling.rb

@@ -1,4 +1,5 @@
-require 'active_support'
+# frozen_string_literal: true
+require 'active_support/core_ext/integer/time'
 require 'active_model/callbacks'
 
 module ActiveEncode
@@ -8,7 +9,7 @@ module ActiveEncode
     POLLING_WAIT_TIME = 10.seconds.freeze
 
     CALLBACKS = [
-        :after_status_update, :after_failed, :after_cancelled, :after_completed
+      :after_status_update, :after_failed, :after_cancelled, :after_completed
     ].freeze
 
     included do

data/lib/active_encode/spec/shared_specs.rb

@@ -0,0 +1,2 @@
+# frozen_string_literal: true
+require 'active_encode/spec/shared_specs/engine_adapter.rb'

data/{spec/shared_specs/engine_adapter_specs.rb → lib/active_encode/spec/shared_specs/engine_adapter.rb}

@@ -21,13 +21,13 @@ RSpec.shared_examples 'an ActiveEncode::EngineAdapter' do |*_flags|
     it 'returns an ActiveEncode::Base object' do
       expect(subject.class).to be ActiveEncode::Base
     end
-    its(:id) { is_expected.not_to be_empty }
+    it { expect(subject.id).to be_present }
     it { is_expected.to be_running }
-    its(:current_operations) { is_expected.to be_empty }
-    its(:percent_complete) { is_expected.to be < 100 }
-    its(:errors) { is_expected.to be_empty }
-    its(:created_at) { is_expected.to be_kind_of Time }
-    its(:updated_at) { is_expected.to be_kind_of Time }
+    it { expect(subject.current_operations).to be_empty }
+    it { expect(subject.percent_complete).to be < 100 }
+    it { expect(subject.errors).to be_empty }
+    it { expect(subject.created_at).to be_kind_of Time }
+    it { expect(subject.updated_at).to be_kind_of Time }
 
     it 'input is a valid ActiveEncode::Input object' do
       expect(subject.input).to be_a ActiveEncode::Input
@@ -50,12 +50,12 @@ RSpec.shared_examples 'an ActiveEncode::EngineAdapter' do |*_flags|
       it 'returns an ActiveEncode::Base object' do
         expect(subject.class).to be ActiveEncode::Base
       end
-      its(:id) { is_expected.to eq 'running-id' }
+      it { expect(subject.id).to be_present }
       it { is_expected.to be_running }
-      its(:percent_complete) { is_expected.to be > 0 }
-      its(:errors) { is_expected.to be_empty }
-      its(:created_at) { is_expected.to be_kind_of Time }
-      its(:updated_at) { is_expected.to be > subject.created_at }
+      it { expect(subject.percent_complete).to be_positive }
+      it { expect(subject.errors).to be_empty }
+      it { expect(subject.created_at).to be_kind_of Time }
+      it { expect(subject.updated_at).to be >= subject.created_at }
 
       it 'input is a valid ActiveEncode::Input object' do
         expect(subject.input).to be_a ActiveEncode::Input
@@ -77,12 +77,12 @@ RSpec.shared_examples 'an ActiveEncode::EngineAdapter' do |*_flags|
       it 'returns an ActiveEncode::Base object' do
         expect(subject.class).to be ActiveEncode::Base
       end
-      its(:id) { is_expected.to eq 'cancelled-id' }
+      it { expect(subject.id).to be_present }
       it { is_expected.to be_cancelled }
-      its(:percent_complete) { is_expected.to be > 0 }
-      its(:errors) { is_expected.to be_empty }
-      its(:created_at) { is_expected.to be_kind_of Time }
-      its(:updated_at) { is_expected.to be >= subject.created_at }
+      it { expect(subject.percent_complete).to be_positive }
+      it { expect(subject.errors).to be_empty }
+      it { expect(subject.created_at).to be_kind_of Time }
+      it { expect(subject.updated_at).to be >= subject.created_at }
 
       it 'input is a valid ActiveEncode::Input object' do
         expect(subject.input).to be_a ActiveEncode::Input
@@ -104,12 +104,12 @@ RSpec.shared_examples 'an ActiveEncode::EngineAdapter' do |*_flags|
       it 'returns an ActiveEncode::Base object' do
         expect(subject.class).to be ActiveEncode::Base
       end
-      its(:id) { is_expected.to eq 'completed-id' }
+      it { expect(subject.id).to be_present }
       it { is_expected.to be_completed }
-      its(:percent_complete) { is_expected.to eq 100 }
-      its(:errors) { is_expected.to be_empty }
-      its(:created_at) { is_expected.to be_kind_of Time }
-      its(:updated_at) { is_expected.to be > subject.created_at }
+      it { expect(subject.percent_complete).to eq 100 }
+      it { expect(subject.errors).to be_empty }
+      it { expect(subject.created_at).to be_kind_of Time }
+      it { expect(subject.updated_at).to be > subject.created_at }
 
       it 'input is a valid ActiveEncode::Input object' do
         expect(subject.input).to be_a ActiveEncode::Input
@@ -130,7 +130,7 @@ RSpec.shared_examples 'an ActiveEncode::EngineAdapter' do |*_flags|
 
       it 'output has technical metadata' do
         subject.output.each do |output|
-          expected_output = completed_output.find {|expected_out| expected_out[:id] == output.id }
+          expected_output = completed_output.find { |expected_out| expected_out[:id] == output.id }
           expect(output.as_json.symbolize_keys).to include expected_output
         end
       end
@@ -142,12 +142,12 @@ RSpec.shared_examples 'an ActiveEncode::EngineAdapter' do |*_flags|
       it 'returns an ActiveEncode::Base object' do
         expect(subject.class).to be ActiveEncode::Base
       end
-      its(:id) { is_expected.to eq 'failed-id' }
+      it { expect(subject.id).to be_present }
       it { is_expected.to be_failed }
-      its(:percent_complete) { is_expected.to be > 0 }
-      its(:errors) { is_expected.not_to be_empty }
-      its(:created_at) { is_expected.to be_kind_of Time }
-      its(:updated_at) { is_expected.to be > subject.created_at }
+      it { expect(subject.percent_complete).to be_positive }
+      it { expect(subject.errors).not_to be_empty }
+      it { expect(subject.created_at).to be_kind_of Time }
+      it { expect(subject.updated_at).to be > subject.created_at }
 
       it 'input is a valid ActiveEncode::Input object' do
         expect(subject.input).to be_a ActiveEncode::Input
@@ -174,12 +174,12 @@ RSpec.shared_examples 'an ActiveEncode::EngineAdapter' do |*_flags|
     it 'returns an ActiveEncode::Base object' do
       expect(subject.class).to be ActiveEncode::Base
     end
-    its(:id) { is_expected.to eq cancelling_job.id }
+    it { expect(subject.id).to eq cancelling_job.id }
     it { is_expected.to be_cancelled }
-    its(:percent_complete) { is_expected.to be > 0 }
-    its(:errors) { is_expected.to be_empty }
-    its(:created_at) { is_expected.to be_kind_of Time }
-    its(:updated_at) { is_expected.to be >= subject.created_at }
+    it { expect(subject.percent_complete).to be_positive }
+    it { expect(subject.errors).to be_empty }
+    it { expect(subject.created_at).to be_kind_of Time }
+    it { expect(subject.updated_at).to be >= subject.created_at }
 
     it 'input is a valid ActiveEncode::Input object' do
       expect(subject.input).to be_a ActiveEncode::Input
@@ -201,12 +201,11 @@ RSpec.shared_examples 'an ActiveEncode::EngineAdapter' do |*_flags|
     it 'returns an ActiveEncode::Base object' do
       expect(subject.class).to be ActiveEncode::Base
     end
-    its(:id) { is_expected.to eq 'running-id' }
-    it { is_expected.to be_running }
-    its(:percent_complete) { is_expected.to be > 0 }
-    its(:errors) { is_expected.to be_empty }
-    its(:created_at) { is_expected.to be_kind_of Time }
-    its(:updated_at) { is_expected.to be > subject.created_at }
+    it { expect(subject.id).to be_present }
+    it { expect(subject.percent_complete).to be_positive }
+    it { expect(subject.errors).to be_empty }
+    it { expect(subject.created_at).to be_kind_of Time }
+    it { expect(subject.updated_at).to be >= subject.created_at }
 
     it 'input is a valid ActiveEncode::Input object' do
       expect(subject.input).to be_a ActiveEncode::Input