active_encode 0.5.0 → 0.8.1

data/lib/active_encode/callbacks.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'active_model/callbacks'
 
 module ActiveEncode

data/lib/active_encode/core.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'active_support'
 require 'active_encode/callbacks'
 
@@ -44,19 +45,19 @@ module ActiveEncode
     end
 
     def initialize(input_url, options = nil)
-      @input = Input.new.tap{ |input| input.url = input_url }
+      @input = Input.new.tap { |input| input.url = input_url }
       @options = self.class.default_options(input_url).merge(Hash(options))
     end
 
     def create!
       run_callbacks :create do
-        merge!(self.class.engine_adapter.create(self.input.url, self.options))
+        merge!(self.class.engine_adapter.create(input.url, options))
       end
     end
 
     def cancel!
       run_callbacks :cancel do
-        merge!(self.class.engine_adapter.cancel(self.id))
+        merge!(self.class.engine_adapter.cancel(id))
       end
     end
 

data/lib/active_encode/engine.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'rails'
 
 module ActiveEncode

data/lib/active_encode/engine_adapter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'active_encode/engine_adapters'
 require 'active_support/core_ext/class/attribute'
 require 'active_support/core_ext/string/inflections'

data/lib/active_encode/engine_adapters.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module ActiveEncode
   # == Active Encode adapters
   #
@@ -13,8 +14,10 @@ module ActiveEncode
     autoload :ElasticTranscoderAdapter
     autoload :TestAdapter
     autoload :FfmpegAdapter
+    autoload :MediaConvertAdapter
+    autoload :PassThroughAdapter
 
-    ADAPTER = 'Adapter'.freeze
+    ADAPTER = 'Adapter'
     private_constant :ADAPTER
 
     class << self

data/lib/active_encode/engine_adapters/elastic_transcoder_adapter.rb

@@ -1,15 +1,15 @@
+# frozen_string_literal: true
 require 'addressable/uri'
-require 'aws-sdk'
+require 'aws-sdk-elastictranscoder'
 require 'file_locator'
 
 module ActiveEncode
   module EngineAdapters
     class ElasticTranscoderAdapter
-
       JOB_STATES = {
         "Submitted" => :running, "Progressing" => :running, "Canceled" => :cancelled,
         "Error" => :failed, "Complete" => :completed
-      }
+      }.freeze
 
       # Require options to include :pipeline_id, :masterfile_bucket and :outputs
       # Example :outputs value:
@@ -29,7 +29,7 @@ module ActiveEncode
         build_encode(job)
       end
 
-      def find(id, opts = {})
+      def find(id, _opts = {})
         build_encode(get_job_details(id))
       end
 
@@ -58,14 +58,14 @@ module ActiveEncode
           return nil if job.nil?
           encode = ActiveEncode::Base.new(convert_input(job), {})
           encode.id = job.id
-        	encode.state = JOB_STATES[job.status]
-        	encode.current_operations = []
-        	encode.percent_complete = convert_percent_complete(job)
-        	encode.created_at = convert_time(job.timing["submit_time_millis"])
-        	encode.updated_at = convert_time(job.timing["finish_time_millis"]) || convert_time(job.timing["start_time_millis"]) || encode.created_at
+          encode.state = JOB_STATES[job.status]
+          encode.current_operations = []
+          encode.percent_complete = convert_percent_complete(job)
+          encode.created_at = convert_time(job.timing["submit_time_millis"])
+          encode.updated_at = convert_time(job.timing["finish_time_millis"]) || convert_time(job.timing["start_time_millis"]) || encode.created_at
 
-        	encode.output = convert_output(job)
-        	encode.errors = job.outputs.select { |o| o.status == "Error" }.collect(&:status_detail).compact
+          encode.output = convert_output(job)
+          encode.errors = job.outputs.select { |o| o.status == "Error" }.collect(&:status_detail).compact
 
           tech_md = convert_tech_metadata(job.input.detected_properties)
           [:width, :height, :frame_rate, :duration, :file_size].each do |field|
@@ -82,7 +82,7 @@ module ActiveEncode
 
         def convert_time(time_millis)
           return nil if time_millis.nil?
-          Time.at(time_millis / 1000)
+          Time.at(time_millis / 1000).utc
         end
 
         def convert_bitrate(rate)
@@ -121,38 +121,38 @@ module ActiveEncode
         end
 
         def convert_input(job)
-          job.input
+          job.input.key
         end
 
-        def copy_to_input_bucket input_url, bucket
+        def copy_to_input_bucket(input_url, bucket)
           case Addressable::URI.parse(input_url).scheme
-          when nil,'file'
+          when nil, 'file'
             upload_to_s3 input_url, bucket
           when 's3'
             check_s3_bucket input_url, bucket
           end
         end
 
-        def check_s3_bucket input_url, source_bucket
+        def check_s3_bucket(input_url, source_bucket)
           # logger.info("Checking `#{input_url}'")
           s3_object = FileLocator::S3File.new(input_url).object
           if s3_object.bucket_name == source_bucket
             # logger.info("Already in bucket `#{source_bucket}'")
             s3_object.key
           else
-            s3_key = File.join(SecureRandom.uuid,s3_object.key)
+            s3_key = File.join(SecureRandom.uuid, s3_object.key)
             # logger.info("Copying to `#{source_bucket}/#{input_url}'")
             target = Aws::S3::Object.new(bucket_name: source_bucket, key: input_url)
-            target.copy_from(s3_object, multipart_copy: s3_object.size > 15728640) # 15.megabytes
+            target.copy_from(s3_object, multipart_copy: s3_object.size > 15_728_640) # 15.megabytes
             s3_key
           end
         end
 
-        def upload_to_s3 input_url, source_bucket
-          original_input = input_url
+        def upload_to_s3(input_url, source_bucket)
+          # original_input = input_url
           bucket = Aws::S3::Resource.new(client: s3client).bucket(source_bucket)
           filename = FileLocator.new(input_url).location
-          s3_key = File.join(SecureRandom.uuid,File.basename(filename))
+          s3_key = File.join(SecureRandom.uuid, File.basename(filename))
           # logger.info("Copying `#{original_input}' to `#{source_bucket}/#{input_url}'")
           obj = bucket.object(s3_key)
           obj.upload_file filename
@@ -161,20 +161,22 @@ module ActiveEncode
         end
 
         def read_preset(id)
-          client.read_preset(id: id).preset
+          @presets ||= {}
+          @presets[id] ||= client.read_preset(id: id).preset
         end
 
         def convert_output(job)
-          pipeline = client.read_pipeline(id: job.pipeline_id).pipeline
+          @pipeline ||= client.read_pipeline(id: job.pipeline_id).pipeline
           job.outputs.collect do |joutput|
             preset = read_preset(joutput.preset_id)
             extension = preset.container == 'ts' ? '.m3u8' : ''
-            tech_md = convert_tech_metadata(joutput, preset).merge({
+            additional_metadata = {
               managed: false,
               id: joutput.id,
               label: joutput.key.split("/", 2).first,
-              url: "s3://#{pipeline.output_bucket}/#{job.output_key_prefix}#{joutput.key}#{extension}"
-            })
+              url: "s3://#{@pipeline.output_bucket}/#{job.output_key_prefix}#{joutput.key}#{extension}"
+            }
+            tech_md = convert_tech_metadata(joutput, preset).merge(additional_metadata)
 
             output = ActiveEncode::Output.new
             output.state = convert_state(joutput)
@@ -194,7 +196,7 @@ module ActiveEncode
           job.outputs.select { |o| o.status == "Error" }.collect(&:status_detail).compact
         end
 
-        def convert_tech_metadata(props, preset=nil)
+        def convert_tech_metadata(props, preset = nil)
           return {} if props.nil? || props.empty?
           metadata_fields = {
             file_size: { key: :file_size, method: :itself },
@@ -216,13 +218,13 @@ module ActiveEncode
           unless preset.nil?
             audio = preset.audio
             video = preset.video
-            metadata.merge!({
+            metadata.merge!(
               audio_codec: audio&.codec,
               audio_channels: audio&.channels,
               audio_bitrate: convert_bitrate(audio&.bit_rate),
               video_codec: video&.codec,
               video_bitrate: convert_bitrate(video&.bit_rate)
-            })
+            )
           end
 
           metadata

data/lib/active_encode/engine_adapters/ffmpeg_adapter.rb

@@ -1,16 +1,31 @@
+# frozen_string_literal: true
 require 'fileutils'
 require 'nokogiri'
+require 'shellwords'
 
 module ActiveEncode
   module EngineAdapters
     class FfmpegAdapter
       WORK_DIR = ENV["ENCODE_WORK_DIR"] || "encodes" # Should read from config
+      MEDIAINFO_PATH = ENV["MEDIAINFO_PATH"] || "mediainfo"
+      FFMPEG_PATH = ENV["FFMPEG_PATH"] || "ffmpeg"
 
       def create(input_url, options = {})
+        # Decode file uris for ffmpeg (mediainfo works either way)
+        case input_url
+        when /^file\:\/\/\//
+          input_url = URI.decode(input_url)
+        when /^s3\:\/\//
+          require 'file_locator'
+
+          s3_object = FileLocator::S3File.new(input_url).object
+          input_url = URI.parse(s3_object.presigned_url(:get))
+        end
+
         new_encode = ActiveEncode::Base.new(input_url, options)
         new_encode.id = SecureRandom.uuid
-        new_encode.created_at = Time.new
-        new_encode.updated_at = Time.new
+        new_encode.created_at = Time.now.utc
+        new_encode.updated_at = Time.now.utc
         new_encode.current_operations = []
         new_encode.output = []
 
@@ -19,18 +34,24 @@ module ActiveEncode
         FileUtils.mkdir_p working_path("outputs", new_encode.id)
 
         # Extract technical metadata from input file
-        `mediainfo --Output=XML --LogFile=#{working_path("input_metadata", new_encode.id)} #{input_url}`
+        curl_option = if options && options[:headers]
+                        headers = options[:headers].map { |k, v| "#{k}: #{v}" }
+                        (["--File_curl=HttpHeader"] + headers).join(",").yield_self { |s| "'#{s}'" }
+                      else
+                        ""
+                      end
+        `#{MEDIAINFO_PATH} #{curl_option} --Output=XML --LogFile=#{working_path("input_metadata", new_encode.id)} "#{input_url}"`
         new_encode.input = build_input new_encode
 
         if new_encode.input.duration.blank?
           new_encode.state = :failed
           new_encode.percent_complete = 1
 
-          if new_encode.input.file_size.blank?
-            new_encode.errors = ["#{input_url} does not exist or is not accessible"]
-          else
-            new_encode.errors = ["Error inspecting input: #{input_url}"]
-          end
+          new_encode.errors = if new_encode.input.file_size.blank?
+                                ["#{input_url} does not exist or is not accessible"]
+                              else
+                                ["Error inspecting input: #{input_url}"]
+                              end
 
           write_errors new_encode
           return new_encode
@@ -42,20 +63,27 @@ module ActiveEncode
 
         # Run the ffmpeg command and save its pid
         command = ffmpeg_command(input_url, new_encode.id, options)
-        pid = Process.spawn(command)
+        pid = Process.spawn(command, err: working_path('error.log', new_encode.id))
         File.open(working_path("pid", new_encode.id), 'w') { |file| file.write pid }
         new_encode.input.id = pid
 
-        # Prevent zombie process
-        Process.detach(pid)
-
         new_encode
+      rescue StandardError => e
+        new_encode.state = :failed
+        new_encode.percent_complete = 1
+        new_encode.errors = [e.full_message]
+        write_errors new_encode
+        return new_encode
+      ensure
+        # Prevent zombie process
+        Process.detach(pid) if pid.present?
       end
 
       # Return encode object from file system
-      def find(id, opts={})
+      def find(id, opts = {})
         encode_class = opts[:cast]
-        encode = ActiveEncode::Base.new(nil, opts)
+        encode_class ||= ActiveEncode::Base
+        encode = encode_class.new(nil, opts)
         encode.id = id
         encode.output = []
         encode.created_at, encode.updated_at = get_times encode.id
@@ -65,28 +93,21 @@ module ActiveEncode
         pid = get_pid(id)
         encode.input.id = pid if pid.present?
 
-        if File.file? working_path("error.log", id)
-          error = File.read working_path("error.log", id)
-          if error.present?
-            encode.state = :failed
-            encode.errors = [error]
-
-            return encode
-          end
-        end
-
-        encode.errors = []
-
         encode.current_operations = []
         encode.created_at, encode.updated_at = get_times encode.id
-
-        if running? pid
+        encode.errors = read_errors(id)
+        if encode.errors.present?
+          encode.state = :failed
+        elsif running? pid
           encode.state = :running
           encode.current_operations = ["transcoding"]
         elsif progress_ended?(encode.id) && encode.percent_complete == 100
-            encode.state = :completed
+          encode.state = :completed
+        elsif cancelled? encode.id
+          encode.state = :cancelled
         elsif encode.percent_complete < 100
-            encode.state = :cancelled
+          encode.errors << "Encoding has completed but the output duration is shorter than the input"
+          encode.state = :failed
         end
 
         encode.output = build_outputs encode if encode.completed?
@@ -96,31 +117,61 @@ module ActiveEncode
 
       # Cancel ongoing encode using pid file
       def cancel(id)
-        pid = get_pid(id)
-        Process.kill 'SIGTERM', pid.to_i
+        encode = find id
+        if encode.running?
+          pid = get_pid(id)
+
+          IO.popen("ps -ef | grep #{pid}") do |pipe|
+            child_pids = pipe.readlines.map do |line|
+              parts = line.split(/\s+/)
+              parts[1] if parts[2] == pid.to_s && parts[1] != pipe.pid.to_s
+            end.compact
+
+            child_pids.each do |cpid|
+              Process.kill 'SIGTERM', cpid.to_i
+            end
+          end
 
-        find id
+          Process.kill 'SIGTERM', pid.to_i
+          File.write(working_path("cancelled", id), "")
+          encode = find id
+        end
+        encode
+      rescue Errno::ESRCH
+        raise NotRunningError
+      rescue StandardError
+        raise CancelError
       end
 
-private
+    private
 
-      def get_times id
+      def get_times(id)
         updated_at = if File.file? working_path("progress", id)
-            File.mtime(working_path("progress", id))
-          elsif File.file? working_path("error.log", id)
-            File.mtime(working_path("error.log", id))
-          else
-            File.mtime(working_path("input_metadata", id))
-          end
-
-        return File.mtime(working_path("input_metadata", id)), updated_at
+                       File.mtime(working_path("progress", id))
+                     elsif File.file? working_path("error.log", id)
+                       File.mtime(working_path("error.log", id))
+                     else
+                       File.mtime(working_path("input_metadata", id))
+                     end
+
+        [File.mtime(working_path("input_metadata", id)), updated_at]
       end
 
-      def write_errors encode
+      def write_errors(encode)
         File.write(working_path("error.log", encode.id), encode.errors.join("\n"))
       end
 
-      def build_input encode
+      def read_errors(id)
+        err_path = working_path("error.log", id)
+        error = File.read(err_path) if File.file? err_path
+        if error.present?
+          [error]
+        else
+          []
+        end
+      end
+
+      def build_input(encode)
         input = ActiveEncode::Input.new
         metadata = get_tech_metadata(working_path("input_metadata", encode.id))
         input.url = metadata[:url]
@@ -132,22 +183,21 @@ private
         input
       end
 
-      def build_outputs encode
+      def build_outputs(encode)
         id = encode.id
         outputs = []
         Dir["#{File.absolute_path(working_path('outputs', id))}/*"].each do |file_path|
           output = ActiveEncode::Output.new
           output.url = "file://#{file_path}"
-          original_extension = File.extname(encode.input.url)
-          original_filename = File.basename(encode.input.url).chomp(original_extension)
-          output.label = file_path[/#{Regexp.quote(original_filename)}\-(.*?)#{Regexp.quote(File.extname(file_path))}$/, 1]
+          sanitized_filename = sanitize_base encode.input.url
+          output.label = file_path[/#{Regexp.quote(sanitized_filename)}\-(.*?)#{Regexp.quote(File.extname(file_path))}$/, 1]
           output.id = "#{encode.input.id}-#{output.label}"
           output.created_at = encode.created_at
           output.updated_at = File.mtime file_path
 
           # Extract technical metadata from output file
           metadata_path = working_path("output_metadata-#{output.label}", id)
-          `mediainfo --Output=XML --LogFile=#{metadata_path} #{output.url}` unless File.file? metadata_path
+          `#{MEDIAINFO_PATH} --Output=XML --LogFile=#{metadata_path} #{output.url}` unless File.file? metadata_path
           output.assign_tech_metadata(get_tech_metadata(metadata_path))
 
           outputs << output
@@ -158,70 +208,75 @@ private
 
       def ffmpeg_command(input_url, id, opts)
         output_opt = opts[:outputs].collect do |output|
-          file_name = "outputs/#{File.basename(input_url, File.extname(input_url))}-#{output[:label]}.#{output[:extension]}"
+          sanitized_filename = sanitize_base input_url
+          file_name = "outputs/#{sanitized_filename}-#{output[:label]}.#{output[:extension]}"
           " #{output[:ffmpeg_opt]} #{working_path(file_name, id)}"
         end.join(" ")
-
-        "ffmpeg -y -loglevel error -progress #{working_path("progress", id)} -i #{input_url} #{output_opt} > #{working_path("error.log", id)} 2>&1"
+        header_opt = Array(opts[:headers]).map do |k, v|
+          "#{k}: #{v}\r\n"
+        end.join
+        header_opt = "-headers '#{header_opt}'" if header_opt.present?
+        "#{FFMPEG_PATH} #{header_opt} -y -loglevel error -progress #{working_path('progress', id)} -i \"#{input_url}\" #{output_opt}"
       end
 
-      def get_pid(id)
-        if File.file? working_path("pid", id)
-          File.read(working_path("pid", id)).remove("\n")
+      def sanitize_base(input_url)
+        if input_url.is_a? URI::HTTP
+          File.basename(input_url.path, File.extname(input_url.path))
         else
-          nil
+          File.basename(input_url, File.extname(input_url)).gsub(/[^0-9A-Za-z.\-]/, '_')
         end
       end
 
+      def get_pid(id)
+        File.read(working_path("pid", id)).remove("\n") if File.file? working_path("pid", id)
+      end
+
       def working_path(path, id)
         File.join(WORK_DIR, id, path)
       end
 
       def running?(pid)
-        begin
-          Process.getpgid pid.to_i
-          true
-        rescue Errno::ESRCH
-          false
-        end
+        Process.getpgid pid.to_i
+        true
+      rescue Errno::ESRCH
+        false
       end
 
-      def calculate_percent_complete encode
+      def calculate_percent_complete(encode)
         data = read_progress encode.id
         if data.blank?
           1
         else
           progress_in_milliseconds = progress_value("out_time_ms=", data).to_i / 1000.0
-          (progress_in_milliseconds / encode.input.duration * 100).round
+          output = (progress_in_milliseconds / encode.input.duration * 100).ceil
+          return 100 if output > 100
+          output
         end
       end
 
-      def read_progress id
-        if File.file? working_path("progress", id)
-          File.read working_path("progress", id)
-        else
-          nil
-        end
+      def cancelled?(id)
+        File.exist? working_path("cancelled", id)
+      end
+
+      def read_progress(id)
+        File.read working_path("progress", id) if File.file? working_path("progress", id)
       end
 
-      def progress_ended? id
+      def progress_ended?(id)
         "end" == progress_value("progress=", read_progress(id))
       end
 
-      def progress_value key, data
-        if data.present? && key.present?
-          ri = data.rindex(key) + key.length
-          data[ri..data.index("\n", ri)-1]
-        else
-          nil
-        end
+      def progress_value(key, data)
+        return nil unless data.present? && key.present?
+        ri = data.rindex(key) + key.length
+        data[ri..data.index("\n", ri) - 1]
       end
 
-      def get_tech_metadata file_path
+      def get_tech_metadata(file_path)
         doc = Nokogiri::XML File.read(file_path)
         doc.remove_namespaces!
         duration = get_xpath_text(doc, '//Duration/text()', :to_f)
-        duration = duration * 1000 unless duration.nil?      # Convert to milliseconds
+        duration *= 1000 unless duration.nil? # Convert to milliseconds
         { url: get_xpath_text(doc, '//media/@ref', :to_s),
           width: get_xpath_text(doc, '//Width/text()', :to_f),
           height: get_xpath_text(doc, '//Height/text()', :to_f),
@@ -234,12 +289,8 @@ private
           video_bitrate: get_xpath_text(doc, '//track[@type="Video"]/BitRate/text()', :to_i) }
       end
 
-      def get_xpath_text doc, xpath, cast_method
-        if doc.xpath(xpath).first
-          doc.xpath(xpath).first.text.send(cast_method)
-        else
-          nil
-        end
+      def get_xpath_text(doc, xpath, cast_method)
+        doc.xpath(xpath).first&.text&.send(cast_method)
       end
     end
   end