active_encode 0.8.0 → 1.0.0

data/lib/active_encode/engine_adapters/media_convert_output.rb

@@ -30,13 +30,32 @@ module ActiveEncode
           "XAVC" => :xavc_settings
         }.freeze
 
-        def tech_metadata(settings, output)
-          url = output.dig('outputFilePaths', 0)
+        # @param output_url [String] url, expected to be `s3://`
+        # @param output_settings [Aws::MediaConvert::Types::Output]
+        # @param output_detail_settings [Aws::MediaConvert::Types::OutputDetail]
+        def tech_metadata_from_settings(output_url:, output_settings:, output_detail_settings:)
           {
-            width: output.dig('videoDetails', 'widthInPx'),
-            height: output.dig('videoDetails', 'heightInPx'),
+            width: output_detail_settings.video_details.width_in_px,
+            height: output_detail_settings.video_details.height_in_px,
+            frame_rate: extract_video_frame_rate(output_settings),
+            duration: output_detail_settings.duration_in_ms,
+            audio_codec: extract_audio_codec(output_settings),
+            video_codec: extract_video_codec(output_settings),
+            audio_bitrate: extract_audio_bitrate(output_settings),
+            video_bitrate: extract_video_bitrate(output_settings),
+            url: output_url,
+            label: (output_url ? File.basename(output_url) : output_settings.name_modifier),
+            suffix: output_settings.name_modifier
+          }
+        end
+
+        def tech_metadata_from_logged(settings, logged_output)
+          url = logged_output.dig('outputFilePaths', 0)
+          {
+            width: logged_output.dig('videoDetails', 'widthInPx'),
+            height: logged_output.dig('videoDetails', 'heightInPx'),
             frame_rate: extract_video_frame_rate(settings),
-            duration: output['durationInMs'],
+            duration: logged_output['durationInMs'],
             audio_codec: extract_audio_codec(settings),
             video_codec: extract_video_codec(settings),
             audio_bitrate: extract_audio_bitrate(settings),
@@ -47,6 +66,49 @@ module ActiveEncode
           }
         end
 
+        # constructs an `s3:` output URL  from the MediaConvert job params, the same
+        # way MediaConvert will.
+        #
+        # @example
+        #   construct_output_filename(
+        #     destination: "s3://bucket/base_name",
+        #     original_filename: "foo.mp3",
+        #     name_modifier: "-1080",
+        #     suffix: "m3u8")
+        #   # =>  "s3://bucket/base_name-1080.m3u8"
+        #
+        # @example
+        #   construct_output_filename(
+        #     destination: "s3://bucket/directory_end_in_slash/",
+        #     original_filename: "original-filename.mp3",
+        #     name_modifier: "-1080",
+        #     suffix: "m3u8")
+        #   # =>  "s3://bucket/directory_end_in_slash/original_filename-1080.m3u8"
+        #
+        # @example
+        #   construct_output_filename(
+        #     destination: "s3://bucket/directory_end_in_slash/",
+        #     original_filename: "original-filename.mp3",
+        #     name_modifier: nil,
+        #     suffix: "m3u8")
+        #   # =>  "s3://bucket/directory_end_in_slash/original_filename.m3u8"
+        def construct_output_url(destination:, file_input_url:, name_modifier:, file_suffix:)
+          output = destination
+
+          # MediaConvert operates such that if you give a destination ending in '/',
+          # it'll use the original file name as part of output url.
+          if output.end_with?('/')
+            # ".*" on the end will strip extension off
+            output += File.basename(file_input_url, '.*')
+          end
+
+          output += name_modifier if name_modifier
+
+          output += "." + file_suffix
+
+          output
+        end
+
         def extract_audio_codec(settings)
           settings.audio_descriptions.first.codec_settings.codec
         rescue

data/lib/active_encode/engine_adapters/pass_through_adapter.rb

@@ -18,7 +18,7 @@ module ActiveEncode
 
       def create(input_url, options = {})
         # Decode file uris for ffmpeg (mediainfo works either way)
-        input_url = URI.decode(input_url) if input_url.starts_with? "file:///"
+        input_url = Addressable::URI.unencode(input_url) if input_url.starts_with? "file:///"
 
         new_encode = ActiveEncode::Base.new(input_url, options)
         new_encode.id = SecureRandom.uuid
@@ -73,7 +73,7 @@ module ActiveEncode
         new_encode.percent_complete = 1
         new_encode.errors = [e.full_message]
         write_errors new_encode
-        return new_encode
+        new_encode
       end
 
       # Return encode object from file system
@@ -110,7 +110,7 @@ module ActiveEncode
         encode.percent_complete = 1
         encode.errors = [e.full_message]
         write_errors encode
-        return encode
+        encode
       end
 
       # Cancel ongoing encode using pid file

data/lib/active_encode/engine_adapters/zencoder_adapter.rb

@@ -19,138 +19,138 @@ module ActiveEncode
 
       private
 
-        def get_job_details(job_id)
-          Zencoder::Job.details(job_id)
-        end
+      def get_job_details(job_id)
+        Zencoder::Job.details(job_id)
+      end
+
+      def get_job_progress(job_id)
+        Zencoder::Job.progress(job_id)
+      end
 
-        def get_job_progress(job_id)
-          Zencoder::Job.progress(job_id)
+      def build_encode(job_details)
+        return nil if job_details.nil?
+        encode = ActiveEncode::Base.new(convert_input(job_details), convert_options(job_details))
+        encode.id = job_details.body["job"]["id"].to_s
+        encode.state = convert_state(get_job_state(job_details))
+        job_progress = get_job_progress(encode.id)
+        encode.current_operations = convert_current_operations(job_progress)
+        encode.percent_complete = convert_percent_complete(job_progress, job_details)
+        encode.created_at = job_details.body["job"]["created_at"]
+        encode.updated_at = job_details.body["job"]["updated_at"]
+        encode.errors = []
+
+        encode.output = convert_output(job_details, job_progress)
+
+        encode.input.id = job_details.body["job"]["input_media_file"]["id"].to_s
+        encode.input.errors = convert_input_errors(job_details)
+        tech_md = convert_tech_metadata(job_details.body["job"]["input_media_file"])
+        [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
+         :audio_bitrate, :video_bitrate, :file_size].each do |field|
+          encode.input.send("#{field}=", tech_md[field])
         end
+        encode.input.state = convert_state(job_details.body["job"]["input_media_file"]["state"])
+        encode.input.created_at = job_details.body["job"]["input_media_file"]["created_at"]
+        encode.input.updated_at = job_details.body["job"]["input_media_file"]["updated_at"]
 
-        def build_encode(job_details)
-          return nil if job_details.nil?
-          encode = ActiveEncode::Base.new(convert_input(job_details), convert_options(job_details))
-          encode.id = job_details.body["job"]["id"].to_s
-          encode.state = convert_state(get_job_state(job_details))
-          job_progress = get_job_progress(encode.id)
-          encode.current_operations = convert_current_operations(job_progress)
-          encode.percent_complete = convert_percent_complete(job_progress, job_details)
-          encode.created_at = job_details.body["job"]["created_at"]
-          encode.updated_at = job_details.body["job"]["updated_at"]
-          encode.errors = []
-
-          encode.output = convert_output(job_details, job_progress)
-
-          encode.input.id = job_details.body["job"]["input_media_file"]["id"].to_s
-          encode.input.errors = convert_input_errors(job_details)
-          tech_md = convert_tech_metadata(job_details.body["job"]["input_media_file"])
-          [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
-           :audio_bitrate, :video_bitrate, :file_size].each do |field|
-            encode.input.send("#{field}=", tech_md[field])
-          end
-          encode.input.state = convert_state(job_details.body["job"]["input_media_file"]["state"])
-          encode.input.created_at = job_details.body["job"]["input_media_file"]["created_at"]
-          encode.input.updated_at = job_details.body["job"]["input_media_file"]["updated_at"]
+        encode
+      end
 
-          encode
+      def convert_state(state)
+        case state
+        when "assigning", "pending", "waiting", "processing" # Should there be a queued state?
+          :running
+        when "cancelled"
+          :cancelled
+        when "failed", "no_input"
+          :failed
+        when "finished"
+          :completed
         end
+      end
 
-        def convert_state(state)
-          case state
-          when "assigning", "pending", "waiting", "processing" # Should there be a queued state?
-            :running
-          when "cancelled"
-            :cancelled
-          when "failed", "no_input"
-            :failed
-          when "finished"
-            :completed
-          end
-        end
+      def get_job_state(job_details)
+        job_details.body["job"]["state"]
+      end
 
-        def get_job_state(job_details)
-          job_details.body["job"]["state"]
-        end
+      def convert_current_operations(job_progress)
+        current_ops = []
+        job_progress.body["outputs"].each { |output| current_ops << output["current_event"] unless output["current_event"].nil? }
+        current_ops
+      end
 
-        def convert_current_operations(job_progress)
-          current_ops = []
-          job_progress.body["outputs"].each { |output| current_ops << output["current_event"] unless output["current_event"].nil? }
-          current_ops
-        end
+      def convert_percent_complete(job_progress, job_details)
+        percent = job_progress.body["progress"]
+        percent ||= 100 if convert_state(get_job_state(job_details)) == :completed
+        percent ||= 0
+        percent
+      end
 
-        def convert_percent_complete(job_progress, job_details)
-          percent = job_progress.body["progress"]
-          percent ||= 100 if convert_state(get_job_state(job_details)) == :completed
-          percent ||= 0
-          percent
-        end
+      def convert_input(job_details)
+        job_details.body["job"]["input_media_file"]["url"]
+      end
 
-        def convert_input(job_details)
-          job_details.body["job"]["input_media_file"]["url"]
-        end
+      def convert_options(_job_details)
+        {}
+      end
 
-        def convert_options(_job_details)
-          {}
-        end
+      def convert_output(job_details, job_progress)
+        job_details.body["job"]["output_media_files"].collect do |o|
+          output = ActiveEncode::Output.new
+          output.id = o["id"].to_s
+          output.label = o["label"]
+          output.url = o["url"]
+          output.errors = Array(o["error_message"])
 
-        def convert_output(job_details, job_progress)
-          job_details.body["job"]["output_media_files"].collect do |o|
-            output = ActiveEncode::Output.new
-            output.id = o["id"].to_s
-            output.label = o["label"]
-            output.url = o["url"]
-            output.errors = Array(o["error_message"])
-
-            tech_md = convert_tech_metadata(o)
-            [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
-             :audio_bitrate, :video_bitrate, :file_size].each do |field|
-              output.send("#{field}=", tech_md[field])
-            end
-            output_progress = job_progress.body["outputs"].find { |out_prog| out_prog["id"] = output.id }
-            output.state = convert_state(output_progress["state"])
-            output.created_at = o["created_at"]
-            output.updated_at = o["updated_at"]
-            output
+          tech_md = convert_tech_metadata(o)
+          [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
+           :audio_bitrate, :video_bitrate, :file_size].each do |field|
+            output.send("#{field}=", tech_md[field])
           end
+          output_progress = job_progress.body["outputs"].find { |out_prog| out_prog["id"] = output.id }
+          output.state = convert_state(output_progress["state"])
+          output.created_at = o["created_at"]
+          output.updated_at = o["updated_at"]
+          output
         end
+      end
 
-        def convert_input_errors(job_details)
-          Array(job_details.body["job"]["input_media_file"]["error_message"])
-        end
+      def convert_input_errors(job_details)
+        Array(job_details.body["job"]["input_media_file"]["error_message"])
+      end
 
-        def convert_tech_metadata(media_file)
-          return {} if media_file.nil?
-
-          metadata = {}
-          media_file.each_pair do |key, value|
-            next if value.blank?
-            case key
-            when "md5_checksum"
-              metadata[:checksum] = value
-            when "format"
-              metadata[:mime_type] = value
-            when "duration_in_ms"
-              metadata[:duration] = value
-            when "audio_codec"
-              metadata[:audio_codec] = value
-            when "channels"
-              metadata[:audio_channels] = value
-            when "audio_bitrate_in_kbps"
-              metadata[:audio_bitrate] = value
-            when "video_codec"
-              metadata[:video_codec] = value
-            when "frame_rate"
-              metadata[:frame_rate] = value
-            when "video_bitrate_in_kbps"
-              metadata[:video_bitrate] = value
-            when "width"
-              metadata[:width] = value
-            when "height"
-              metadata[:height] = value
-            end
+      def convert_tech_metadata(media_file)
+        return {} if media_file.nil?
+
+        metadata = {}
+        media_file.each_pair do |key, value|
+          next if value.blank?
+          case key
+          when "md5_checksum"
+            metadata[:checksum] = value
+          when "format"
+            metadata[:mime_type] = value
+          when "duration_in_ms"
+            metadata[:duration] = value
+          when "audio_codec"
+            metadata[:audio_codec] = value
+          when "channels"
+            metadata[:audio_channels] = value
+          when "audio_bitrate_in_kbps"
+            metadata[:audio_bitrate] = value
+          when "video_codec"
+            metadata[:video_codec] = value
+          when "frame_rate"
+            metadata[:frame_rate] = value
+          when "video_bitrate_in_kbps"
+            metadata[:video_bitrate] = value
+          when "width"
+            metadata[:width] = value
+          when "height"
+            metadata[:height] = value
           end
-          metadata
         end
+        metadata
+      end
     end
   end
 end

data/lib/active_encode/errors.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-module ActiveEncode #:nodoc:
+module ActiveEncode # :nodoc:
   class NotFound < RuntimeError; end
   class NotRunningError < RuntimeError; end
   class CancelError < RuntimeError; end

data/lib/active_encode/persistence.rb

@@ -27,25 +27,25 @@ module ActiveEncode
 
     private
 
-      def persist(encode_attributes)
-        model = ActiveEncode::EncodeRecord.find_or_initialize_by(global_id: encode_attributes[:global_id])
-        model.update(encode_attributes) # Don't fail if persisting doesn't succeed?
-      end
+    def persist(encode_attributes)
+      model = ActiveEncode::EncodeRecord.find_or_initialize_by(global_id: encode_attributes[:global_id])
+      model.update(encode_attributes) # Don't fail if persisting doesn't succeed?
+    end
 
-      def persistence_model_attributes(encode, create_options = nil)
-        attributes = {
-          global_id: encode.to_global_id.to_s,
-          state: encode.state,
-          adapter: encode.class.engine_adapter.class.name,
-          title: encode.input.url.to_s,
-          # Need to ensure that these values come through or else validations will fail
-          created_at: encode.created_at,
-          updated_at: encode.updated_at,
-          raw_object: encode.to_json,
-          progress: encode.percent_complete
-        }
-        attributes[:create_options] = create_options.to_json if create_options.present?
-        attributes
-      end
+    def persistence_model_attributes(encode, create_options = nil)
+      attributes = {
+        global_id: encode.to_global_id.to_s,
+        state: encode.state,
+        adapter: encode.class.engine_adapter.class.name,
+        title: encode.input.url.to_s,
+        # Need to ensure that these values come through or else validations will fail
+        created_at: encode.created_at,
+        updated_at: encode.updated_at,
+        raw_object: encode.to_json,
+        progress: encode.percent_complete
+      }
+      attributes[:create_options] = create_options.to_json if create_options.present?
+      attributes
+    end
   end
 end

data/lib/active_encode/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module ActiveEncode
-  VERSION = '0.8.0'
+  VERSION = '1.0.0'
 end

data/lib/file_locator.rb

@@ -10,8 +10,8 @@ class FileLocator
 
     def initialize(uri)
       uri = Addressable::URI.parse(uri)
-      @bucket = URI.decode(uri.host)
-      @key = URI.decode(uri.path).sub(%r{^/*(.+)/*$}, '\1')
+      @bucket = Addressable::URI.unencode(uri.host)
+      @key = Addressable::URI.unencode(uri.path).sub(%r{^/*(.+)/*$}, '\1')
     end
 
     def object
@@ -26,21 +26,21 @@ class FileLocator
   def uri
     if @uri.nil?
       if source.is_a? File
-        @uri = Addressable::URI.parse("file://#{URI.encode(File.expand_path(source))}")
+        @uri = Addressable::URI.parse("file://#{Addressable::URI.encode(File.expand_path(source))}")
       else
         encoded_source = source
         begin
           @uri = Addressable::URI.parse(encoded_source)
         rescue URI::InvalidURIError
           if encoded_source == source
-            encoded_source = URI.encode(encoded_source)
+            encoded_source = Addressable::URI.encode(encoded_source)
             retry
           else
             raise
           end
         end
 
-        @uri = Addressable::URI.parse("file://#{URI.encode(File.expand_path(source))}") if @uri.scheme.nil?
+        @uri = Addressable::URI.parse("file://#{Addressable::URI.encode(File.expand_path(source))}") if @uri.scheme.nil?
       end
     end
     @uri
@@ -51,7 +51,7 @@ class FileLocator
     when 's3'
       S3File.new(uri).object.presigned_url(:get)
     when 'file'
-      URI.decode(uri.path)
+      Addressable::URI.unencode(uri.path)
     else
       @uri.to_s
     end

data/spec/fixtures/ffmpeg/cancelled-id/exit_status.code

@@ -0,0 +1 @@
+143

data/spec/fixtures/ffmpeg/completed-id/exit_status.code

@@ -0,0 +1 @@
+0

data/spec/fixtures/ffmpeg/completed-with-warnings-id/error.log

@@ -0,0 +1,3 @@
+[mpeg2video @ 0x6e48440] slice mismatch
+[mpeg2video @ 0x6e48440] Warning MVs not available
+[mpeg2video @ 0x8f89a0] concealing 45 DC, 45 AC, 45 MV errors in P frame

data/spec/fixtures/ffmpeg/completed-with-warnings-id/exit_status.code

@@ -0,0 +1 @@
+0

data/spec/fixtures/ffmpeg/completed-with-warnings-id/input_metadata

@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<MediaInfo
+    xmlns="https://mediaarea.net/mediainfo"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="https://mediaarea.net/mediainfo https://mediaarea.net/mediainfo/mediainfo_2_0.xsd"
+    version="2.0">
+<creatingLibrary version="18.05" url="https://mediaarea.net/MediaInfo">MediaInfoLib</creatingLibrary>
+<media ref="/home/pdinh/Downloads/videoshort.mp4">
+<track type="General">
+<VideoCount>1</VideoCount>
+<AudioCount>1</AudioCount>
+<FileExtension>mp4</FileExtension>
+<Format>MPEG-4</Format>
+<Format_Profile>Base Media</Format_Profile>
+<CodecID>mp42</CodecID>
+<FileSize>199160</FileSize>
+<Duration>6.315</Duration>
+<OverallBitRate_Mode>VBR</OverallBitRate_Mode>
+<OverallBitRate>252301</OverallBitRate>
+<FrameRate>23.719</FrameRate>
+<FrameCount>149</FrameCount>
+<StreamSize>5679</StreamSize>
+<HeaderSize>160</HeaderSize>
+<DataSize>193489</DataSize>
+<FooterSize>5511</FooterSize>
+<IsStreamable>No</IsStreamable>
+<Encoded_Date>UTC 2010-09-23 00:37:25</Encoded_Date>
+<Tagged_Date>UTC 2010-09-23 00:37:27</Tagged_Date>
+<File_Modified_Date>UTC 2017-12-14 19:29:35</File_Modified_Date>
+<File_Modified_Date_Local>2017-12-14 14:29:35</File_Modified_Date_Local>
+<Encoded_Application>HandBrake 0.9.4 2009112300</Encoded_Application>
+</track>
+<track type="Video">
+<StreamOrder>0</StreamOrder>
+<ID>1</ID>
+<Format>AVC</Format>
+<Format_Profile>Main</Format_Profile>
+<Format_Level>1.1</Format_Level>
+<Format_Settings_CABAC>Yes</Format_Settings_CABAC>
+<Format_Settings_RefFrames>2</Format_Settings_RefFrames>
+<CodecID>avc1</CodecID>
+<Duration>6.282</Duration>
+<BitRate>74477</BitRate>
+<Width>200</Width>
+<Height>110</Height>
+<Stored_Width>208</Stored_Width>
+<Stored_Height>112</Stored_Height>
+<Sampled_Width>200</Sampled_Width>
+<Sampled_Height>110</Sampled_Height>
+<PixelAspectRatio>1.000</PixelAspectRatio>
+<DisplayAspectRatio>1.818</DisplayAspectRatio>
+<Rotation>0.000</Rotation>
+<FrameRate_Mode>VFR</FrameRate_Mode>
+<FrameRate>23.719</FrameRate>
+<FrameRate_Minimum>12.500</FrameRate_Minimum>
+<FrameRate_Maximum>24.390</FrameRate_Maximum>
+<FrameRate_Original>24.000</FrameRate_Original>
+<FrameCount>149</FrameCount>
+<ColorSpace>YUV</ColorSpace>
+<ChromaSubsampling>4:2:0</ChromaSubsampling>
+<BitDepth>8</BitDepth>
+<ScanType>Progressive</ScanType>
+<StreamSize>58482</StreamSize>
+<Encoded_Library>x264 - core 79 r1347 5ddd61b</Encoded_Library>
+<Encoded_Library_Name>x264</Encoded_Library_Name>
+<Encoded_Library_Version>core 79 r1347 5ddd61b</Encoded_Library_Version>
+<Encoded_Library_Settings>cabac=1 / ref=2 / deblock=1:0:0 / analyse=0x1:0x111 / me=hex / subme=6 / psy=1 / psy_rd=1.0:0.0 / mixed_ref=0 / me_range=16 / chroma_me=1 / trellis=0 / 8x8dct=0 / cqm=0 / deadzone=21,11 / chroma_qp_offset=-2 / threads=3 / nr=0 / decimate=1 / mbaff=0 / constrained_intra=0 / bframes=2 / b_pyramid=0 / b_adapt=1 / b_bias=0 / direct=1 / wpredb=0 / wpredp=2 / keyint=240 / keyint_min=24 / scenecut=40 / rc_lookahead=40 / rc=crf / mbtree=1 / crf=25.5 / qcomp=0.60 / qpmin=10 / qpmax=51 / qpstep=4 / ip_ratio=1.40 / aq=1:1.00</Encoded_Library_Settings>
+<Encoded_Date>UTC 2010-09-23 00:37:25</Encoded_Date>
+<Tagged_Date>UTC 2010-09-23 00:37:27</Tagged_Date>
+<colour_range>Limited</colour_range>
+<colour_description_present>Yes</colour_description_present>
+<colour_primaries>BT.601 NTSC</colour_primaries>
+<transfer_characteristics>BT.709</transfer_characteristics>
+<matrix_coefficients>BT.601</matrix_coefficients>
+</track>
+<track type="Audio">
+<StreamOrder>1</StreamOrder>
+<ID>2</ID>
+<Format>AAC</Format>
+<Format_Profile>LC</Format_Profile>
+<CodecID>mp4a-40-2</CodecID>
+<Duration>6.315</Duration>
+<BitRate_Mode>VBR</BitRate_Mode>
+<BitRate>171030</BitRate>
+<BitRate_Maximum>201736</BitRate_Maximum>
+<Channels>1</Channels>
+<ChannelPositions>Front: C</ChannelPositions>
+<ChannelLayout>C</ChannelLayout>
+<SamplesPerFrame>1024</SamplesPerFrame>
+<SamplingRate>48000</SamplingRate>
+<SamplingCount>303120</SamplingCount>
+<FrameRate>46.875</FrameRate>
+<FrameCount>296</FrameCount>
+<Compression_Mode>Lossy</Compression_Mode>
+<StreamSize>134999</StreamSize>
+<StreamSize_Proportion>0.67784</StreamSize_Proportion>
+<Title>Stereo</Title>
+<Encoded_Date>UTC 2010-09-23 00:37:25</Encoded_Date>
+<Tagged_Date>UTC 2010-09-23 00:37:27</Tagged_Date>
+</track>
+</media>
+</MediaInfo>