active_encode 1.2.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa6172de1470349b8dd8c2422c51049402168ca45eba77aa62ab6b7d9637f4ae
-  data.tar.gz: 999ec2e97a500f3bb258db4d4fbfa6e5fa4132dfefb4a82da1667eefa6875cff
+  metadata.gz: 4f90e9c3cedb039d22d5576047ff97f27007e1ab0a4a42be49a8af6332152237
+  data.tar.gz: cd525b293617c059017568d2bd9d70817213f4b81e9331cf08b330412c8f35c7
 SHA512:
-  metadata.gz: a78df8af781cf5f4f198a9b4d9e994108294fd0248725d9b6553a96fb829de41afc7a7ccc69cadacafe8d6cb4ac809cb4331c2331c968b7429b7306e19cf3f51
-  data.tar.gz: 358557321351790685a68c4604d6ca43592c928523085e96f5353603ffe8ad0bfc70b6bcde119b2275cb3d3bbec7b6be78c74b84c3f65900dcd2f0e8f3c38421
+  metadata.gz: 7cdfba6f2f6f5712ecb9c54f8f7fd61c679559ca07930c43430cc26a85d96e7469d6b1d33061922dc05abf2a287b92676a7c4cf9bc66f4216607cafdc7fa86b3
+  data.tar.gz: 9c7ce5b39d81c746eda7bf54e023c6893300aa6a5bd4743f5afefe430189e62cd7419ff065d86bd3be4368b44bf4005ba18241472d1374f74744700905272ed5

data/.circleci/config.yml

@@ -46,6 +46,16 @@ jobs:
 
       - samvera/cached_checkout
 
+      - run:
+          name: Check for a branch named 'master'
+          command: |
+            git fetch --all --quiet --prune --prune-tags
+            if [[ -n "$(git branch --all --list master */master)" ]]; then
+              echo "A branch named 'master' was found. Please remove it."
+              echo "$(git branch --all --list master */master)"
+            fi
+            [[ -z "$(git branch --all --list master */master)" ]]
+
       - samvera/bundle:
           ruby_version: << parameters.ruby_version >>
           bundler_version: << parameters.bundler_version >>
@@ -65,30 +75,38 @@ workflows:
   ci:
     jobs:
       - bundle_and_test:
-          name: "ruby3-2_rails7-0"
-          ruby_version: "3.2.0"
-          rails_version: "7.0.4.2"
+          name: "ruby3-4_rails8-0"
+          ruby_version: "3.4.1"
+          rails_version: "8.0.1"
+      - bundle_and_test:
+          name: "ruby3-4_rails7-2"
+          ruby_version: "3.4.1"
+          rails_version: "7.2.2.1"
+      - bundle_and_test:
+          name: "ruby3-4_rails7-1"
+          ruby_version: "3.4.1"
+          rails_version: "7.1.5.1"
       - bundle_and_test:
-          name: "ruby3-1_rails7-0"
-          ruby_version: "3.1.3"
-          rails_version: "7.0.4.2"
+          name: "ruby3-3_rails8-0"
+          ruby_version: "3.3.7"
+          rails_version: "8.0.1"
       - bundle_and_test:
-          name: "ruby3-0_rails7-0"
-          ruby_version: "3.0.5"
-          rails_version: "7.0.4.2"
+          name: "ruby3-3_rails7-2"
+          ruby_version: "3.3.7"
+          rails_version: "7.2.2.1"
       - bundle_and_test:
-          name: "ruby3-0_rails6-1"
-          ruby_version: "3.0.5"
-          rails_version: "6.1.7.2"
+          name: "ruby3-3_rails7-1"
+          ruby_version: "3.3.7"
+          rails_version: "7.1.5.1"
       - bundle_and_test:
-          name: "ruby3-0_rails6-0"
-          ruby_version: "3.0.5"
-          rails_version: "6.0.6.1"
+          name: "ruby3-2_rails8-0"
+          ruby_version: "3.2.7"
+          rails_version: "8.0.1"
       - bundle_and_test:
-          name: "ruby2-7_rails6-0"
-          ruby_version: "2.7.7"
-          rails_version: "6.0.6.1"
+          name: "ruby3-2_rails7-2"
+          ruby_version: "3.2.7"
+          rails_version: "7.2.2.1"
       - bundle_and_test:
-          name: "ruby2-7_rails5-2"
-          ruby_version: "2.7.7"
-          rails_version: "5.2.8.1"
+          name: "ruby3-2_rails7-1"
+          ruby_version: "3.2.7"
+          rails_version: "7.1.5.1"

data/.rubocop_todo.yml

@@ -8,6 +8,7 @@ Metrics/AbcSize:
 Metrics/BlockLength:
   Exclude:
     - 'lib/active_encode/spec/shared_specs/*'
+    - 'lib/active_encode/engine_adapters/media_convert_adapter.rb'
     - 'spec/**/*'
 
 Metrics/BlockNesting:
@@ -26,12 +27,11 @@ Metrics/CyclomaticComplexity:
   Exclude:
     - 'lib/active_encode/engine_adapters/elastic_transcoder_adapter.rb'
     - 'lib/active_encode/engine_adapters/ffmpeg_adapter.rb'
-    - 'lib/active_encode/engine_adapters/zencoder_adapter.rb'
     - 'lib/active_encode/engine_adapters/media_convert_adapter.rb'
+    - 'lib/active_encode/engine_adapters/media_convert_output.rb'
 
 Layout/LineLength:
   Exclude:
-    - 'lib/active_encode/engine_adapters/matterhorn_adapter.rb'
     - 'spec/**/*'
 
 Metrics/MethodLength:
@@ -44,6 +44,7 @@ Metrics/PerceivedComplexity:
   Exclude:
     - 'lib/active_encode/engine_adapters/ffmpeg_adapter.rb'
     - 'lib/active_encode/engine_adapters/media_convert_adapter.rb'
+    - 'lib/active_encode/engine_adapters/media_convert_output.rb'
     - 'lib/file_locator.rb'
 
 RSpec/AnyInstance:
@@ -61,7 +62,6 @@ RSpec/InstanceVariable:
 
 RSpec/MessageSpies:
   Exclude:
-    - 'spec/integration/matterhorn_adapter_spec.rb'
     - 'spec/integration/ffmpeg_adapter_spec.rb'
 
 RSpec/NamedSubject:

data/CONTRIBUTING.md

@@ -68,7 +68,7 @@ further details.
 * Fork the repository on GitHub
 * Create a topic branch from where you want to base your work.
   * This is usually the `main` branch.
-  * To quickly create a topic branch based on `main`; `git branch fix/master/my_contribution main`
+  * To quickly create a topic branch based on `main`; `git branch fix/main/my_contribution main`
   * Then checkout the new branch with `git checkout fix/main/my_contribution`.
   * Please avoid working directly on the `main` branch.
   * Please do not create a branch called `master`. (See note below.)

data/Gemfile

@@ -1,15 +1,12 @@
 # frozen_string_literal: true
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in hydra-transcoder.gemspec
+# Specify your gem's dependencies in active_encode.gemspec
 gemspec
 
 gem 'aws-sdk-elastictranscoder'
 gem 'aws-sdk-s3'
 gem 'byebug'
-gem 'rubyhorn', git: "https://github.com/avalonmediasystem/rubyhorn.git"
-gem 'shingoncoder'
-gem 'zencoder'
 
 # BEGIN ENGINE_CART BLOCK
 # engine_cart: 2.4.0
@@ -33,16 +30,6 @@ else
     else
       gem 'rails', ENV['RAILS_VERSION']
     end
-
-    case ENV['RAILS_VERSION']
-    when /^6.0/
-      gem 'sass-rails', '>= 6'
-      gem 'webpacker', '~> 4.0'
-    when /^5.[12]/
-      gem 'sass-rails', '~> 5.0'
-      gem 'sprockets', '~> 3.7'
-      gem 'thor', '~> 0.20'
-    end
   end
 end
 # END ENGINE_CART BLOCK

data/README.md

@@ -1,8 +1,8 @@
 # ActiveEncode
 
 Code: [![Version](https://badge.fury.io/rb/active_encode.png)](http://badge.fury.io/rb/active_encode)
-[![Build Status](https://travis-ci.org/samvera-labs/active_encode.png?branch=master)](https://travis-ci.org/samvera-labs/active_encode)
-[![Coverage Status](https://coveralls.io/repos/github/samvera-labs/active_encode/badge.svg?branch=master)](https://coveralls.io/github/samvera-labs/active_encode?branch=master)
+[![CircleCI](https://dl.circleci.com/status-badge/img/gh/samvera-labs/active_encode/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/samvera-labs/active_encode/tree/main)
+[![Coverage Status](https://coveralls.io/repos/github/samvera-labs/active_encode/badge.svg?branch=main)](https://coveralls.io/github/samvera-labs/active_encode?branch=main)
 
 Docs: [![Contribution Guidelines](http://img.shields.io/badge/CONTRIBUTING-Guidelines-blue.svg)](./CONTRIBUTING.md)
 [![Apache 2.0 License](http://img.shields.io/badge/APACHE2-license-blue.svg)](./LICENSE)
@@ -11,7 +11,7 @@ Jump in: [![Slack Status](http://slack.samvera.org/badge.svg)](http://slack.samv
 
 # What is ActiveEncode?
 
-ActiveEncode serves as the basis for the interface between a Ruby (Rails) application and a provider of encoding services such as [FFmpeg](https://www.ffmpeg.org/), [Amazon Elastic Transcoder](http://aws.amazon.com/elastictranscoder/), [AWS Elemental MediaConvert](https://aws.amazon.com/mediaconvert/), and [Zencoder](http://zencoder.com).
+ActiveEncode serves as the basis for the interface between a Ruby (Rails) application and a provider of encoding services such as [FFmpeg](https://www.ffmpeg.org/), [Amazon Elastic Transcoder](http://aws.amazon.com/elastictranscoder/), and [AWS Elemental MediaConvert](https://aws.amazon.com/mediaconvert/).
 
 # Help
 
@@ -247,6 +247,12 @@ RSpec.describe MyCustomAdapter do
 end
 ```
 
+## Contributing 
+
+If you're working on PR for this project, create a feature branch off of `main`. 
+
+This repository follows the [Samvera Community Code of Conduct](https://samvera.atlassian.net/wiki/spaces/samvera/pages/405212316/Code+of+Conduct) and [language recommendations](https://github.com/samvera/maintenance/blob/main/templates/CONTRIBUTING.md#language).  Please ***do not*** create a branch called `master` for this repository or as part of your pull request; the branch will either need to be removed or renamed before it can be considered for inclusion in the code base and history of this repository.
+
 # Acknowledgments
 
 This software has been developed by and is brought to you by the Samvera community.  Learn more at the

data/active_encode.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Michael Klein, Chris Colvard, Phuong Dinh"]
   spec.email         = ["mbklein@gmail.com, chris.colvard@gmail.com, phuongdh@gmail.com"]
   spec.summary       = 'Declare encode job classes that can be run by a variety of encoding services'
-  spec.description   = 'This gem provides an interface to transcoding services such as Ffmpeg, Amazon Elastic Transcoder, or Zencoder.'
+  spec.description   = 'This gem provides an interface to transcoding services such as Ffmpeg, Amazon Elastic Transcoder, or Amazon Elemental MediaConvert.'
   spec.homepage      = "https://github.com/samvera-labs/active_encode"
   spec.license       = "Apache-2.0"
 
@@ -22,11 +22,13 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "rails"
   spec.add_dependency "addressable", "~> 2.8"
+  spec.add_dependency "retriable"
 
   spec.add_development_dependency "aws-sdk-cloudwatchevents"
   spec.add_development_dependency "aws-sdk-cloudwatchlogs"
+  spec.add_development_dependency "aws-sdk-core", "<= 3.220.0"
   spec.add_development_dependency "aws-sdk-elastictranscoder"
-  spec.add_development_dependency "aws-sdk-mediaconvert"
+  spec.add_development_dependency "aws-sdk-mediaconvert", ">= 1.157.0"
   spec.add_development_dependency "aws-sdk-s3"
   spec.add_development_dependency "bixby", '~> 5.0', '>= 5.0.2'
   spec.add_development_dependency "bundler"

data/lib/active_encode/engine.rb

@@ -4,5 +4,17 @@ require 'rails'
 module ActiveEncode
   class Engine < ::Rails::Engine
     isolate_namespace ActiveEncode
+
+    config.before_configuration do
+      # rubocop:disable Style/IfUnlessModifier
+      # see https://github.com/fxn/zeitwerk#for_gem
+      # Blacklight puts a generator into LOCAL APP lib/generators, so tell
+      # zeitwerk to ignore the whole directory? If we're using zeitwerk
+      #
+      # See: https://github.com/cbeer/engine_cart/issues/117
+      if ::Rails.try(:autoloaders).try(:main).respond_to?(:ignore)
+        ::Rails.autoloaders.main.ignore(::Rails.root.join('lib', 'generators'))
+      end
+    end
   end
 end

data/lib/active_encode/engine_adapters/ffmpeg_adapter/cleaner.rb

@@ -53,7 +53,11 @@ module ActiveEncode
         def remove_empty_directories(directories)
           directories_to_delete = directories.select { |d| Dir.empty?(d) }
           non_empty_directories = directories - directories_to_delete
-          directories_to_delete += non_empty_directories.select { |ned| Dir.children(ned) == ["outputs"] && directories_to_delete.include?(File.join(ned, "outputs")) }
+          directories_to_delete += non_empty_directories.select do |ned|
+            Dir.children(ned).sort == ["outputs", "supplemental_files"] &&
+              directories_to_delete.include?(File.join(ned, "outputs")) &&
+              directories_to_delete.include?(File.join(ned, "supplemental_files"))
+          end
           FileUtils.rmdir(directories_to_delete) unless directories_to_delete.empty?
         end
 

data/lib/active_encode/engine_adapters/ffmpeg_adapter.rb

@@ -42,6 +42,7 @@ module ActiveEncode
         # Create a working directory that holds all output files related to the encode
         FileUtils.mkdir_p working_path("", new_encode.id)
         FileUtils.mkdir_p working_path("outputs", new_encode.id)
+        FileUtils.mkdir_p working_path("supplemental_files", new_encode.id)
 
         # Extract technical metadata from input file
         curl_option = if options && options[:headers]
@@ -91,12 +92,14 @@ module ActiveEncode
           new_encode.input.duration = fixed_duration(working_path("duration_input_metadata", new_encode.id))
         end
 
+        subtitle_count = new_encode.input.subtitles.length if new_encode.input.subtitles.present?
+
         new_encode.state = :running
         new_encode.percent_complete = 1
         new_encode.errors = []
 
         # Run the ffmpeg command and save its pid
-        command = ffmpeg_command(input_url, new_encode.id, options)
+        command = ffmpeg_command(input_url, new_encode.id, options, subtitle_count)
         # Capture the exit status in a file in order to differentiate warning output in stderr between real process failure
         exit_status_file = working_path("exit_status.code", new_encode.id)
         command = "#{command}; echo $? > #{exit_status_file}"
@@ -150,6 +153,7 @@ module ActiveEncode
         end
 
         encode.output = build_outputs encode if encode.completed?
+        encode.output += build_supplemental_outputs encode if encode.completed?
 
         encode
       end
@@ -198,6 +202,7 @@ module ActiveEncode
 
       def write_errors(encode)
         File.write(working_path("error.log", encode.id), encode.errors.join("\n"))
+        File.write(working_path("exit_status.code", encode.id), "1") unless File.exist?(working_path("exit_status.code", encode.id))
       end
 
       def read_errors(id)
@@ -251,17 +256,51 @@ module ActiveEncode
         outputs
       end
 
-      def ffmpeg_command(input_url, id, opts)
+      def build_supplemental_outputs(encode)
+        id = encode.id
+        files = []
+        Dir["#{File.absolute_path(working_path('supplemental_files', id))}/*"].each_with_index do |file_path, index|
+          file = ActiveEncode::Output.new
+          file.url = "file://#{file_path}"
+          file.id = "#{encode.input.id}-#{File.basename(file_path)}"
+          file.created_at = encode.created_at
+          file.updated_at = File.mtime file_path
+          file.label = encode.input.subtitles[index][:label]
+          file.language = encode.input.subtitles[index][:language]
+          file.format = 'vtt'
+
+          files << file
+        end
+
+        files
+      end
+
+      def ffmpeg_command(input_url, id, opts, subtitle_count = nil)
+        sanitized_filename = ActiveEncode.sanitize_base input_url
         output_opt = opts[:outputs].collect do |output|
-          sanitized_filename = ActiveEncode.sanitize_base input_url
           file_name = "outputs/#{sanitized_filename}-#{output[:label]}.#{output[:extension]}"
           " #{output[:ffmpeg_opt]} #{working_path(file_name, id)}"
         end.join(" ")
+
+        supplemental_file_opt = if opts[:extract_subtitles] && subtitle_count.present?
+                                  caption_extraction_options(sanitized_filename, subtitle_count, id)
+                                end
+
         header_opt = Array(opts[:headers]).map do |k, v|
           "#{k}: #{v}\r\n"
         end.join
         header_opt = "-headers '#{header_opt}'" if header_opt.present?
-        "#{FFMPEG_PATH} #{header_opt} -y -loglevel level+fatal -progress #{working_path('progress', id)} -i \"#{input_url}\" #{output_opt}"
+        "#{FFMPEG_PATH} #{header_opt} -y -loglevel level+fatal -progress #{working_path('progress', id)} -i \"#{input_url}\" #{supplemental_file_opt} #{output_opt}"
+      end
+
+      def caption_extraction_options(filename, count, id)
+        opts = ""
+        (0..count - 1).each do |i|
+          subtitle_filename = "supplemental_files/#{filename}-caption#{i}.vtt"
+          opts += " -map 0:s:#{i} -c:s webvtt #{working_path(subtitle_filename, id)}"
+        end
+
+        opts
       end
 
       def get_pid(id)
@@ -273,6 +312,8 @@ module ActiveEncode
       end
 
       def running?(pid)
+        return false if pid.nil?
+
         Process.getpgid pid.to_i
         true
       rescue Errno::ESRCH
@@ -314,22 +355,39 @@ module ActiveEncode
         doc.remove_namespaces!
         duration = get_xpath_text(doc, '//Duration/text()', :to_f)
         duration *= 1000 unless duration.nil? # Convert to milliseconds
+        audio_codec = get_xpath_text(doc, '//track[@type="Audio"]/CodecID/text()', :to_s)
+        if get_xpath_text(doc, '//track[@type="Audio"]/Format/text()', :to_s) == "MPEG Audio" &&
+           get_xpath_text(doc, '//track[@type="Audio"]/Format_Profile/text()', :to_s) == "Layer 3"
+          audio_codec ||= "mp3"
+        end
+
         { url: get_xpath_text(doc, '//media/@ref', :to_s),
           width: get_xpath_text(doc, '//Width/text()', :to_f),
           height: get_xpath_text(doc, '//Height/text()', :to_f),
           frame_rate: get_xpath_text(doc, '//FrameRate/text()', :to_f),
           duration: duration,
           file_size: get_xpath_text(doc, '//FileSize/text()', :to_i),
-          audio_codec: get_xpath_text(doc, '//track[@type="Audio"]/CodecID/text()', :to_s),
+          audio_codec: audio_codec,
           audio_bitrate: get_xpath_text(doc, '//track[@type="Audio"]/BitRate/text()', :to_i),
           video_codec: get_xpath_text(doc, '//track[@type="Video"]/CodecID/text()', :to_s),
-          video_bitrate: get_xpath_text(doc, '//track[@type="Video"]/BitRate/text()', :to_i) }
+          video_bitrate: get_xpath_text(doc, '//track[@type="Video"]/BitRate/text()', :to_i),
+          subtitles: get_subtitle_tech_metadata(doc).compact }
       end
 
       def get_xpath_text(doc, xpath, cast_method)
         doc.xpath(xpath).first&.text&.send(cast_method)
       end
 
+      def get_subtitle_tech_metadata(doc)
+        doc.xpath("//track[@type='Text' and Format='Timed Text']").collect do |track|
+          {
+            format: get_xpath_text(track, "./CodecID/text()", :to_s),
+            label: get_xpath_text(track, "./Title/text()", :to_s),
+            language: get_xpath_text(track, "./Language/text()", :to_s)
+          }
+        end
+      end
+
       def fixed_duration(path)
         get_tech_metadata(path)[:duration]
       end

data/lib/active_encode/engine_adapters/media_convert_adapter.rb

@@ -6,6 +6,7 @@ require 'aws-sdk-cloudwatchevents'
 require 'aws-sdk-cloudwatchlogs'
 require 'aws-sdk-mediaconvert'
 require 'file_locator'
+require 'retriable'
 
 require 'active_support/json'
 require 'active_support/time'
@@ -46,12 +47,16 @@ module ActiveEncode
     #
     #      ActiveEncode::Base.engine_adapter.setup!
     #
-    # **OR**, there is experimental functionality to get what we can directly from the job without
-    # requiring a CloudWatch log -- this is expected to be complete only for  HLS output at present.
-    # It seems to work well for HLS output. To opt-in, and not require CloudWatch logs:
+    # **OR** There is functionality to get what we can directly from the job without requiring
+    # a CloudWatch log -- this only works for HLS and file outputs at present.
+    # To opt-in, and not require CloudWatch logs:
     #
     #     ActiveEncode::Base.engine_adapter.direct_output_lookup = true
     #
+    # MediaConvert also provides a probe endpoint to extract technical metadata about a file stored
+    # in s3.  This can be used to gather output metadata when using direct_output_lookup instead of
+    # having to look in the CloudWatch logs.
+    #
     # ## Example
     #
     #     ActiveEncode::Base.engine_adapter = :media_convert
@@ -102,16 +107,42 @@ module ActiveEncode
         end
       end
 
+      class RetriableClient
+        def initialize
+          @retry_params = { on: Aws::MediaConvert::Errors::ServiceError, base_interval: 1 }
+          endpoint = Retriable.retriable(@retry_params) { Aws::MediaConvert::Client.new.describe_endpoints.endpoints.first.url }
+          @client = Aws::MediaConvert::Client.new(endpoint: endpoint)
+        end
+
+        def method_missing(method, *args, &block)
+          if @client.respond_to?(method)
+            Retriable.retriable(@retry_params) { @client.send(method, *args, &block) }
+          else
+            super
+          end
+        end
+
+        def respond_to_missing?(method_name, include_private = false)
+          @client.respond_to?(method_name) || super
+        end
+      end
+
       # @!attribute [rw] role simple name of AWS role to pass to MediaConvert, eg `my-role-name`
       # @!attribute [rw] output_bucket simple bucket name to write output to
       # @!attribute [rw] direct_output_lookup if true, do NOT get output information from cloudwatch,
       #                  instead retrieve and construct it only from job itself. Currently
-      #                  working only for HLS output. default false.
-      attr_accessor :role, :output_bucket, :direct_output_lookup
+      #                  working only for HLS output. (default: false)
+      # @!attribute [rw] use_probe if true use probe endpoint to get technical metadata about input and outputs (default: false)
+      attr_accessor :role, :output_bucket, :direct_output_lookup, :use_probe
 
       # @!attribute [w] log_group log_group_name that is being used to capture output
       # @!attribute [w] queue name of MediaConvert queue to use.
-      attr_writer :log_group, :queue
+      # @!attribute [w] output_id_format sprintf format for output ids (default: "%{job_id}-output%{suffix}")
+      #                  Available string variables are job_id and the keys in the MediaConvertOutput tech meatdata hash
+      # @!attribute [w] output_label_format sprintf format for output ids (default: "%{basename}" if output_url is preset
+      #                  otherwise "%{suffix}")
+      #                  Available string variables are job_id, basename, and the keys in the MediaConvertOutput tech metadata hash
+      attr_writer :log_group, :queue, :output_id_format, :output_label_format
 
       # Creates a [CloudWatch Logs]
       # (https://aws.amazon.com/cloudwatch/) log group and an EventBridge rule to forward status
@@ -274,6 +305,10 @@ module ActiveEncode
 
         encode.input.created_at = encode.created_at
         encode.input.updated_at = encode.updated_at
+        if use_probe
+          tech_md = MediaConvertOutput.tech_metadata_from_probe(url: encode.input.url, probe_response: probe(encode.input.url))
+          encode.input.assign_tech_metadata(tech_md)
+        end
 
         encode = complete_encode(encode, job) if encode.state == :completed
         encode
@@ -338,35 +373,42 @@ module ActiveEncode
 
         output_group_details  = job.dig("output_group_details", 0, "output_details")
         file_input_url        = job.dig("settings", "inputs", 0, "file_input")
+        output_group_type     = output_group_settings.type.downcase.to_sym
+        output_destination    = output_group_settings[output_group_type].destination
 
         outputs = output_group_details.map.with_index do |output_group_detail, index|
-          # Right now we only know how to get a URL for hls output, although
-          # the others should be possible and very analagous, just not familiar with them.
-          if output_group_settings.type == "HLS_GROUP_SETTINGS"
+          # Only HLS and file output groups have been tested with this approach
+          if [:hls_group_settings, :file_group_settings].include? output_group_type
             output_url = MediaConvertOutput.construct_output_url(
-              destination: output_group_settings.hls_group_settings.destination,
+              destination: output_destination,
               file_input_url: file_input_url,
               name_modifier: output_settings[index].name_modifier,
-              file_suffix: "m3u8"
+              file_suffix: output_settings[index].container_settings.container.downcase
             )
           end
 
-          tech_md = MediaConvertOutput.tech_metadata_from_settings(
-            output_url: output_url,
-            output_settings: output_settings[index],
-            output_detail_settings: output_group_detail
-          )
+          tech_md = if use_probe
+                      MediaConvertOutput.tech_metadata_from_probe(
+                        url: output_url,
+                        output_settings: output_settings[index],
+                        probe_response: probe(output_url)
+                      )
+                    else
+                      MediaConvertOutput.tech_metadata_from_settings(
+                        output_url: output_url,
+                        output_settings: output_settings[index],
+                        output_detail_settings: output_group_detail
+                      )
+                    end
 
           output = ActiveEncode::Output.new
-
+          output.url = output_url
           output.created_at = job.timing.submit_time
           output.updated_at = job.timing.finish_time || job.timing.start_time || output.created_at
-
-          [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
-           :audio_bitrate, :video_bitrate, :file_size, :label, :url, :id].each do |field|
-            output.send("#{field}=", tech_md[field])
-          end
-          output.id ||= "#{job.id}-output#{tech_md[:suffix]}"
+          output.assign_tech_metadata(tech_md)
+          file_basename = File.basename(output.url)
+          output.id = format(output_id_format, tech_md.merge(job_id: job.id))
+          output.label = format(output_label_format(file_basename), tech_md.merge(job_id: job.id, basename: file_basename))
           output
         end
 
@@ -382,15 +424,13 @@ module ActiveEncode
           )
 
           output = ActiveEncode::Output.new
+          output.url = adaptive_playlist_url
           output.created_at = job.timing.submit_time
           output.updated_at = job.timing.finish_time || job.timing.start_time || output.created_at
-          output.id = "#{job.id}-output-auto"
-
-          [:duration, :audio_codec, :video_codec].each do |field|
-            output.send("#{field}=", outputs.first.send(field))
-          end
-          output.label = File.basename(adaptive_playlist_url)
-          output.url = adaptive_playlist_url
+          output.assign_tech_metadata(outputs.first.tech_metadata)
+          file_basename = File.basename(output.url)
+          output.id = format(output_id_format, output.tech_metadata.merge(job_id: job.id, suffix: '-auto'))
+          output.label = format(output_label_format(file_basename), output.tech_metadata.merge(job_id: job.id, suffix: '-auto', basename: file_basename))
           outputs << output
         end
 
@@ -411,30 +451,26 @@ module ActiveEncode
         outputs = logged_results.dig('detail', 'outputGroupDetails', 0, 'outputDetails').map.with_index do |logged_detail, index|
           tech_md = MediaConvertOutput.tech_metadata_from_logged(output_settings[index], logged_detail)
           output = ActiveEncode::Output.new
-
+          output.url = tech_md[:url]
           output.created_at = job.timing.submit_time
           output.updated_at = job.timing.finish_time || job.timing.start_time || output.created_at
-
-          [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
-           :audio_bitrate, :video_bitrate, :file_size, :label, :url, :id].each do |field|
-            output.send("#{field}=", tech_md[field])
-          end
-          output.id ||= "#{job.id}-output#{tech_md[:suffix]}"
+          output.assign_tech_metadata(tech_md)
+          file_basename = File.basename(output.url)
+          output.id = format(output_id_format, tech_md.merge(job_id: job.id))
+          output.label = format(output_label_format(file_basename), tech_md.merge(job_id: job.id, basename: file_basename))
           output
         end
 
         adaptive_playlist = logged_results.dig('detail', 'outputGroupDetails', 0, 'playlistFilePaths', 0)
         unless adaptive_playlist.nil?
           output = ActiveEncode::Output.new
+          output.url = adaptive_playlist
           output.created_at = job.timing.submit_time
           output.updated_at = job.timing.finish_time || job.timing.start_time || output.created_at
-          output.id = "#{job.id}-output-auto"
-
-          [:duration, :audio_codec, :video_codec].each do |field|
-            output.send("#{field}=", outputs.first.send(field))
-          end
-          output.label = File.basename(adaptive_playlist)
-          output.url = adaptive_playlist
+          output.assign_tech_metadata(outputs.first.tech_metadata)
+          file_basename = File.basename(output.url)
+          output.id = format(output_id_format, output.tech_metadata.merge(job_id: job.id, suffix: '-auto'))
+          output.label = format(output_label_format(file_basename), output.tech_metadata.merge(job_id: job.id, suffix: '-auto', basename: file_basename))
           outputs << output
         end
         outputs
@@ -476,10 +512,21 @@ module ActiveEncode
       end
 
       def mediaconvert
-        @mediaconvert ||= begin
-          endpoint = Aws::MediaConvert::Client.new.describe_endpoints.endpoints.first.url
-          Aws::MediaConvert::Client.new(endpoint: endpoint)
-        end
+        @mediaconvert ||= RetriableClient.new
+      end
+
+      def probe(url)
+        # NOTE: certain audio files don't return any track information
+        @probe_cache ||= {}
+        @probe_cache[url] ||= mediaconvert.probe({ input_files: [{ file_url: url }] })&.probe_results&.first
+      end
+
+      def output_id_format
+        @output_id_format || "%{job_id}-output%{suffix}"
+      end
+
+      def output_label_format(file_basename)
+        @output_label_format || (file_basename.present? ? "%{basename}" : "%{suffix}")
       end
 
       def s3client