active_encode 0.4.1 → 0.8.0

data/lib/active_encode/engine_adapters/ffmpeg_adapter.rb

@@ -1,16 +1,31 @@
+# frozen_string_literal: true
 require 'fileutils'
 require 'nokogiri'
+require 'shellwords'
 
 module ActiveEncode
   module EngineAdapters
     class FfmpegAdapter
       WORK_DIR = ENV["ENCODE_WORK_DIR"] || "encodes" # Should read from config
+      MEDIAINFO_PATH = ENV["MEDIAINFO_PATH"] || "mediainfo"
+      FFMPEG_PATH = ENV["FFMPEG_PATH"] || "ffmpeg"
 
       def create(input_url, options = {})
+        # Decode file uris for ffmpeg (mediainfo works either way)
+        case input_url
+        when /^file\:\/\/\//
+          input_url = URI.decode(input_url)
+        when /^s3\:\/\//
+          require 'file_locator'
+
+          s3_object = FileLocator::S3File.new(input_url).object
+          input_url = URI.parse(s3_object.presigned_url(:get))
+        end
+
         new_encode = ActiveEncode::Base.new(input_url, options)
         new_encode.id = SecureRandom.uuid
-        new_encode.created_at = Time.new
-        new_encode.updated_at = Time.new
+        new_encode.created_at = Time.now.utc
+        new_encode.updated_at = Time.now.utc
         new_encode.current_operations = []
         new_encode.output = []
 
@@ -19,18 +34,24 @@ module ActiveEncode
         FileUtils.mkdir_p working_path("outputs", new_encode.id)
 
         # Extract technical metadata from input file
-        `mediainfo --Output=XML --LogFile=#{working_path("input_metadata", new_encode.id)} #{input_url}`
+        curl_option = if options && options[:headers]
+                        headers = options[:headers].map { |k, v| "#{k}: #{v}" }
+                        (["--File_curl=HttpHeader"] + headers).join(",").yield_self { |s| "'#{s}'" }
+                      else
+                        ""
+                      end
+        `#{MEDIAINFO_PATH} #{curl_option} --Output=XML --LogFile=#{working_path("input_metadata", new_encode.id)} "#{input_url}"`
         new_encode.input = build_input new_encode
 
         if new_encode.input.duration.blank?
           new_encode.state = :failed
           new_encode.percent_complete = 1
 
-          if new_encode.input.file_size.blank?
-            new_encode.errors = ["#{input_url} does not exist or is not accessible"]
-          else
-            new_encode.errors = ["Error inspecting input: #{input_url}"]
-          end
+          new_encode.errors = if new_encode.input.file_size.blank?
+                                ["#{input_url} does not exist or is not accessible"]
+                              else
+                                ["Error inspecting input: #{input_url}"]
+                              end
 
           write_errors new_encode
           return new_encode
@@ -42,20 +63,27 @@ module ActiveEncode
 
         # Run the ffmpeg command and save its pid
         command = ffmpeg_command(input_url, new_encode.id, options)
-        pid = Process.spawn(command)
+        pid = Process.spawn(command, err: working_path('error.log', new_encode.id))
         File.open(working_path("pid", new_encode.id), 'w') { |file| file.write pid }
         new_encode.input.id = pid
 
-        # Prevent zombie process
-        Process.detach(pid)
-
         new_encode
+      rescue StandardError => e
+        new_encode.state = :failed
+        new_encode.percent_complete = 1
+        new_encode.errors = [e.full_message]
+        write_errors new_encode
+        return new_encode
+      ensure
+        # Prevent zombie process
+        Process.detach(pid) if pid.present?
       end
 
       # Return encode object from file system
-      def find(id, opts={})
+      def find(id, opts = {})
         encode_class = opts[:cast]
-        encode = ActiveEncode::Base.new(nil, opts)
+        encode_class ||= ActiveEncode::Base
+        encode = encode_class.new(nil, opts)
         encode.id = id
         encode.output = []
         encode.created_at, encode.updated_at = get_times encode.id
@@ -65,28 +93,21 @@ module ActiveEncode
         pid = get_pid(id)
         encode.input.id = pid if pid.present?
 
-        if File.file? working_path("error.log", id)
-          error = File.read working_path("error.log", id)
-          if error.present?
-            encode.state = :failed
-            encode.errors = [error]
-
-            return encode
-          end
-        end
-
-        encode.errors = []
-
         encode.current_operations = []
         encode.created_at, encode.updated_at = get_times encode.id
-
-        if running? pid
+        encode.errors = read_errors(id)
+        if encode.errors.present?
+          encode.state = :failed
+        elsif running? pid
           encode.state = :running
           encode.current_operations = ["transcoding"]
         elsif progress_ended?(encode.id) && encode.percent_complete == 100
-            encode.state = :completed
+          encode.state = :completed
+        elsif cancelled? encode.id
+          encode.state = :cancelled
         elsif encode.percent_complete < 100
-            encode.state = :cancelled
+          encode.errors << "Encoding has completed but the output duration is shorter than the input"
+          encode.state = :failed
         end
 
         encode.output = build_outputs encode if encode.completed?
@@ -96,31 +117,61 @@ module ActiveEncode
 
       # Cancel ongoing encode using pid file
       def cancel(id)
-        pid = get_pid(id)
-        Process.kill 'SIGTERM', pid.to_i
+        encode = find id
+        if encode.running?
+          pid = get_pid(id)
+
+          IO.popen("ps -ef | grep #{pid}") do |pipe|
+            child_pids = pipe.readlines.map do |line|
+              parts = line.split(/\s+/)
+              parts[1] if parts[2] == pid.to_s && parts[1] != pipe.pid.to_s
+            end.compact
+
+            child_pids.each do |cpid|
+              Process.kill 'SIGTERM', cpid.to_i
+            end
+          end
 
-        find id
+          Process.kill 'SIGTERM', pid.to_i
+          File.write(working_path("cancelled", id), "")
+          encode = find id
+        end
+        encode
+      rescue Errno::ESRCH
+        raise NotRunningError
+      rescue StandardError
+        raise CancelError
       end
 
-private
+    private
 
-      def get_times id
+      def get_times(id)
         updated_at = if File.file? working_path("progress", id)
-            File.mtime(working_path("progress", id))
-          elsif File.file? working_path("error.log", id)
-            File.mtime(working_path("error.log", id))
-          else
-            File.mtime(working_path("input_metadata", id))
-          end
-
-        return File.mtime(working_path("input_metadata", id)), updated_at
+                       File.mtime(working_path("progress", id))
+                     elsif File.file? working_path("error.log", id)
+                       File.mtime(working_path("error.log", id))
+                     else
+                       File.mtime(working_path("input_metadata", id))
+                     end
+
+        [File.mtime(working_path("input_metadata", id)), updated_at]
       end
 
-      def write_errors encode
+      def write_errors(encode)
         File.write(working_path("error.log", encode.id), encode.errors.join("\n"))
       end
 
-      def build_input encode
+      def read_errors(id)
+        err_path = working_path("error.log", id)
+        error = File.read(err_path) if File.file? err_path
+        if error.present?
+          [error]
+        else
+          []
+        end
+      end
+
+      def build_input(encode)
         input = ActiveEncode::Input.new
         metadata = get_tech_metadata(working_path("input_metadata", encode.id))
         input.url = metadata[:url]
@@ -132,22 +183,21 @@ private
         input
       end
 
-      def build_outputs encode
+      def build_outputs(encode)
         id = encode.id
         outputs = []
         Dir["#{File.absolute_path(working_path('outputs', id))}/*"].each do |file_path|
           output = ActiveEncode::Output.new
           output.url = "file://#{file_path}"
-          original_extension = File.extname(encode.input.url)
-          original_filename = File.basename(encode.input.url).chomp(original_extension)
-          output.label = file_path[/#{Regexp.quote(original_filename)}\-(.*?)#{Regexp.quote(File.extname(file_path))}$/, 1]
+          sanitized_filename = sanitize_base encode.input.url
+          output.label = file_path[/#{Regexp.quote(sanitized_filename)}\-(.*?)#{Regexp.quote(File.extname(file_path))}$/, 1]
           output.id = "#{encode.input.id}-#{output.label}"
           output.created_at = encode.created_at
           output.updated_at = File.mtime file_path
 
           # Extract technical metadata from output file
           metadata_path = working_path("output_metadata-#{output.label}", id)
-          `mediainfo --Output=XML --LogFile=#{metadata_path} #{output.url}` unless File.file? metadata_path
+          `#{MEDIAINFO_PATH} --Output=XML --LogFile=#{metadata_path} #{output.url}` unless File.file? metadata_path
           output.assign_tech_metadata(get_tech_metadata(metadata_path))
 
           outputs << output
@@ -158,72 +208,80 @@ private
 
       def ffmpeg_command(input_url, id, opts)
         output_opt = opts[:outputs].collect do |output|
-          file_name = "outputs/#{File.basename(input_url, File.extname(input_url))}-#{output[:label]}.#{output[:extension]}"
+          sanitized_filename = sanitize_base input_url
+          file_name = "outputs/#{sanitized_filename}-#{output[:label]}.#{output[:extension]}"
           " #{output[:ffmpeg_opt]} #{working_path(file_name, id)}"
         end.join(" ")
-
-        "ffmpeg -y -loglevel error -progress #{working_path("progress", id)} -i #{input_url} #{output_opt} > #{working_path("error.log", id)} 2>&1"
+        header_opt = Array(opts[:headers]).map do |k, v|
+          "#{k}: #{v}\r\n"
+        end.join
+        header_opt = "-headers '#{header_opt}'" if header_opt.present?
+        "#{FFMPEG_PATH} #{header_opt} -y -loglevel error -progress #{working_path('progress', id)} -i \"#{input_url}\" #{output_opt}"
       end
 
-      def get_pid(id)
-        if File.file? working_path("pid", id)
-          File.read(working_path("pid", id)).remove("\n")
+      def sanitize_base(input_url)
+        if input_url.is_a? URI::HTTP
+          File.basename(input_url.path, File.extname(input_url.path))
         else
-          nil
+          File.basename(input_url, File.extname(input_url)).gsub(/[^0-9A-Za-z.\-]/, '_')
         end
       end
 
+      def get_pid(id)
+        File.read(working_path("pid", id)).remove("\n") if File.file? working_path("pid", id)
+      end
+
       def working_path(path, id)
         File.join(WORK_DIR, id, path)
       end
 
       def running?(pid)
-        begin
-          Process.getpgid pid.to_i
-          true
-        rescue Errno::ESRCH
-          false
-        end
+        Process.getpgid pid.to_i
+        true
+      rescue Errno::ESRCH
+        false
       end
 
-      def calculate_percent_complete encode
+      def calculate_percent_complete(encode)
         data = read_progress encode.id
         if data.blank?
           1
         else
-          (progress_value("out_time_ms=", data).to_i * 0.0001 / encode.input.duration).round
+          progress_in_milliseconds = progress_value("out_time_ms=", data).to_i / 1000.0
+          output = (progress_in_milliseconds / encode.input.duration * 100).ceil
+          return 100 if output > 100
+          output
         end
       end
 
-      def read_progress id
-        if File.file? working_path("progress", id)
-          File.read working_path("progress", id)
-        else
-          nil
-        end
+      def cancelled?(id)
+        File.exist? working_path("cancelled", id)
+      end
+
+      def read_progress(id)
+        File.read working_path("progress", id) if File.file? working_path("progress", id)
       end
 
-      def progress_ended? id
+      def progress_ended?(id)
         "end" == progress_value("progress=", read_progress(id))
       end
 
-      def progress_value key, data
-        if data.present? && key.present?
-          ri = data.rindex(key) + key.length
-          data[ri..data.index("\n", ri)-1]
-        else
-          nil
-        end
+      def progress_value(key, data)
+        return nil unless data.present? && key.present?
+        ri = data.rindex(key) + key.length
+        data[ri..data.index("\n", ri) - 1]
       end
 
-      def get_tech_metadata file_path
+      def get_tech_metadata(file_path)
         doc = Nokogiri::XML File.read(file_path)
         doc.remove_namespaces!
+        duration = get_xpath_text(doc, '//Duration/text()', :to_f)
+        duration *= 1000 unless duration.nil? # Convert to milliseconds
         { url: get_xpath_text(doc, '//media/@ref', :to_s),
           width: get_xpath_text(doc, '//Width/text()', :to_f),
           height: get_xpath_text(doc, '//Height/text()', :to_f),
           frame_rate: get_xpath_text(doc, '//FrameRate/text()', :to_f),
-          duration: get_xpath_text(doc, '//Duration/text()', :to_f),
+          duration: duration,
           file_size: get_xpath_text(doc, '//FileSize/text()', :to_i),
           audio_codec: get_xpath_text(doc, '//track[@type="Audio"]/CodecID/text()', :to_s),
           audio_bitrate: get_xpath_text(doc, '//track[@type="Audio"]/BitRate/text()', :to_i),
@@ -231,12 +289,8 @@ private
           video_bitrate: get_xpath_text(doc, '//track[@type="Video"]/BitRate/text()', :to_i) }
       end
 
-      def get_xpath_text doc, xpath, cast_method
-        if doc.xpath(xpath).first
-          doc.xpath(xpath).first.text.send(cast_method)
-        else
-          nil
-        end
+      def get_xpath_text(doc, xpath, cast_method)
+        doc.xpath(xpath).first&.text&.send(cast_method)
       end
     end
   end

data/lib/active_encode/engine_adapters/matterhorn_adapter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'rubyhorn'
 
 module ActiveEncode
@@ -11,7 +12,7 @@ module ActiveEncode
         build_encode(get_workflow(workflow_om))
       end
 
-      def find(id, opts = {})
+      def find(id, _opts = {})
         build_encode(fetch_workflow(id))
       end
 
@@ -145,7 +146,7 @@ module ActiveEncode
 
         def convert_created_at(workflow)
           created_at = workflow.xpath('mediapackage/@start').last.to_s
-          created_at.present? ? Time.parse(created_at) : nil
+          created_at.present? ? Time.parse(created_at).utc : nil
         end
 
         def convert_updated_at(workflow)
@@ -156,13 +157,13 @@ module ActiveEncode
         def convert_output_created_at(track, workflow)
           quality = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
           created_at = workflow.xpath("//operation[@id=\"compose\"][configurations/configuration[@key=\"target-tags\" and contains(text(), \"#{quality}\")]]/started/text()").to_s
-          created_at.present? ? Time.at(created_at.to_i / 1000.0) : nil
+          created_at.present? ? Time.at(created_at.to_i / 1000.0).utc : nil
         end
 
         def convert_output_updated_at(track, workflow)
           quality = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
           updated_at = workflow.xpath("//operation[@id=\"compose\"][configurations/configuration[@key=\"target-tags\" and contains(text(), \"#{quality}\")]]/completed/text()").to_s
-          updated_at.present? ? Time.at(updated_at.to_i / 1000.0) : nil
+          updated_at.present? ? Time.at(updated_at.to_i / 1000.0).utc : nil
         end
 
         def convert_options(workflow)

data/lib/active_encode/engine_adapters/media_convert_adapter.rb

@@ -0,0 +1,372 @@
+# frozen_string_literal: true
+require 'active_encode/engine_adapters/media_convert_output.rb'
+require 'active_support/core_ext/integer/time'
+require 'addressable/uri'
+require 'aws-sdk-cloudwatchevents'
+require 'aws-sdk-cloudwatchlogs'
+require 'aws-sdk-mediaconvert'
+require 'file_locator'
+
+require 'active_support/json'
+require 'active_support/time'
+
+module ActiveEncode
+  module EngineAdapters
+    class MediaConvertAdapter
+      # [AWS Elemental MediaConvert](https://aws.amazon.com/mediaconvert/) doesn't provide detailed
+      # output information in the job description that can be pulled directly from the service.
+      # Instead, it provides that information along with the job status notification when the job
+      # status changes to `COMPLETE`. The only way to capture that notification is through an [Amazon
+      # Eventbridge](https://aws.amazon.com/eventbridge/) rule that forwards the status change
+      # notification to another service for capture and/or handling.
+      #
+      # `ActiveEncode::EngineAdapters::MediaConvert` does this by creating a [CloudWatch Logs]
+      # (https://aws.amazon.com/cloudwatch/) log group and an EventBridge rule to forward status
+      # change notifications to the log group. It can then find the log entry containing the output
+      # details later when the job is complete. This is accomplished by calling the idempotent
+      # `#setup!` method.
+      #
+      # The AWS user/role calling the `#setup!` method will require permissions to create the
+      # necessary CloudWatch and EventBridge resources, and the role passed to the engine adapter
+      # will need access to any S3 buckets where files will be read from or written to during
+      # transcoding.
+      #
+      # Configuration example:
+      #
+      #     ActiveEncode::Base.engine_adapter = :media_convert
+      #     ActiveEncode::Base.engine_adapter.role = 'arn:aws:iam::123456789012:role/service-role/MediaConvert_Default_Role'
+      #     ActiveEncode::Base.engine_adapter.output_bucket = 'output-bucket'
+      #     ActiveEncode::Base.engine_adapter.setup!
+
+      JOB_STATES = {
+        "SUBMITTED" => :running, "PROGRESSING" => :running, "CANCELED" => :cancelled,
+        "ERROR" => :failed, "COMPLETE" => :completed
+      }.freeze
+
+      OUTPUT_GROUP_TEMPLATES = {
+        hls: { min_segment_length: 0, segment_control: "SEGMENTED_FILES", segment_length: 10 },
+        dash_iso: { fragment_length: 2, segment_control: "SEGMENTED_FILES", segment_length: 30 },
+        file: {},
+        ms_smooth: { fragment_length: 2 },
+        cmaf: { fragment_length: 2, segment_control: "SEGMENTED_FILES", segment_length: 10 }
+      }.freeze
+
+      attr_accessor :role, :output_bucket
+      attr_writer :log_group, :queue
+
+      def setup!
+        rule_name = "active-encode-mediaconvert-#{queue}"
+        return true if event_rule_exists?(rule_name)
+
+        queue_arn = mediaconvert.get_queue(name: queue).queue.arn
+
+        event_pattern = {
+          source: ["aws.mediaconvert"],
+          "detail-type": ["MediaConvert Job State Change"],
+          detail: {
+            queue: [queue_arn]
+          }
+        }
+
+        log_group_arn = create_log_group(log_group).arn
+
+        cloudwatch_events.put_rule(
+          name: rule_name,
+          event_pattern: event_pattern.to_json,
+          state: "ENABLED",
+          description: "Forward MediaConvert job state changes from queue #{queue} to #{log_group}"
+        )
+
+        cloudwatch_events.put_targets(
+          rule: rule_name,
+          targets: [
+            {
+              id: "Id#{SecureRandom.uuid}",
+              arn: log_group_arn
+            }
+          ]
+        )
+        true
+      end
+
+      # Required options:
+      #
+      # * `output_prefix`: The S3 key prefix to use as the base for all outputs.
+      #
+      # * `outputs`: An array of `{preset, modifier}` options defining how to transcode and name the outputs.
+      #
+      # Optional options:
+      #
+      # * `masterfile_bucket`: The bucket to which file-based inputs will be copied before
+      #                        being passed to MediaConvert. Also used for S3-based inputs
+      #                        unless `use_original_url` is specified.
+      #
+      # * `use_original_url`: If `true`, any S3 URL passed in as input will be passed directly to
+      #                       MediaConvert as the file input instead of copying the source to
+      #                       the `masterfile_bucket`.
+      #
+      # Example:
+      # {
+      #   output_prefix: "path/to/output/files",
+      #   outputs: [
+      #       {preset: "System-Avc_16x9_1080p_29_97fps_8500kbps", modifier: "-1080"},
+      #       {preset: "System-Avc_16x9_720p_29_97fps_5000kbps", modifier: "-720"},
+      #       {preset: "System-Avc_16x9_540p_29_97fps_3500kbps", modifier: "-540"}
+      #     ]
+      #   }
+      # }
+      def create(input_url, options = {})
+        input_url = s3_uri(input_url, options)
+
+        input = options[:media_type] == :audio ? make_audio_input(input_url) : make_video_input(input_url)
+
+        create_job_params = {
+          role: role,
+          settings: {
+            inputs: [input],
+            output_groups: make_output_groups(options)
+          }
+        }
+
+        response = mediaconvert.create_job(create_job_params)
+        job = response.job
+        build_encode(job)
+      end
+
+      def find(id, _opts = {})
+        response = mediaconvert.get_job(id: id)
+        job = response.job
+        build_encode(job)
+      rescue Aws::MediaConvert::Errors::NotFound
+        raise ActiveEncode::NotFound, "Job #{id} not found"
+      end
+
+      def cancel(id)
+        mediaconvert.cancel_job(id: id)
+        find(id)
+      end
+
+      def log_group
+        @log_group ||= "/aws/events/active-encode/mediaconvert/#{queue}"
+      end
+
+      def queue
+        @queue ||= "Default"
+      end
+
+      private
+
+        def build_encode(job)
+          return nil if job.nil?
+          encode = ActiveEncode::Base.new(job.settings.inputs.first.file_input, {})
+          encode.id = job.id
+          encode.input.id = job.id
+          encode.state = JOB_STATES[job.status]
+          encode.current_operations = [job.current_phase].compact
+          encode.created_at = job.timing.submit_time
+          encode.updated_at = job.timing.finish_time || job.timing.start_time || encode.created_at
+          encode.percent_complete = convert_percent_complete(job)
+          encode.errors = [job.error_message].compact
+
+          encode.input.created_at = encode.created_at
+          encode.input.updated_at = encode.updated_at
+
+          encode.output = encode.state == :completed ? convert_output(job) : []
+          encode
+        end
+
+        def convert_percent_complete(job)
+          case job.status
+          when "SUBMITTED"
+            5
+          when "PROGRESSING"
+            job.job_percent_complete
+          when "CANCELED", "ERROR"
+            50
+          when "COMPLETE"
+            100
+          else
+            0
+          end
+        end
+
+        def convert_output(job)
+          results = get_encode_results(job)
+          settings = job.settings.output_groups.first.outputs
+
+          outputs = results.dig('detail', 'outputGroupDetails', 0, 'outputDetails').map.with_index do |detail, index|
+            tech_md = MediaConvertOutput.tech_metadata(settings[index], detail)
+            output = ActiveEncode::Output.new
+
+            output.created_at = job.timing.submit_time
+            output.updated_at = job.timing.finish_time || job.timing.start_time || output.created_at
+
+            [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
+             :audio_bitrate, :video_bitrate, :file_size, :label, :url, :id].each do |field|
+              output.send("#{field}=", tech_md[field])
+            end
+            output.id ||= "#{job.id}-output#{tech_md[:suffix]}"
+            output
+          end
+
+          adaptive_playlist = results.dig('detail', 'outputGroupDetails', 0, 'playlistFilePaths', 0)
+          unless adaptive_playlist.nil?
+            output = ActiveEncode::Output.new
+            output.created_at = job.timing.submit_time
+            output.updated_at = job.timing.finish_time || job.timing.start_time || output.created_at
+            output.id = "#{job.id}-output-auto"
+
+            [:duration, :audio_codec, :video_codec].each do |field|
+              output.send("#{field}=", outputs.first.send(field))
+            end
+            output.label = File.basename(adaptive_playlist)
+            output.url = adaptive_playlist
+            outputs << output
+          end
+          outputs
+        end
+
+        def get_encode_results(job)
+          start_time = job.timing.submit_time
+          end_time = job.timing.finish_time || Time.now.utc
+
+          response = cloudwatch_logs.start_query(
+            log_group_name: log_group,
+            start_time: start_time.to_i,
+            end_time: end_time.to_i,
+            limit: 1,
+            query_string: "fields @message | filter detail.jobId = '#{job.id}' | filter detail.status = 'COMPLETE' | sort @ingestionTime desc"
+          )
+          query_id = response.query_id
+          response = cloudwatch_logs.get_query_results(query_id: query_id)
+          until response.status == "Complete"
+            sleep(0.5)
+            response = cloudwatch_logs.get_query_results(query_id: query_id)
+          end
+          raise ActiveEncode::NotFound, "Unable to load progress for job #{job.id}" if response.results.empty?
+
+          JSON.parse(response.results.first.first.value)
+        end
+
+        def cloudwatch_events
+          @cloudwatch_events ||= Aws::CloudWatchEvents::Client.new
+        end
+
+        def cloudwatch_logs
+          @cloudwatch_logs ||= Aws::CloudWatchLogs::Client.new
+        end
+
+        def mediaconvert
+          endpoint = Aws::MediaConvert::Client.new.describe_endpoints.endpoints.first.url
+          @mediaconvert ||= Aws::MediaConvert::Client.new(endpoint: endpoint)
+        end
+
+        def s3_uri(url, options = {})
+          bucket = options[:masterfile_bucket]
+
+          case Addressable::URI.parse(url).scheme
+          when nil, 'file'
+            upload_to_s3 url, bucket
+          when 's3'
+            return url if options[:use_original_url]
+            check_s3_bucket url, bucket
+          else
+            raise ArgumentError, "Cannot handle source URL: #{url}"
+          end
+        end
+
+        def check_s3_bucket(input_url, source_bucket)
+          # logger.info("Checking `#{input_url}'")
+          s3_object = FileLocator::S3File.new(input_url).object
+          if s3_object.bucket_name == source_bucket
+            # logger.info("Already in bucket `#{source_bucket}'")
+            s3_object.key
+          else
+            s3_key = File.join(SecureRandom.uuid, s3_object.key)
+            # logger.info("Copying to `#{source_bucket}/#{input_url}'")
+            target = Aws::S3::Object.new(bucket_name: source_bucket, key: input_url)
+            target.copy_from(s3_object, multipart_copy: s3_object.size > 15_728_640) # 15.megabytes
+            s3_key
+          end
+        end
+
+        def upload_to_s3(input_url, source_bucket)
+          # original_input = input_url
+          bucket = Aws::S3::Resource.new(client: s3client).bucket(source_bucket)
+          filename = FileLocator.new(input_url).location
+          s3_key = File.join(SecureRandom.uuid, File.basename(filename))
+          # logger.info("Copying `#{original_input}' to `#{source_bucket}/#{input_url}'")
+          obj = bucket.object(s3_key)
+          obj.upload_file filename
+
+          s3_key
+        end
+
+        def event_rule_exists?(rule_name)
+          rule = cloudwatch_events.list_rules(name_prefix: rule_name).rules.find do |existing_rule|
+            existing_rule.name == rule_name
+          end
+          !rule.nil?
+        end
+
+        def find_log_group(name)
+          cloudwatch_logs.describe_log_groups(log_group_name_prefix: name).log_groups.find do |group|
+            group.log_group_name == name
+          end
+        end
+
+        def create_log_group(name)
+          result = find_log_group(name)
+
+          return result unless result.nil?
+
+          cloudwatch_logs.create_log_group(log_group_name: name)
+          find_log_group(name)
+        end
+
+        def make_audio_input(input_url)
+          {
+            audio_selectors: { "Audio Selector 1" => { default_selection: "DEFAULT" } },
+            audio_selector_groups: {
+              "Audio Selector Group 1" => {
+                audio_selector_names: ["Audio Selector 1"]
+              }
+            },
+            file_input: input_url,
+            timecode_source: "ZEROBASED"
+          }
+        end
+
+        def make_video_input(input_url)
+          {
+            audio_selectors: { "Audio Selector 1" => { default_selection: "DEFAULT" } },
+            file_input: input_url,
+            timecode_source: "ZEROBASED",
+            video_selector: {}
+          }
+        end
+
+        def make_output_groups(options)
+          output_type = options[:output_type] || :hls
+          raise ArgumentError, "Unknown output type: #{output_type.inspect}" unless OUTPUT_GROUP_TEMPLATES.keys.include?(output_type)
+          output_group_settings_key = "#{output_type}_group_settings".to_sym
+          output_group_settings = OUTPUT_GROUP_TEMPLATES[output_type].merge(destination: "s3://#{output_bucket}/#{options[:output_prefix]}")
+
+          outputs = options[:outputs].map do |output|
+            {
+              preset: output[:preset],
+              name_modifier: output[:modifier]
+            }
+          end
+
+          [{
+            output_group_settings: {
+              type: output_group_settings_key.upcase,
+              output_group_settings_key => output_group_settings
+            },
+            outputs: outputs
+          }]
+        end
+    end
+  end
+end