active_encode 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5aed6c18b1e8d3b2a0ecb4e8dd296660f2af65ca00973edad791b0315964da4c
-  data.tar.gz: 2ce90e1367403bc7d45ffb3d3ba5ada6991ca1f80d0a1b59944abeb63d47d372
+  metadata.gz: e36b69edd440383a39009cc2a331f6cda32974e62fde4961353e5abd80e2bc0b
+  data.tar.gz: ce26e5a14a08c96b708d6fea08ff127606f9da7023a674444037374cf9812ef9
 SHA512:
-  metadata.gz: a1edfa784257928dc9d06463434a4132a7ab07995a5679aca9051aa68335dd1ed644cd2c5f9544dc439911f29087d2af1b358c839ef2692a29a0478fae5e5ab2
-  data.tar.gz: f3702f56f989045165a70b29850fc7e480f02c85877912db3115b4a9909e39673090d79fe23127a0a1a30d5b0be96c15f6bba4ea04af751b388531b864786e33
+  metadata.gz: d82bbc86cfc93cc33d699fb943c5b1caab27bc76b8149a599a61514f5257572aa3f0239ab617062f5b9ed4ef5b3874b9764f6786b0700fe2fa08625b917ead07
+  data.tar.gz: 53aabadb2c2337ac2cf8725ca0476b5afb9446aa8102aa156e93913d978317ceece57facf2bce0775fb8d5e3cf6e5a9ad47538c15006aed40d7bb94b98e301ce

data/.rubocop.yml

@@ -8,6 +8,7 @@ AllCops:
   TargetRubyVersion: 2.4
   DisplayCopNames: true
   Exclude:
+    - 'active_encode.gemspec'
     - 'vendor/**/*'
   Include:
     - '**/Rakefile'

data/.rubocop_todo.yml

@@ -7,6 +7,7 @@ Metrics/AbcSize:
 
 Metrics/BlockLength:
   Exclude:
+    - 'lib/active_encode/spec/shared_specs/*'
     - 'spec/**/*'
 
 Metrics/BlockNesting:

data/Gemfile

@@ -5,7 +5,8 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'addressable'
-gem 'aws-sdk'
+gem 'aws-sdk-elastictranscoder'
+gem 'aws-sdk-s3'
 gem 'byebug'
 gem 'rubyhorn', git: "https://github.com/avalonmediasystem/rubyhorn.git"
 gem 'shingoncoder'

data/README.md

@@ -124,6 +124,75 @@ end
 
 Engine adapters are shims between ActiveEncode and the back end encoding service.  You can add an additional engine by creating an engine adapter class that implements `:create`, `:find`, and `:cancel` and passes the shared specs.
 
+For example:
+```ruby
+# In your application at:
+# lib/active_encode/engine_adapters/my_custom_adapter.rb
+module ActiveEncode
+  module EngineAdapters
+    class MyCustomAdapter
+      def create(input_url, options = {})
+        # Start a new encoding job. This may be an external service, or a
+        # locally queued job.
+
+        # Return an instance ActiveEncode::Base (or subclass) that represents
+        # the encoding job that was just started.        
+      end
+
+      def find(id, opts = {})
+        # Find the encoding job for the given parameters.
+
+        # Return an instance of ActiveEncode::Base (or subclass) that represents
+        # the found encoding job.
+      end
+
+      def cancel(id)
+        # Cancel the encoding job for the given id.
+
+        # Return an instance of ActiveEncode::Base (or subclass) that represents
+        # the canceled job.
+      end
+    end
+  end
+end
+```
+Then, use the shared specs...
+```ruby
+# In your application at...
+# spec/lib/active_encode/engine_adapters/my_custom_adapter_spec.rb
+require 'spec_helper'
+require 'active_encode/spec/shared_specs'
+RSpec.describe MyCustomAdapter do
+  let(:created_job) {
+    # an instance of ActiveEncode::Base represented a newly created encode job
+  }
+  let(:running_job) {
+    # an instance of ActiveEncode::Base represented a running encode job
+  }
+  let(:canceled_job) {
+    # an instance of ActiveEncode::Base represented a canceled encode job
+  }
+  let(:completed_job) {
+    # an instance of ActiveEncode::Base represented a completed encode job
+  }
+  let(:failed_job) {
+    # an instance of ActiveEncode::Base represented a failed encode job
+  }
+  let(:completed_tech_metadata) {
+    # a hash representing completed technical metadata
+  }
+  let(:completed_output) {
+    # data representing completed output
+  }
+  let(:failed_tech_metadata) {
+    # a hash representing failed technical metadata
+  }
+
+  # Run the shared specs.
+  it_behaves_like 'an ActiveEncode::EngineAdapter'
+end
+```
+
 # Acknowledgments
 
 This software has been developed by and is brought to you by the Samvera community.  Learn more at the

data/active_encode.gemspec

@@ -22,7 +22,11 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "rails"
 
-  spec.add_development_dependency "aws-sdk"
+  spec.add_development_dependency "aws-sdk-cloudwatchevents"
+  spec.add_development_dependency "aws-sdk-cloudwatchlogs"
+  spec.add_development_dependency "aws-sdk-elastictranscoder"
+  spec.add_development_dependency "aws-sdk-mediaconvert"
+  spec.add_development_dependency "aws-sdk-s3"
   spec.add_development_dependency "bixby", '~> 1.0.0'
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "coveralls"
@@ -30,7 +34,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "engine_cart", "~> 2.2"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
-  spec.add_development_dependency "rspec-its"
   spec.add_development_dependency 'rspec_junit_formatter'
   spec.add_development_dependency "rspec-rails"
 

data/lib/active_encode/engine_adapters.rb

@@ -14,6 +14,7 @@ module ActiveEncode
     autoload :ElasticTranscoderAdapter
     autoload :TestAdapter
     autoload :FfmpegAdapter
+    autoload :MediaConvertAdapter
     autoload :PassThroughAdapter
 
     ADAPTER = 'Adapter'

data/lib/active_encode/engine_adapters/elastic_transcoder_adapter.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 require 'addressable/uri'
-require 'aws-sdk'
+require 'aws-sdk-elastictranscoder'
 require 'file_locator'
 
 module ActiveEncode

data/lib/active_encode/engine_adapters/ffmpeg_adapter.rb

@@ -34,7 +34,13 @@ module ActiveEncode
         FileUtils.mkdir_p working_path("outputs", new_encode.id)
 
         # Extract technical metadata from input file
-        `#{MEDIAINFO_PATH} --Output=XML --LogFile=#{working_path("input_metadata", new_encode.id)} "#{input_url}"`
+        curl_option = if options && options[:headers]
+                        headers = options[:headers].map { |k, v| "#{k}: #{v}" }
+                        (["--File_curl=HttpHeader"] + headers).join(",").yield_self { |s| "'#{s}'" }
+                      else
+                        ""
+                      end
+        `#{MEDIAINFO_PATH} #{curl_option} --Output=XML --LogFile=#{working_path("input_metadata", new_encode.id)} "#{input_url}"`
         new_encode.input = build_input new_encode
 
         if new_encode.input.duration.blank?
@@ -206,7 +212,11 @@ module ActiveEncode
           file_name = "outputs/#{sanitized_filename}-#{output[:label]}.#{output[:extension]}"
           " #{output[:ffmpeg_opt]} #{working_path(file_name, id)}"
         end.join(" ")
-        "#{FFMPEG_PATH} -y -loglevel error -progress #{working_path('progress', id)} -i \"#{input_url}\" #{output_opt}"
+        header_opt = Array(opts[:headers]).map do |k, v|
+          "#{k}: #{v}\r\n"
+        end.join
+        header_opt = "-headers '#{header_opt}'" if header_opt.present?
+        "#{FFMPEG_PATH} #{header_opt} -y -loglevel error -progress #{working_path('progress', id)} -i \"#{input_url}\" #{output_opt}"
       end
 
       def sanitize_base(input_url)

data/lib/active_encode/engine_adapters/media_convert_adapter.rb

@@ -0,0 +1,372 @@
+# frozen_string_literal: true
+require 'active_encode/engine_adapters/media_convert_output.rb'
+require 'active_support/core_ext/integer/time'
+require 'addressable/uri'
+require 'aws-sdk-cloudwatchevents'
+require 'aws-sdk-cloudwatchlogs'
+require 'aws-sdk-mediaconvert'
+require 'file_locator'
+
+require 'active_support/json'
+require 'active_support/time'
+
+module ActiveEncode
+  module EngineAdapters
+    class MediaConvertAdapter
+      # [AWS Elemental MediaConvert](https://aws.amazon.com/mediaconvert/) doesn't provide detailed
+      # output information in the job description that can be pulled directly from the service.
+      # Instead, it provides that information along with the job status notification when the job
+      # status changes to `COMPLETE`. The only way to capture that notification is through an [Amazon
+      # Eventbridge](https://aws.amazon.com/eventbridge/) rule that forwards the status change
+      # notification to another service for capture and/or handling.
+      #
+      # `ActiveEncode::EngineAdapters::MediaConvert` does this by creating a [CloudWatch Logs]
+      # (https://aws.amazon.com/cloudwatch/) log group and an EventBridge rule to forward status
+      # change notifications to the log group. It can then find the log entry containing the output
+      # details later when the job is complete. This is accomplished by calling the idempotent
+      # `#setup!` method.
+      #
+      # The AWS user/role calling the `#setup!` method will require permissions to create the
+      # necessary CloudWatch and EventBridge resources, and the role passed to the engine adapter
+      # will need access to any S3 buckets where files will be read from or written to during
+      # transcoding.
+      #
+      # Configuration example:
+      #
+      #     ActiveEncode::Base.engine_adapter = :media_convert
+      #     ActiveEncode::Base.engine_adapter.role = 'arn:aws:iam::123456789012:role/service-role/MediaConvert_Default_Role'
+      #     ActiveEncode::Base.engine_adapter.output_bucket = 'output-bucket'
+      #     ActiveEncode::Base.engine_adapter.setup!
+
+      JOB_STATES = {
+        "SUBMITTED" => :running, "PROGRESSING" => :running, "CANCELED" => :cancelled,
+        "ERROR" => :failed, "COMPLETE" => :completed
+      }.freeze
+
+      OUTPUT_GROUP_TEMPLATES = {
+        hls: { min_segment_length: 0, segment_control: "SEGMENTED_FILES", segment_length: 10 },
+        dash_iso: { fragment_length: 2, segment_control: "SEGMENTED_FILES", segment_length: 30 },
+        file: {},
+        ms_smooth: { fragment_length: 2 },
+        cmaf: { fragment_length: 2, segment_control: "SEGMENTED_FILES", segment_length: 10 }
+      }.freeze
+
+      attr_accessor :role, :output_bucket
+      attr_writer :log_group, :queue
+
+      def setup!
+        rule_name = "active-encode-mediaconvert-#{queue}"
+        return true if event_rule_exists?(rule_name)
+
+        queue_arn = mediaconvert.get_queue(name: queue).queue.arn
+
+        event_pattern = {
+          source: ["aws.mediaconvert"],
+          "detail-type": ["MediaConvert Job State Change"],
+          detail: {
+            queue: [queue_arn]
+          }
+        }
+
+        log_group_arn = create_log_group(log_group).arn
+
+        cloudwatch_events.put_rule(
+          name: rule_name,
+          event_pattern: event_pattern.to_json,
+          state: "ENABLED",
+          description: "Forward MediaConvert job state changes from queue #{queue} to #{log_group}"
+        )
+
+        cloudwatch_events.put_targets(
+          rule: rule_name,
+          targets: [
+            {
+              id: "Id#{SecureRandom.uuid}",
+              arn: log_group_arn
+            }
+          ]
+        )
+        true
+      end
+
+      # Required options:
+      #
+      # * `output_prefix`: The S3 key prefix to use as the base for all outputs.
+      #
+      # * `outputs`: An array of `{preset, modifier}` options defining how to transcode and name the outputs.
+      #
+      # Optional options:
+      #
+      # * `masterfile_bucket`: The bucket to which file-based inputs will be copied before
+      #                        being passed to MediaConvert. Also used for S3-based inputs
+      #                        unless `use_original_url` is specified.
+      #
+      # * `use_original_url`: If `true`, any S3 URL passed in as input will be passed directly to
+      #                       MediaConvert as the file input instead of copying the source to
+      #                       the `masterfile_bucket`.
+      #
+      # Example:
+      # {
+      #   output_prefix: "path/to/output/files",
+      #   outputs: [
+      #       {preset: "System-Avc_16x9_1080p_29_97fps_8500kbps", modifier: "-1080"},
+      #       {preset: "System-Avc_16x9_720p_29_97fps_5000kbps", modifier: "-720"},
+      #       {preset: "System-Avc_16x9_540p_29_97fps_3500kbps", modifier: "-540"}
+      #     ]
+      #   }
+      # }
+      def create(input_url, options = {})
+        input_url = s3_uri(input_url, options)
+
+        input = options[:media_type] == :audio ? make_audio_input(input_url) : make_video_input(input_url)
+
+        create_job_params = {
+          role: role,
+          settings: {
+            inputs: [input],
+            output_groups: make_output_groups(options)
+          }
+        }
+
+        response = mediaconvert.create_job(create_job_params)
+        job = response.job
+        build_encode(job)
+      end
+
+      def find(id, _opts = {})
+        response = mediaconvert.get_job(id: id)
+        job = response.job
+        build_encode(job)
+      rescue Aws::MediaConvert::Errors::NotFound
+        raise ActiveEncode::NotFound, "Job #{id} not found"
+      end
+
+      def cancel(id)
+        mediaconvert.cancel_job(id: id)
+        find(id)
+      end
+
+      def log_group
+        @log_group ||= "/aws/events/active-encode/mediaconvert/#{queue}"
+      end
+
+      def queue
+        @queue ||= "Default"
+      end
+
+      private
+
+        def build_encode(job)
+          return nil if job.nil?
+          encode = ActiveEncode::Base.new(job.settings.inputs.first.file_input, {})
+          encode.id = job.id
+          encode.input.id = job.id
+          encode.state = JOB_STATES[job.status]
+          encode.current_operations = [job.current_phase].compact
+          encode.created_at = job.timing.submit_time
+          encode.updated_at = job.timing.finish_time || job.timing.start_time || encode.created_at
+          encode.percent_complete = convert_percent_complete(job)
+          encode.errors = [job.error_message].compact
+
+          encode.input.created_at = encode.created_at
+          encode.input.updated_at = encode.updated_at
+
+          encode.output = encode.state == :completed ? convert_output(job) : []
+          encode
+        end
+
+        def convert_percent_complete(job)
+          case job.status
+          when "SUBMITTED"
+            5
+          when "PROGRESSING"
+            job.job_percent_complete
+          when "CANCELED", "ERROR"
+            50
+          when "COMPLETE"
+            100
+          else
+            0
+          end
+        end
+
+        def convert_output(job)
+          results = get_encode_results(job)
+          settings = job.settings.output_groups.first.outputs
+
+          outputs = results.dig('detail', 'outputGroupDetails', 0, 'outputDetails').map.with_index do |detail, index|
+            tech_md = MediaConvertOutput.tech_metadata(settings[index], detail)
+            output = ActiveEncode::Output.new
+
+            output.created_at = job.timing.submit_time
+            output.updated_at = job.timing.finish_time || job.timing.start_time || output.created_at
+
+            [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
+             :audio_bitrate, :video_bitrate, :file_size, :label, :url, :id].each do |field|
+              output.send("#{field}=", tech_md[field])
+            end
+            output.id ||= "#{job.id}-output#{tech_md[:suffix]}"
+            output
+          end
+
+          adaptive_playlist = results.dig('detail', 'outputGroupDetails', 0, 'playlistFilePaths', 0)
+          unless adaptive_playlist.nil?
+            output = ActiveEncode::Output.new
+            output.created_at = job.timing.submit_time
+            output.updated_at = job.timing.finish_time || job.timing.start_time || output.created_at
+            output.id = "#{job.id}-output-auto"
+
+            [:duration, :audio_codec, :video_codec].each do |field|
+              output.send("#{field}=", outputs.first.send(field))
+            end
+            output.label = File.basename(adaptive_playlist)
+            output.url = adaptive_playlist
+            outputs << output
+          end
+          outputs
+        end
+
+        def get_encode_results(job)
+          start_time = job.timing.submit_time
+          end_time = job.timing.finish_time || Time.now.utc
+
+          response = cloudwatch_logs.start_query(
+            log_group_name: log_group,
+            start_time: start_time.to_i,
+            end_time: end_time.to_i,
+            limit: 1,
+            query_string: "fields @message | filter detail.jobId = '#{job.id}' | filter detail.status = 'COMPLETE' | sort @ingestionTime desc"
+          )
+          query_id = response.query_id
+          response = cloudwatch_logs.get_query_results(query_id: query_id)
+          until response.status == "Complete"
+            sleep(0.5)
+            response = cloudwatch_logs.get_query_results(query_id: query_id)
+          end
+          raise ActiveEncode::NotFound, "Unable to load progress for job #{job.id}" if response.results.empty?
+
+          JSON.parse(response.results.first.first.value)
+        end
+
+        def cloudwatch_events
+          @cloudwatch_events ||= Aws::CloudWatchEvents::Client.new
+        end
+
+        def cloudwatch_logs
+          @cloudwatch_logs ||= Aws::CloudWatchLogs::Client.new
+        end
+
+        def mediaconvert
+          endpoint = Aws::MediaConvert::Client.new.describe_endpoints.endpoints.first.url
+          @mediaconvert ||= Aws::MediaConvert::Client.new(endpoint: endpoint)
+        end
+
+        def s3_uri(url, options = {})
+          bucket = options[:masterfile_bucket]
+
+          case Addressable::URI.parse(url).scheme
+          when nil, 'file'
+            upload_to_s3 url, bucket
+          when 's3'
+            return url if options[:use_original_url]
+            check_s3_bucket url, bucket
+          else
+            raise ArgumentError, "Cannot handle source URL: #{url}"
+          end
+        end
+
+        def check_s3_bucket(input_url, source_bucket)
+          # logger.info("Checking `#{input_url}'")
+          s3_object = FileLocator::S3File.new(input_url).object
+          if s3_object.bucket_name == source_bucket
+            # logger.info("Already in bucket `#{source_bucket}'")
+            s3_object.key
+          else
+            s3_key = File.join(SecureRandom.uuid, s3_object.key)
+            # logger.info("Copying to `#{source_bucket}/#{input_url}'")
+            target = Aws::S3::Object.new(bucket_name: source_bucket, key: input_url)
+            target.copy_from(s3_object, multipart_copy: s3_object.size > 15_728_640) # 15.megabytes
+            s3_key
+          end
+        end
+
+        def upload_to_s3(input_url, source_bucket)
+          # original_input = input_url
+          bucket = Aws::S3::Resource.new(client: s3client).bucket(source_bucket)
+          filename = FileLocator.new(input_url).location
+          s3_key = File.join(SecureRandom.uuid, File.basename(filename))
+          # logger.info("Copying `#{original_input}' to `#{source_bucket}/#{input_url}'")
+          obj = bucket.object(s3_key)
+          obj.upload_file filename
+
+          s3_key
+        end
+
+        def event_rule_exists?(rule_name)
+          rule = cloudwatch_events.list_rules(name_prefix: rule_name).rules.find do |existing_rule|
+            existing_rule.name == rule_name
+          end
+          !rule.nil?
+        end
+
+        def find_log_group(name)
+          cloudwatch_logs.describe_log_groups(log_group_name_prefix: name).log_groups.find do |group|
+            group.log_group_name == name
+          end
+        end
+
+        def create_log_group(name)
+          result = find_log_group(name)
+
+          return result unless result.nil?
+
+          cloudwatch_logs.create_log_group(log_group_name: name)
+          find_log_group(name)
+        end
+
+        def make_audio_input(input_url)
+          {
+            audio_selectors: { "Audio Selector 1" => { default_selection: "DEFAULT" } },
+            audio_selector_groups: {
+              "Audio Selector Group 1" => {
+                audio_selector_names: ["Audio Selector 1"]
+              }
+            },
+            file_input: input_url,
+            timecode_source: "ZEROBASED"
+          }
+        end
+
+        def make_video_input(input_url)
+          {
+            audio_selectors: { "Audio Selector 1" => { default_selection: "DEFAULT" } },
+            file_input: input_url,
+            timecode_source: "ZEROBASED",
+            video_selector: {}
+          }
+        end
+
+        def make_output_groups(options)
+          output_type = options[:output_type] || :hls
+          raise ArgumentError, "Unknown output type: #{output_type.inspect}" unless OUTPUT_GROUP_TEMPLATES.keys.include?(output_type)
+          output_group_settings_key = "#{output_type}_group_settings".to_sym
+          output_group_settings = OUTPUT_GROUP_TEMPLATES[output_type].merge(destination: "s3://#{output_bucket}/#{options[:output_prefix]}")
+
+          outputs = options[:outputs].map do |output|
+            {
+              preset: output[:preset],
+              name_modifier: output[:modifier]
+            }
+          end
+
+          [{
+            output_group_settings: {
+              type: output_group_settings_key.upcase,
+              output_group_settings_key => output_group_settings
+            },
+            outputs: outputs
+          }]
+        end
+    end
+  end
+end