envoi-mam-agent 1.2.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3542672342cb1a7bf58f0ab4889f5241d722592f
-  data.tar.gz: 4d74581af836bf987ada79e4250526a3d8a40b15
+  metadata.gz: 786d46404e4cba2dc315f28715568437da1f5b7a
+  data.tar.gz: ee6601a7376a73e74ab0f7278a5a766bbb9d0db5
 SHA512:
-  metadata.gz: 5df25d66d00dd8d77dca9c76b404d3c8b2248e19be8f2df2dc1ffe4141134136559856c0df042cec913ce1c22b80f96dc813e9548b83a2911fc2e762586c609b
-  data.tar.gz: 8149920ac1f94e68d246da8b5d0758198201ccaf4d9388e8fdefd7eba90ef0081e7267fdf54f4c0d28040d768b11483ad2809fbd00d2b884a238c702dc678b48
+  metadata.gz: 22b74933cae7aa8dbaf193434d193feca691a35f3b61b5ad290df4e4babcb01bb76e51edc9e51d89195ab3fcae9f494e422541a3bdc8f8b3b57cb9e0fc856694
+  data.tar.gz: b35dd88fe050844c25e0da41dd28070de6b05184b8bc70c27e240be3a466ed2590bf48422bb410bcc11604d46032d51cb246a75b1f02de382a6a19c16dee65f7

data/envoi-mam-agent.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'asperalm', '~> 0.9'
   spec.add_runtime_dependency 'aws-sdk-s3', '~> 1'
   spec.add_runtime_dependency 'aws-sdk-sqs', '~> 1'
+  spec.add_runtime_dependency 'aws-sdk-mediaconvert', '~> 1'
   # spec.add_runtime_dependency 'shoryuken' '~> 4'
   spec.add_runtime_dependency 'daemons', '~> 1.3'
   # spec.add_runtime_dependency 'faraday', '~> 0.15'

data/lib/envoi/mam/agent/cli/commands/mediaconvert-retry.rb

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
+$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
+require 'rubygems'
+require 'optparse'
+require 'time'
+
+require 'envoi/mam/agent/cli'
+require 'envoi/utils/mediaconvert_enumerator'
+
+DEFAULT_LIST_JOBS_STATUS = 'ERROR'
+
+args = {
+  job_status: DEFAULT_LIST_JOBS_STATUS
+}
+op = OptionParser.new
+op.accept(Time) { |time_in| Time.parse(time_in) }
+op.on('--aws-profile PROFILENAME', 'AWS credentials file profile name to use.') { |v| args[:aws_profile] = v }
+op.on('--aws-access-key KEY', 'AWS access ley') { |v| args[:aws_access_key_id] = v }
+op.on('--aws-secret-key KEY', 'AWS secret access key') { |v| args[:aws_secret_access_key] = v }
+op.on('--aws-region REGION', 'The AWS region the jobs are located.') { |v| args[:aws_region] = v }
+op.on('--new-role ROLE', 'The new role arn to use when resubmitting jobs.') { |v| args[:new_role] = v }
+op.on('--new-queue QUEUE', 'The new queue arn to use when resubmitting jobs.') { |v| args[:new_queue] = v }
+op.on('--[no-]job-status [STATUS]', 'A job status to query for.',
+      "default: '#{args[:job_status]}'") { |v| args[:job_status] = v }
+op.on('--from-date DATE', Time, 'The earliest date to filter jobs to.') { |v| args[:from_date] = v }
+op.on('--to-date DATE', Time,'The latest date to filter jobs to.') { |v| args[:to_date] = v }
+op.on('--limit LIMIT', 'Limit the number of jobs to process.') { |v| args[:limit] = v }
+op.on('--verbose', '') { |v| args[:verbose] = v }
+op.on('--[no-]preview', 'Output paths with summary and stop.',
+      "default: #{args[:preview_mode] ? 'true' : 'false'}") { |v| args[:preview_only] = v }
+op.on('--[no-]summary', 'Output the paths and calculate the number of assets.',
+      "default: #{args[:show_summary] ? 'true' : 'false'}",) { |v| args[:show_summary] = v }
+
+op.on('-h', '--help', 'Print help (this message) and exit') { puts op; exit }
+op.parse!
+
+new_queue = args[:new_queue]
+new_role = args[:new_role]
+
+mc_enum = MediaConvertEnumerator.new(args)
+mc_enum.run
+exit if args[:preview_only]
+
+puts "\n\n"
+
+jobs_to_retry = [ ]
+mc_enum.jobs_by_error_code.each do |error_code, jobs|
+  jobs.each do |job|
+    jobs_to_retry << job
+  end
+end
+
+verbose = args[:verbose]
+
+job_keys_to_keep = [ :queue, :role, :settings, :user_metadata ]
+job_counter = 0
+total_jobs_to_retry = jobs_to_retry.length
+
+jobs_to_retry.each do |job|
+  job_counter += 1
+  input_file = job.settings.inputs.first.file_input rescue ''
+  puts "Processing #{job_counter} of #{total_jobs_to_retry} #{job.arn} '#{input_file}'"
+  new_job = job.to_h
+  new_job.keep_if { |k,v| job_keys_to_keep.include?(k) }
+  new_job[:queue] = new_queue if new_queue
+  new_job[:role] = new_role if new_role
+
+  resp = mc_enum.media_convert.create_job(new_job)
+  puts "Response: #{resp}" if verbose
+end
+

data/lib/envoi/mam/agent/version.rb

@@ -1,7 +1,7 @@
 module Envoi
   module Mam
     class Agent
-      VERSION = '1.2.0'.freeze
+      VERSION = '1.3.1'.freeze
     end
   end
 end

data/lib/envoi/utils/mediaconvert_enumerator.rb

@@ -0,0 +1,167 @@
+require 'aws-sdk-mediaconvert'
+
+class MediaConvertEnumerator
+
+  attr_reader :media_convert
+
+  attr_reader :jobs
+
+  attr_reader :preview_only, :should_show_summary
+
+  attr_reader :limit, :max_keys
+
+  attr_accessor :from_date, :to_date, :job_status
+
+  attr_reader :jobs_by_status, :jobs_by_error_code
+
+  def initialize(args = {})
+    aws_access_key_id = args[:aws_access_key_id]
+    aws_secret_access_key = args[:aws_secret_access_key]
+    aws_region = args[:aws_region]
+    aws_profile = args.fetch(:aws_profile,
+                             args.fetch(:aws_profile_name,
+                                        args.fetch(:profile,
+                                                   args.fetch(:profile_name, nil))))
+    aws_config = {}
+    aws_config[:credentials] = (aws_access_key_id || aws_secret_access_key) ?
+                                   Aws::Credentials.new(aws_access_key_id, aws_secret_access_key) :
+                                   Aws::SharedCredentials.new(profile_name: aws_profile)
+    aws_config[:region] ||= aws_region if aws_region
+
+    @media_convert = Aws::MediaConvert::Client.new(aws_config)
+
+    resp = @media_convert.describe_endpoints
+    endpoint = resp.endpoints.first
+    @media_convert = Aws::MediaConvert::Client.new(aws_config.merge(endpoint: endpoint.url))
+
+    @preview_only = args[:preview_only]
+    @should_show_summary = args.fetch(:show_summary, preview_only)
+
+    @job_status = args[:job_status]
+
+    @limit = args[:limit]
+    @limit = @limit.to_i if @limit
+
+    @from_date = args[:from_date]
+    @to_date = args[:to_date]
+
+    @jobs = []
+    @jobs_by_status = Hash.new { |h, k| h[k] = [] }
+    @jobs_by_error_code = Hash.new { |h, k| h[k] = [] }
+  end
+
+  def concat_jobs(jobs, new_jobs)
+    jobs.concat new_jobs
+    jobs
+  end
+
+  def retrieve_jobs(options = {}, &block)
+    @jobs = []
+
+    list_jobs_args = { }
+    list_jobs_args[:status] = job_status if job_status #||= DEFAULT_LIST_JOBS_STATUS
+
+    resp = @media_convert.list_jobs(list_jobs_args)
+    # pp resp
+    loop do
+      concat_jobs(jobs, resp.jobs)
+      job_count = jobs.length
+      puts job_count
+
+      break unless resp.next_token && (!limit || job_count < limit)
+      created_date = resp.jobs.last.created_at
+      break if from_date && created_date <= from_date
+
+      resp = @media_convert.list_jobs(list_jobs_args.merge(next_token: resp.next_token))
+    end
+
+    if from_date || to_date
+      jobs.delete_if { |job| (from_date && job.created_at < from_date) || (to_date && job.created_at > to_date) }
+    end
+    @jobs = jobs.first(limit) if limit
+
+  end
+
+  def jobs_by_status
+    build_grouped_data unless @group_data_initialized
+    @jobs_by_status
+  end
+
+  def jobs_by_error_code
+    build_grouped_data unless @group_data_initialized
+    @jobs_by_error_code
+  end
+
+  def build_grouped_data
+    jobs.each do |job|
+      status = job.status
+      @jobs_by_status[status] << job
+      @jobs_by_error_code[job.error_code] << job if status == 'ERROR'
+    end
+    @group_data_initialized = true
+  end
+
+  def print_table(data, options = { })
+    first_row = data.first
+    table = first_row.is_a?(Hash) ? [first_row.keys] + data.map(&:values) : data
+
+    widths = []
+    table.each do |line|
+      line.each_with_index do |col, idx|
+        cur_col_width = widths[idx]
+        if cur_col_width
+          col_len = col.to_s.length
+          widths[idx] = col_len if col_len > cur_col_width
+        else
+          widths[idx] = col.to_s.length
+        end
+      end
+    end
+
+    # header separator
+    separator_ary = widths.map { |n| '-' * n }
+    table.insert(1, separator_ary)
+    table.insert(-2, separator_ary) if options[:has_totals]
+
+    format = widths.collect { |n| "%-#{n}s"}.join(' | ')
+    table.each { |line| printf "| #{format} |\n", *line }
+  end
+
+  def run(&block)
+    retrieve_jobs(&block)
+    show_summary if @should_show_summary
+  end
+
+  def human_readable_bytes_short(human_readable_number)
+    "#{human_readable_number.split(',').first} #{HUMAN_READABLE_SHORT_SUFFIX[human_readable_number.count(',')]}"
+  end
+
+  def humanize_number(number)
+    number.to_s.chars.to_a.reverse.each_slice(3).map(&:join).join(',').reverse
+  end
+
+  def grouped_data_to_table(collection, key_name)
+    table_data =  [ [ key_name, 'Count' ] ]
+    collection.each do |key, objects|
+      row_values = [ key, humanize_number(objects.length) ]
+      table_data << row_values
+    end
+    table_data
+  end
+
+  def show_summary
+    jobs_by_status_table_data = grouped_data_to_table(jobs_by_status, 'Status')
+    jobs_by_error_code_table_data = grouped_data_to_table(jobs_by_error_code.sort, 'Error Code')
+
+    row_data = [ 'TOTAL', humanize_number(jobs.length)]
+    jobs_by_status_table_data << row_data
+
+    puts "\n\n--- Jobs Count by Status ---"
+    print_table(jobs_by_status_table_data, { has_totals: true })
+
+    puts "\n\n--- Jobs Count by Error Code ---\n| https://docs.aws.amazon.com/mediaconvert/latest/ug/mediaconvert_error_codes.html\n"
+
+    print_table(jobs_by_error_code_table_data, { has_totals: false })
+  end
+
+end

data/lib/envoi/utils/s3_enumerator.rb

@@ -0,0 +1,204 @@
+require 'aws-sdk-s3'
+
+class S3Enumerator
+
+  attr_reader :s3, :s3_bucket_name, :s3_object_key_prefix, :s3_bucket_region
+
+  attr_reader :objects
+
+  attr_reader :preview_only, :should_show_summary
+
+  attr_reader :limit, :max_keys
+
+  attr_reader  :objects_by_ext, :objects_by_storage_class, :total_bytes, :directories, :files, :largest_file
+
+  HUMAN_READABLE_SHORT_SUFFIX = %w(Bytes KB MB GB TB PB EB ZB YB)
+
+  def initialize(args = { })
+    aws_access_key_id = args[:aws_access_key_id]
+    aws_secret_access_key = args[:aws_secret_access_key]
+    aws_region = args[:aws_region]
+
+    aws_config = {}
+    aws_config[:credentials] = (aws_access_key_id || aws_secret_access_key) ?
+                                   Aws::Credentials.new(aws_access_key_id, aws_secret_access_key) :
+                                   Aws::SharedCredentials.new(profile_name: args[:aws_profile])
+    aws_config[:region] = aws_region if aws_region
+    Aws.config.update(aws_config) unless aws_config.empty?
+
+    @preview_only = args[:preview_only]
+    @should_show_summary = args.fetch(:show_summary, preview_only)
+
+    @limit = args[:limit]
+    @limit = @limit.to_i if @limit
+
+    @max_keys = 1000
+    @max_keys = @limit if @limit && @limit < @max_keys
+
+    @s3_bucket_name = args[:s3_bucket_name] || args[:bucket_name]
+    @s3_object_key_prefix = args[:s3_object_key_prefix] || args[:object_key_prefix] || ''
+    @s3_object_key_prefix = @s3_object_key_prefix[1..-1] if @s3_object_key_prefix.start_with?('/')
+
+    @s3 = Aws::S3::Client.new
+    @s3_bucket_region = s3.get_bucket_location(bucket: @s3_bucket_name).location_constraint
+    @s3_bucket_region = 'us-east-1' if @s3_bucket_region.empty?
+
+    @objects = []
+    @files = []
+    @ignored = []
+    @directories = []
+    @objects_by_ext = Hash.new { |h, k| h[k] = [] }
+  end
+
+  def concat_objects(objects, resp)
+    objects.concat resp.contents
+    objects
+  end
+
+  def total_object_count
+    objects.length
+  end
+
+  def retrieve_objects(&block)
+    @objects = []
+
+    resp = s3.list_objects_v2(bucket: s3_bucket_name, prefix: s3_object_key_prefix, max_keys: max_keys)
+    loop do
+      concat_objects(objects, resp)
+      puts objects.length
+      break if !resp.next_page? || (limit && total_object_count >= limit)
+      resp = resp.next_page
+    end
+    @objects = objects.first(limit) if limit
+
+    @files = []
+    @directories = []
+    @objects_by_ext = Hash.new { |h, k| h[k] = [] }
+    @objects_by_storage_class = Hash.new { |h, k| h[k] = [] }
+    #
+    @total_bytes = 0
+    @largest_file = nil
+    objects.each do |object|
+      if object.key.end_with?('/')
+        @directories << object
+      else
+        filename = File.basename(object.key)
+        should_ignore = filename.start_with?('.')
+        if should_ignore
+          @ignored << object
+        else
+          @files << object
+          @total_bytes += object.size
+          @largest_file = object unless @largest_file && @largest_file.size > object.size
+          filename_ext = File.extname(filename).downcase
+          objects_by_ext[filename_ext] << object
+          objects_by_storage_class[object.storage_class] << object
+        end
+      end
+
+      yield object, self if block_given?
+    end
+  end
+
+
+  def print_table(data, options = { })
+    first_row = data.first
+    table = first_row.is_a?(Hash) ? [first_row.keys] + data.map(&:values) : data
+
+    widths = []
+    table.each do |line|
+      line.each_with_index do |col, idx|
+        cur_col_width = widths[idx]
+        if cur_col_width
+          col_len = col.to_s.length
+          widths[idx] = col_len if col_len > cur_col_width
+        else
+          widths[idx] = col.to_s.length
+        end
+      end
+    end
+
+    # header separator
+    separator_ary = widths.map { |n| '-' * n }
+    table.insert(1, separator_ary)
+    table.insert(-2, separator_ary) if options[:has_totals]
+
+    format = widths.collect { |n| "%-#{n}s"}.join(' | ')
+    table.each { |line| printf "| #{format} |\n", *line }
+  end
+
+  def run(&block)
+    retrieve_objects(&block)
+    show_summary if @should_show_summary
+  end
+
+  def human_readable_bytes_short(human_readable_number)
+    "#{human_readable_number.split(',').first} #{HUMAN_READABLE_SHORT_SUFFIX[human_readable_number.count(',')]}"
+  end
+
+  def humanize_number(number)
+    number.to_s.chars.to_a.reverse.each_slice(3).map(&:join).join(',').reverse
+  end
+
+  def grouped_data_to_table(collection, key_name)
+    table_data =  [ [ key_name, 'Count', 'Short Bytes', 'Total Bytes' ] ]
+    collection.each do |key, objects|
+      group_total_size = objects.reduce(0) { |s, o| s + o.size }
+      human_readable_group_total_size = humanize_number(group_total_size)
+      human_readable_group_total_size_short = human_readable_bytes_short(human_readable_group_total_size)
+      row_values = [ key, humanize_number(objects.length), human_readable_group_total_size_short, human_readable_group_total_size ]
+      table_data << row_values
+    end
+    table_data
+  end
+
+  def show_summary
+    objects_by_ext_table_data = grouped_data_to_table(objects_by_ext, 'File Ext')
+    objects_by_storage_class_table_data = grouped_data_to_table(objects_by_storage_class, 'Storage Class')
+
+    human_readable_total_bytes = humanize_number(total_bytes)
+    human_readable_total_bytes_short = human_readable_bytes_short(human_readable_total_bytes)
+    row_data = [ 'TOTAL', humanize_number(objects.length), human_readable_total_bytes_short,  human_readable_total_bytes]
+    objects_by_ext_table_data << row_data
+    objects_by_storage_class_table_data << row_data
+
+    puts "\n\n--- Summary ---"
+    puts "Bucket Name: #{s3_bucket_name}"
+    puts "Bucket Region: #{s3_bucket_region}"
+    puts "Object Key Prefix: #{s3_object_key_prefix}"
+    puts "Total Objects: #{humanize_number(total_object_count)}"
+    puts "Total Directories: #{humanize_number(directories.length)}"
+    puts "Total Files: #{humanize_number(files.length)}"
+    puts "Total Size (in bytes): #{human_readable_total_bytes} (#{human_readable_total_bytes_short})"
+    puts "Largest File: #{largest_file}"
+    puts "\n"
+    print_table(objects_by_ext_table_data, { has_totals: true })
+    puts "\n"
+    print_table(objects_by_storage_class_table_data, { has_totals: true })
+  end
+
+  def process_files(args = { })
+    process_objects(files, { total_bytes: total_bytes })
+  end
+
+  def process_objects(objects, args = {})
+    return objects unless block_given?
+    return objects.map { |object| yield object, self } if args[:quiet]
+
+    bytes_remaining = args[:total_bytes] || objects.reduce(0) { |s, o| s + o.size }
+    _total_object_count = objects.length
+    counter = 0
+    objects.each do |object|
+      counter += 1
+      human_readable_bytes_remaining = humanize_number(bytes_remaining)
+      human_readable_bytes_remaining_short = human_readable_bytes_short(human_readable_bytes_remaining)
+      human_readable_object_bytes = humanize_number(object.size)
+      human_readable_object_bytes_short = human_readable_bytes_short(human_readable_object_bytes)
+      puts "Processing #{humanize_number(counter)} of #{humanize_number(_total_object_count)} #{human_readable_object_bytes} (#{human_readable_object_bytes_short}) of #{human_readable_bytes_remaining} (#{human_readable_bytes_remaining_short})  #{object.key}"
+      yield object, self if block_given?
+      bytes_remaining -= object.size
+    end
+  end
+
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: envoi-mam-agent
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.1
 platform: ruby
 authors:
 - John Whitson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-06 00:00:00.000000000 Z
+date: 2019-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asperalm
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-mediaconvert
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: daemons
   requirement: !ruby/object:Gem::Requirement
@@ -248,6 +262,7 @@ files:
 - lib/envoi/mam/agent/cli.rb
 - lib/envoi/mam/agent/cli/commands.rb
 - lib/envoi/mam/agent/cli/commands/iconik.rb
+- lib/envoi/mam/agent/cli/commands/mediaconvert-retry.rb
 - lib/envoi/mam/agent/cli/commands/mediasilo.rb
 - lib/envoi/mam/agent/cli/commands/restore.rb
 - lib/envoi/mam/agent/cli/commands/vidispine.rb
@@ -265,6 +280,8 @@ files:
 - lib/envoi/restore/agent.rb
 - lib/envoi/restore/glacier-restore-event-handler.rb
 - lib/envoi/restore/sqs-queue-worker.rb
+- lib/envoi/utils/mediaconvert_enumerator.rb
+- lib/envoi/utils/s3_enumerator.rb
 homepage: http://www.github.com/XPlatform-Consulting/envoi-mam-agent
 licenses:
 - MIT