envoi-mam-agent 1.3.4 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9e39a1126b74c5cef31e17223c4eda416441fee4
-  data.tar.gz: 95761df9c0ae153134ce68d4ed75be5e649baf12
+  metadata.gz: cd5eb9858be86d36b652b9760314d6664c57f440
+  data.tar.gz: 59c7b8dfc9f82e6350f8a8c77d3cadd168b150c7
 SHA512:
-  metadata.gz: 81b822772c0f696e98765bdf4fd8242a9000f9ab17990a260d55fc4f01b9f46e48c7d7df163e7caa57d4485dd5257d72eddb36a3eefe8847c6675b8b9368bb27
-  data.tar.gz: be1e49c65bf70c27d1c62f0cc833a90d006b8379bafe1f0795f885d6cf2adae4ad8a999c496fd9d6a58e619e7ec65f0fac797c97bb743b49dd68cc8e85033d94
+  metadata.gz: 9a9aefa6eace13a770f3b952931a024bd2e959f590f3300149bfeba6a045c582531db683b32106005c098e1b4bade5f4ff3acc7fb76290a2ee2bb30bfcb0abcd
+  data.tar.gz: f8157b85c98363e723a22d553953be922cada020c7aa93fe355944f998eb1b14aac845b4961008026d0836c52dfe988cd77e7f03d04063f3790966ff4c819b49

data/lib/envoi/ingest_service/client.rb

@@ -0,0 +1,130 @@
+require 'logger'
+require 'net/https'
+require 'json'
+
+module Envoi
+
+  class IngestService
+
+    class Client
+
+      attr_accessor :logger, :http, :uri, :request, :response, :headers
+      attr_accessor :log_request_body, :log_response_body, :log_pretty_print_body
+
+      def initialize(args = {})
+        initialize_logger(args)
+
+        url = args[:url]
+        app_name = args[:app_name]
+        url = "#{url}?appname=#{app_name}"
+
+        @default_source_bucket_name = args[:s3_source_bucket_name] || args[:source_bucket_name]
+        @default_target_bucket_name = args[:s3_target_bucket_name] || args[:target_bucket_name]
+        @default_allow_reimport = args.fetch(:allow_reimport, false)
+
+        @uri = URI.parse(url)
+        @headers = {
+            'Accept' => 'application/json',
+            'Content-Type' => 'application/json'
+        }
+        @http = Net::HTTP.new(uri.host, uri.port)
+        @http.use_ssl = true
+
+        @default_app_name = args[:app_name]
+
+        @log_pretty_print_body = true
+        @log_request_body = true
+        @log_response_body = true
+        @parse_response = true
+      end
+
+      def initialize_logger(args = { })
+        @logger = args[:logger] || Logger.new(STDOUT)
+      end
+
+      def job_create(path, source_bucket_name = @default_source_bucket_name, target_bucket_name = @default_target_bucket_name, options = { })
+        allow_reimport = options.fetch(:allow_reimport, @default_allow_reimport)
+
+        args_out = { path: path, sourceBucket: source_bucket_name, targetBucket: target_bucket_name }
+        args_out[:allowReimport] = allow_reimport unless allow_reimport.nil?
+
+        request = Net::HTTP::Post.new(uri.request_uri, headers)
+        request.body = JSON.generate(args_out)
+        send_request(request)
+        response.code == '200'
+      end
+
+      def job_create_old(path)
+        args_out = [ path ]
+        request = Net::HTTP::Post.new(uri.request_uri, headers)
+        request.body = JSON.generate(args_out)
+        send_request(request)
+        response.code == '200'
+      end
+
+      # Formats a HTTPRequest or HTTPResponse body for log output.
+      # @param [HTTPRequest|HTTPResponse] obj
+      # @return [String]
+      def format_body_for_log_output(obj)
+        if obj.content_type == 'application/json'
+          if @log_pretty_print_body
+            _body = obj.body
+            output = JSON.pretty_generate(JSON.parse(_body)) rescue _body
+            return output
+          else
+            return obj.body
+          end
+        elsif obj.content_type == 'application/xml'
+          return obj.body
+        else
+          return obj.body.inspect
+        end
+      end
+
+      def response_parsed
+        @response_parsed ||= begin
+          response_body = response.respond_to?(:body) ? response.body : ''
+          logger.debug { "Parsing Response." }
+
+          case response.content_type
+          when 'application/json'
+            response_body.empty? ? response_body : JSON.parse(response_body) # rescue response
+          else
+            response_body
+          end
+        end
+      end
+
+
+      # @param [HTTPRequest] request
+      def send_request(request)
+        @response_parsed = nil
+        @request = request
+        logger.debug { %(REQUEST: #{request.method} http#{http.use_ssl? ? 's' : ''}://#{http.address}:#{http.port}#{request.path} HEADERS: #{request.to_hash.inspect} #{log_request_body and request.request_body_permitted? ? "\n-- BODY BEGIN --\n#{format_body_for_log_output(request)}\n-- BODY END --" : ''}) }
+        max_retries = 3
+        retries = 0
+        loop do
+          should_retry = false
+
+          @request_time_start = Time.now
+          @response = http.request(request)
+          @request_time_end = Time.now
+          logger.debug { %(RESPONSE: #{response.inspect} HEADERS: #{response.to_hash.inspect} #{log_response_body and response.respond_to?(:body) ? "\n-- BODY BEGIN --\n#{format_body_for_log_output(response)}\n-- BODY END--" : ''}\nTook: #{@request_time_end - @request_time_start} seconds) }
+          #logger.debug { "Parse Response? #{@parse_response}" }
+
+          if retries < max_retries && ['504'].include?(@response.code)
+            retries += 1
+            should_retry = true
+          end
+
+          break unless should_retry
+        end
+        @parse_response ? response_parsed : response.body
+      end
+
+
+    end
+
+  end
+
+end

data/lib/envoi/mam/agent/cli/commands/s3-to-eis.rb

@@ -0,0 +1,66 @@
+#! /usr/bin/env ruby
+# Installation
+# `gem install aws-sdk-s3`
+#
+require 'optparse'
+require 'pp'
+require 'csv'
+
+require 'envoi/utils/s3_enumerator'
+require 'envoi/ingest_service/client'
+
+
+ARGV << '--help' if ARGV.empty?
+args = {
+  show_summary: true,
+  eis_url: 'https://6bxyr9mdti.execute-api.us-east-1.amazonaws.com/latest/createElemental',
+  s3_object_key_prefix: '',
+  preview_only: false
+}
+op = OptionParser.new
+op.on('--app-name [APPNAME]', '' ) { |v| args[:app_name] = v }
+op.on('--source-bucket [BUCKETNAME]', 'The bucket to get objects from.') { |v| args[:s3_source_bucket_name] = v }
+op.on('--target-bucket [BUCKETNAME]', '') { |v| args[:s3_target_bucket_name] = v }
+op.on('--aws-access-key [KEY]', '') { |v| args[:aws_access_key_id] = v }
+op.on('--aws-secret-key [KEY]', '') { |v| args[:aws_secret_access_key] = v }
+op.on('--aws-profile [PROFILENAME]', '') { |v| args[:aws_profile] = v }
+op.on('--aws-region [REGION]', 'The AWS region what the bucket is located.') { |v| args[:aws_region] = v }
+op.on('--limit [LIMIT]', 'Limit the number of results') { |v| args[:limit] = v }
+op.on('--key-prefix [KEYPREFIX]', 'The object key prefix to limit the results by.') { |v| args[:s3_object_key_prefix] = v }
+op.on('--eis-url [URL]', 'The URL for the Envoi Ingest Service.',
+      "default: #{args[:eis_url]}") { |v| args[:eis_url] = v }
+op.on('--[no-]allow-reimport', 'Determines if files existing in the database will be re-imported.') { |v| args[:allow_reimport] = v }
+op.on('--[no-]preview', 'Output paths with summary and stop.',
+      "default: #{args[:preview_mode] ? 'true' : 'false'}") { |v| args[:preview_only] = v }
+op.on('--[no-]summary', 'Output the paths and calculate the number of assets.',
+      "default: #{args[:show_summary] ? 'true' : 'false'}",) { |v| args[:show_summary] = v }
+
+op.on('-h', '--help', 'Print help (this message) and exit') { puts op; exit }
+op.parse!
+
+s3_enum = S3Enumerator.new(args)
+s3_enum.run
+exit if args[:preview_only]
+
+app_name = args[:app_name]
+eis_url = args[:eis_url]
+s3_source_bucket_name = args[:s3_source_bucket_name]
+s3_target_bucket_name = args[:s3_target_bucket_name]
+
+eis_args = {
+  url: eis_url,
+  app_name: app_name,
+  source_bucket_name: s3_source_bucket_name,
+  target_bucket_name: s3_target_bucket_name,
+  allow_reimport: args[:allow_reimport]
+}
+
+eis = Envoi::IngestService::Client.new(eis_args)
+
+s3_enum.process_files do |object|
+  full_path = File.join(s3_enum.s3_bucket_region, s3_source_bucket_name, object.key)
+  raise "Error Creating Job for '#{full_path}':\n#{eis.response.body}" unless eis.job_create(object.key)
+end
+
+
+

data/lib/envoi/mam/agent/version.rb

@@ -1,7 +1,7 @@
 module Envoi
   module Mam
     class Agent
-      VERSION = '1.3.4'.freeze
+      VERSION = '1.4.0'.freeze
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: envoi-mam-agent
 version: !ruby/object:Gem::Version
-  version: 1.3.4
+  version: 1.4.0
 platform: ruby
 authors:
 - John Whitson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-13 00:00:00.000000000 Z
+date: 2019-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asperalm
@@ -258,6 +258,7 @@ files:
 - lib/cantemo/portal/api/client.rb
 - lib/envoi/aspera/watch_service/client.rb
 - lib/envoi/aspera/watch_service/watch_folder.rb
+- lib/envoi/ingest_service/client.rb
 - lib/envoi/mam/agent.rb
 - lib/envoi/mam/agent/cli.rb
 - lib/envoi/mam/agent/cli/commands.rb
@@ -265,6 +266,7 @@ files:
 - lib/envoi/mam/agent/cli/commands/mediaconvert-retry.rb
 - lib/envoi/mam/agent/cli/commands/mediasilo.rb
 - lib/envoi/mam/agent/cli/commands/restore.rb
+- lib/envoi/mam/agent/cli/commands/s3-to-eis.rb
 - lib/envoi/mam/agent/cli/commands/vidispine.rb
 - lib/envoi/mam/agent/cli/commands/wiredrive.rb
 - lib/envoi/mam/agent/config_service_client.rb