envoi-mam-agent 1.1.0 → 1.2.0

data/lib/envoi/mam/vidispine/agent.rb

@@ -34,18 +34,23 @@ module Envoi
         def initialize_api_client(args = { })
           _vidispine_config = vidispine_config
           @api_client = args[:vidispine_api_client] || begin
-    
+
             vidispine_host = _vidispine_config['host']
+            vidispine_port = _vidispine_config['port']
+            vidispine_host_use_ssl = _vidispine_config['use_ssl']
             vidispine_username = _vidispine_config['username']
             vidispine_password = _vidispine_config['password']
+
             client_args = { }
             client_args[:http_host_address] = vidispine_host if vidispine_host
+            client_args[:http_host_port] = vidispine_port if vidispine_port
+            client_args[:http_host_use_ssl] = vidispine_host_use_ssl if vidispine_host_use_ssl
             client_args[:username] = vidispine_username if vidispine_username
             client_args[:password] =  vidispine_password if vidispine_password
             ::Vidispine::API::Utilities.new(client_args)
           end
     
-          @default_vidispine_shape_tag = args[:default_shape_tag] || _vidispine_config['default_shape_tag'] || _vidispine_config['shape_tag'] || DEFAULT_SHAPE_TAG
+          @default_vidispine_original_shape_tag = args[:default_shape_tag] || _vidispine_config['default_shape_tag'] || _vidispine_config['shape_tag'] || DEFAULT_SHAPE_TAG
     
         end
     

data/lib/envoi/restore/agent.rb

@@ -0,0 +1,73 @@
+require 'envoi/mam/agent'
+require 'envoi/mam/agent/transfer_client/aspera'
+require 'envoi/mam/agent/transfer_client/s3'
+
+require 'envoi/restore/sqs-queue-worker'
+require 'envoi/restore/glacier-restore-event-handler'
+
+module Envoi
+
+  class Restore
+
+    class Agent < Envoi::Mam::Agent
+
+      DEFAULT_ASPERA_ARGS = '-v -k3 --overwrite=diff -P 33001'
+      DEFAULT_DESTINATION_PATH = '.'
+
+      attr_accessor :default_aspera_ascp_args,
+                    :default_aspera_ascp_path
+
+      def after_initialize
+        args = initial_args
+        @running = false
+        @should_stop = false
+        @default_aspera_ascp_path   = args[:default_aspera_ascp_path]
+        @default_aspera_args = args[:default_ascp_args] || DEFAULT_ASPERA_ARGS
+        initialize_queue_handler
+        initialize_event_handler
+      end
+
+      def system_config
+        @system_config ||= config['restore'] || { }
+      end
+
+      def sqs_config
+        @sqs_config ||= system_config['sqs'] || {}
+      end
+
+      def initialize_queue_handler
+        @queue_handler = SQSQueueWorker.new(config: sqs_config)
+        @queue_handler.poller.before_request do |stats|
+          throw :stop_polling if @should_stop
+        end
+      end
+
+      def initialize_event_handler
+        @event_handler = GlacierRestoreEventHandler.new(config: sqs_config)
+      end
+
+      def run
+        @running = true
+        @queue_handler.poll do |messages|
+          [*messages].each do |msg|
+            break if @should_stop
+            msg_body_parsed = JSON.parse(msg.body)
+            event = msg_body_parsed
+            @event_handler.process_event(event)
+          end
+          throw :stop_polling if @should_stop
+        end
+        @running = false
+      end
+
+      def stop
+        @should_stop = true
+      end
+
+      # Agent
+    end
+
+    # Restore
+  end
+
+end

data/lib/envoi/restore/glacier-restore-event-handler.rb

@@ -0,0 +1,94 @@
+require 'logger'
+require 'envoi/mam/agent'
+require 'envoi/mam/agent/transfer_client/aspera'
+require 'envoi/mam/agent/transfer_client/s3'
+
+module Envoi
+
+  class Restore
+
+    class GlacierRestoreEventHandler
+
+      attr_accessor :logger, :config, :agent
+
+      DEFAULT_DESTINATION_PATH = '.'
+
+      def initialize(args = {})
+        @config = args[:config]
+        initialize_logger(args)
+      end
+
+      def initialize_logger(args = {})
+        @logger = args[:logger] ||= begin
+          log_to  = args[:log_to] || STDOUT
+          log_age = args[:log_age]
+          Logger.new(log_to, log_age)
+        end
+
+        log_level = args[:log_level] ||= Logger::DEBUG
+        @logger.level = log_level
+        @logger
+      end
+
+      def system_config
+        @system_config ||= config
+      end
+
+      def process_event(event)
+        event_s3_data = event['s3']
+
+        event_bucket_data = event_s3_data['bucket']
+        event_bucket_name = event_bucket_data['name']
+
+        event_object_data = event_s3_data['object']
+        event_object_key  = event_object_data['key']
+
+        s3_config = system_config['s3']
+        @current_config = s3_config[event_bucket_name] || s3_config
+
+        download_file_args = {
+            :destination_path => config['destination_path'] || DEFAULT_DESTINATION_PATH
+        }
+
+        file = {
+            'path' => event_object_key
+        }
+
+        download_file(download_file_args, file)
+      end
+
+      def download_file(args, file)
+        logger.debug { "File: #{file}" }
+        transfer_type = args[:transfer_type]
+
+        file_path = file['path']
+
+        preserve_path = args.fetch(:preserve_path, true)
+
+        destination_path = args[:destination_path] || DEFAULT_DESTINATION_PATH
+        relative_path = preserve_path ? File.dirname(file_path) : nil
+        relative_path = nil if relative_path == '.'
+
+        target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
+        target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
+
+        aspera_config = @current_config['aspera']
+        if (transfer_type.nil? || transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
+          client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
+          return client.download(aspera_config, file_path, target_path)
+        end
+
+        s3_config = @current_config['s3']
+        if (transfer_type.nil? || transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
+          target_path = File.expand_path(target_path) if target_path == '.'
+          target_path = File.join(target_path, File.basename(file_path))
+          client      = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
+          return client.download(s3_config, file_path, target_path)
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/envoi/restore/sqs-queue-worker.rb

@@ -0,0 +1,68 @@
+require 'aws-sdk-sqs'
+
+module Envoi
+  class Restore
+    class SQSQueueWorker
+
+      attr_reader :logger, :config, :client, :poller
+
+      def initialize(args = {})
+        @config = args[:config] || args
+        sqs_config = config
+        @sqs_client_options = sqs_config['client_options'] || sqs_config
+        @sqs_client_options = symbolize_keys(@sqs_client_options, true) if @sqs_client_options
+
+        @sqs_poller_options = sqs_config['poller_options'] || {}
+        @sqs_poller_options = symbolize_keys(@sqs_poller_options, true) if @sqs_poller_options
+
+        @sqs_poll_options = sqs_config['poll_options'] || {}
+        @sqs_poll_options = symbolize_keys(@sqs_poll_options, true) if @sqs_poll_options
+
+        @sqs_poller_options[:queue_url] ||= sqs_config['queue_url']
+
+        # puts "Client Options: #{@sqs_client_options}"
+        # puts "Poller Options: #{@sqs_poller_options}"
+        initialize_client
+        initialize_poller
+      end
+
+      def initialize_client(args = @sqs_client_options)
+        client_options = args[:client_options] || {}
+        aws_access_key_id = client_options.delete(:access_key_id)
+        aws_secret_access_key = client_options.delete(:secret_access_key)
+        aws_profile_name = args[:aws_profile_name] || args[:aws_profile] || client_options.delete(:profile_name)
+        client_options[:credentials] = (aws_access_key_id || aws_secret_access_key) ?
+                                           Aws::Credentials.new(aws_access_key_id, aws_secret_access_key) :
+                                           Aws::SharedCredentials.new(profile_name: aws_profile_name)
+        @client = Aws::SQS::Client.new(client_options)
+      end
+
+      def initialize_poller(args = @sqs_poller_options)
+        queue_url = args.delete(:queue_url)
+        args[:client] ||= @client if @client
+        @poller = Aws::SQS::QueuePoller.new(queue_url, args)
+      end
+
+      def poll(options = @sqs_poll_options, &block)
+        @poller.poll(options, &block)
+      end
+
+      def symbolize_keys (value, recursive = true)
+        case value
+        when Hash
+          Hash[value.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym : k, recursive ? symbolize_keys(v, true) : v ] }]
+        when Array
+          value.map { |v| symbolize_keys(v, recursive) }
+        else
+          value
+        end
+      end
+
+      def self.poll(args, &block)
+        handler = self.new(args)
+        handler.poll(&block)
+      end
+
+    end
+  end
+end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: envoi-mam-agent
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - John Whitson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-24 00:00:00.000000000 Z
+date: 2019-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: adobe_media_encoder
+  name: asperalm
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.1
+        version: '0.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.1
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
@@ -38,34 +38,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: daemons
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: faraday
+  name: listen
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '3.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: ubiquity-envoi
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +156,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -201,8 +229,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".rbenv-gemsets"
-- ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -211,13 +237,19 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- docs/restore.md
 - envoi-mam-agent.gemspec
 - exe/envoi-mam-agent
+- lib/cantemo/portal/agent/version.rb
+- lib/cantemo/portal/api/client.rb
+- lib/envoi/aspera/watch_service/client.rb
+- lib/envoi/aspera/watch_service/watch_folder.rb
 - lib/envoi/mam/agent.rb
 - lib/envoi/mam/agent/cli.rb
 - lib/envoi/mam/agent/cli/commands.rb
 - lib/envoi/mam/agent/cli/commands/iconik.rb
 - lib/envoi/mam/agent/cli/commands/mediasilo.rb
+- lib/envoi/mam/agent/cli/commands/restore.rb
 - lib/envoi/mam/agent/cli/commands/vidispine.rb
 - lib/envoi/mam/agent/cli/commands/wiredrive.rb
 - lib/envoi/mam/agent/config_service_client.rb
@@ -225,10 +257,14 @@ files:
 - lib/envoi/mam/agent/transfer_client/aspera.rb
 - lib/envoi/mam/agent/transfer_client/s3.rb
 - lib/envoi/mam/agent/version.rb
+- lib/envoi/mam/cantemo/agent.rb
 - lib/envoi/mam/iconik/agent.rb
 - lib/envoi/mam/mediasilo/agent.rb
 - lib/envoi/mam/vidispine/agent.rb
 - lib/envoi/mam/wiredrive/agent.rb
+- lib/envoi/restore/agent.rb
+- lib/envoi/restore/glacier-restore-event-handler.rb
+- lib/envoi/restore/sqs-queue-worker.rb
 homepage: http://www.github.com/XPlatform-Consulting/envoi-mam-agent
 licenses:
 - MIT

data/.rbenv-gemsets

@@ -1,3 +0,0 @@
-.gems
-envoi-mam-agent
--global

data/.ruby-version

@@ -1 +0,0 @@
-2.3.3