envoi-mam-agent 1.1.0 → 1.2.0

data/lib/envoi/mam/agent/cli/commands/iconik.rb

@@ -10,7 +10,7 @@ require 'envoi/mam/iconik/agent'
 current_command = ARGV.shift
 ARGV << '--help' if ARGV.empty?
 
-default_config_file_paths = Envoi::Mam::Agent::Cli::CONFIG_FILE_PATHS
+default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
 
 aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
 default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first

data/lib/envoi/mam/agent/cli/commands/mediasilo.rb

@@ -11,7 +11,7 @@ require 'envoi/mam/mediasilo/agent'
 current_command = ARGV.shift
 ARGV << '--help' if ARGV.empty?
 
-default_config_file_paths = Envoi::Mam::Agent::Cli::CONFIG_FILE_PATHS
+default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
 
 aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
 default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first

data/lib/envoi/mam/agent/cli/commands/restore.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
+$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
+require 'rubygems'
+require 'optparse'
+require 'open3'
+
+require 'envoi/mam/agent/cli'
+require 'envoi/restore/agent'
+
+default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
+
+aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
+default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
+
+current_command = ARGV.shift
+args = {
+    :config_file_path  => default_config_file_paths,
+    :default_ascp_path => default_aspera_ascp_path,
+    :dry_run           => false,
+    :operation         => :download,
+    :preserve_path     => true,
+    :transfer_type     => '',
+}
+
+args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
+args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
+args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
+
+op = OptionParser.new
+op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
+      "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
+# op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
+# op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
+# op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
+
+op.on('-h', '--help', 'Show this message.') { puts op; exit }
+op.load
+op.parse!
+
+begin
+  agent = Envoi::Restore::Agent.load_config_and_init(args)
+rescue => e
+  abort(e.message)
+end
+
+begin
+  agent.run
+rescue Interrupt
+  agent.stop
+end
+

data/lib/envoi/mam/agent/cli/commands/vidispine.rb

@@ -8,7 +8,7 @@ require 'open3'
 require 'envoi/mam/agent/cli'
 require 'envoi/mam/vidispine/agent'
 
-default_config_file_paths = Envoi::Mam::Agent::Cli::CONFIG_FILE_PATHS
+default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
 
 aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
 default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first

data/lib/envoi/mam/agent/cli/commands/wiredrive.rb

@@ -11,7 +11,7 @@ require 'envoi/mam/iconik/agent'
 current_command = ARGV.shift
 ARGV << '--help' if ARGV.empty?
 
-default_config_file_paths = Envoi::Mam::Agent::Cli::CONFIG_FILE_PATHS
+default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
 # test_args = %w(--download --send-to-envoi --destination-path /tmp)
 # ARGV.concat test_args
 args = {

data/lib/envoi/mam/agent/transfer_client/aspera.rb

@@ -1,4 +1,12 @@
 require 'envoi/mam/agent/transfer_client'
+require 'asperalm/fasp/local'
+
+module Asperalm
+  class Log
+    def logger=(new_logger) @logger = new_logger end
+  end
+end
+
 
 module Envoi
   module Mam
@@ -11,12 +19,12 @@ module Envoi
           DEFAULT_ASCP_ARGS = '-v -k3 --overwrite=diff -P 33001'
 
           ASCP_PATHS = [
-              '/usr/local/bin/ascp',
-              '/usr/bin/ascp',
-              '/Library/Aspera/bin/ascp',
-              '~/Applications/Aspera CLI/bin/ascp',
-              '~/Applications/Aspera Connect.app/Contents/Resources/ascp',
-              '/Applications/Aspera Connect.app/Contents/Resources/ascp'
+            '/usr/local/bin/ascp',
+            '/usr/bin/ascp',
+            '/Library/Aspera/bin/ascp',
+            '~/Applications/Aspera CLI/bin/ascp',
+            '~/Applications/Aspera Connect.app/Contents/Resources/ascp',
+            '/Applications/Aspera Connect.app/Contents/Resources/ascp'
           ]
           ASCP_PATHS.map { |v| File.expand_path(v) }
 
@@ -32,52 +40,166 @@ module Envoi
             File.executable?(@ascp_path)
           end
 
-          def build_ascp_command(config, mode, source_path, target_path)
+          def build_asperala_transfer_args(config, mode, source_path, target_path)
+
             aspera_host_address = config['host_address'] || config['host']
             aspera_username = config['username']
             aspera_password = config['password']
+            aspera_ssh_port = config['ssh_port']
+
+            aspera_url = config['url'] || config['uri']
+            if aspera_url
+              aspera_uri = URI(aspera_url)
+              aspera_host_address ||= aspera_uri.host if aspera_uri
+              un, pw = (aspera_uri.userinfo || '').split(':')
+              aspera_username ||= un if un
+              aspera_password ||= pw if pw
+              aspera_ssh_port = aspera_uri.port
+            end
+
+            force_username_password = config.fetch('force_username_password', false)
+
             aspera_token = config['aspera_transfer_token'] || config['aspera_token'] || config['token'] || config['transfer_token']
+            if aspera_username && aspera_password && !force_username_password && (aspera_token.nil? || aspera_token.empty?)
+              aspera_token = %(Basic #{["#{aspera_username}:#{aspera_password}"]
+                                            .pack('m')
+                                            .delete("\r\n")})
+              aspera_username = 'xfer'
+              aspera_password = nil
+            end
+
+            # @ascp_path = config['ascp_path'] || default_ascp_path
+            # @ascp_path = File.expand_path(@ascp_path)
+            # aspera_ascp_path = @ascp_path
+
+            env_vars = { }
+            env_vars['ASPERA_SCP_PASS'] = aspera_password if aspera_password
+            # env_vars['ASPERA_SCP_TOKEN'] = aspera_token if aspera_token
+
+            ascp_args = config['ascp_args'] ||
+                default_ascp_args ||
+                (agent.respond_to?(:default_ascp_args) ? agent.default_ascp_args : '')
+
+            tags = config['tags'] ||= { }
+            aspera_tags = tags['aspera'] ||= { }
+            aspera_tags['xfer_id'] ||= SecureRandom.uuid
+
+            cmdline_args = [
+              '--mode', mode,
+              '--host', aspera_host_address,
+              '--user', aspera_username,
+              '--tags64', Base64.strict_encode64(JSON.generate(tags)),
+            ]
+            cmdline_args.concat [ '-P', aspera_ssh_port ] if aspera_ssh_port
+            cmdline_args.concat ascp_args.is_a?(Array) ? ascp_args : ascp_args.split(' ') if ascp_args && !ascp_args.empty?
+            if aspera_token && !aspera_token.empty?
+              cmdline_args.concat ['-W', aspera_token, '-i', 'asperaweb_id_dsa.openssh' ]
+            end
+            cmdline_args.concat [ source_path.gsub('"', '\"'), target_path.gsub('"', '\"') ]
+
+            { env: env_vars, args: cmdline_args, ascp_version: :ascp }
+          end
+
+          def build_ascp_command(config, mode, source_path, target_path)
+            aspera_host_address = config['host_address'] || config['host']
+            aspera_username = config['username']
+            aspera_password = config['password']
+            aspera_ssh_port = config['ssh_port']
+
+            aspera_url = config['url'] || config['uri']
+            if aspera_url
+              aspera_uri = URI(aspera_url)
+              aspera_host_address ||= aspera_uri.host if aspera_uri
+              un, pw = (aspera_uri.userinfo || '').split(':')
+              aspera_username ||= un if un
+              aspera_password ||= pw if pw
+              aspera_ssh_port ||= aspera_uri.port
+            end
 
+            aspera_token = config['aspera_transfer_token'] || config['aspera_token'] || config['token'] || config['transfer_token']
+            if aspera_username && aspera_password && (aspera_token.nil? || aspera_token.empty?)
+              aspera_token = %(Basic #{["#{aspera_username}:#{aspera_password}"]
+                                            .pack('m')
+                                            .delete("\r\n")})
+              aspera_username = 'xfer'
+            end
             @ascp_path = config['ascp_path'] || default_ascp_path
             @ascp_path = File.expand_path(@ascp_path)
 
             aspera_ascp_path = @ascp_path
 
+            tags = config['tags'] || { }
+            aspera_tags = tags['aspera'] || { }
+            aspera_tags['xfer_id'] ||= SecureRandom.uuid
+
+
             ascp_args = config['ascp_args'] || default_ascp_args || agent.default_ascp_args
+
             # command = %(export ASPERA_SCP_PASS="#{aspera_password}"; "#{aspera_ascp_path}" #{ascp_args} --host="#{aspera_host_address}" --user=#{aspera_username} --mode=#{mode} "#{source_path}" "#{target_path}" )
             command = ''
             command << %(export ASPERA_SCP_PASS="#{aspera_password}"; ) if aspera_password
             command << %("#{aspera_ascp_path}" --mode=#{mode} --host="#{aspera_host_address}" --user="#{aspera_username}")
-            command << %( #{ascp_args}) if ascp_args && !ascp_args.empty?
+            command << %( -P #{aspera_ssh_port}) if aspera_ssh_port
+            command << %(--tags64 #{Base64.strict_encode64(JSON.generate(tags))}) if tags && !tags.empty?
+            if ascp_args && !ascp_args.empty?
+              command << (ascp_args.is_a?(Array)) ? ascp_args.join(' ') : ascp_args
+            end
             command << %( -W "#{aspera_token}") if aspera_token && !aspera_token.empty?
             command << %( "#{source_path.gsub('"', '\"')}" "#{target_path.gsub('"', '\"')}")
           end
 
           def download(config, path, destination_path = DEFAULT_DESTINATION_PATH)
             aspera_base_path = config['base_path'] || ''
-            source_path = File.join(aspera_base_path, path)
-            command = build_ascp_command(config, 'recv', source_path, destination_path)
+            source_path = aspera_base_path.empty? ? path : File.join(aspera_base_path, path)
 
-            unless ascp_path_exists?
-              warn "ASCP not found. '#{ascp_path}'"
-              return false
-            end
-            agent.shell_execute(command)
+            mode = 'recv'
+            transfer(config, mode, source_path, destination_path)
           end
 
           def upload(config, path, destination_path = DEFAULT_DESTINATION_PATH)
             aspera_base_path = config['base_path'] || ''
             source_path = path
             destination_path = aspera_base_path.empty? ? destination_path : File.join(aspera_base_path, destination_path)
-            command = build_ascp_command(config, 'send', source_path, destination_path)
+
+            mode = 'send'
+            transfer(config, mode, source_path, destination_path)
+          end
+
+          def transfer(config, mode, source_path, destination_path)
+            if true
+              transfer_using_asperala(config, mode, source_path, destination_path)
+            else
+              transfer_using_shell_execute(config, mode, source_path, destination_path)
+            end
+          end
+
+          def transfer_using_asperala(config, mode, source_path, destination_path)
+            args_out = build_asperala_transfer_args(config, mode, source_path, destination_path)
+            @fasp ||= Asperalm::Fasp::Local.instance
+            Asperalm::Log.instance.logger = logger
+            # Asperalm::Log.instance.level = :debug #logger.level
+            @fasp.start_transfer_with_args_env(args_out, {})
+            { success: true }
+          end
+
+          def transfer_using_shell_execute(config, mode, source_path, destination_path)
+            command = build_ascp_command(config, mode, source_path, destination_path)
 
             unless ascp_path_exists?
-              warn "ASCP not found. '#{ascp_path}'"
-              return false
+              msg = "ASCP not found. '#{ascp_path}'"
+              warn msg
+              return { message: msg, success: false }
             end
+
             agent.shell_execute(command)
           end
 
+          def shutdown(graceful = true)
+            if @fasp && @fasp.respond_to?(:shutdown)
+              @fasp.shutdown(graceful)
+            end
+          end
+
           # AsperaTransferClient
         end
 

data/lib/envoi/mam/agent/transfer_client/s3.rb

@@ -17,12 +17,20 @@ module Envoi
             region = config['region']
             access_key_id = config['access_key_id']
             secret_access_key = config['secret_access_key']
+            profile_name = config['profile'] || config['aws_profile']
 
-            resource_args = { }
-            resource_args[:region] = region if region && !region.empty?
-            resource_args[:access_key_id] = access_key_id if access_key_id && !access_key_id.empty?
-            resource_args[:secret_access_key] = secret_access_key if secret_access_key && !secret_access_key.empty?
-            @s3 = Aws::S3::Resource.new(resource_args)
+            aws_config = {}
+            aws_config[:credentials] = (access_key_id || secret_access_key) ?
+                                           Aws::Credentials.new(access_key_id, secret_access_key) :
+                                           Aws::SharedCredentials.new(profile_name: profile_name)
+            aws_config[:region] = region if region
+
+            # resource_args = { }
+            # resource_args[:region] = region if region && !region.empty?
+            # resource_args[:access_key_id] = access_key_id if access_key_id && !access_key_id.empty?
+            # resource_args[:secret_access_key] = secret_access_key if secret_access_key && !secret_access_key.empty?
+            # @s3 = Aws::S3::Resource.new(resource_args)
+            @s3 = Aws::S3::Resource.new(aws_config)
           end
 
           def download(config, path, destination_path = DEFAULT_DESTINATION_PATH)

data/lib/envoi/mam/agent/version.rb

@@ -1,7 +1,7 @@
 module Envoi
   module Mam
     class Agent
-      VERSION = '1.1.0'.freeze
+      VERSION = '1.2.0'.freeze
     end
   end
 end

data/lib/envoi/mam/cantemo/agent.rb

@@ -0,0 +1,353 @@
+require 'cantemo/portal/api/client'
+require 'envoi/mam/agent/transfer_client/aspera'
+require 'envoi/mam/agent/transfer_client/s3'
+
+module Envoi
+
+  module Mam
+
+    module Cantemo
+
+      class Agent < Envoi::Mam::Agent
+
+        DEFAULT_SHAPE_TAG          = 'original'
+        DEFAULT_DESTINATION_PATH   = '.'
+        DEFAULT_PRESERVE_FILE_PATH = false
+
+        attr_accessor :default_aspera_ascp_args,
+                      :default_aspera_ascp_path,
+                      :default_vidispine_shape_tag,
+                      :default_preserve_file_path
+
+        def after_initialize
+          args                        = initial_args
+          @default_aspera_ascp_path   = args[:default_aspera_ascp_path]
+          @default_aspera_args        = args.fetch(:default_ascp_args, Envoi::Mam::Agent::TransferClient::Aspera::DEFAULT_ASCP_ARGS)
+          @default_preserve_file_path = args.fetch(:default_preserve_file_path, DEFAULT_PRESERVE_FILE_PATH)
+        end
+
+        def dry_run?;
+          @dry_run
+        end
+
+        def agent_config
+          @agent_config ||= config['cantemo'] || config || {}
+        end
+
+        def agent_config_storages
+          agent_config['storages']
+        end
+
+        def initialize_api_client(args = {})
+          api_config  = agent_config
+          @api_client = args[:api_client] || begin
+            client_args = Hash[api_config.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym.downcase : k, v ]}]
+            client_args[:logger] = logger
+            _client = ::Cantemo::Portal::API::Client.new(client_args)
+
+            begin
+              _client.version
+            rescue => e
+              raise "Error connecting to Portal: #{e.message}"
+            end
+
+            _client
+          end
+
+          @default_vidispine_shape_tag = args[:default_shape_tag] || api_config['default_shape_tag'] || api_config['shape_tag'] || DEFAULT_SHAPE_TAG
+
+        end
+
+        def item_get_shape_by_tag(item_id, shape_tag)
+          item_shapes_get_response = api_client.item_shapes_get(:item_id => item_id, :tag => shape_tag)
+          shape_id                 = item_shapes_get_response['uri'].first
+        end
+
+        def download(args = {})
+
+          item_id  = args[:item_id]
+          shape_id = args[:shape_id]
+          unless shape_id && !shape_id.empty?
+            shape_tag = args[:shape_tag] || default_vidispine_shape_tag
+            shape_id  = item_get_shape_by_tag(item_id, shape_tag)
+          end
+
+          logger.info { "Getting file path for Item ID: #{item_id} Shape ID: #{shape_id}" }
+          item_shape_get_response = api_client.item_shape_get(:item_id => item_id, :shape_id => shape_id)
+
+          files = item_shape_get_response['containerComponent']['file']
+          logger.debug { "Files: #{files}" }
+
+          # file = files.first
+          files = [files.first] # just do the first file for now
+          files.each do |file|
+            begin
+              download_file(args, file)
+            rescue => e
+              logger.warn { "Exception: #{$!}" }
+            end
+          end
+          logger.info { 'DONE' }
+        end
+
+        def download_file(args, file)
+          logger.debug { "File: #{file}" }
+          transfer_type = args[:transfer_type]
+
+          file_storage_id = file['storage']
+          file_path       = file['path']
+
+          file_storage_config = agent_config_storages[file_storage_id]
+
+          unless file_storage_config && !file_storage_config.empty?
+            raise Exception, "No configuration found for storage '#{file_storage_id}'"
+          end
+
+          logger.info { "Transferring File Path: '#{file_path}'" }
+          preserve_path = args.fetch(:preserve_path, file_storage_config.fetch('preserve_path', default_preserve_file_path))
+
+          destination_path = args[:destination_path] || file_storage_config['destination_path'] || DEFAULT_DESTINATION_PATH
+          relative_path    = preserve_path ? File.dirname(file_path) : nil
+          relative_path    = nil if relative_path == '.'
+
+          target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
+          target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
+
+          aspera_config = file_storage_config['aspera']
+          if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
+            client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
+            return client.download(aspera_config, file_path, target_path)
+            # download_using_aspera(aspera_config, file_path, target_path)
+          end
+
+          s3_config = file_storage_config['s3']
+          if (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
+            target_path = File.expand_path(target_path) if target_path == '.'
+            target_path = File.join(target_path, File.basename(file_path))
+            client      = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
+            return client.download(s3_config, file_path, target_path)
+          end
+
+          logger.warn { "No Supported TransferClient Configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}Found in Storage Configuration." }
+        end
+
+        # @param [Hash] args
+        # @option args [String] :file_path
+        # @option args [String] :storage_id
+        # @option args [String] :transfer_type ('')
+        # @option args [Boolean] :import_file (true)
+        # @option args [Boolean] :preserve_path (storage_config['preserve_path'] || default_preserve_path)
+        #
+        # @return [Hash]
+        def upload(args = {})
+          _response = {}
+
+          file_path = args[:file_path]
+          raise ArgumentError, "Path not found: '#{file_path}'" unless File.exists?(file_path)
+
+          if File.directory?(file_path)
+            # Non-recursive directory upload
+            file_paths = Dir.glob(File.join(file_path, '*.*'))
+            logger.debug { "File Paths: #{file_paths}" }
+            file_paths.map { |fp| upload(args.merge(file_path: fp)) }
+            return file_paths
+          end
+          logger.debug { "Preparing to upload '#{file_path}'" }
+
+          transfer_type            = args[:transfer_type] || ''
+          storage_id               = args[:storage_id]
+          storage_config = agent_config_storages[storage_id]
+
+          unless storage_config && !storage_config.empty?
+            raise "No configuration found for storage '#{storage_id}'"
+          end
+
+          should_import_file = args.fetch(:import_file, storage_config.fetch('import', true))
+
+          should_preserve_path = args.fetch(:preserve_path,
+                                            storage_config.fetch('preserve_path', default_preserve_file_path))
+
+          destination_path = args[:destination_path] || storage_config['destination_path'] || '/'
+          relative_path    = should_preserve_path ? File.dirname(file_path) : nil
+          relative_path    = File.expand_path(relative_path) if relative_path == '.'
+
+          target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
+          target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
+
+
+          # upload file
+
+          transfer_response = begin
+            response      = nil
+            aspera_config = storage_config['aspera']
+            begin
+              if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
+                client   = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
+                response = client.upload(aspera_config, file_path, target_path)
+              end
+            rescue => e
+              logger.error { "Aspera Transfer Failed. '#{e.message}'\n#{e.backtrace.first}" }
+            end
+
+            s3_config = storage_config['s3']
+            begin
+              if !response && (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
+                _target_path = target_path
+                _target_path = File.expand_path(_target_path) if target_path == '.'
+                _target_path = File.join(_target_path, File.basename(file_path))
+                client       = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
+                response     = client.upload(s3_config, file_path, _target_path)
+              end
+            rescue => e
+              logger.error { "S3 Transfer Failed. '#{e.message}'" }
+            end
+
+            response
+          end
+          transfer_response = { success: transfer_response } if transfer_response == true || transfer_response == false
+
+          logger.debug { "Transfer Response: #{transfer_response}" }
+          _response[:transfer_response] = transfer_response
+
+          if transfer_response.nil?
+            logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}'" : ''} found in storage configuration." }
+            _response[:success] = false
+            return _response
+          end
+
+          unless transfer_response[:success]
+            logger.error { "Error transferring file." }
+            _response[:success] = false
+            return _response
+          end
+
+          unless should_import_file
+            _response[:success] = transfer_response[:success]
+            return _response
+          end
+
+
+          ### IMPORT - START
+          import_file_args                   = args.dup
+          import_file_args[:response_object] = _response
+          import_file_args[:target_path]     = target_path
+          import_file(import_file_args)
+          _response[:success] = true unless _response[:success] === false
+          ### IMPORT - END
+
+          _response
+        rescue => e
+          logger.error { "Exception: #{e.message}" }
+          _response[:exception] = e
+          return _response
+        end
+
+        # @param [Hash]  args
+        # @option args [String] :file_path
+        # @option args [Hash] :import_args ({})
+        # @option args [Hash] :import_options ({})
+        # @option args [String] :item_id
+        # @option args [Hash] :response_object ({})
+        # @option args [String] :shape_tag (@default_vidispine_shape_tag)
+        # @option args [String] :storage_id
+        # @option args [String] :target_path
+        #
+        # @return [Hash{ :success->Boolean, :file_create_response->{}, :import_response->{} }]
+        def import_file(args = {})
+          logger.debug { "Agent - Import File - Args: #{args}"}
+          _response = args[:response_object] || {}
+
+          file_path  = args[:file_path]
+          storage_id = args[:storage_id]
+
+          target_path = args[:target_path]
+
+          item_id   = args[:item_id]
+          shape_tag = args[:shape_tag] || default_vidispine_shape_tag
+
+          item_add_args_in = args[:item_add_args] || { }
+          item_add_options_in = args[:item_add_options] || { }
+
+          # attach file to item as shape
+          path_on_storage = File.join(target_path, File.basename(file_path))
+          path_on_storage = path_on_storage[1..-1] if path_on_storage.start_with?('/')
+          # file_create_response = api_client.storage_file_create(storage_id: storage_id,
+          #                                                       path: path_on_storage, state: 'CLOSED')
+          file_create_response             = api_client.storage_file_get_or_create(storage_id, path_on_storage, { :extended_response => true })
+          _response[:file_create_response] = file_create_response
+          file                             = file_create_response[:file]
+
+          file_id = file['id']
+
+          unless file_id
+            _file = file.dup
+            _file.keep_if { |k, v| v }
+            logger.error { "Failed to create file. #{_file}" }
+            _response[:success] = false
+            return _response
+          end
+
+          item = (file['item'] || []).first
+
+          if item
+            shape = (item['shape'] || []).first
+            msg   = "File already exist and is associated to item #{item['id']} as shape #{shape['id']}."
+            logger.warn { "#{msg} #{file}" }
+            _response[:error]   = { :message => msg }
+            _response[:success] = false
+            return _response
+          end
+
+          if item_id
+            item_shape_import_response = api_client.item_shape_import(item_id: item_id,
+                                                                      tag:     shape_tag, file_id: file_id)
+          else
+            item_add_args = {
+              storage_id:       storage_id,
+              file_path:        path_on_storage,
+              file_id:           file_id,
+              storage_path_map: { '/' => storage_id }
+            }
+            item_add_args.merge!(item_add_args_in) if item_add_args_in.is_a?(Hash)
+
+            item_add_options = { }
+            item_add_options.merge!(item_add_options_in) if item_add_options_in.is_a?(Hash)
+
+            logger.debug { "Executing Item Add/Import. ARGS: #{item_add_args} OPTS: #{item_add_options}" }
+            item_shape_import_response = api_client.item_add_using_file_path(item_add_args, item_add_options)
+          end
+          _response[:import_response] = item_shape_import_response
+
+          _response
+        end
+
+        def ingest_file
+          # 1) export ASPERA_SCP_TOKEN="ATB3_AEA9dUI5dd2u1k8XBMVD0qInR-Lyylkz8AylTXLVt5_SMVGR8SO7Sdc0lj6RkoHK_DvAAEWac8bllF_Yan1NbbDTyPj_3BTA"
+          # 2) ascp -i asperaweb_id_dsa.openssh -P 33001 -l 20m /Users/nicholasstokes/Desktop/CantemoAgent.mov xfer@ats-aws-us-west-2.aspera.io:
+          #
+          # single line equivalent
+          # ascp -i asperaweb_id_dsa.openssh -W 'ATB3_AEAiBdrf-7sMQu782sPoQ4Q7Yh4LadgvYK9BP4Kt1hcVlZGR8SO7Sdc0lj6RkoHK_DvAAG5XlBBIZGEINicYOxMRE7g_3BTA' -P 33001 -l 20m /Users/nicholasstokes/Desktop/CantemoAgent.mov xfer@ats-aws-us-west-2.aspera.io:
+
+          # Aspera:
+          # `GET /aspera/ping/` - to check if Aspera upload is available
+          # `POST /vs/item/new/` to create a placeholder to upload to
+          # `POST /aspera/upload_setup/` - Aspera token
+          # ... upload the file with Aspera ...
+          # `POST /API/v1/placeholder/VX-3012/associatefile/?filepath=Screen%20Shot%202018-08-07%20at%2013.06.58.png&storageid=VX-11&platform=mac`  - associate new file to placeholder
+
+          # HTTP upload:
+          # `POST /vs/item/new/` - create a placeholder
+          # then streamind POST (?) requests to `VSAPI/item/VX-3014/shape/essence/raw?transferId=2D30030A-6EC3-4364-9598-5BF20972A218&throttle=false&filename=Screen%20Shot%202018-08-07%20at%2014.34.15.png&jobmetadata=portal_groups:StringArray%3dAdmin`  to upload the data.
+          # that end-point for HTTP is documented in http://apidoc.vidispine.com/latest/ref/item/shape.html#import-a-shape-using-the-request-body
+          #
+        end
+
+        # Agent
+      end
+
+      # Cantemo
+    end
+
+  end
+
+end