cantemo-portal-agent 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e52bfb1fe0a32298dab4b64bf2aca6f18fe0f6ab
-  data.tar.gz: 3d442e6e485752b2dba2c7bcdaef210dd9398163
+  metadata.gz: b1a098feb7ef559fc883d25d942ee7f501599684
+  data.tar.gz: 2b90e43a0ccdd4ad7e2283b8ddb09495a15998e8
 SHA512:
-  metadata.gz: 9bd60ea14fba7a17e322228e074e948165aebb750a264e035cf2728d68d8690158674754040ffd5e1dcf26dd4a2d89df69259b138d52f802798999b8d63f6060
-  data.tar.gz: 9db178a801b0656b6a3e737cc904a46ec21ac2d0bc1d30380ed1ff85daa0e753ea1e66aec7309f7f32f9512000ce59f97171e569a2f4a7117e66e830550dff76
+  metadata.gz: 853bd438e02b10f756fb8524671c2bfa3884bf6d8fe4cfde3b189117d8681deb5f230b290ca28df524b1ce7e9ede16d07ce13044a38ac97d1ac611beea1c57f6
+  data.tar.gz: '078cc907795490344fa612380e21fc4f240ab6edbbf4118d7d2063ea438796564e908352c525106e865cd215d3d011e5efc7763b78d990793fb503ad2269a855'

data/lib/cantemo/portal/agent/cli/commands/watch_folders-working.rb

@@ -226,10 +226,10 @@ class Watcher
     w.run
   end
 
-  def self.run_as_daemon(args)
+  def self.run_as_daemon(args, options = { })
     # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
     require 'daemons'
-    Daemons.run_proc('cantemo-portal-agent-watch-folders') { self.run(args) }
+    Daemons.run_proc('cantemo-portal-agent-watch-folders', options) { self.run(args) }
   end
 
 end

data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb

@@ -10,7 +10,7 @@ require 'pp'
 require 'envoi/mam/agent/cli'
 require 'envoi/mam/cantemo/agent'
 # require 'envoi/aspera/watch_service/watch_folder'
-require 'envoi/mam/cantemo/agent/watch_folder_handler'
+require 'envoi/mam/cantemo/agent/watch_folder_manager'
 
 Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.clear
 Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.concat [
@@ -32,7 +32,7 @@ ARGV << 'run' if ARGV.empty?
     :config_file_path  => default_config_file_paths,
     :dry_run           => false,
     :operation         => :upload,
-    :preserve_path     => true,
+    :preserve_path     => false,
     :transfer_type     => '',
 }
 def args; @args end
@@ -49,15 +49,31 @@ op.parse!
 config_file_path = args[:config_file_path]
 args[:config_file_path].map! { |v| File.expand_path(v) } if config_file_path.is_a?(Array)
 
-control_command_present = %w(start stop restart run zap killall status).include? ARGV.first
+command = ARGV.first.dup
 
-daemonize = control_command_present || begin
-  _d = args[:daemonize]
+control_command_present = command && begin
+  command.downcase!
+  %w(start stop restart run zap killall status).include?(command)
 end
 
+if command
+  case command
+  when 'install'
+    puts
+  when 'uninstall'
+    puts
+  else
+    puts
+  end
+end
+
+
+daemonize = (control_command_present && %(start restart).include?(command)) || args[:daemonize]
+
 # puts "#{__FILE__}:#{__LINE__ } #{args}"
 # next_command = ARGV.shift
 # puts "COMMAND: #{next_command}"
 
-class Watcher < Envoi::Mam::Cantemo::Agent::WatchFolderHandler; end
-daemonize ? Watcher.run_as_daemon(args) : Watcher.run(args)
+class Watcher < Envoi::Mam::Cantemo::Agent::WatchFolderManager; end
+# daemonize ? Watcher.run_as_daemon(args, { force: true }) : Watcher.run(args)
+daemonize ? Watcher.run_as_daemon(args) : Watcher.run(args)

data/lib/cantemo/portal/agent/version.rb

@@ -1,7 +1,7 @@
 module Cantemo
   module Portal
     class Agent
-      VERSION = '1.1.0'.freeze
+      VERSION = '1.1.1'.freeze
     end
   end
 end

data/lib/envoi/mam/agent.rb

@@ -48,10 +48,13 @@ module Envoi
         end
       end
 
+      # @param [String] command
+      # @param [Boolean] dry_run
+      # @return [Hash]
       def shell_execute(command, dry_run = @dry_run)
         if dry_run
           logger.debug { "Skipping Execution of Command: '#{command}' " }
-          return
+          return { }
         end
         logger.debug { "Executing Command: '#{command}'" }
 
@@ -74,7 +77,7 @@ module Envoi
           success = thread.value == 0 ? true : false
         end
 
-        success
+        { success: success }
       end
 
       def self.load_from_config_file(args)

data/lib/envoi/mam/agent/transfer_client/aspera.rb

@@ -105,6 +105,14 @@ module Envoi
             end
 
             aspera_token = config['aspera_transfer_token'] || config['aspera_token'] || config['token'] || config['transfer_token']
+            if aspera_username && aspera_password && (aspera_token.nil? || aspera_token.empty?)
+              _token_string = %(Basic #{["#{aspera_username}:#{aspera_password}"]
+                                            .pack('m')
+                                            .delete("\r\n")})
+              aspera_token = _token_string
+              aspera_password = nil
+              aspera_username = 'xfer'
+            end
 
             # @ascp_path = config['ascp_path'] || default_ascp_path
             # @ascp_path = File.expand_path(@ascp_path)
@@ -112,11 +120,12 @@ module Envoi
 
             env_vars = { }
             env_vars['ASPERA_SCP_PASS'] = aspera_password if aspera_password
+            # env_vars['ASPERA_SCP_TOKEN'] = aspera_token if aspera_token
 
             ascp_args = config['ascp_args'] || default_ascp_args || agent.default_ascp_args
 
-            tags = config['tags'] || { }
-            aspera_tags = tags['aspera'] || { }
+            tags = config['tags'] ||= { }
+            aspera_tags = tags['aspera'] ||= { }
             aspera_tags['xfer_id'] ||= SecureRandom.uuid
 
             cmdline_args = [
@@ -127,7 +136,9 @@ module Envoi
             ]
             cmdline_args.concat [ '-P', aspera_ssh_port ] if aspera_ssh_port
             cmdline_args.concat ascp_args.is_a?(Array) ? ascp_args : ascp_args.split(' ') if ascp_args && !ascp_args.empty?
-            cmdline_args.concat [ '-W', aspera_token ] if aspera_token && !aspera_token.empty?
+            if aspera_token && !aspera_token.empty?
+              cmdline_args.concat ['-W', aspera_token, '-i', 'asperaweb_id_dsa.openssh' ]
+            end
             cmdline_args.concat [ source_path.gsub('"', '\"'), target_path.gsub('"', '\"') ]
 
             { env: env_vars, args: cmdline_args, ascp_version: :ascp }
@@ -150,7 +161,12 @@ module Envoi
             end
 
             aspera_token = config['aspera_transfer_token'] || config['aspera_token'] || config['token'] || config['transfer_token']
-
+            if aspera_username && aspera_password && (aspera_token.nil? || aspera_token.empty?)
+              _token_string = %(Basic #{["#{aspera_username}:#{aspera_password}"]
+                                            .pack('m')
+                                            .delete("\r\n")})
+              aspera_username = 'xfer'
+            end
             @ascp_path = config['ascp_path'] || default_ascp_path
             @ascp_path = File.expand_path(@ascp_path)
 
@@ -169,14 +185,16 @@ module Envoi
             command << %("#{aspera_ascp_path}" --mode=#{mode} --host="#{aspera_host_address}" --user="#{aspera_username}")
             command << %( -P #{aspera_ssh_port}) if aspera_ssh_port
             command << %(--tags64 #{Base64.strict_encode64(JSON.generate(tags))}) if tags && !tags.empty?
-            command << %( #{ascp_args}) if ascp_args && !ascp_args.empty?
+            if ascp_args && !ascp_args.empty?
+              command << (ascp_args.is_a?(Array)) ? ascp_args.join(' ') : ascp_args
+            end
             command << %( -W "#{aspera_token}") if aspera_token && !aspera_token.empty?
             command << %( "#{source_path.gsub('"', '\"')}" "#{target_path.gsub('"', '\"')}")
           end
 
           def download(config, path, destination_path = DEFAULT_DESTINATION_PATH)
             aspera_base_path = config['base_path'] || ''
-            source_path = File.join(aspera_base_path, path)
+            source_path = aspera_base_path.empty? ? path : File.join(aspera_base_path, path)
 
             mode = 'recv'
             transfer(config, mode, source_path, destination_path)
@@ -192,7 +210,7 @@ module Envoi
           end
 
           def transfer(config, mode, source_path, destination_path)
-            if false
+            if true
               transfer_using_asperala(config, mode, source_path, destination_path)
             else
               transfer_using_shell_execute(config, mode, source_path, destination_path)
@@ -201,21 +219,30 @@ module Envoi
 
           def transfer_using_asperala(config, mode, source_path, destination_path)
             args_out = build_asperala_transfer_args(config, mode, source_path, destination_path)
-            fasp = Asperalm::Fasp::Local.instance
-            Asperalm::Log.instance.level = :debug
-            fasp.start_transfer_with_args_env(args_out, {})
+            @fasp ||= Asperalm::Fasp::Local.instance
+            Asperalm::Log.instance.level = :debug #logger.level
+            @fasp.start_transfer_with_args_env(args_out, {})
+            { success: true }
           end
 
           def transfer_using_shell_execute(config, mode, source_path, destination_path)
             command = build_ascp_command(config, mode, source_path, destination_path)
 
             unless ascp_path_exists?
-              warn "ASCP not found. '#{ascp_path}'"
-              return false
+              msg = "ASCP not found. '#{ascp_path}'"
+              warn msg
+              return { message: msg, success: false }
             end
+
             agent.shell_execute(command)
           end
 
+          def shutdown(graceful = true)
+            if @fasp && @fasp.respond_to?(:shutdown)
+              @fasp.shutdown(graceful)
+            end
+          end
+
           # AsperaTransferClient
         end
 

data/lib/envoi/mam/cantemo/agent.rb

@@ -13,15 +13,18 @@ module Envoi
     
         DEFAULT_SHAPE_TAG = 'original'
         DEFAULT_DESTINATION_PATH = '.'
+        DEFAULT_PRESERVE_FILE_PATH = true
 
         attr_accessor :default_aspera_ascp_args,
                       :default_aspera_ascp_path,
-                      :default_vidispine_shape_tag
+                      :default_vidispine_shape_tag,
+                      :default_preserve_file_path
     
         def after_initialize
           args = initial_args
-          @default_aspera_ascp_path   = args[:default_aspera_ascp_path]
-          @default_aspera_args = args[:default_ascp_args] || Envoi::Mam::Agent::TransferClient::Aspera::DEFAULT_ASCP_ARGS
+          @default_aspera_ascp_path = args[:default_aspera_ascp_path]
+          @default_aspera_args = args.fetch(:default_ascp_args, Envoi::Mam::Agent::TransferClient::Aspera::DEFAULT_ASCP_ARGS)
+          @default_preserve_file_path = args.fetch(:default_preserve_file_path, DEFAULT_PRESERVE_FILE_PATH)
         end
         
         def dry_run?; @dry_run end
@@ -35,8 +38,8 @@ module Envoi
           @api_client = args[:api_client] || begin
     
             api_host = api_config['host']
-            api_host_use_ssl = api_config['use_ssl']
             api_port = api_config['port']
+            api_host_use_ssl = api_config['ssl']
             api_username = api_config['username']
             api_password = api_config['password']
             api_auth_token = api_config['api_auth_token']
@@ -54,13 +57,17 @@ module Envoi
                 api_username ||= _api_username
                 api_password ||= _api_password
               end
+              api_host_use_ssl = api_uri.scheme == 'https' if api_host_use_ssl.nil?
               api_uri_query = api_uri.query
               api_default_query_data ||= Hash[api_uri_query.split('&').map { |kp| kp.split('=') }]
               api_base_path ||= api_uri.path
             end
 
             api_port ||= (api_host_use_ssl ? 443 : 80)
-            api_base_path ||= 'VSAPI/'
+            api_base_path ||= '/'
+
+            api_endpoint_prefix = 'VSAPI'
+            api_noauth_endpoint_prefix = 'APInoauth'
 
             client_args = { }
             client_args[:http_host_address] = api_host if api_host
@@ -70,6 +77,8 @@ module Envoi
             client_args[:password] =  api_password if api_password
             client_args[:default_base_path] = api_base_path if api_base_path
             client_args[:default_query_data] = api_default_query_data if api_default_query_data
+            client_args[:api_endpoint_prefix] = api_endpoint_prefix
+            client_args[:api_noauth_endpoint_prefix] = api_noauth_endpoint_prefix
 
             if api_auth_token
               # Cantemo Portal supports an auth token for authentication, replace basic auth with auth-token
@@ -78,10 +87,16 @@ module Envoi
             end
 
             _client = ::Vidispine::API::Utilities.new(client_args)
-            _client
-          end
 
+            begin
+              _client.version
+            rescue => e
+              e.message = "Error connecting to Portal: #{e.message}"
+              raise e
+            end
 
+            _client
+          end
 
           @default_vidispine_shape_tag = args[:default_shape_tag] || api_config['default_shape_tag'] || api_config['shape_tag'] || DEFAULT_SHAPE_TAG
     
@@ -133,7 +148,7 @@ module Envoi
           end
 
           logger.info { "Transferring File Path: '#{file_path}'" }
-          preserve_path = args.fetch(:preserve_path, file_storage_config.fetch('preserve_path', true))
+          preserve_path = args.fetch(:preserve_path, file_storage_config.fetch('preserve_path', default_preserve_file_path))
 
           destination_path = args[:destination_path] || file_storage_config['destination_path'] || DEFAULT_DESTINATION_PATH
           relative_path = preserve_path ? File.dirname(file_path) : nil
@@ -161,6 +176,8 @@ module Envoi
         end
 
         def upload(args = { })
+          _response = { }
+
           file_path = args[:file_path]
           raise ArgumentError, "Path not found: '#{file_path}'" unless File.exists?(file_path)
 
@@ -183,7 +200,7 @@ module Envoi
 
           should_import_file = args.fetch(:import_file, vidispine_storage_config.fetch('import', true))
 
-          should_preserve_path = args.fetch(:preserve_path, vidispine_storage_config.fetch('preserve_path', true))
+          should_preserve_path = args.fetch(:preserve_path, vidispine_storage_config.fetch('preserve_path', default_preserve_file_path))
     
           destination_path = args[:destination_path] || vidispine_storage_config['destination_path'] || '/'
           relative_path = should_preserve_path ? File.dirname(file_path) : nil
@@ -194,7 +211,7 @@ module Envoi
     
     
           # upload file
-    
+
           transfer_response = begin
             response = nil
             aspera_config = vidispine_storage_config['aspera']
@@ -204,7 +221,7 @@ module Envoi
                 response = client.upload(aspera_config, file_path, target_path)
               end
             rescue => e
-              logger.error { "Aspera Transfer Failed. '#{e.message}'" }
+              logger.error { "Aspera Transfer Failed. '#{e.message}'\n#{e.backtrace.first}" }
             end
     
             s3_config = vidispine_storage_config['s3']
@@ -222,28 +239,95 @@ module Envoi
 
             response
           end
-    
-          logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}found in storage configuration." } unless transfer_response != nil
+          transfer_response = { success: transfer_response } if transfer_response == true || transfer_response == false
+
+          logger.debug { "Transfer Response: #{transfer_response}" }
+          _response[:transfer_response] = transfer_response
 
-          _response = { transfer_response: transfer_response }
+          if transfer_response.nil?
+            logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}found in storage configuration." }
+            _response[:success] = false
+            return _response
+          end
+
+          unless transfer_response[:success]
+            logger.error { "Error transferring file." }
+            _response[:success] = false
+            return _response
+          end
 
-          return _response unless should_import_file
+          unless should_import_file
+            _response[:success] = transfer_response[:success]
+            return _response
+          end
+
+
+          ### IMPORT - START
+          import_file_args = args.dup
+          import_file_args[:response_object] = _response
+          import_file_args[:target_path] = target_path
+          import_file(import_file_args)
+          _response[:success] = true unless _response[:success] === false
+          ### IMPORT - END
+
+          _response
+        rescue => e
+          logger.error { "Exception: #{e.message}" }
+          _response[:exception] = e
+          return _response
+        end
+
+        def import_file(args = { })
+          _response = args[:response_object] || { }
+
+          file_path = args[:file_path]
+          storage_id = args[:storage_id]
+
+          target_path = args[:target_path]
 
           item_id = args[:item_id]
           shape_tag = args[:shape_tag] || default_vidispine_shape_tag
 
           # attach file to item as shape
           path_on_storage = File.join(target_path, File.basename(file_path))
-          file_create_response = api_client.storage_file_create storage_id: storage_id, path: path_on_storage, state: 'CLOSED'
-          file_id = file_create_response['id']
+          path_on_storage = path_on_storage[1..-1] if path_on_storage.start_with?('/')
+          # file_create_response = api_client.storage_file_create(storage_id: storage_id,
+          #                                                       path: path_on_storage, state: 'CLOSED')
+          file_create_response = api_client.storage_file_get_or_create(storage_id, path_on_storage, { :extended_response => true })
+          _response[:file_create_response] = file_create_response
+          file = file_create_response[:file]
+
+          file_id = file['id']
+
+          unless file_id
+            _file = file.dup
+            _file.keep_if { |k,v| v }
+            logger.error { "Failed to create file. #{_file}" }
+            _response[:success] = false
+            return _response
+          end
+
+          item = (file['item'] || []).first
+
+          if item
+            shape = (item['shape'] || []).first
+            msg = "File already exist and is associated to item #{item['id']} as shape #{shape['id']}."
+            logger.warn { "#{msg} #{file}" }
+            _response[:error] = { :message => msg }
+            _response[:success] = false
+            return _response
+          end
 
           if item_id
-            item_shape_import_response = api_client.item_shape_import item_id: item_id, tag: shape_tag, fileId: file_id
+            item_shape_import_response = api_client.item_shape_import(item_id: item_id,
+                                                                      tag: shape_tag, fileId: file_id)
+
+
           else
-            item_shape_import_response = api_client.item_add_using_file_path storage_id: storage_id,
-                                                                         file_path: file_path,
-                                                                         file_id: file_id,
-                                                                         storage_path_map: { '/' => storage_id }
+            item_shape_import_response = api_client.item_add_using_file_path(storage_id: storage_id,
+                                                                             file_path: path_on_storage,
+                                                                             fileId: file_id,
+                                                                             storage_path_map: { '/' => storage_id })
           end
           _response[:import_response] = item_shape_import_response