cantemo-portal-agent 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1a098feb7ef559fc883d25d942ee7f501599684
-  data.tar.gz: 2b90e43a0ccdd4ad7e2283b8ddb09495a15998e8
+  metadata.gz: 6d670e7ef575f0152b7bf5d77d2763de6a64e363
+  data.tar.gz: e76ed955968ebd2182b49e29badbbff6e7b606bd
 SHA512:
-  metadata.gz: 853bd438e02b10f756fb8524671c2bfa3884bf6d8fe4cfde3b189117d8681deb5f230b290ca28df524b1ce7e9ede16d07ce13044a38ac97d1ac611beea1c57f6
-  data.tar.gz: '078cc907795490344fa612380e21fc4f240ab6edbbf4118d7d2063ea438796564e908352c525106e865cd215d3d011e5efc7763b78d990793fb503ad2269a855'
+  metadata.gz: 4e52761626f536e564d71856e361f16b3d2d1418da3e464723256ffad5459cf512a67b684e9628da912c78031ca520e407748881c1804eb79123cac37846c917
+  data.tar.gz: f8a9a3ba1f3621feb3e8958c51a0068bcaf6e81264bca425605e1d2ade8fe75adcf3891a04a2ae353296138401457b330ce59278650a54491d99e93ecc6790af

data/lib/cantemo/portal/agent/cli/commands/watch_folder.rb

@@ -0,0 +1 @@
+require_relative('watch_folders')

data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb

@@ -51,7 +51,7 @@ args[:config_file_path].map! { |v| File.expand_path(v) } if config_file_path.is_
 
 command = ARGV.first.dup
 
-control_command_present = command && begin
+daemon_control_command_present = command && begin
   command.downcase!
   %w(start stop restart run zap killall status).include?(command)
 end
@@ -68,7 +68,7 @@ if command
 end
 
 
-daemonize = (control_command_present && %(start restart).include?(command)) || args[:daemonize]
+daemonize = (daemon_control_command_present && %w(start restart).include?(command)) || args[:daemonize]
 
 # puts "#{__FILE__}:#{__LINE__ } #{args}"
 # next_command = ARGV.shift
@@ -76,4 +76,5 @@ daemonize = (control_command_present && %(start restart).include?(command)) || a
 
 class Watcher < Envoi::Mam::Cantemo::Agent::WatchFolderManager; end
 # daemonize ? Watcher.run_as_daemon(args, { force: true }) : Watcher.run(args)
-daemonize ? Watcher.run_as_daemon(args) : Watcher.run(args)
+# daemon_control_command_present ? Watcher.run_as_daemon(args) : Watcher.run(args)
+Watcher.run_as_daemon(args)

data/lib/cantemo/portal/agent/version.rb

@@ -1,7 +1,7 @@
 module Cantemo
   module Portal
     class Agent
-      VERSION = '1.1.1'.freeze
+      VERSION = '1.2.0'.freeze
     end
   end
 end

data/lib/envoi/mam/agent.rb

@@ -12,7 +12,7 @@ module Envoi
 
       def initialize(args = { })
         @initial_args = args.clone
-        @config = args[:config]
+        @config = args[:config] || { }
         @notifiers = args[:notifiers] || [ ]
 
         @dry_run = args.fetch(:dry_run, false)
@@ -80,29 +80,36 @@ module Envoi
         { success: success }
       end
 
-      def self.load_from_config_file(args)
+      def self.load_config_from_file(args)
         config_file_path = args[:config_file_path]
+        config_file_path = File.expand_path(config_file_path)
         config_file_path = config_file_path.find { |v| File.exists?(v) } if config_file_path.is_a?(Array)
-        abort("Missing Config File. '#{config_file_path}'") unless config_file_path && !config_file_path.empty? && File.exists?(config_file_path)
+        raise "Missing Config File. '#{config_file_path}'" unless config_file_path && !config_file_path.empty? && File.exists?(config_file_path)
 
         begin
           config = JSON.parse(File.read(config_file_path))
         rescue => e
-          abort("Config File Failed to Load. '#{$!}'")
+          raise "Failed to Load Config File. '#{config_file_path}' '#{e.message}'"
         end
-        args[:config] = config
-
-        self.new(args.merge({ :config => config }))
+        config
       end
 
-      def self.load_from_config_service(args)
+      def self.load_config_from_service(args)
         args_out = { }
         args_out[:app_id] = args[:config_service_app_id]
         args_out[:token] = args[:config_service_app_token]
         args_out[:api_url] = args[:config_service_app_url]
         config = Envoi::Mam::Agent::ConfigServiceClient.config_get(args_out)
-        args[:config] = config
+        config
+      end
 
+      def self.load_from_config_file(args)
+        config = load_config_from_file(args)
+        self.new(args.merge({ :config => config }))
+      end
+
+      def self.load_from_config_service(args)
+        config = load_config_from_service(args)
         self.new(args.merge({ :config => config }))
       end
 

data/lib/envoi/mam/agent/cli/commands/cantemo-agent.rb

@@ -57,6 +57,4 @@ args[:file_path] ||= ARGV.shift
 args[:destination_path] ||= ARGV.shift unless ARGV.empty?
 
 agent = Envoi::Mam::Cantemo::Agent.load_config_and_init(args)
-
-exit
 agent.run_operation

data/lib/envoi/mam/agent/cli/commands/cantemo.rb

@@ -1,4 +1,4 @@
-if (ARGV.first || '').tr('_-', '') == 'watchfolders'
+if [ 'watchfolder', 'watchfolders'].include?((ARGV.first || '').tr('_-', ''))
   require 'cantemo/portal/agent/cli/commands/watch_folders'
 else
   require 'envoi/mam/agent/cli/commands/cantemo-agent'

data/lib/envoi/mam/cantemo/agent.rb

@@ -6,83 +6,89 @@ require 'vidispine/api/utilities'
 module Envoi
 
   module Mam
-    
+
     module Cantemo
 
       class Agent < Envoi::Mam::Agent
-    
-        DEFAULT_SHAPE_TAG = 'original'
-        DEFAULT_DESTINATION_PATH = '.'
+
+        DEFAULT_SHAPE_TAG          = 'original'
+        DEFAULT_DESTINATION_PATH   = '.'
         DEFAULT_PRESERVE_FILE_PATH = true
 
         attr_accessor :default_aspera_ascp_args,
                       :default_aspera_ascp_path,
                       :default_vidispine_shape_tag,
                       :default_preserve_file_path
-    
+
         def after_initialize
-          args = initial_args
-          @default_aspera_ascp_path = args[:default_aspera_ascp_path]
-          @default_aspera_args = args.fetch(:default_ascp_args, Envoi::Mam::Agent::TransferClient::Aspera::DEFAULT_ASCP_ARGS)
+          args                        = initial_args
+          @default_aspera_ascp_path   = args[:default_aspera_ascp_path]
+          @default_aspera_args        = args.fetch(:default_ascp_args, Envoi::Mam::Agent::TransferClient::Aspera::DEFAULT_ASCP_ARGS)
           @default_preserve_file_path = args.fetch(:default_preserve_file_path, DEFAULT_PRESERVE_FILE_PATH)
         end
-        
-        def dry_run?; @dry_run end
-    
+
+        def dry_run?;
+          @dry_run
+        end
+
         def agent_config
-          @agent_config ||= config['cantemo'] || config['vidispine'] || { }
+          @agent_config ||= config['cantemo'] || config || {}
         end
-    
-        def initialize_api_client(args = { })
-          api_config = agent_config
+
+        def agent_config_storages
+          agent_config['storages']
+        end
+
+        def initialize_api_client(args = {})
+          api_config  = agent_config
           @api_client = args[:api_client] || begin
-    
-            api_host = api_config['host']
-            api_port = api_config['port']
-            api_host_use_ssl = api_config['ssl']
-            api_username = api_config['username']
-            api_password = api_config['password']
-            api_auth_token = api_config['api_auth_token']
-            api_base_path = api_config['api_base_path']
+
+            api_host               = api_config['host']
+            api_port               = api_config['port']
+            api_host_use_ssl       = api_config['ssl']
+            api_username           = api_config['username']
+            api_password           = api_config['password']
+            api_auth_token         = api_config['api_auth_token']
+            api_base_path          = api_config['api_base_path']
             api_default_query_data = api_config['default_query_data']
 
             api_url = api_config['url'] || api_config['uri']
             if api_url
-              api_uri = URI(api_url)
-              api_host ||= api_uri.host
-              api_port ||= api_uri.port
+              api_uri      = URI(api_url)
+              api_host     ||= api_uri.host
+              api_port     ||= api_uri.port
               api_userinfo = api_uri.userinfo
               if api_userinfo
                 _api_username, _api_password = api_userinfo.split(':')
-                api_username ||= _api_username
-                api_password ||= _api_password
+                api_username                 ||= _api_username
+                api_password                 ||= _api_password
               end
-              api_host_use_ssl = api_uri.scheme == 'https' if api_host_use_ssl.nil?
-              api_uri_query = api_uri.query
+              api_host_use_ssl       = api_uri.scheme == 'https' if api_host_use_ssl.nil?
+              api_uri_query          = api_uri.query
               api_default_query_data ||= Hash[api_uri_query.split('&').map { |kp| kp.split('=') }]
-              api_base_path ||= api_uri.path
+              api_base_path          ||= api_uri.path
             end
 
-            api_port ||= (api_host_use_ssl ? 443 : 80)
+            api_port      ||= (api_host_use_ssl ? 443 : 80)
             api_base_path ||= '/'
 
-            api_endpoint_prefix = 'VSAPI'
+            api_endpoint_prefix        = 'VSAPI'
             api_noauth_endpoint_prefix = 'APInoauth'
 
-            client_args = { }
-            client_args[:http_host_address] = api_host if api_host
-            client_args[:http_host_port] = api_port if api_port
-            client_args[:http_host_use_ssl] = api_host_use_ssl if api_host_use_ssl
-            client_args[:username] = api_username if api_username
-            client_args[:password] =  api_password if api_password
-            client_args[:default_base_path] = api_base_path if api_base_path
-            client_args[:default_query_data] = api_default_query_data if api_default_query_data
-            client_args[:api_endpoint_prefix] = api_endpoint_prefix
+            client_args                              = {}
+            client_args[:http_host_address]          = api_host if api_host
+            client_args[:http_host_port]             = api_port if api_port
+            client_args[:http_host_use_ssl]          = api_host_use_ssl if api_host_use_ssl
+            client_args[:username]                   = api_username if api_username
+            client_args[:password]                   = api_password if api_password
+            client_args[:default_base_path]          = api_base_path if api_base_path
+            client_args[:default_query_data]         = api_default_query_data if api_default_query_data
+            client_args[:api_endpoint_prefix]        = api_endpoint_prefix
             client_args[:api_noauth_endpoint_prefix] = api_noauth_endpoint_prefix
 
             if api_auth_token
               # Cantemo Portal supports an auth token for authentication, replace basic auth with auth-token
-              client_args[:authorization_header_key] = 'Auth-Token'
+              client_args[:authorization_header_key]   = 'Auth-Token'
               client_args[:authorization_header_value] = api_auth_token
             end
 
@@ -91,39 +97,38 @@ module Envoi
             begin
               _client.version
             rescue => e
-              e.message = "Error connecting to Portal: #{e.message}"
-              raise e
+              raise "Error connecting to Portal: #{e.message}"
             end
 
             _client
           end
 
           @default_vidispine_shape_tag = args[:default_shape_tag] || api_config['default_shape_tag'] || api_config['shape_tag'] || DEFAULT_SHAPE_TAG
-    
+
         end
-    
+
         def item_get_shape_by_tag(item_id, shape_tag)
           item_shapes_get_response = api_client.item_shapes_get(:item_id => item_id, :tag => shape_tag)
-          shape_id = item_shapes_get_response['uri'].first
+          shape_id                 = item_shapes_get_response['uri'].first
         end
 
-        def download(args = { })
+        def download(args = {})
 
-          item_id = args[:item_id]
+          item_id  = args[:item_id]
           shape_id = args[:shape_id]
           unless shape_id && !shape_id.empty?
             shape_tag = args[:shape_tag] || default_vidispine_shape_tag
-            shape_id = item_get_shape_by_tag(item_id, shape_tag)
+            shape_id  = item_get_shape_by_tag(item_id, shape_tag)
           end
 
-          logger.info { "Getting file path for Item ID: #{item_id} Shape ID: #{shape_id}"}
+          logger.info { "Getting file path for Item ID: #{item_id} Shape ID: #{shape_id}" }
           item_shape_get_response = api_client.item_shape_get(:item_id => item_id, :shape_id => shape_id)
 
           files = item_shape_get_response['containerComponent']['file']
-          logger.debug { "Files: #{files}"}
+          logger.debug { "Files: #{files}" }
 
           # file = files.first
-          files = [ files.first ] # just do the first file for now
+          files = [files.first] # just do the first file for now
           files.each do |file|
             begin
               download_file(args, file)
@@ -135,13 +140,13 @@ module Envoi
         end
 
         def download_file(args, file)
-          logger.debug { "File: #{file}"}
+          logger.debug { "File: #{file}" }
           transfer_type = args[:transfer_type]
 
           file_storage_id = file['storage']
-          file_path = file['path']
+          file_path       = file['path']
 
-          file_storage_config = agent_config['storages'][file_storage_id]
+          file_storage_config = agent_config_storages[file_storage_id]
 
           unless file_storage_config && !file_storage_config.empty?
             raise Exception, "No configuration found for storage '#{file_storage_id}'"
@@ -151,8 +156,8 @@ module Envoi
           preserve_path = args.fetch(:preserve_path, file_storage_config.fetch('preserve_path', default_preserve_file_path))
 
           destination_path = args[:destination_path] || file_storage_config['destination_path'] || DEFAULT_DESTINATION_PATH
-          relative_path = preserve_path ? File.dirname(file_path) : nil
-          relative_path = nil if relative_path == '.'
+          relative_path    = preserve_path ? File.dirname(file_path) : nil
+          relative_path    = nil if relative_path == '.'
 
           target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
           target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
@@ -168,15 +173,15 @@ module Envoi
           if (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
             target_path = File.expand_path(target_path) if target_path == '.'
             target_path = File.join(target_path, File.basename(file_path))
-            client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
+            client      = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
             return client.download(s3_config, file_path, target_path)
           end
 
           logger.warn { "No Supported TransferClient Configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}Found in Storage Configuration." }
         end
 
-        def upload(args = { })
-          _response = { }
+        def upload(args = {})
+          _response = {}
 
           file_path = args[:file_path]
           raise ArgumentError, "Path not found: '#{file_path}'" unless File.exists?(file_path)
@@ -184,16 +189,16 @@ module Envoi
           if File.directory?(file_path)
             # Non-recursive directory upload
             file_paths = Dir.glob(File.join(file_path, '*.*'))
-            logger.debug { "File Paths: #{file_paths}"}
-            file_paths.map { |fp| upload(args.merge(file_path: fp))}
+            logger.debug { "File Paths: #{file_paths}" }
+            file_paths.map { |fp| upload(args.merge(file_path: fp)) }
             return file_paths
           end
           logger.debug { "Preparing to upload '#{file_path}'" }
 
-          transfer_type = args[:transfer_type] || ''
-          storage_id = args[:storage_id]
-          vidispine_storage_config = agent_config['storages'][storage_id]
-    
+          transfer_type            = args[:transfer_type] || ''
+          storage_id               = args[:storage_id]
+          vidispine_storage_config = agent_config_storages[storage_id]
+
           unless vidispine_storage_config && !vidispine_storage_config.empty?
             raise "No configuration found for storage '#{storage_id}'"
           end
@@ -201,37 +206,37 @@ module Envoi
           should_import_file = args.fetch(:import_file, vidispine_storage_config.fetch('import', true))
 
           should_preserve_path = args.fetch(:preserve_path, vidispine_storage_config.fetch('preserve_path', default_preserve_file_path))
-    
+
           destination_path = args[:destination_path] || vidispine_storage_config['destination_path'] || '/'
-          relative_path = should_preserve_path ? File.dirname(file_path) : nil
-          relative_path = File.expand_path(relative_path) if relative_path == '.'
-    
+          relative_path    = should_preserve_path ? File.dirname(file_path) : nil
+          relative_path    = File.expand_path(relative_path) if relative_path == '.'
+
           target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
           target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
-    
-    
+
+
           # upload file
 
           transfer_response = begin
-            response = nil
+            response      = nil
             aspera_config = vidispine_storage_config['aspera']
             begin
               if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
-                client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
+                client   = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
                 response = client.upload(aspera_config, file_path, target_path)
               end
             rescue => e
               logger.error { "Aspera Transfer Failed. '#{e.message}'\n#{e.backtrace.first}" }
             end
-    
+
             s3_config = vidispine_storage_config['s3']
             begin
               if !response && (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
                 _target_path = target_path
                 _target_path = File.expand_path(_target_path) if target_path == '.'
                 _target_path = File.join(_target_path, File.basename(file_path))
-                client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
-                response = client.upload(s3_config, file_path, _target_path)
+                client       = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
+                response     = client.upload(s3_config, file_path, _target_path)
               end
             rescue => e
               logger.error { "S3 Transfer Failed. '#{e.message}'" }
@@ -263,9 +268,9 @@ module Envoi
 
 
           ### IMPORT - START
-          import_file_args = args.dup
+          import_file_args                   = args.dup
           import_file_args[:response_object] = _response
-          import_file_args[:target_path] = target_path
+          import_file_args[:target_path]     = target_path
           import_file(import_file_args)
           _response[:success] = true unless _response[:success] === false
           ### IMPORT - END
@@ -277,31 +282,46 @@ module Envoi
           return _response
         end
 
-        def import_file(args = { })
-          _response = args[:response_object] || { }
-
-          file_path = args[:file_path]
+        # @param [Hash]  args
+        # @option args [String] :file_path
+        # @option args [Hash] :import_args ({})
+        # @option args [Hash] :import_options ({})
+        # @option args [String] :item_id
+        # @option args [Hash] :response_object ({})
+        # @option args [String] :shape_id (@default_vidispine_shape_tag)
+        # @option args [String] :storage_id
+        # @option args [String] :target_path
+        #
+        # @return [Hash{ :success->Boolean, :file_create_response->{}, :import_response->{} }]
+        def import_file(args = {})
+          _response = args[:response_object] || {}
+
+          file_path  = args[:file_path]
           storage_id = args[:storage_id]
 
           target_path = args[:target_path]
 
-          item_id = args[:item_id]
+          item_id   = args[:item_id]
           shape_tag = args[:shape_tag] || default_vidispine_shape_tag
 
+          import_args_in = args[:import_args]
+          import_options = args[:import_options] || {}
+
+
           # attach file to item as shape
           path_on_storage = File.join(target_path, File.basename(file_path))
           path_on_storage = path_on_storage[1..-1] if path_on_storage.start_with?('/')
           # file_create_response = api_client.storage_file_create(storage_id: storage_id,
           #                                                       path: path_on_storage, state: 'CLOSED')
-          file_create_response = api_client.storage_file_get_or_create(storage_id, path_on_storage, { :extended_response => true })
+          file_create_response             = api_client.storage_file_get_or_create(storage_id, path_on_storage, { :extended_response => true })
           _response[:file_create_response] = file_create_response
-          file = file_create_response[:file]
+          file                             = file_create_response[:file]
 
           file_id = file['id']
 
           unless file_id
             _file = file.dup
-            _file.keep_if { |k,v| v }
+            _file.keep_if { |k, v| v }
             logger.error { "Failed to create file. #{_file}" }
             _response[:success] = false
             return _response
@@ -311,23 +331,28 @@ module Envoi
 
           if item
             shape = (item['shape'] || []).first
-            msg = "File already exist and is associated to item #{item['id']} as shape #{shape['id']}."
+            msg   = "File already exist and is associated to item #{item['id']} as shape #{shape['id']}."
             logger.warn { "#{msg} #{file}" }
-            _response[:error] = { :message => msg }
+            _response[:error]   = { :message => msg }
             _response[:success] = false
             return _response
           end
 
           if item_id
             item_shape_import_response = api_client.item_shape_import(item_id: item_id,
-                                                                      tag: shape_tag, fileId: file_id)
+                                                                      tag:     shape_tag, fileId: file_id)
 
 
           else
-            item_shape_import_response = api_client.item_add_using_file_path(storage_id: storage_id,
-                                                                             file_path: path_on_storage,
-                                                                             fileId: file_id,
-                                                                             storage_path_map: { '/' => storage_id })
+            import_args = {
+                storage_id:       storage_id,
+                file_path:        path_on_storage,
+                fileId:           file_id,
+                storage_path_map: { '/' => storage_id }
+            }
+            import_args.merge!(import_args_in) if import_args_in
+
+            item_shape_import_response = api_client.item_add_using_file_path(import_args, import_options)
           end
           _response[:import_response] = item_shape_import_response
 
@@ -339,7 +364,6 @@ module Envoi
         end
 
 
-
         def ingest_file
           # 1) export ASPERA_SCP_TOKEN="ATB3_AEA9dUI5dd2u1k8XBMVD0qInR-Lyylkz8AylTXLVt5_SMVGR8SO7Sdc0lj6RkoHK_DvAAEWac8bllF_Yan1NbbDTyPj_3BTA"
           # 2) ascp -i asperaweb_id_dsa.openssh -P 33001 -l 20m /Users/nicholasstokes/Desktop/CantemoAgent.mov xfer@ats-aws-us-west-2.aspera.io:
@@ -363,10 +387,10 @@ module Envoi
 
         # Agent
       end
-    
+
       # Cantemo
     end
-    
+
   end
 
 end