cantemo-portal-agent 1.2.4 → 1.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: de359aa5f03562ec1007a7bdb3a9dc0af6362216
-  data.tar.gz: 1fb3d59042129c18561dd1504a9ace1a5508a385
+  metadata.gz: f0df20d26170e018fbb101b896771dda9a848a8a
+  data.tar.gz: 977b72cf9a04ce9c70b31dc03436a57c78a5675a
 SHA512:
-  metadata.gz: 4a75c0ac6f20dedffbfb5cd7575828ea43ea29c75b1c4fea6f56c029ac4ef89dcd7d740d2b048bb04f3c455ef3420f4b7c16264ac799d7f0d4b6334dda0f87b2
-  data.tar.gz: b793017b7a68087dd13d1a278c9f37ca2cbf77bbf4a29455b9dd7f70f97ac137e4c3304698027c8cb37bb988855ff560f46da54782be700ede429b4750ee10a3
+  metadata.gz: 5173fe6dbe088d8ab5195c59c3666887d957cd2876e86c65463dd3da4386594973252ddb93fc714ef6831c0685cda06db0cbd8413424e9e46b8c6646b233938f
+  data.tar.gz: 4ad8f5e06305f2cd174c3bcb6a86c7959414ba257dcd59580cfd32dd9fe7ef072aedef4033f840e732987237738e8a4616f9b0586b57d8266cdaf7c8c4fffd4a

data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb

@@ -35,19 +35,22 @@ def args; @args end
 op = OptionParser.new
 op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
       "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
-op.on('-b', '--[no-]background', 'Tells the application to run in the background.') { |v| args[:daemonize] = v }
-op.on('-v', '--version', 'Output the version and exit.') { puts "Version #{Cantemo::Portal::Agent::VERSION}"; exit }
-op.on('--help', 'Show this message.') { puts op; exit }
+# op.on('-b', '--[no-]background', 'Tells the application to run in the background.') { |v| args[:daemonize] = v }
+op.on_tail('-t', '--log-to FILEPATH', 'Log file to log to') { |log_to| args[:log_to] = log_to }
+op.on_tail('-l', '--log-level LEVEL', ['fatal', 'error', 'warn', 'info', 'debug'], 'Select logging level') do |log_level|
+  args[:log_level] = log_level.upcase
+end
+op.on_tail('-v', '--version', 'Output the version and exit.') { puts "Version #{Cantemo::Portal::Agent::VERSION}"; exit }
+op.on_tail('--help', 'Show this message.') { puts op; exit }
 op.load
 op.parse!
 
 config_file_path = args[:config_file_path]
 args[:config_file_path].map! { |v| File.expand_path(v) } if config_file_path.is_a?(Array)
 
-command = ARGV.first.dup
-
+command = ARGV.first
 daemon_control_command_present = command && begin
-  command.downcase!
+  command = command.dup.downcase if command
   %w(start stop restart run zap killall status).include?(command)
 end
 
@@ -71,9 +74,11 @@ should_daemonize = (daemon_control_command_present && %w(start restart).include?
 #
 #
 
-
-
 class Watcher < Envoi::Mam::Cantemo::Agent::WatchFolderManager; end
 # daemonize ? Watcher.run_as_daemon(args, { force: true }) : Watcher.run(args)
 # daemon_control_command_present ? Watcher.run_as_daemon(args) : Watcher.run(args)
-Watcher.run_with_process_manager(args)
+begin
+  Watcher.run_with_process_manager(args)
+rescue => e
+  abort("An error occurred. #{e.message}\n#{e.backtrace}")
+end

data/lib/cantemo/portal/agent/version.rb

@@ -1,7 +1,7 @@
 module Cantemo
   module Portal
     class Agent
-      VERSION = '1.2.4'.freeze
+      VERSION = '1.2.5'.freeze
     end
   end
 end

data/lib/envoi/mam/agent/cli/commands/cantemo.rb

@@ -1,4 +1,4 @@
-if [ 'watchfolder', 'watchfolders'].include?((ARGV.first || '').tr('_-', ''))
+if [ 'watchfolder', 'watchfolders'].include?((ARGV.first || '').tr('_-', '').downcase)
   require 'cantemo/portal/agent/cli/commands/watch_folders'
 else
   require 'envoi/mam/agent/cli/commands/cantemo-agent'

data/lib/envoi/mam/cantemo/agent.rb

@@ -132,6 +132,15 @@ module Envoi
           logger.warn { "No Supported TransferClient Configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}Found in Storage Configuration." }
         end
 
+        # @param [Hash] args
+        # @option args [String] :file_path
+        # @option args [String] :storage_id
+        # @option args [String] :transfer_type ('')
+        # @option args [Boolean] :import_file (true)
+        # @option args [Boolean] :preserve_path (storage_config['preserve_path'] || default_preserve_path)
+        # @return [Hash]
+        #
+        #
         def upload(args = {})
           _response = {}
 
@@ -149,17 +158,18 @@ module Envoi
 
           transfer_type            = args[:transfer_type] || ''
           storage_id               = args[:storage_id]
-          vidispine_storage_config = agent_config_storages[storage_id]
+          storage_config = agent_config_storages[storage_id]
 
-          unless vidispine_storage_config && !vidispine_storage_config.empty?
+          unless storage_config && !storage_config.empty?
             raise "No configuration found for storage '#{storage_id}'"
           end
 
-          should_import_file = args.fetch(:import_file, vidispine_storage_config.fetch('import', true))
+          should_import_file = args.fetch(:import_file, storage_config.fetch('import', true))
 
-          should_preserve_path = args.fetch(:preserve_path, vidispine_storage_config.fetch('preserve_path', default_preserve_file_path))
+          should_preserve_path = args.fetch(:preserve_path,
+                                            storage_config.fetch('preserve_path', default_preserve_file_path))
 
-          destination_path = args[:destination_path] || vidispine_storage_config['destination_path'] || '/'
+          destination_path = args[:destination_path] || storage_config['destination_path'] || '/'
           relative_path    = should_preserve_path ? File.dirname(file_path) : nil
           relative_path    = File.expand_path(relative_path) if relative_path == '.'
 
@@ -171,7 +181,7 @@ module Envoi
 
           transfer_response = begin
             response      = nil
-            aspera_config = vidispine_storage_config['aspera']
+            aspera_config = storage_config['aspera']
             begin
               if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
                 client   = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
@@ -181,7 +191,7 @@ module Envoi
               logger.error { "Aspera Transfer Failed. '#{e.message}'\n#{e.backtrace.first}" }
             end
 
-            s3_config = vidispine_storage_config['s3']
+            s3_config = storage_config['s3']
             begin
               if !response && (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
                 _target_path = target_path
@@ -202,7 +212,7 @@ module Envoi
           _response[:transfer_response] = transfer_response
 
           if transfer_response.nil?
-            logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}found in storage configuration." }
+            logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}'" : ''} found in storage configuration." }
             _response[:success] = false
             return _response
           end
@@ -240,7 +250,7 @@ module Envoi
         # @option args [Hash] :import_options ({})
         # @option args [String] :item_id
         # @option args [Hash] :response_object ({})
-        # @option args [String] :shape_id (@default_vidispine_shape_tag)
+        # @option args [String] :shape_tag (@default_vidispine_shape_tag)
         # @option args [String] :storage_id
         # @option args [String] :target_path
         #
@@ -256,9 +266,8 @@ module Envoi
           item_id   = args[:item_id]
           shape_tag = args[:shape_tag] || default_vidispine_shape_tag
 
-          import_args_in = args[:import_args]
-          import_options = args[:import_options] || {}
-
+          item_add_args_in = args[:item_add_args] || { }
+          item_add_options_in = args[:item_add_options] || { }
 
           # attach file to item as shape
           path_on_storage = File.join(target_path, File.basename(file_path))
@@ -292,28 +301,26 @@ module Envoi
 
           if item_id
             item_shape_import_response = api_client.item_shape_import(item_id: item_id,
-                                                                      tag:     shape_tag, fileId: file_id)
+                                                                      tag:     shape_tag, file_id: file_id)
           else
-            import_args = {
-                storage_id:       storage_id,
-                file_path:        path_on_storage,
-                fileId:           file_id,
-                storage_path_map: { '/' => storage_id }
+            item_add_args = {
+              storage_id:       storage_id,
+              file_path:        path_on_storage,
+              file_id:           file_id,
+              storage_path_map: { '/' => storage_id }
             }
-            import_args.merge!(import_args_in) if import_args_in
+            item_add_args.merge(item_add_args_in) if item_add_args_in.is_a?(Hash)
 
-            item_shape_import_response = api_client.item_add_using_file_path(import_args, import_options)
+            item_add_options = { }
+            item_add_options.merge!(item_add_options_in) if item_add_options_in.is_a?(Hash)
+
+            item_shape_import_response = api_client.item_add_using_file_path(item_add_args, item_add_options)
           end
           _response[:import_response] = item_shape_import_response
 
           _response
         end
 
-        def upload_transfer_spec_get(file_paths)
-
-        end
-
-
         def ingest_file
           # 1) export ASPERA_SCP_TOKEN="ATB3_AEA9dUI5dd2u1k8XBMVD0qInR-Lyylkz8AylTXLVt5_SMVGR8SO7Sdc0lj6RkoHK_DvAAEWac8bllF_Yan1NbbDTyPj_3BTA"
           # 2) ascp -i asperaweb_id_dsa.openssh -P 33001 -l 20m /Users/nicholasstokes/Desktop/CantemoAgent.mov xfer@ats-aws-us-west-2.aspera.io:

data/lib/envoi/mam/cantemo/agent/watch_folder_manager.rb

@@ -42,6 +42,8 @@ module Envoi::Mam::Cantemo
         args[:default_preserve_file_path] = args.fetch(:default_preserve_file_path, false)
 
         @config = Envoi::Mam::Cantemo::Agent.load_config_from_file(args)
+        initialize_logger_from_config
+
         cantemo_config = config[:cantemo] || config['cantemo']
 
         @default_agent_class = Envoi::Mam::Cantemo::Agent
@@ -63,7 +65,7 @@ module Envoi::Mam::Cantemo
         @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
         @ignored_file_paths_lock            = Mutex.new
 
-        @threaded = args.fetch(:threaded, true)
+        @threaded = args.fetch(:threaded, config.fetch(:threaded, config.fetch('threaded', true)))
 
         @default_maximum_active_processors = DEFAULT_WATCH_FOLDER_PROCESSOR_LIMIT
         @processors_by_watch_folder = Hash.new { |h, k| h[k] = {} }
@@ -73,7 +75,7 @@ module Envoi::Mam::Cantemo
 
       def initialize_logger(args = {})
         @logger   = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
-        log_level = args[:log_level] = Logger::DEBUG
+        log_level = args[:log_level] = Logger::INFO
         if log_level
           @logger.level = log_level
           args[:logger] = @logger
@@ -81,6 +83,18 @@ module Envoi::Mam::Cantemo
         @logger
       end
 
+      def initialize_logger_from_config(_config = @config)
+        logger_args = { }
+        log_to = _config['log_to']
+        logger_args[:log_to] = log_to if log_to
+        log_level = _config['log_level']
+        if log_level
+          _log_level = ['fatal', 'error', 'warn', 'info', 'debug'].find(log_level.downcase)
+          logger_args[:log_level] = _log_level.upcase if _log_level
+        end
+        initialize_logger(logger_args) unless logger_args.empty?
+      end
+
       # @param [Hash] watch_folder_def
       # @option watch_folder_def [String] path
       # @option watch_folder_def [String] upload_to_storage_id
@@ -140,7 +154,7 @@ module Envoi::Mam::Cantemo
       end
 
       # @param [Object] file
-      # @return [FalseClass]
+      # @return [Hash]
       def process_file(file)
         file.processing = true
         file_name       = file.name || file.path
@@ -150,20 +164,26 @@ module Envoi::Mam::Cantemo
         agent ||= default_agent
 
         storage_id                = watch_folder.definition['upload_to_storage_id']
-        quarantine_directory_path = watch_folder.definition['quarantine_directory_path']
-        completed_directory_path  = watch_folder.definition['completed_directory_path']
-        import_args               = watch_folder.definition['import_args']
-        import_options            = watch_folder.definition['import_options']
 
-        # full_file_path = File.join(watch_folder.path, file.path)
-        full_file_path = file.path
         unless storage_id
           logger.warn { "Skipping processing of file because of missing storage ID." }
           return { success: false, message: 'Missing storage ID.' }
         end
 
-        _response = agent.upload(file_path: full_file_path,
-                                 storage_id: storage_id, import_args: import_args, import_options: import_options)
+        quarantine_directory_path = watch_folder.definition['quarantine_directory_path']
+        completed_directory_path  = watch_folder.definition['completed_directory_path']
+        watch_folder_upload_args  = watch_folder.definition['upload_args']
+
+        # full_file_path = File.join(watch_folder.path, file.path)
+        full_file_path = file.path
+
+        upload_args = {
+          file_path: full_file_path,
+          storage_id: storage_id
+        }
+        upload_args.merge!(watch_folder_upload_args) if watch_folder_upload_args.is_a?(Hash)
+
+        _response = agent.upload(upload_args)
         _response = { success: _response } if _response == true || _response == false
 
         if _response[:success]
@@ -219,7 +239,6 @@ module Envoi::Mam::Cantemo
           ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
         end
 
-
         stable_files.each do |file|
           next if file.respond_to?(:ignore?) ? file.ignore? : ignored_file_paths.include?(file.path)
           next if file.processing || file.processed
@@ -240,9 +259,9 @@ module Envoi::Mam::Cantemo
 
           if @threaded
             active_processors.keep_if { |k, v| k.processing }
-            if active_processors.length > maximum_active_processors
+            if active_processors.length >= maximum_active_processors
               logger.debug { "Maximum number of active processors reached for watch folder. #{wf.name || wf.paths}" }
-              next
+              break
             end
             t = Thread.new(file) do |file|
               wf = file.watch_folder
@@ -302,7 +321,8 @@ module Envoi::Mam::Cantemo
         end
         logger.info { 'Exiting...' }
       rescue => e
-        logger.error { "An error occurred.\n#{e.message}\n#{e.backtrace.join("\n")}\n#{e.message}" }
+        logger.debug { "EXCEPTION #{e.message} \n#{e.backtrace.join("\n")}\n" }
+        logger.error { "An error occurred. #{e.message}" }
         raise e
       ensure
         watch_folders.each { |wf| wf.stop if wf.respond_to?(:stop) }
@@ -331,8 +351,6 @@ module Envoi::Mam::Cantemo
       def self.run(args)
         w = self.new(args)
         w.run
-      rescue => e
-        abort("An error occurred. #{e.message}\n#{e.backtrace.join("\n")}")
       end
 
       def self.daemons_run_proc_with_cleanup(options, &block)

data/lib/envoi/watch_folder_utility/watch_folder.rb

@@ -49,7 +49,7 @@ module Envoi
 
       def initialize_logger(args = {})
         @logger   = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
-        log_level = args[:log_level] = Logger::DEBUG
+        log_level = args[:log_level] = Logger::INFO
         if log_level
           @logger.level = log_level
           args[:logger] = @logger
@@ -220,13 +220,20 @@ module Envoi
         end
         logger.debug { "Parameter 'upload to storage id' initialized." }
 
-        logger.debug { "Initializing parameter 'import args'." }
-        import_args = watch_folder_def['import_args'] || { }
-        import_options = watch_folder_def['import_options'] || { }
+        logger.debug { "Initializing parameter 'upload/import arguments'." }
+        upload_args               = watch_folder_def['upload_args']
+        item_add_args             = watch_folder_def['item_add_args']
+        item_add_options          = watch_folder_def['item_add_options']
+        import_args               = watch_folder_def['import_args'] || { }
+        import_options            = watch_folder_def['import_options'] || { }
+
         import_args = Hash[import_args.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym.downcase : k, v ] }]
         import_options = Hash[import_options.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym.downcase : k, v ] }]
 
-        add_item_to_collection = import_options.fetch(:add_item_to_collection, import_args[:add_item_to_collection])
+        # Allow adding to collection to be overridden
+        add_item_to_collection = import_options.fetch(:add_item_to_collection,
+                                                      import_args.fetch(:add_item_to_collection,
+                                                                        watch_folder_def['add_item_to_collection']))
         if add_item_to_collection.nil? || add_item_to_collection
           _add_item_to_collection = false
           collection_id = watch_folder_def['collection_id']
@@ -256,10 +263,36 @@ module Envoi
         if metadata
           import_args[:metadata] = metadata
           watch_folder_def.delete('metadata')
-          metadata_map = watch_folder_def['metadata_map']
+        end
+
+        metadata_map = watch_folder_def['metadata_map']
+        metadata_group = watch_folder_def['metadata_group']
+        if metadata_map || metadata_group
+          ((metadata_map ||= { })[:__build_metadata_document_options] ||= {})[:parent_group_name] = metadata_group if metadata_group
           import_options[:metadata_map] ||= metadata_map if metadata_map
         end
-        logger.debug { "Parameter 'import args' initialized." }
+
+        ingest_group = watch_folder_def['ingest_group']
+        if ingest_group
+          job_metadata = (import_args[:jobmetadata] ||= '')
+          job_metadata += ',' unless job_metadata.empty?
+          job_metadata += "portal_groups:StringArray=#{ingest_group}"
+          import_args[:jobmetadata] = job_metadata
+        end
+
+        item_add_args = symbolize_keys(item_add_args)
+        ((item_add_args ||= {})[:import_args] ||= {}).merge! import_args if import_args.is_a?(Hash)
+        ((item_add_args ||= {})[:import_options] ||= {}).merge! import_options if import_options.is_a?(Hash)
+
+        upload_args = symbolize_keys(upload_args)
+        ((upload_args ||= {})[:item_add_args] ||= {}).merge! item_add_args if item_add_args.is_a?(Hash)
+        ((upload_args ||= {})[:item_add_options] ||= {}).merge! symbolize_keys(item_add_options) if item_add_options.is_a?(Hash)
+
+        watch_folder_def.delete('import_args')
+        watch_folder_def.delete('import_options')
+
+        watch_folder_def['upload_args'] = upload_args
+        logger.debug { "Parameter 'upload/import arguments' initialized." }
 
 
         maximum_active_processors = watch_folder_def['maximum_active_processors']
@@ -305,6 +338,21 @@ module Envoi
         handler.stable_files
       end
 
+      # Converts hash keys to symbols
+      #
+      # @param [Hash] value hash
+      # @param [Boolean] recursive Will recurse into any values that are hashes or arrays
+      def symbolize_keys (value, recursive = true)
+        case value
+        when Hash
+          Hash[value.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym : k, recursive ? symbolize_keys(v, true) : v ] }]
+        when Array
+          value.map { |v| symbolize_keys(v, recursive) }
+        else
+          value
+        end
+      end
+
     end
 
   end

data/lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb

@@ -121,7 +121,7 @@ module Envoi
           end
 
           def process_deleted_path(path)
-            return path.each { |path| process_deleted_path(path, event_type) } if path.is_a?(Array)
+            return path.each { |path| process_deleted_path(path) } if path.is_a?(Array)
             logger.debug { "PATH #{:deleted} '#{path}'" }
             file = known_path_map[path]
             known_path_map.delete(path)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cantemo-portal-agent
 version: !ruby/object:Gem::Version
-  version: 1.2.4
+  version: 1.2.5
 platform: ruby
 authors:
 - John Whitson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-02 00:00:00.000000000 Z
+date: 2019-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asperalm