cantemo-portal-agent 1.2.6 → 1.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f0d27e760275d391be6c1ddcf25ef719b0dfc3c
-  data.tar.gz: 48467f20279721c70111f4cb9f5df80434449e64
+  metadata.gz: edd73213f17ac2ccac314ef21efd8ee07e68bd77
+  data.tar.gz: f31df7db4af1f985a4f53a870b278c5f0c94b5f9
 SHA512:
-  metadata.gz: 2f5839f0f15c28da04609aa8e8b4c699b252a0fc16915cc68c935a2fa801b01fd1ff99089ff6bf1320e6236747a159d380c55df35fd84d8ac40f2d1df29b1592
-  data.tar.gz: a27477de9b6679772c3b45bec647a45588d4567765b628180bdf4897c4534df99da9ef4ca9a089885fd101e7b1b7aac03c5e48508ac88d9c3f1fdbbbef713439
+  metadata.gz: 1baaacd2944586fd81f65f7f4e58fb978ed7e641481d064fd3d76378478a93d1c13575fea5ba676fe8d6bd714a9e7fd8a9e0fa8fa722ad22729db331f62cf7c9
+  data.tar.gz: 294f8ee74c1aaad3f5053c1cc8c1338c08c0a45147529aeac48956cd52bb9f65af45c3ee0ce8fc36d8a06f6ad67c0162b76f321d6912791da58a967b3e71af3f

data/lib/cantemo/portal/agent/version.rb

@@ -1,7 +1,7 @@
 module Cantemo
   module Portal
     class Agent
-      VERSION = '1.2.6'.freeze
+      VERSION = '1.2.7'.freeze
     end
   end
 end

data/lib/envoi/mam/agent/transfer_client/aspera.rb

@@ -3,58 +3,69 @@ require 'asperalm/fasp/local'
 
 
 
+# module Asperalm
+#   module Fasp
+#     module InstallationExtensions
+#
+#       PRODUCTS = {
+#         unix: {
+#           :aspera_cli_local => {
+#             :expected=>'Aspera CLI',
+#             :exe_ext=>'',
+#             :app_root=>File.join(Dir.home,'.aspera','cli'),
+#             :run_root=>File.join(Dir.home,'.aspera','cli'),
+#             :sub_bin=>'bin',
+#             :sub_keys=>'etc',
+#             :dsa=>'asperaweb_id_dsa.openssh'
+#           }
+#         }
+#       }
+#
+#       def find_missing_products(os_type, common_places)
+#         missing_products = []
+#         case os_type
+#         when :unix
+#           extended_products = PRODUCTS[os_type] || { }
+#           extended_products.each do |product_name, product|
+#             product_found = common_places.find do |cp|
+#               cp[:expected] == product[:expected] && cp[:app_root] == product[:app_root]
+#             end
+#             missing_products.push product unless product_found
+#           end
+#         end
+#         missing_products
+#       end
+#
+#       def add_missing_products(os_type, common_places)
+#         missing_products = find_missing_products(os_type, common_places)
+#         common_places.concat missing_products unless missing_products.empty?
+#       end
+#
+#       def product_locations
+#         common_places = super
+#         add_missing_products(OpenApplication.current_os_type, common_places)
+#         common_places
+#       end
+#
+#     end
+#
+#     class Installation
+#       include InstallationExtensions
+#     end
+#   end
+# end
+
 module Asperalm
-  module Fasp
-    module InstallationExtensions
-
-      PRODUCTS = {
-        unix: {
-          :aspera_cli_local => {
-            :expected=>'Aspera CLI',
-            :exe_ext=>'',
-            :app_root=>File.join(Dir.home,'.aspera','cli'),
-            :run_root=>File.join(Dir.home,'.aspera','cli'),
-            :sub_bin=>'bin',
-            :sub_keys=>'etc',
-            :dsa=>'asperaweb_id_dsa.openssh'
-          }
-        }
-      }
-
-      def find_missing_products(os_type, common_places)
-        missing_products = []
-        case os_type
-        when :unix
-          extended_products = PRODUCTS[os_type] || { }
-          extended_products.each do |product_name, product|
-            product_found = common_places.find do |cp|
-              cp[:expected] == product[:expected] && cp[:app_root] == product[:app_root]
-            end
-            missing_products.push product unless product_found
-          end
-        end
-        missing_products
-      end
 
-      def add_missing_products(os_type, common_places)
-        missing_products = find_missing_products(os_type, common_places)
-        common_places.concat missing_products unless missing_products.empty?
-      end
+  class Log
 
-      def product_locations
-        common_places = super
-        add_missing_products(OpenApplication.current_os_type, common_places)
-        common_places
-      end
+    def logger=(new_logger) @logger = new_logger end
 
-    end
-
-    class Installation
-      include InstallationExtensions
-    end
   end
+
 end
 
+
 module Envoi
   module Mam
     class Agent
@@ -122,7 +133,9 @@ module Envoi
             env_vars['ASPERA_SCP_PASS'] = aspera_password if aspera_password
             # env_vars['ASPERA_SCP_TOKEN'] = aspera_token if aspera_token
 
-            ascp_args = config['ascp_args'] || default_ascp_args || agent.default_ascp_args
+            ascp_args = config['ascp_args'] ||
+                default_ascp_args ||
+                (agent.respond_to?(:default_ascp_args) ? agent.default_ascp_args : '')
 
             tags = config['tags'] ||= { }
             aspera_tags = tags['aspera'] ||= { }
@@ -220,7 +233,8 @@ module Envoi
           def transfer_using_asperala(config, mode, source_path, destination_path)
             args_out = build_asperala_transfer_args(config, mode, source_path, destination_path)
             @fasp ||= Asperalm::Fasp::Local.instance
-            Asperalm::Log.instance.level = :debug #logger.level
+            Asperalm::Log.instance.logger = logger
+            # Asperalm::Log.instance.level = :debug #logger.level
             @fasp.start_transfer_with_args_env(args_out, {})
             { success: true }
           end

data/lib/envoi/mam/cantemo/agent/watch_folder_manager.rb

@@ -27,6 +27,25 @@ module Envoi::Mam::Cantemo
 
     class WatchFolderManager
 
+      class MultiIO
+        def initialize(*targets)
+          @targets = targets
+        end
+
+        def write(*args)
+          @targets.each {|t| t.write(*args)}
+        end
+
+        def close
+          @targets.each(&:close)
+        end
+
+        def add_target(*target)
+          @targets.concat target
+        end
+
+      end
+
       # AWF = Envoi::Aspera::WatchService::WatchFolder # Aspera Watch Folder
       LWF = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen # Listen Watch Folder
 
@@ -45,39 +64,51 @@ module Envoi::Mam::Cantemo
         initialize_logger_from_config
         args[:logger] = @logger
 
-        cantemo_config = config[:cantemo] || config['cantemo']
+        @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
+        @ignored_file_paths_lock            = Mutex.new
+
+        @threaded = args.fetch(:threaded, config.fetch(:threaded, config.fetch('threaded', true)))
+
+        @default_maximum_active_processors = DEFAULT_WATCH_FOLDER_PROCESSOR_LIMIT
+        @processors_by_watch_folder = Hash.new { |h, k| h[k] = {} }
+
+        @watch_folder_defs = config[:watch_folders] || config['watch_folders']
 
         @default_agent_class = Envoi::Mam::Cantemo::Agent
 
+        cantemo_config = config[:cantemo] || config['cantemo']
         if cantemo_config
           logger.debug { 'Initializing Default Cantemo Portal Agent.' }
           @default_agent = @default_agent_class.new(args.merge({ config: cantemo_config }))
           logger.debug { 'Default Cantemo Portal Agent Initialized.' }
           @default_storages = @default_agent.agent_config_storages
+
+          if !@watch_folder_defs
+            @watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
+          end
         else
           @default_agent = nil
         end
 
-        @watch_folder_defs = config[:watch_folders] || config['watch_folders']
-        if !@watch_folder_defs && cantemo_config
-          @watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
-        end
-
-        @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
-        @ignored_file_paths_lock            = Mutex.new
-
-        @threaded = args.fetch(:threaded, config.fetch(:threaded, config.fetch('threaded', true)))
-
-        @default_maximum_active_processors = DEFAULT_WATCH_FOLDER_PROCESSOR_LIMIT
-        @processors_by_watch_folder = Hash.new { |h, k| h[k] = {} }
-
         process_watch_folder_defs
       end
 
       def initialize_logger(args = {})
-        @logger   = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
+        @logger   = args[:logger] ||= begin
+          _log_to = MultiIO.new(STDOUT)
+          log_to = args[:log_to]
+          log_age = args[:log_age] || 'daily'
+          _log_to.add_target(File.open(log_to, 'a')) if log_to
+          _logger = Logger.new(_log_to, log_age)
+        end
+
         log_level = args[:log_level] ||= Logger::INFO
         if log_level
+          if log_level.is_a?(String)
+            log_level.downcase!
+            _log_level = %w(fatal error warn info debug).find { |v| v == log_level }
+            log_level = _log_level ? Logger::Severity.const_get(_log_level.to_sym.upcase) : Logger::INFO
+          end
           @logger.level = log_level
         end
         @logger
@@ -85,13 +116,7 @@ module Envoi::Mam::Cantemo
 
       def initialize_logger_from_config(_config = @config)
         logger_args = { }
-        log_to = _config['log_to']
-        logger_args[:log_to] = log_to if log_to
-        log_level = _config['log_level']
-        if log_level
-          _log_level = ['fatal', 'error', 'warn', 'info', 'debug'].find { |v| v = log_level.downcase }
-          logger_args[:log_level] = _log_level.upcase if _log_level
-        end
+        [ :log_to, :log_level, :log_age ].each { |k| v = _config[k] || _config[k.to_s]; logger_args[k] = v if v }
         initialize_logger(logger_args) unless logger_args.empty?
       end
 
@@ -227,6 +252,7 @@ module Envoi::Mam::Cantemo
         end
       end
 
+      # This should be part of the watch folder but it is here to track active processors globally
       def process_watch_folder_stable_files(wf, stable_files)
         active_processors         = @processors_by_watch_folder[wf]
         maximum_active_processors = wf.definition['maximum_active_processors']
@@ -269,7 +295,6 @@ module Envoi::Mam::Cantemo
               begin
                 process_file(file)
               rescue => e
-                logger.error { "Exception '#{e.message}' in thread for `` `#{file.path}`. " }
                 logger.error { "Exception '#{e.message}' in thread for `#{wf.name || wf.paths}` `#{file.path}`. " }
                 raise e
               ensure

data/lib/envoi/watch_folder_utility/watch_folder.rb

@@ -13,13 +13,24 @@ module Envoi
 
       attr_accessor :logger, :definition, :agent, :handler
 
-      attr_accessor :processors
+      attr_accessor :processors, :active_processors
 
       attr_accessor :last_poll_time,
                     :min_stable_poll_count,
                     :min_stable_time,
                     :poll_interval
 
+
+      class MultiIO
+        def initialize(*targets) @targets = targets end
+
+        def write(*args) @targets.each {|t| t.write(*args)} end
+
+        def close; @targets.each(&:close) end
+
+        def add_target(*target) @targets.concat target end
+      end
+
       def initialize(args = { })
         initialize_logger(args)
         @definition = args[:definition].dup
@@ -48,11 +59,22 @@ module Envoi
       end
 
       def initialize_logger(args = {})
-        @logger   = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
+        @logger   = args[:logger] ||= begin
+          _log_to = MultiIO.new(STDOUT)
+          log_to = args[:log_to]
+          log_age = args[:log_age] || 'daily'
+          _log_to.add_target(File.open(log_to, 'a')) if log_to
+          _logger = Logger.new(_log_to, log_age)
+        end
+
         log_level = args[:log_level] ||= Logger::INFO
         if log_level
+          if log_level.is_a?(String)
+            log_level.downcase!
+            _log_level = %w(fatal error warn info debug).find { |v| v == log_level }
+            log_level = _log_level ? Logger::Severity.const_get(_log_level.to_sym.upcase) : Logger::INFO
+          end
           @logger.level = log_level
-          args[:logger] = @logger
         end
         @logger
       end
@@ -61,7 +83,7 @@ module Envoi
 
         @agent ||= begin
           logger.debug { "Initializing Agent. #{@default_agent_class} #{args}" }
-          _agent = @default_agent_class.new(config: args)
+          _agent = @default_agent_class.new(config: args, logger: logger, default_preserve_file_path: false)
           logger.debug { "Agent Instance created." }
           _agent
         end
@@ -105,12 +127,7 @@ module Envoi
 
       def initialize_handler(watch_folder_def = @definition)
         args_out              = {}
-        logging               = watch_folder_def['logging'] || watch_folder_def
-        log_to                = logging['log_to']
-        log_level             = logging['log_level']
-        args_out[:log_to]     ||= log_to if log_to && !log_to.empty?
-        args_out[:log_level]  ||= log_level if log_level && !log_level.empty?
-        args_out[:logger]     ||= logger.dup unless log_to
+        args_out[:logger]     ||= logger.dup
         args_out[:definition] = watch_folder_def
 
         handler_class = @default_handler_class
@@ -337,6 +354,67 @@ module Envoi
         handler.stable_files
       end
 
+      def process_stable_files
+        active_processors         = @processors_by_watch_folder[wf]
+        maximum_active_processors = wf.definition['maximum_active_processors']
+
+        includes = wf.definition['includes']
+        excludes = wf.definition['excludes']
+
+        ignored_file_paths = wf.ignored_file_paths if wf.respond_to?(:ingored_file_paths)
+        unless ignored_file_paths
+          ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
+        end
+
+        stable_files.each do |file|
+          file.watch_folder ||= wf
+          next if file.respond_to?(:ignore?) ? file.ignore? : ignored_file_paths.include?(file.path)
+          next if file.processing || file.processed
+
+          if includes && !includes.empty?
+            should_include = find_in_patterns(includes, file)
+            unless should_include
+              add_to_ignore(file)
+              next
+            end
+          end
+
+          should_exclude = find_in_patterns(excludes, file)
+          if should_exclude
+            add_to_ignore(file)
+            next
+          end
+
+          if @threaded
+            active_processors.keep_if { |k, v| k.processing }
+            if active_processors.length >= maximum_active_processors
+              logger.debug { "Maximum number of active processors reached for watch folder. #{wf.name || wf.paths}" }
+              break
+            end
+            t = Thread.new(file) do |file|
+              wf = file.watch_folder
+              begin
+                process_file(file)
+              rescue => e
+                logger.error { "Exception '#{e.message}' in thread for `#{wf.name || wf.paths}` `#{file.path}`. " }
+                raise e
+              ensure
+                file.processing = false rescue nil
+              end
+            end
+            t.join
+            active_processors[file] = t if file.processing
+          else
+            process_file(file)
+          end
+
+        end
+      end
+
+      def process_stable_file(file)
+
+      end
+
       # Converts hash keys to symbols
       #
       # @param [Hash] value hash

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cantemo-portal-agent
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 1.2.7
 platform: ruby
 authors:
 - John Whitson