cantemo-portal-agent 1.1.0 → 1.1.1

data/lib/envoi/mam/cantemo/agent/watch_folder_handler-working.rb

@@ -35,6 +35,7 @@ module Envoi::Mam::Cantemo
       end
 
       def add_to_ignore(wf, file)
+        logger.debug { "Adding path to ignore cache: '#{file.path}'" }
         @ignored_file_paths_by_watch_folder[wf] << file.path
       end
 
@@ -64,8 +65,9 @@ module Envoi::Mam::Cantemo
           if exclude
             next if ignored_files.include?(file.path)
             if [*exclude].find { |ep| File.fnmatch(ep, file.path) }
-              logger.debug { "Adding File to Ignore Cache: '#{file.path}'"}
-              ignored_files << file.path
+              add_to_ignore(wf, file)
+              # logger.debug { "Adding File to Ignore Cache: '#{file.path}'"}
+              # ignored_files << file.path
               next
             end
           end
@@ -93,6 +95,7 @@ module Envoi::Mam::Cantemo
         AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
       end
 
+      # Initialize then run
       def self.run(args)
         w = self.new(args)
         w.run

data/lib/envoi/mam/cantemo/agent/watch_folder_manager.rb

@@ -0,0 +1,426 @@
+require 'envoi/mam/cantemo/agent'
+require 'envoi/aspera/watch_service/watch_folder'
+require 'envoi/watch_folder_utility/watch_folder/handler/listen'
+
+module Envoi::Mam::Cantemo
+
+  class Agent
+
+    class WatchFolderManager
+
+      # AWF = Envoi::Aspera::WatchService::WatchFolder # Aspera Watch Folder
+      LWF = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen # Listen Watch Folder
+
+      DEFAULT_WATCH_FOLDER_PROCESSOR_LIMIT = 10
+
+      attr_accessor :logger, :agent, :config, :watch_folder_defs, :watch_folders
+
+      def initialize(args = {})
+        initialize_logger(args)
+
+        logger.debug { 'Initializing Agent Watch Folder Manager.' }
+
+        logger.debug { 'Initializing Cantemo Portal Agent.' }
+        args[:default_preserve_file_path] = args.fetch(:default_preserve_file_path, false)
+        @agent                            = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
+        logger.debug { 'Cantemo Portal Agent Initialized.' }
+
+        @config            = agent.config
+        cantemo_config     = config[:cantemo] || config['cantemo']
+        @watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
+
+        @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
+        @ignored_file_paths_lock            = Mutex.new
+
+        @threaded = args.fetch(:threaded, true)
+
+        @default_maximum_active_processors = DEFAULT_WATCH_FOLDER_PROCESSOR_LIMIT
+        @processors_by_watch_folder = Hash.new { |h, k| h[k] = {} }
+
+        process_watch_folder_defs
+      end
+
+      def initialize_logger(args = {})
+        @logger   = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
+        log_level = args[:log_level] = Logger::DEBUG
+        if log_level
+          @logger.level = log_level
+          args[:logger] = @logger
+        end
+        @logger
+      end
+
+      # @TODO Move most of this code into a watch folder class
+      #
+      # @param [Hash] watch_folder_def
+      # @option watch_folder_def [String] path
+      # @option watch_folder_def [String] upload_to_storage_id
+      # @option watch_folder_def [String] name (path)
+      # @option watch_folder_def [Array<String>] paths ([path])
+      # @option watch_folder_def [String] exclude ('**/.*')
+      # @option watch_folder_def [Array<string>] excludes ([exclude])
+      # @option watch_folder_def [String] include
+      # @option watch_folder_def [Array<String>] includes ([include])
+      # @option watch_folder_def [String] quarantine_directory_path
+      # @option watch_folder_def [String] completed_directory_path
+      # @option watch_folder_def [Integer|False] maximum_active_processors (@default_maximum_active_processors)
+      # @option watch_folder_def [Hash] logging
+      def process_watch_folder_def(watch_folder_def)
+        logger.debug { "Initializing Watch Folder #{watch_folder_def.inspect}" }
+
+        logger.debug { "Initializing parameter 'paths'." }
+        name = watch_folder_def['name']
+
+        path = watch_folder_def['path']
+
+        paths = watch_folder_def['paths'] ||= []
+        paths = [ paths ] if paths.is_a?(String)
+        paths.concat [*path] if path
+        paths.map! { |p| File.expand_path(p) }
+        if paths.empty?
+          name_as_path = File.expand_path(name)
+          paths.concat name_as_path if Dir.exist?(name_as_path)
+        end
+        paths.uniq!
+        watch_folder_def['paths'] = paths
+        # watch_folder_def['path'] ||= paths.first if paths.length == 1
+        watch_folder_def.delete('path')
+
+        if paths.empty?
+          logger.error { "Failed to initialize watch folder. No path found in watch folder definition." }
+          return false
+        end
+        logger.debug { "Parameter 'paths' initialized." }
+
+        logger.debug { "Initializing parameter 'includes'." }
+        include  = watch_folder_def['include']
+        includes = (watch_folder_def['includes'] ||= [])
+        includes.concat [*include] if include
+        includes.uniq!
+        includes.map! { |e| Regexp.try_convert(e) || e }
+        watch_folder_def['includes'] = includes
+        watch_folder_def.delete('include')
+        logger.debug { "Parameter `includes` initialized." }
+
+        logger.debug { "Initializing parameter 'excludes'." }
+        exclude  = watch_folder_def['exclude']
+        exclude  ||= '**/.*'
+        excludes = (watch_folder_def['excludes'] ||= [])
+        excludes.concat [*exclude] if exclude
+        excludes.uniq!
+        excludes.map! { |e| Regexp.try_convert(e) || e }
+        watch_folder_def['excludes'] = excludes
+        watch_folder_def.delete('exclude')
+        logger.debug { "Parameter `excludes` initialized." }
+
+
+        logger.debug { "Initializing parameter `quarantine directory path`." }
+        quarantine_directory_path = watch_folder_def['quarantine_directory_path'] || watch_folder_def['quarantine_path']
+        if quarantine_directory_path
+          quarantine_directory_path                     = File.expand_path(quarantine_directory_path)
+          watch_folder_def['quarantine_directory_path'] = quarantine_directory_path
+
+          unless Dir.exist?(quarantine_directory_path)
+            logger.warn { "Quarantine directory path '#{quarantine_directory_path}' does not exist. Files will be ignored instead." }
+          end
+        end
+        watch_folder_def.delete('quarantine_path')
+        logger.debug { "Parameter `quarantine directory path` initialized." }
+
+        logger.debug { "Initializing parameter 'completed directory path'." }
+        completed_directory_path = watch_folder_def['completed_directory_path'] || watch_folder_def['completed_path']
+        if completed_directory_path
+          completed_directory_path                     = File.expand_path(completed_directory_path)
+          watch_folder_def['completed_directory_path'] = completed_directory_path
+
+          unless Dir.exist?(completed_directory_path)
+            logger.warn { "Completed directory path '#{completed_directory_path}' does not exist. File will be ignored instead." }
+          end
+        end
+        watch_folder_def.delete('completed_path')
+        logger.debug { "Parameter 'completed directory path' initialized." }
+
+
+        logger.debug { "Initializing parameter `upload to storage id`." }
+        storage_id = watch_folder_def['upload_to_storage_id'] || watch_folder_def['storage_id']
+        watch_folder_def['upload_to_storage_id'] ||= storage_id
+        watch_folder_def.delete('storage_id')
+        unless storage_id
+          logger.warn { "No `upload to storage id` specified. Uploading will be skipped for this watch folder." }
+        end
+        logger.debug { "Parameter 'upload to storage id' initialized." }
+
+        maximum_active_processors = watch_folder_def['maximum_active_processors']
+        if maximum_active_processors.nil?
+          maximum_active_processors = @default_maximum_active_processors
+          watch_folder_def['maximum_active_processors'] = maximum_active_processors
+        end
+
+        args_out              = {}
+        logging               = watch_folder_def['logging'] || watch_folder_def
+        log_to                = logging['log_to']
+        log_level             = logging['log_level']
+        args_out[:log_to]     ||= log_to if log_to && !log_to.empty?
+        args_out[:log_level]  ||= log_level if log_level && !log_level.empty?
+        args_out[:logger]     ||= logger.dup unless log_to
+        args_out[:definition] = watch_folder_def
+
+        logger.debug { "Initializing Watch Folder Instance from Def. #{watch_folder_def}" }
+        watch_folder = LWF.new(args_out)
+        logger.debug { "Watch Folder Instance Created." }
+        watch_folder
+      end
+
+      # Iterates through watch_folder_defs and populates @watch_folders with watch folders initialized from the watch
+      # folder definitions
+      def process_watch_folder_defs
+        logger.debug { 'Processing watch folder definitions.' }
+        if watch_folder_defs.is_a?(Array)
+          @watch_folders = watch_folder_defs.map { |watch_folder_def| process_watch_folder_def(watch_folder_def) }
+        elsif watch_folder_defs.is_a?(Hash)
+          @watch_folders = watch_folder_defs.map do |name, watch_folder_def|
+            watch_folder_def['name'] ||= name
+            process_watch_folder_def(watch_folder_def)
+          end
+        else
+          raise "Unhandled format: #{watch_folder_defs.class.name}"
+        end
+        @watch_folders.keep_if { |wf| wf }
+        logger.debug { 'Processing of watch folder definitions completed.' }
+      end
+
+      # @param [Object] watch_folder
+      # @param [Object] file
+      def add_to_ignore(watch_folder, file)
+        logger.debug { "Adding File to Ignore Cache: '#{file.path}'" }
+        @ignored_file_paths_lock.synchronize do
+          @ignored_file_paths_by_watch_folder[watch_folder] << file.path
+          file.ignore if file.respond_to?(:ignore)
+        end
+      end
+
+      # @param [Object] watch_folder
+      # @param [Object] file
+      # @return [FalseClass]
+      def process_file(watch_folder, file)
+        file.processing = true
+        file_name       = file.name || file.path
+        logger.debug { "Processing File '#{file_name}'" }
+
+        storage_id                = watch_folder.definition['upload_to_storage_id']
+        quarantine_directory_path = watch_folder.definition['quarantine_directory_path']
+        completed_directory_path  = watch_folder.definition['completed_directory_path']
+
+        # full_file_path = File.join(watch_folder.path, file.path)
+        full_file_path = file.path
+        unless storage_id
+          logger.warn { "Skipping processing of file because of missing storage ID." }
+          return { success: false, message: 'Missing storage ID.' }
+        end
+
+        _response = agent.upload(file_path: full_file_path, storage_id: storage_id)
+        _response = { success: _response } if _response == true || _response == false
+
+        if _response[:success]
+          if completed_directory_path
+            if Dir.exist?(completed_directory_path)
+              logger.debug { "Moving '#{full_file_path}' to completed directory path '#{completed_directory_path}'" }
+              FileUtils.mv full_file_path, completed_directory_path
+            else
+              logger.warn { "Completed directory path not found: '#{completed_directory_path}'" }
+              add_to_ignore(watch_folder, file)
+            end
+          else
+            FileUtils.rm full_file_path
+          end
+        else
+          if quarantine_directory_path && Dir.exist?(quarantine_directory_path)
+            logger.warn { "Moving '#{full_file_path}' to quarantine directory path '#{quarantine_directory_path}'" }
+            FileUtils.mv full_file_path, quarantine_directory_path
+          else
+            logger.warn { "Adding '#{full_file_path}' to the temporary ignore list." }
+            add_to_ignore(watch_folder, file)
+          end
+        end
+
+        file.processed = true
+
+        _response
+      rescue => e
+        file.exception = e
+        raise e
+      ensure
+        file.processing = false
+      end
+
+      # Used to compare file to patterns
+      def find_in_patterns(patterns, file)
+        patterns.find do |pattern|
+            matched = pattern.is_a?(Regexp) ? pattern.match(file.path) : File.fnmatch(pattern, file.path)
+            logger.debug { "#{pattern} #{matched ? 'matched' : "didn't match"} #{file.path}" }
+            matched
+        end
+      end
+
+      def process_watch_folder_stable_files(wf, stable_files)
+        active_processors         = @processors_by_watch_folder[wf]
+        maximum_active_processors = wf.definition['maximum_active_processors']
+
+        includes = wf.definition['includes']
+        excludes = wf.definition['excludes']
+
+        ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
+        stable_files.each do |file|
+          next if ignored_file_paths.include?(file.path)
+          next if file.processing
+          next if file.processed
+
+          if includes && !includes.empty?
+            should_include = find_in_patterns(includes, file)
+            unless should_include
+              add_to_ignore(wf, file)
+              next
+            end
+          end
+
+          should_exclude = find_in_patterns(excludes, file)
+          if should_exclude
+            add_to_ignore(wf, file)
+            next
+          end
+
+          if @threaded
+            active_processors.keep_if { |k, v| k.processing }
+            if active_processors.length > maximum_active_processors
+              logger.debug { "Maximum number of active processors reached for watch folder. #{wf.name || wf.paths}" }
+              next
+            end
+            t = Thread.new(wf, file) do |wf, file|
+              begin
+                process_file(wf, file)
+              rescue => e
+                logger.error { "Exception '#{e.message}' in thread for `#{wf.name || wf.paths}` `#{file.path}`. " }
+                raise e
+              ensure
+                file.processing = false rescue nil
+              end
+            end
+            t.join
+            active_processors[file] = t if file.processing
+          else
+            process_file(wf, file)
+          end
+
+        end
+
+      end
+
+      # @TODO Thread this
+      def poll
+        stable_files_by_watch_folder = {} # Hash.new { |h, k| h[k] = [] }
+        watch_folders.each do |watch_folder|
+          if !watch_folder.last_poll_time || (Time.now - watch_folder.last_poll_time) >= watch_folder.poll_interval
+            stable_files                               = watch_folder.poll
+            stable_files_by_watch_folder[watch_folder] = stable_files
+          end
+        end
+        stable_files_by_watch_folder.each do |watch_folder, stable_files|
+          process_watch_folder_stable_files(watch_folder, stable_files)
+        end
+      end
+
+      # The main execution method
+      def run
+        # AWF.run_once(watch_folders) { |wf| pp wf }
+        # AWF.run(watch_folders) { |wf| process_watch_folder(wf) }
+
+        logger.info { 'Running...' }
+        watch_folders.map { |wf| wf.run if wf.respond_to?(:run) }
+        loop do
+          begin
+            poll
+            sleep 1
+          rescue Interrupt, SystemExit => e
+            logger.debug { "Received Signal: #{e.class.name}" }
+            break
+          end
+        end
+        logger.info { 'Exiting...' }
+      rescue => e
+        logger.error { "An error occurred.\n#{e.message}\n#{e.backtrace.join("\n")}\n#{e.message}" }
+        raise e
+      ensure
+        watch_folders.each { |wf| wf.stop if wf.respond_to?(:stop) }
+      end
+
+      def run_once
+        # AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
+      end
+
+      def self.run(args)
+        w = self.new(args)
+        w.run
+      rescue => e
+        abort("An error occurred.\n#{e.message}\n#{e.message}")
+      end
+
+      def self.daemons_run_proc_with_cleanup(options, &block)
+        if block_given?
+          options[:mode] = :proc
+          options[:proc] = block
+        end
+
+        controller                            = Daemons::Controller.new(options, ARGV)
+        _command, _controller_part, _app_part = controller.class.split_argv(ARGV)
+
+        controller_group                 = Daemons::ApplicationGroup.new(controller.app_name, controller.options)
+        controller_group.controller_argv = _controller_part
+        controller_group.app_argv        = _app_part
+
+        controller_group.setup
+        applications = controller_group.applications
+
+        is_running = applications.find { |a| a.running? }
+        if !applications.empty?
+          puts "Found #{applications.length} existing pid file(s) #{applications.map { |a| a.pid.pid }}"
+          should_zap_all = !is_running || (applications.length == 1 && applications.first.pid.pid == 0)
+          if should_zap_all
+            warn "Found stale pid file(s)"
+            controller_group.zap_all
+            controller_group.options[:force] = true
+            # controller_group.applications = []
+
+            controller.options[:force] = true
+          end
+        end
+
+        Daemons.run_proc(options[:app_name], options, &block)
+        # controller.catch_exceptions do
+        #   controller.run
+        # end
+
+      end
+
+      def self.run_as_daemon(args, options = {})
+        # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
+        require 'daemons'
+        # Daemons.run_proc('cantemo-portal-watch-folders', { force: true }) { self.run(args) }
+        app_name = 'cantemo-portal-watch-folders'
+        proc     = Proc.new { self.run(args) }
+
+        options[:app_name] = app_name
+        options[:mode]     = :proc
+        options[:dir_mode] = :normal
+        options[:dir]      = File.split(__FILE__)[0]
+
+        daemons_run_proc_with_cleanup(options, &proc)
+
+        # app_controller = Daemons::Controller.new(options, ARGV)
+        # Daemons.run_proc(app_name, options, &proc)
+      end
+
+    end
+
+  end
+
+end