cantemo-portal-agent 1.1.0 → 1.1.1

data/lib/envoi/mam/vidispine/agent.rb

@@ -50,7 +50,7 @@ module Envoi
             ::Vidispine::API::Utilities.new(client_args)
           end
     
-          @default_vidispine_shape_tag = args[:default_shape_tag] || _vidispine_config['default_shape_tag'] || _vidispine_config['shape_tag'] || DEFAULT_SHAPE_TAG
+          @default_vidispine_original_shape_tag = args[:default_shape_tag] || _vidispine_config['default_shape_tag'] || _vidispine_config['shape_tag'] || DEFAULT_SHAPE_TAG
     
         end
     

data/lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb

@@ -1,48 +1,113 @@
 require 'listen' # https://github.com/guard/listen
 require 'pp'
+Pathname
 module Envoi
   module WatchFolderUtility
     class WatchFolder
       class Handler
         class Listen
 
-          attr_accessor :logger, :definition,
+          attr_accessor :logger,
+                        :definition,
+                        :paths,
                         :known_path_map,
+                        :ignored_files_map,
                         :last_poll_time,
                         :min_stable_poll_count,
-                        :paths,
+                        :min_stable_time,
                         :poll_interval,
 
                         :listener, :lock,
 
-
                         :quarantine_directory_path
 
+          class DiscoveredPath < OpenStruct
+
+            def min_stable_time
+              self[:monitor].min_stable_time
+            end
+
+            def min_stable_poll_count
+              self[:monitor].min_stable_poll_count
+            end
+
+            def stable?
+              return false if deleted?
+              return false if !exist?
+
+              _min_stable_time = min_stable_time
+              if _min_stable_time
+                 return ((Time.now - self[:event_timestamp]) >= _min_stable_time)
+              else
+                return (self[:stable_poll_count] >= min_stable_poll_count)
+              end
+            end
+
+            def exist?
+              File.exist?(self[:path])
+            end
+
+            def deleted?
+              self[:event_type] == :deleted
+            end
+
+            def summary
+              _table = @table.dup
+              _table.delete(:monitor)
+              _table
+            end
+
+            def watch_folder
+              self[:monitor]
+            end
+
+            def watch_folder=(wf)
+              self[:monitor] = wf
+            end
+
+            def ignore
+              self[:monitor].add_to_ignore(self)
+            end
+
+            def ignore?
+              self[:monitor].ignored_files_map[self[:path]] == self
+            end
+
+          end
+
           def initialize(args = { })
             initialize_logger(args)
-            logger.debug { "Initializing Work Folder. Args: #{args}" }
+            logger.debug { "Initializing Watch Folder. Args: #{args}" }
 
             @definition = args[:definition]
             raise ArgumentError, "Definition is a required argument." unless definition
 
+            @known_path_map = { }
+            @ignored_files_map = { }
+
+            @lock = Mutex.new
+
+            @processing = { }
 
+            process_definition(@definition)
+            initialize_listener
+          end
 
+          def process_definition(work_folder_def)
             # @paths = [ File.expand_path('~/watch_folders/aspera') ]
             @paths = definition['paths'] || []
             path = definition['path']
-            @paths << path if path
+            @paths << path if path && !@paths.include?(path)
 
-            @known_path_map = { }
-            @ignored_files_map = { }
-
-            @lock = Mutex.new
             # @quarantine_directory_path = definition['quarantine_directory_path'] || definition['quarantine_path']
             @poll_interval = definition['poll_interval'] || 15
-            @min_stable_poll_count = 3
-            @processing_limit = 10
-            @processing = { }
+            @min_stable_poll_count = definition['minimum_stable_poll_count'] || 3
+            @min_stable_time = definition['minimum_stable_time']
+            if @min_stable_time.nil? && @min_stable_poll_count && @min_stable_time
+              @min_stable_time = @poll_interval * @min_stable_poll_count
+            end
+            @processing_limit = definition['processing_limit'] || 10
 
-            initialize_listener
           end
 
           def initialize_logger(args = { })
@@ -55,10 +120,27 @@ module Envoi
             @logger
           end
 
+          def process_deleted_path(path)
+            return path.each { |path| process_deleted_path(path, event_type) } if path.is_a?(Array)
+            logger.debug { "PATH #{:deleted} '#{path}'" }
+            file = known_path_map[path]
+            known_path_map.delete(path)
+            return unless file
+            file.event_type = :deleted
+            file.event_timestamp = Time.now
+            file.stable_poll_count = 0
+            file.deleted = true
+          end
+
           def process_add_or_modified_path(path, event_type)
             return path.each { |path| process_add_or_modified_path(path, event_type) } if path.is_a?(Array)
             logger.debug { "PATH #{event_type} '#{path}'" }
-            lock.synchronize { known_path_map[path] = OpenStruct.new({ type: event_type, path: path, stable_poll_count: 0, timestamp: Time.now}) }
+            # lock.synchronize { known_path_map[path] = OpenStruct.new({ type: event_type, path: path, stable_poll_count: 0, timestamp: Time.now}) }
+            file = known_path_map[path] ||= DiscoveredPath.new({ path: path, processed: false, monitor: self })
+            file.event_type = event_type
+            file.event_timestamp = Time.now
+            file.stable_poll_count = 0
+            file
           end
 
           def initialize_listener(args = { })
@@ -70,7 +152,7 @@ module Envoi
             @listener = ::Listen.to(*paths, {}) do |m, a, r|
               if !r.empty?
                 log "#{r.length} file(s) removed."
-                lock.synchronize { r.each { |f| known_path_map.delete(f) } }
+                lock.synchronize { r.each { |f| process_deleted_path(f) } }
               end
 
               if !m.empty?
@@ -123,28 +205,75 @@ module Envoi
           #   end
           # end
 
-          def poll
+          def poll(options = { })
+            should_increment_counter = options.fetch(:should_increment_counter, true)
+
             @previous_poll_time = @last_poll_time
             @last_poll_time = Time.now
-            stable_files = []
             lock.synchronize do
               @known_path_map.each do |fp, f|
-                logger.debug { "Incrementing Stable Count: #{f.inspect}" }
-                f[:stable_poll_count] += 1
-                if f[:stable_poll_count] >= @min_stable_poll_count
-                  stable_files << f
-                  # handle_stable_file(f)
-                end
+                next if f.ignore?
+                next unless f.exist?
+
+                logger.debug { "Incrementing Stable Stats: #{f.summary}" }
+
+                f.last_poll_time ||= Time.now
+                f[:stable_poll_count] += 1 if should_increment_counter
+
+                #
+                # if @min_stable_time
+                #   if (Time.now - f.event_timestamp) >= @min_stable_time
+                #     is_stable = true
+                #   end
+                # else
+                #   if f[:stable_poll_count] >= @min_stable_poll_count
+                #     is_stable = true
+                #     # handle_stable_file(f)
+                #   end
+                # end
               end
             end
             stable_files
           end
 
+          def stable_files
+            @known_path_map.select { |p,f| f.stable? }.values
+          end
+
           def run(options = { })
             logger.info { "Starting handler for #{paths}" }
             listener.start
           end
 
+          def stop
+            listener.stop if listener && listener.respond_to?(:stop)
+          end
+
+          InspectKey = :__inspect_key__ # :nodoc:
+
+          def inspect
+            str = "#<#{self.class}"
+
+            ids = (Thread.current[InspectKey] ||= [])
+            if ids.include?(object_id)
+              return str << ' ...>'
+            end
+
+            ids << object_id
+            # begin
+            #   first = true
+            #   for k,v in @table
+            #     str << "," unless first
+            #     first = false
+            #     str << " #{k}=#{v.inspect}"
+            #   end
+            #   return str << '>'
+            # ensure
+            #   ids.pop
+            # end
+          end
+          alias :to_s :inspect
+
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cantemo-portal-agent
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - John Whitson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-02 00:00:00.000000000 Z
+date: 2018-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asperalm
@@ -181,8 +181,8 @@ files:
 - lib/envoi/mam/agent/watch_folder_utility/foreman.rb
 - lib/envoi/mam/cantemo/agent.rb
 - lib/envoi/mam/cantemo/agent/watch_folder_handler-working.rb
-- lib/envoi/mam/cantemo/agent/watch_folder_handler.rb
 - lib/envoi/mam/cantemo/agent/watch_folder_handler_aspera.rb
+- lib/envoi/mam/cantemo/agent/watch_folder_manager.rb
 - lib/envoi/mam/iconik/agent.rb
 - lib/envoi/mam/mediasilo/agent.rb
 - lib/envoi/mam/vdms/agent.rb

data/lib/envoi/mam/cantemo/agent/watch_folder_handler.rb

@@ -1,176 +0,0 @@
-require 'envoi/mam/cantemo/agent'
-require 'envoi/aspera/watch_service/watch_folder'
-require 'envoi/watch_folder_utility/watch_folder/handler/listen'
-
-module Envoi::Mam::Cantemo
-
-  class Agent
-
-    class WatchFolderHandler
-
-      AWF = Envoi::Aspera::WatchService::WatchFolder
-      LWF = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen
-
-      attr_accessor :logger, :agent, :config, :watch_folder_defs, :watch_folders
-
-      def initialize(args = { })
-        initialize_logger(args)
-
-        @agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
-        @config = agent.config
-        cantemo_config = config[:cantemo] || config['cantemo']
-        @watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
-
-        @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
-        process_watch_folder_defs
-      end
-
-      def process_watch_folder_def(watch_folder_def)
-        name = watch_folder_def['name']
-
-        path = watch_folder_def['path']
-        path ||= name
-        (watch_folder_def['paths'] ||= []).concat [*path]
-        watch_folder_def['paths'].uniq!
-        paths = watch_folder_def['paths']
-        watch_folder_def['path'] ||= paths.first if path.length == 1
-
-        exclude = watch_folder_def['exclude']
-        exclude ||= '**/.*'
-        (watch_folder_def['excludes'] ||= []).concat [*exclude]
-        watch_folder_def['excludes'].uniq!
-
-        args_out = { }
-        logging = watch_folder_def['logging'] || watch_folder_def
-        log_to = logging['log_to']
-        log_level = logging['log_level']
-        args_out[:log_to] ||= log_to if log_to && !log_to.empty?
-        args_out[:log_level] ||= log_level if log_level && !log_level.empty?
-        args_out[:logger] ||= logger.dup unless log_to
-        args_out[:definition] = watch_folder_def
-
-        LWF.new(args_out)
-      end
-
-      def process_watch_folder_defs
-        if watch_folder_defs.is_a?(Array)
-          @watch_folders = watch_folder_defs.map { |watch_folder_def| process_watch_folder_def(watch_folder_def) }
-        elsif watch_folder_defs.is_a?(Hash)
-          @watch_folders = watch_folder_defs.map do |name, watch_folder_def|
-            watch_folder_def['name'] ||= name
-
-            process_watch_folder_def(watch_folder_def)
-          end
-        else
-          raise "Unhandled format: #{watch_folder_defs.class.name}"
-        end
-      end
-
-      def initialize_logger(args = { })
-        @logger = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
-        log_level = args[:log_level]
-        if log_level
-          @logger.level = log_level
-          args[:logger] = @logger
-        end
-        @logger
-      end
-
-      def add_to_ignore(watch_folder, file)
-        logger.debug { "Adding File to Ignore Cache: '#{file.path}'"}
-        @ignored_file_paths_by_watch_folder[watch_folder] << file.path
-      end
-
-      def process_file(watch_folder, file, storage_id = nil, quarantine_directory_path = nil)
-        # full_file_path = File.join(watch_folder.path, file.path)
-        full_file_path = file.path
-        if storage_id && agent.upload(file_path: full_file_path, storage_id: storage_id)
-          FileUtils.rm full_file_path
-        else
-          if Dir.exist?(quarantine_directory_path)
-            FileUtils.mv full_file_path, quarantine_directory_path
-          else
-            add_to_ignore(watch_folder, file)
-          end
-        end
-      end
-
-      def process_watch_folder_stable_files(wf, stable_files)
-        storage_id = wf.definition['upload_to_storage_id'] || wf.definition['storage_id']
-        quarantine_directory_path = wf.definition['quarantine_path']
-        exclude = wf.definition['excludes']
-
-        ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
-        stable_files.each do |file|
-          next if ignored_file_paths.include?(file.path)
-
-          if exclude && [*exclude].find do |ep|
-            excluded = (r = Regexp.try_convert(ep) ? r.match(ep) : File.fnmatch(ep, file.path))
-            logger.debug { "#{ep} #{excluded ? 'matched' : "didn't match"} #{file.path}" }
-            excluded
-          end
-            add_to_ignore(wf, file)
-            next
-          end
-
-          process_file(wf, file, storage_id, quarantine_directory_path)
-        end
-
-      end
-
-      def poll
-        stable_files_by_watch_folder = {} # Hash.new { |h, k| h[k] = [] }
-        watch_folders.each do |watch_folder|
-          if !watch_folder.last_poll_time || (Time.now - watch_folder.last_poll_time) >= watch_folder.poll_interval
-            stable_files = watch_folder.poll
-            stable_files_by_watch_folder[watch_folder] = stable_files
-          end
-        end
-        stable_files_by_watch_folder.each do |watch_folder, stable_files|
-          process_watch_folder_stable_files(watch_folder, stable_files)
-        end
-      end
-
-      def run
-        # AWF.run_once(watch_folders) { |wf| pp wf }
-        # AWF.run(watch_folders) { |wf| process_watch_folder(wf) }
-
-        logger.info { 'Running...' }
-        watch_folders.map { |wf| wf.run }
-        loop do
-          begin
-            poll
-            sleep 1
-          rescue SystemExit, Interrupt
-            break
-          end
-        end
-        logger.info { 'Exiting...' }
-      rescue => e
-        logger.error { "An error occurred.\n#{e.message}\n#{e.backtrace.join("\n")}\n#{e.message}" }
-        raise e
-      end
-
-      def run_once
-        # AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
-      end
-
-      def self.run(args)
-        w = self.new(args)
-        w.run
-      rescue => e
-        abort("An error occurred.\n#{e.message}\n#{e.message}")
-      end
-
-      def self.run_as_daemon(args)
-        # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
-        require 'daemons'
-        # Daemons.run_proc('cantemo-portal-watch-folders', { force: true }) { self.run(args) }
-        Daemons.run_proc('cantemo-portal-watch-folders') { self.run(args) }
-      end
-
-    end
-
-  end
-
-end