cantemo-portal-agent 1.3.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: deef6be201da32030d9b495463c2c5b2a1a55086
-  data.tar.gz: 89ca91ba9fadf0f204cb9180ca28d3c3110766e7
+  metadata.gz: 9b3f1ea75c5103dd91d6b863538cb5bd4b6ff7c2
+  data.tar.gz: '0796314e40e182e785a2d8cc8290be0e0a653545'
 SHA512:
-  metadata.gz: 7f24e5db79bed7d457b59e0d845aff26e4f388e651ea5378a889ffcde36d3e5d1618a26ef570851501f391fead8ab3779017324ac4b2be668cdd07c7da53db14
-  data.tar.gz: 01d9ab468ba03278a8db51c057210cb8109ab96c37ff6ba719fb2bc581d08f0782695cb8e1d8f80dc99290e14e1547b1138313d236c4f3aa572872bb481a357e
+  metadata.gz: a57aeb7f14fceb67dfbbbfe091593c49ef57491fb5968b3f4b46e9a3beccff7964153a8b06d88e13268b10de315102db28a147647a5c9455090c4a7fe46af748
+  data.tar.gz: 68487f27150376d370b6254df75a339b49385d3957eb490e90e79bc9c3054dc7b77d8e5fe50dae5ad0ee42c9eea71d5fee803306e7378338c18bb6ae00e18083

data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb

@@ -37,12 +37,12 @@ op = OptionParser.new
 op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
       "Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
 # op.on('-b', '--[no-]background', 'Tells the application to run in the background.') { |v| args[:daemonize] = v }
-op.on_tail('-t', '--log-to FILEPATH', 'Log file to log to') { |log_to| args[:log_to] = log_to }
-op.on_tail('-l', '--log-level LEVEL', ['fatal', 'error', 'warn', 'info', 'debug'], 'Select logging level') do |log_level|
-  args[:log_level] = log_level.upcase
+op.on_tail('-t', '--log-to FILEPATH', 'Log file to log to') { |v| args[:log_to] = v }
+op.on_tail('-l', '--log-level LEVEL', %w(fatal error warn info debug), 'Select logging level') do |v|
+  args[:log_level] = v
 end
 op.on_tail('-v', '--version', 'Output the version and exit.') { puts "Version #{Cantemo::Portal::Agent::VERSION}"; exit }
-op.on_tail('--help', 'Show this message.') { puts op; exit }
+op.on_tail('-h', '--help', 'Show this message.') { puts op; exit }
 op.load
 op.parse!
 
@@ -53,27 +53,26 @@ if sub_command == 'help'
 end
 
 config_file_path = args[:config_file_path]
-args[:config_file_path].map! { |v| File.expand_path(v) } if config_file_path.is_a?(Array)
-
-daemon_control_command_present = sub_command && begin
-  sub_command = sub_command.dup.downcase if sub_command
-  %w(start stop restart zap killall status).include?(sub_command)
-end
-should_daemonize = (daemon_control_command_present && %w(start restart).include?(sub_command)) || args[:daemonize]
-
-if sub_command
-  case sub_command
-  when 'install'
-    puts
-  when 'uninstall'
-    puts
-  else
-    puts
-  end
+if config_file_path.is_a?(Array)
+  args[:config_file_path].map! { |v| File.expand_path(v) }
+  config_file_path = config_file_path.find { |file_path| File.exist?(file_path) }
 end
 
+
+# daemon_control_command_present = sub_command && begin
+#   sub_command = sub_command.dup.downcase if sub_command
+#   %w(start stop restart zap killall status).include?(sub_command)
+# end
+# should_daemonize = (daemon_control_command_present && %w(start restart).include?(sub_command)) || args[:daemonize]
+
+WATCH_FOLDER_MANAGER_CLASS = Envoi::Mam::Cantemo::Agent::WatchFolderManager
+
 if sub_command == 'run'
-  Envoi::Mam::Cantemo::Agent::WatchFolderManager.run(args)
+  begin
+    WATCH_FOLDER_MANAGER_CLASS.run(args)
+  rescue => e
+    abort("An exception occurred: #{e.message}#{args[:log_level] == 'debug' ? " - #{e.backtrace.first}" : ''}")
+  end
   exit
 end
 
@@ -110,16 +109,90 @@ module Application
       end
 
       def service_main(*args)
-        @WatchFolderManager = Envoi::Mam::Cantemo::Agent::WatchFolderManager.run(args)
+        @WatchFolderManager = WATCH_FOLDER_MANAGER_CLASS.run(args)
       end
 
       def service_stop
         @WatchFolderManager.stop
       end
 
+      def service_pause
+
+      end
+
+      def service_resume
+
+      end
+
+      # Watcher - Windows
     end
   else
-    class Watcher < Envoi::Mam::Cantemo::Agent::WatchFolderManager; end
+    class Watcher
+      APP_NAME = 'cantemo-portal-watch-folders'
+
+      def self.run(args)
+        WATCH_FOLDER_MANAGER_CLASS.run(args)
+      end
+
+      def self.daemons_run_proc_with_cleanup(options, &block)
+        options[:dir_mode] = :normal
+        options[:dir]      = File.split(__FILE__)[0]
+
+        if block_given?
+          options[:mode] = :proc
+          options[:proc] = block
+        end
+
+        controller                            = Daemons::Controller.new(options, ARGV)
+        _command, _controller_part, _app_part = controller.class.split_argv(ARGV)
+
+        controller_group                 = Daemons::ApplicationGroup.new(controller.app_name, controller.options)
+        controller_group.controller_argv = _controller_part
+        controller_group.app_argv        = _app_part
+
+        controller_group.setup
+        applications = controller_group.applications
+
+        is_running = applications.find { |a| a.running? }
+        if !applications.empty?
+          puts "Found #{applications.length} existing pid file(s) #{applications.map { |a| a.pid.pid }}"
+          should_zap_all = !is_running || (applications.length == 1 && applications.first.pid.pid == 0)
+          if should_zap_all
+            warn "Found stale pid file(s)"
+            controller_group.zap_all
+            controller_group.options[:force] = true
+            # controller_group.applications = []
+
+            controller.options[:force] = true
+          end
+        end
+
+        Daemons.run_proc(options[:app_name], options, &block)
+        # controller.catch_exceptions do
+        #   controller.run
+        # end
+
+      end
+
+      def self.run_with_process_manager(args, options = {})
+        # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
+        self.run(args)
+        require 'daemons'
+        proc     = Proc.new { self.run(args) }
+        app_name = APP_NAME
+
+        # options[:app_name] = app_name
+        # daemons_run_proc_with_cleanup(options, &proc)
+
+        # options[:force] = true
+        Daemons.run_proc(app_name, options, &proc)
+      end
+
+      def self.run_in_foreground(args, options = { })
+        self.run(args)
+      end
+      # Watcher - *nix
+    end
   end
 
 
@@ -139,18 +212,16 @@ end
 
 if !Application::OS.windows?
   begin
-    Watcher.run_with_process_manager(args)
+    Application.run_with_process_manager(args)
     exit
   rescue => e
-    abort("An error occurred. #{e.message}")
+    abort("An exception occurred: #{e.message}#{args[:log_level] == 'debug' ? " - #{e.backtrace.first}" : ''}")
   end
 end
 
 # Options:
-#    install    - Installs the service.  The service name is "DemoSvc"
-#                 and the display name is "Demo".
-#    start      - Starts the service.  Make sure you stop it at some point or
-#                 you will eventually fill up your filesystem!.
+#    install    - Installs the service.
+#    start      - Starts the service.
 #    stop       - Stops the service.
 #    pause      - Pauses the service.
 #    resume     - Resumes the service.
@@ -161,31 +232,18 @@ require 'rbconfig'
 include Win32
 include RbConfig
 
-# Make sure you're using the version you think you're using.
-puts 'VERSION: ' + Service::VERSION
-
 SERVICE_NAME = 'cantemo_portal_watch_folder_agent'
 SERVICE_DISPLAYNAME = 'Cantemo Portal Watch Folder Agent'
 EXECUTABLE_NAME = __FILE__
 
 
 # You must provide at least one argument.
-raise ArgumentError, 'No argument provided' unless sub_command
+abort('No argument provided.') unless sub_command
+abort('Service not installed.') unless Service.exists?(SERVICE_NAME) && sub_command == 'install'
 
-case sub_command
-when 'install'
-  # Quote the full path to deal with possible spaces in the path name.
-  ruby = File.join(CONFIG['bindir'], CONFIG['ruby_install_name']).tr('/', '\\')
-  path = %Q(#{File.dirname(File.expand_path($0)).tr('/', '\\')}\\#{EXECUTABLE_NAME})
-  cmd = %Q("#{ruby}" "#{path}" run --config-file-path "#{config_file_path}")
 
-  Service.new(
-      :service_name     => SERVICE_NAME,
-      :display_name     => SERVICE_DISPLAYNAME,
-      :description      => 'Watch Folder Agent for Cantemo Portal',
-      :binary_path_name => cmd
-  )
-  puts 'Service ' + SERVICE_NAME + ' installed'
+case sub_command
+when 'runasservice'
 when 'start'
   if Service.status(SERVICE_NAME).current_state != 'running'
     Service.start(SERVICE_NAME, nil)
@@ -208,16 +266,6 @@ when 'stop'
   else
     puts 'Already stopped'
   end
-when 'uninstall', 'delete'
-  if Service.status(SERVICE_NAME).current_state != 'stopped'
-    Service.stop(SERVICE_NAME)
-  end
-  while Service.status(SERVICE_NAME).current_state != 'stopped'
-    puts 'One moment...' + Service.status(SERVICE_NAME).current_state
-    sleep 1
-  end
-  Service.delete(SERVICE_NAME)
-  puts 'Service ' + SERVICE_NAME + ' deleted'
 when 'pause'
   if Service.status(SERVICE_NAME).current_state != 'paused'
     Service.pause(SERVICE_NAME)
@@ -242,6 +290,30 @@ when 'resume'
   end
 when 'status'
   puts Service.status(SERVICE_NAME)
+when 'install'
+  # Quote the full path to deal with possible spaces in the path name.
+  ruby = File.join(CONFIG['bindir'], CONFIG['ruby_install_name']).tr('/', '\\')
+  path = %Q(#{File.dirname(File.expand_path($0)).tr('/', '\\')}\\#{EXECUTABLE_NAME})
+  cmd = %Q("#{ruby}" "#{path}" runasservice --config-file-path "#{config_file_path}")
+
+  Service.new(
+      :service_name     => SERVICE_NAME,
+      :display_name     => SERVICE_DISPLAYNAME,
+      :description      => 'Watch Folder Agent for Cantemo Portal',
+      :binary_path_name => cmd
+  )
+  puts 'Service ' + SERVICE_NAME + ' installed'
+when 'uninstall', 'delete'
+  if Service.status(SERVICE_NAME).current_state != 'stopped'
+    Service.stop(SERVICE_NAME)
+  end
+  while Service.status(SERVICE_NAME).current_state != 'stopped'
+    puts 'One moment...' + Service.status(SERVICE_NAME).current_state
+    sleep 1
+  end
+  Service.delete(SERVICE_NAME)
+  puts 'Service ' + SERVICE_NAME + ' deleted'
+
 else
   raise ArgumentError, 'unknown option: ' + sub_command
 end

data/lib/cantemo/portal/agent/version.rb

@@ -1,7 +1,7 @@
 module Cantemo
   module Portal
     class Agent
-      VERSION = '1.3.0'.freeze
+      VERSION = '1.3.1'.freeze
     end
   end
 end

data/lib/envoi/mam/cantemo/agent/watch_folder_manager.rb

@@ -5,48 +5,10 @@ require 'envoi/watch_folder_utility/watch_folder/handler/listen'
 
 module Envoi::Mam::Cantemo
 
-  module OS
-    def OS.windows?
-      (/cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM) != nil
-    end
-
-    def OS.mac?
-      (/darwin/ =~ RUBY_PLATFORM) != nil
-    end
-
-    def OS.unix?
-      !OS.windows?
-    end
-
-    def OS.linux?
-      OS.unix? and not OS.mac?
-    end
-  end
-
   class Agent
 
     class WatchFolderManager
 
-
-      class MultiIO
-        def initialize(*targets)
-          @targets = targets
-        end
-
-        def write(*args)
-          @targets.each {|t| t.write(*args)}
-        end
-
-        def close
-          @targets.each(&:close)
-        end
-
-        def add_target(*target)
-          @targets.concat target
-        end
-
-      end
-
       class MultiLogger
         attr_accessor :targets
 
@@ -340,6 +302,10 @@ module Envoi::Mam::Cantemo
         end
       end
 
+      def run_once
+        # AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
+      end
+
       # The main execution method
       def run
         @should_run = true
@@ -367,18 +333,28 @@ module Envoi::Mam::Cantemo
         logger.info { 'Exiting...' }
       end
 
-      def run_once
-        # AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
-      end
-
       def stop
         if @should_run
           @should_run = false
-          logger.info { "Stopping..." }
+          logger.info { 'Stopping...' }
           watch_folders.each { |wf| wf.stop if wf.respond_to?(:stop) }
         end
       end
 
+      def pause
+        if @should_run
+          logger.info { 'Pausing...' }
+          watch_folders.each { |wf| wf.pause if wf.respond_to?(:pause) }
+        end
+      end
+
+      def resume
+        if @should_run
+          logger.info { 'Resuming...' }
+          watch_folders.each { |wf| wf.resume if wf.respond_to?(:resume) }
+        end
+      end
+
       # Converts hash keys to symbols
       #
       # @param [Hash] value hash
@@ -398,67 +374,7 @@ module Envoi::Mam::Cantemo
       def self.run(args)
         w = self.new(args)
         w.run
-      end
-
-      def self.daemons_run_proc_with_cleanup(options, &block)
-        options[:dir_mode] = :normal
-        options[:dir]      = File.split(__FILE__)[0]
-
-        if block_given?
-          options[:mode] = :proc
-          options[:proc] = block
-        end
-
-        controller                            = Daemons::Controller.new(options, ARGV)
-        _command, _controller_part, _app_part = controller.class.split_argv(ARGV)
-
-        controller_group                 = Daemons::ApplicationGroup.new(controller.app_name, controller.options)
-        controller_group.controller_argv = _controller_part
-        controller_group.app_argv        = _app_part
-
-        controller_group.setup
-        applications = controller_group.applications
-
-        is_running = applications.find { |a| a.running? }
-        if !applications.empty?
-          puts "Found #{applications.length} existing pid file(s) #{applications.map { |a| a.pid.pid }}"
-          should_zap_all = !is_running || (applications.length == 1 && applications.first.pid.pid == 0)
-          if should_zap_all
-            warn "Found stale pid file(s)"
-            controller_group.zap_all
-            controller_group.options[:force] = true
-            # controller_group.applications = []
-
-            controller.options[:force] = true
-          end
-        end
-
-        Daemons.run_proc(options[:app_name], options, &block)
-        # controller.catch_exceptions do
-        #   controller.run
-        # end
-
-      end
-
-      def self.run_with_process_manager(args, options = {})
-        # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
-        if OS.windows?
-          self.run(args)
-        else
-          require 'daemons'
-          proc     = Proc.new { self.run(args) }
-          app_name = 'cantemo-portal-watch-folders'
-
-          # options[:app_name] = app_name
-          # daemons_run_proc_with_cleanup(options, &proc)
-
-          # options[:force] = true
-          Daemons.run_proc(app_name, options, &proc)
-        end
-      end
-
-      def self.run_in_foreground(args, options = { })
-        self.run(args)
+        w
       end
 
     end

data/lib/envoi/watch_folder_utility/watch_folder.rb

@@ -229,8 +229,8 @@ module Envoi
 
         logger.debug { "Initializing parameter 'upload/import arguments'." }
         upload_args               = watch_folder_def['upload_args']
-        item_add_args             = watch_folder_def['item_add_args']
-        item_add_options          = watch_folder_def['item_add_options']
+        item_add_args             = watch_folder_def['item_add_args'] || { }
+        item_add_options          = watch_folder_def['item_add_options'] || { }
         import_args               = watch_folder_def['import_args'] || { }
         import_options            = watch_folder_def['import_options'] || { }
 
@@ -238,26 +238,26 @@ module Envoi
         import_options = Hash[import_options.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym.downcase : k, v ] }]
 
         # Allow adding to collection to be overridden
-        add_item_to_collection = import_options.fetch(:add_item_to_collection,
-                                                      import_args.fetch(:add_item_to_collection,
+        add_item_to_collection = item_add_args.fetch(:add_item_to_collection,
+                                                     item_add_options.fetch(:add_item_to_collection,
                                                                         watch_folder_def['add_item_to_collection']))
         if add_item_to_collection.nil? || add_item_to_collection
           _add_item_to_collection = false
           collection_id = watch_folder_def['collection_id']
           if collection_id
-            import_args[:collection_id] ||= collection_id
+            item_add_args[:collection_id] ||= collection_id
             _add_item_to_collection = true
             watch_folder_def.delete('collection_id')
           else
             collection_name = watch_folder_def['collection_name']
             if collection_name
-              import_args[:collection_name] ||= collection_name
+              item_add_args[:collection_name] ||= collection_name
               _add_item_to_collection = true
               watch_folder_def.delete('collection_name')
             else
               file_path_collection_name_position = watch_folder_def['file_path_collection_name_position']
               if file_path_collection_name_position
-                import_args[:file_path_collection_name_position] = file_path_collection_name_position
+                item_add_args[:file_path_collection_name_position] = file_path_collection_name_position
                 _add_item_to_collection = true
                 watch_folder_def.delete('file_path_collection_name_position')
               end
@@ -268,13 +268,14 @@ module Envoi
 
         metadata = watch_folder_def['metadata']
         if metadata
-          import_args[:metadata] = metadata
+          item_add_args[:metadata] = metadata
           watch_folder_def.delete('metadata')
         end
 
-        metadata_group = watch_folder_def['metadata_group'] || watch_folder_def['field_group']
-        if metadata_group
-          import_options[:field_group] = metadata_group
+        field_group = watch_folder_def['field_group']
+        if field_group
+          item_add_args[:field_group] = field_group
+          watch_folder_def.delete('field_group')
         end
 
         ingest_group = watch_folder_def['ingest_group']
@@ -283,11 +284,12 @@ module Envoi
           job_metadata += ',' unless job_metadata.empty?
           job_metadata += "portal_groups:StringArray=#{ingest_group}"
           import_args[:jobmetadata] = job_metadata
+          watch_folder_def.delete('ingest_group')
         end
 
         item_add_args = symbolize_keys(item_add_args)
-        ((item_add_args ||= {})[:import_args] ||= {}).merge! import_args if import_args.is_a?(Hash)
-        ((item_add_args ||= {})[:import_options] ||= {}).merge! import_options if import_options.is_a?(Hash)
+        (item_add_args[:import_args] ||= {}).merge! import_args if import_args.is_a?(Hash)
+        (item_add_args[:import_options] ||= {}).merge! import_options if import_options.is_a?(Hash)
 
         upload_args = symbolize_keys(upload_args)
         ((upload_args ||= {})[:item_add_args] ||= {}).merge! item_add_args if item_add_args.is_a?(Hash)
@@ -398,8 +400,67 @@ module Envoi
         end
       end
 
-      def process_stable_file(file)
+      # @param [Object] file
+      # @return [Hash]
+      def process_file(file)
+        file.processing = true
+        file_name       = file.name || file.path
+        logger.debug { "Processing File '#{file_name}'" }
 
+        storage_id                = watch_folder.definition['upload_to_storage_id']
+
+        unless storage_id
+          logger.warn { "Skipping processing of file because of missing storage ID." }
+          return { success: false, message: 'Missing storage ID.' }
+        end
+
+        quarantine_directory_path = definition['quarantine_directory_path']
+        completed_directory_path  = definition['completed_directory_path']
+        watch_folder_upload_args  = definition['upload_args']
+
+        # full_file_path = File.join(watch_folder.path, file.path)
+        full_file_path = file.path
+
+        upload_args = {
+            file_path: full_file_path,
+            storage_id: storage_id
+        }
+        upload_args.merge!(watch_folder_upload_args) if watch_folder_upload_args.is_a?(Hash)
+
+        logger.debug { "Executing Upload. #{upload_args}" }
+        _response = agent.upload(upload_args)
+        _response = { success: _response } if _response == true || _response == false
+
+        if _response[:success]
+          if completed_directory_path
+            if Dir.exist?(completed_directory_path)
+              logger.debug { "Moving '#{full_file_path}' to completed directory path '#{completed_directory_path}'" }
+              FileUtils.mv full_file_path, completed_directory_path
+            else
+              logger.warn { "Completed directory path not found: '#{completed_directory_path}'" }
+              add_to_ignore(file)
+            end
+          else
+            FileUtils.rm full_file_path
+          end
+        else
+          if quarantine_directory_path && Dir.exist?(quarantine_directory_path)
+            logger.warn { "Moving '#{full_file_path}' to quarantine directory path '#{quarantine_directory_path}'" }
+            FileUtils.mv full_file_path, quarantine_directory_path
+          else
+            logger.warn { "Adding '#{full_file_path}' to the temporary ignore list." }
+            add_to_ignore(file)
+          end
+        end
+
+        file.processed = true
+
+        _response
+      rescue => e
+        file.exception = e
+        raise e
+      ensure
+        file.processing = false
       end
 
       # Converts hash keys to symbols

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cantemo-portal-agent
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.1
 platform: ruby
 authors:
 - John Whitson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-11 00:00:00.000000000 Z
+date: 2019-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asperalm
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.2
+        version: 1.6.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.2
+        version: 1.6.4
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement