cantemo-portal-agent 1.2.7 → 1.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: edd73213f17ac2ccac314ef21efd8ee07e68bd77
-  data.tar.gz: f31df7db4af1f985a4f53a870b278c5f0c94b5f9
+  metadata.gz: 65d5d27c1b9a329675a2f56b9a1fbd2e12a0f6eb
+  data.tar.gz: 535b93e6dbd17809b3fa48c3b4d5b2cd4df1ab0d
 SHA512:
-  metadata.gz: 1baaacd2944586fd81f65f7f4e58fb978ed7e641481d064fd3d76378478a93d1c13575fea5ba676fe8d6bd714a9e7fd8a9e0fa8fa722ad22729db331f62cf7c9
-  data.tar.gz: 294f8ee74c1aaad3f5053c1cc8c1338c08c0a45147529aeac48956cd52bb9f65af45c3ee0ce8fc36d8a06f6ad67c0162b76f321d6912791da58a967b3e71af3f
+  metadata.gz: b626b4dd6e2f3b54a708213ccb41442513ea2606ce4ae603030a2b577f6d42d02e8681371bf6dc262e433a816bdda96d14e5e76499c3397e5373afa30492b0ed
+  data.tar.gz: d0b2c5ba1d5a5d5fde11308928bc1cd89ccd5c1dc106123284b6179db8504b156fa57474e94a5082c167173d1f25b3b06de2e3349ff517f01c5fd80c256dfa95

data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb

@@ -11,6 +11,7 @@ require 'envoi/mam/agent/cli'
 require 'envoi/mam/cantemo/agent'
 require 'envoi/mam/cantemo/agent/watch_folder_manager'
 
+
 Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.clear
 Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.concat [
    './cantemo-portal-agent-config.json',
@@ -45,15 +46,27 @@ op.on_tail('--help', 'Show this message.') { puts op; exit }
 op.load
 op.parse!
 
+command = (ARGV.first || 'run').downcase
+if command == 'help'
+  puts op
+  exit
+end
+
 config_file_path = args[:config_file_path]
 args[:config_file_path].map! { |v| File.expand_path(v) } if config_file_path.is_a?(Array)
 
-command = ARGV.first
 daemon_control_command_present = command && begin
   command = command.dup.downcase if command
-  %w(start stop restart run zap killall status).include?(command)
+  %w(start stop restart zap killall status).include?(command)
+end
+
+if command == 'run'
+  Envoi::Mam::Cantemo::Agent::WatchFolderManager.run(args)
+  exit
 end
 
+
+
 if command
   case command
   when 'install'
@@ -73,10 +86,69 @@ should_daemonize = (daemon_control_command_present && %w(start restart).include?
 # puts "COMMAND: #{next_command}"
 #
 #
+#
+
+module Application
+
+  module OS
+    def OS.windows?
+      (/cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM) != nil
+    end
+
+    def OS.mac?
+      (/darwin/ =~ RUBY_PLATFORM) != nil
+    end
+
+    def OS.unix?
+      !OS.windows?
+    end
+
+    def OS.linux?
+      OS.unix? and not OS.mac?
+    end
+  end
+
+  class ProcessManager
+
+  end
+
+  if OS.windows?
+    require 'win32/daemon'
+    class Watcher < Win32::Daemon
+
+      def service_init
+
+      end
+
+      def service_main(*args)
+        @WatchFolderManager = Envoi::Mam::Cantemo::Agent::WatchFolderManager.run(args)
+      end
+
+      def service_stop
+        @WatchFolderManager.stop
+      end
+
+    end
+  else
+    class Watcher < Envoi::Mam::Cantemo::Agent::WatchFolderManager; end
+  end
+
+
+  def run_in_background(args)
+
+  end
+
+  def run_in_foreground(args)
+    Watcher.run(args)
+  end
+
+  def run(args)
+
+  end
+
+end
+
 
-class Watcher < Envoi::Mam::Cantemo::Agent::WatchFolderManager; end
-# daemonize ? Watcher.run_as_daemon(args, { force: true }) : Watcher.run(args)
-# daemon_control_command_present ? Watcher.run_as_daemon(args) : Watcher.run(args)
 begin
   Watcher.run_with_process_manager(args)
 rescue => e

data/lib/cantemo/portal/agent/version.rb

@@ -1,7 +1,7 @@
 module Cantemo
   module Portal
     class Agent
-      VERSION = '1.2.7'.freeze
+      VERSION = '1.2.8'.freeze
     end
   end
 end

data/lib/envoi/mam/agent/transfer_client/aspera.rb

@@ -1,68 +1,10 @@
 require 'envoi/mam/agent/transfer_client'
 require 'asperalm/fasp/local'
 
-
-
-# module Asperalm
-#   module Fasp
-#     module InstallationExtensions
-#
-#       PRODUCTS = {
-#         unix: {
-#           :aspera_cli_local => {
-#             :expected=>'Aspera CLI',
-#             :exe_ext=>'',
-#             :app_root=>File.join(Dir.home,'.aspera','cli'),
-#             :run_root=>File.join(Dir.home,'.aspera','cli'),
-#             :sub_bin=>'bin',
-#             :sub_keys=>'etc',
-#             :dsa=>'asperaweb_id_dsa.openssh'
-#           }
-#         }
-#       }
-#
-#       def find_missing_products(os_type, common_places)
-#         missing_products = []
-#         case os_type
-#         when :unix
-#           extended_products = PRODUCTS[os_type] || { }
-#           extended_products.each do |product_name, product|
-#             product_found = common_places.find do |cp|
-#               cp[:expected] == product[:expected] && cp[:app_root] == product[:app_root]
-#             end
-#             missing_products.push product unless product_found
-#           end
-#         end
-#         missing_products
-#       end
-#
-#       def add_missing_products(os_type, common_places)
-#         missing_products = find_missing_products(os_type, common_places)
-#         common_places.concat missing_products unless missing_products.empty?
-#       end
-#
-#       def product_locations
-#         common_places = super
-#         add_missing_products(OpenApplication.current_os_type, common_places)
-#         common_places
-#       end
-#
-#     end
-#
-#     class Installation
-#       include InstallationExtensions
-#     end
-#   end
-# end
-
 module Asperalm
-
   class Log
-
     def logger=(new_logger) @logger = new_logger end
-
   end
-
 end
 
 
@@ -77,12 +19,12 @@ module Envoi
           DEFAULT_ASCP_ARGS = '-v -k3 --overwrite=diff -P 33001'
 
           ASCP_PATHS = [
-              '/usr/local/bin/ascp',
-              '/usr/bin/ascp',
-              '/Library/Aspera/bin/ascp',
-              '~/Applications/Aspera CLI/bin/ascp',
-              '~/Applications/Aspera Connect.app/Contents/Resources/ascp',
-              '/Applications/Aspera Connect.app/Contents/Resources/ascp'
+            '/usr/local/bin/ascp',
+            '/usr/bin/ascp',
+            '/Library/Aspera/bin/ascp',
+            '~/Applications/Aspera CLI/bin/ascp',
+            '~/Applications/Aspera Connect.app/Contents/Resources/ascp',
+            '/Applications/Aspera Connect.app/Contents/Resources/ascp'
           ]
           ASCP_PATHS.map { |v| File.expand_path(v) }
 
@@ -117,12 +59,11 @@ module Envoi
 
             aspera_token = config['aspera_transfer_token'] || config['aspera_token'] || config['token'] || config['transfer_token']
             if aspera_username && aspera_password && (aspera_token.nil? || aspera_token.empty?)
-              _token_string = %(Basic #{["#{aspera_username}:#{aspera_password}"]
+              aspera_token = %(Basic #{["#{aspera_username}:#{aspera_password}"]
                                             .pack('m')
                                             .delete("\r\n")})
-              aspera_token = _token_string
-              aspera_password = nil
               aspera_username = 'xfer'
+              aspera_password = nil
             end
 
             # @ascp_path = config['ascp_path'] || default_ascp_path
@@ -175,7 +116,7 @@ module Envoi
 
             aspera_token = config['aspera_transfer_token'] || config['aspera_token'] || config['token'] || config['transfer_token']
             if aspera_username && aspera_password && (aspera_token.nil? || aspera_token.empty?)
-              _token_string = %(Basic #{["#{aspera_username}:#{aspera_password}"]
+              aspera_token = %(Basic #{["#{aspera_username}:#{aspera_password}"]
                                             .pack('m')
                                             .delete("\r\n")})
               aspera_username = 'xfer'

data/lib/envoi/mam/cantemo/agent.rb

@@ -2,8 +2,6 @@ require 'cantemo/portal/api/client'
 require 'envoi/mam/agent/transfer_client/aspera'
 require 'envoi/mam/agent/transfer_client/s3'
 
-require 'vidispine/api/utilities'
-
 module Envoi
 
   module Mam
@@ -14,7 +12,7 @@ module Envoi
 
         DEFAULT_SHAPE_TAG          = 'original'
         DEFAULT_DESTINATION_PATH   = '.'
-        DEFAULT_PRESERVE_FILE_PATH = true
+        DEFAULT_PRESERVE_FILE_PATH = false
 
         attr_accessor :default_aspera_ascp_args,
                       :default_aspera_ascp_path,
@@ -139,9 +137,8 @@ module Envoi
         # @option args [String] :transfer_type ('')
         # @option args [Boolean] :import_file (true)
         # @option args [Boolean] :preserve_path (storage_config['preserve_path'] || default_preserve_path)
-        # @return [Hash]
-        #
         #
+        # @return [Hash]
         def upload(args = {})
           _response = {}
 
@@ -257,6 +254,7 @@ module Envoi
         #
         # @return [Hash{ :success->Boolean, :file_create_response->{}, :import_response->{} }]
         def import_file(args = {})
+          logger.debug { "Agent - Import File - Args: #{args}"}
           _response = args[:response_object] || {}
 
           file_path  = args[:file_path]
@@ -310,11 +308,12 @@ module Envoi
               file_id:           file_id,
               storage_path_map: { '/' => storage_id }
             }
-            item_add_args.merge(item_add_args_in) if item_add_args_in.is_a?(Hash)
+            item_add_args.merge!(item_add_args_in) if item_add_args_in.is_a?(Hash)
 
             item_add_options = { }
             item_add_options.merge!(item_add_options_in) if item_add_options_in.is_a?(Hash)
 
+            logger.debug { "Executing Item Add/Import. ARGS: #{item_add_args} OPTS: #{item_add_options}" }
             item_shape_import_response = api_client.item_add_using_file_path(item_add_args, item_add_options)
           end
           _response[:import_response] = item_shape_import_response

data/lib/envoi/mam/cantemo/agent/watch_folder_manager.rb

@@ -27,6 +27,7 @@ module Envoi::Mam::Cantemo
 
     class WatchFolderManager
 
+
       class MultiIO
         def initialize(*targets)
           @targets = targets
@@ -46,6 +47,20 @@ module Envoi::Mam::Cantemo
 
       end
 
+      class MultiLogger
+        attr_accessor :targets
+
+        def initialize(*targets)
+          @targets = targets
+        end
+
+        %w(log debug info warn error fatal unknown level=).each do |m|
+          define_method(m) do |*args, &block|
+            @targets.map { |t| t.send(m, *args, &block) }
+          end
+        end
+      end
+
       # AWF = Envoi::Aspera::WatchService::WatchFolder # Aspera Watch Folder
       LWF = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen # Listen Watch Folder
 
@@ -56,8 +71,7 @@ module Envoi::Mam::Cantemo
       def initialize(args = {})
         initialize_logger(args)
 
-        logger.debug { 'Initializing Agent Watch Folder Manager.' }
-
+        logger.info { 'Initializing Agent Watch Folder Manager.' }
         args[:default_preserve_file_path] = args.fetch(:default_preserve_file_path, false)
 
         @config = Envoi::Mam::Cantemo::Agent.load_config_from_file(args)
@@ -94,23 +108,18 @@ module Envoi::Mam::Cantemo
       end
 
       def initialize_logger(args = {})
-        @logger   = args[:logger] ||= begin
-          _log_to = MultiIO.new(STDOUT)
+        @logger = MultiLogger.new(Logger.new(STDOUT))
+
+        _logger = args[:logger] ||= begin
           log_to = args[:log_to]
-          log_age = args[:log_age] || 'daily'
-          _log_to.add_target(File.open(log_to, 'a')) if log_to
-          _logger = Logger.new(_log_to, log_age)
-        end
+          log_age = args.fetch(:log_age, 'daily')
 
-        log_level = args[:log_level] ||= Logger::INFO
-        if log_level
-          if log_level.is_a?(String)
-            log_level.downcase!
-            _log_level = %w(fatal error warn info debug).find { |v| v == log_level }
-            log_level = _log_level ? Logger::Severity.const_get(_log_level.to_sym.upcase) : Logger::INFO
-          end
-          @logger.level = log_level
+          log_to ? Logger.new(log_to, log_age) : nil
         end
+        @logger.targets << _logger if _logger
+
+        log_level     = args[:log_level] ||= Logger::INFO
+        @logger.level = log_level if log_level
         @logger
       end
 
@@ -138,9 +147,9 @@ module Envoi::Mam::Cantemo
         logging               = watch_folder_def['logging'] || watch_folder_def
         log_to                = logging['log_to']
         log_level             = logging['log_level']
-        args_out[:log_to]     ||= log_to if log_to && !log_to.empty?
-        args_out[:log_level]  ||= log_level if log_level && !log_level.empty?
-        args_out[:logger]     ||= logger.dup unless log_to
+        # args_out[:log_to]     = log_to if log_to && !log_to.empty?
+        # args_out[:log_level]  = log_level if log_level && !log_level.empty?
+        args_out[:logger]     = logger # unless log_to
         args_out[:default_agent] = default_agent
         args_out[:default_agent_class] = default_agent_class
         args_out[:definition] = watch_folder_def
@@ -208,6 +217,7 @@ module Envoi::Mam::Cantemo
         }
         upload_args.merge!(watch_folder_upload_args) if watch_folder_upload_args.is_a?(Hash)
 
+        logger.debug { "Executing Upload. #{upload_args}" }
         _response = agent.upload(upload_args)
         _response = { success: _response } if _response == true || _response == false
 
@@ -253,7 +263,8 @@ module Envoi::Mam::Cantemo
       end
 
       # This should be part of the watch folder but it is here to track active processors globally
-      def process_watch_folder_stable_files(wf, stable_files)
+      def process_watch_folder_stable_files(wf)
+        stable_files = wf.stable_files
         active_processors         = @processors_by_watch_folder[wf]
         maximum_active_processors = wf.definition['maximum_active_processors']
 
@@ -325,19 +336,20 @@ module Envoi::Mam::Cantemo
         end
 
         stable_files_by_watch_folder.each do |watch_folder, stable_files|
-          process_watch_folder_stable_files(watch_folder, stable_files)
+          process_watch_folder_stable_files(watch_folder)
         end
       end
 
       # The main execution method
       def run
+        @should_run = true
+
         # AWF.run_once(watch_folders) { |wf| pp wf }
         # AWF.run(watch_folders) { |wf| process_watch_folder(wf) }
-
         logger.info { 'Running...' }
         watch_folders.map { |wf| wf.respond_to?(:run) ? wf.run : logger.debug { "Skip run for #{wf}" } }
         logger.debug { 'Initial Run Complete.' }
-        loop do
+        while(@should_run) do
           begin
             poll
             sleep 1
@@ -346,19 +358,27 @@ module Envoi::Mam::Cantemo
             break
           end
         end
-        logger.info { 'Exiting...' }
       rescue => e
         logger.debug { "EXCEPTION #{e.message} \n#{e.backtrace.join("\n")}\n" }
         logger.error { "An error occurred. #{e.message}" }
         raise e
       ensure
-        watch_folders.each { |wf| wf.stop if wf.respond_to?(:stop) }
+        stop
+        logger.info { 'Exiting...' }
       end
 
       def run_once
         # AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
       end
 
+      def stop
+        if @should_run
+          @should_run = false
+          logger.info { "Stopping..." }
+          watch_folders.each { |wf| wf.stop if wf.respond_to?(:stop) }
+        end
+      end
+
       # Converts hash keys to symbols
       #
       # @param [Hash] value hash

data/lib/envoi/watch_folder_utility/watch_folder.rb

@@ -20,17 +20,6 @@ module Envoi
                     :min_stable_time,
                     :poll_interval
 
-
-      class MultiIO
-        def initialize(*targets) @targets = targets end
-
-        def write(*args) @targets.each {|t| t.write(*args)} end
-
-        def close; @targets.each(&:close) end
-
-        def add_target(*target) @targets.concat target end
-      end
-
       def initialize(args = { })
         initialize_logger(args)
         @definition = args[:definition].dup
@@ -59,23 +48,24 @@ module Envoi
       end
 
       def initialize_logger(args = {})
-        @logger   = args[:logger] ||= begin
-          _log_to = MultiIO.new(STDOUT)
-          log_to = args[:log_to]
-          log_age = args[:log_age] || 'daily'
-          _log_to.add_target(File.open(log_to, 'a')) if log_to
-          _logger = Logger.new(_log_to, log_age)
-        end
-
-        log_level = args[:log_level] ||= Logger::INFO
-        if log_level
-          if log_level.is_a?(String)
-            log_level.downcase!
-            _log_level = %w(fatal error warn info debug).find { |v| v == log_level }
-            log_level = _log_level ? Logger::Severity.const_get(_log_level.to_sym.upcase) : Logger::INFO
-          end
-          @logger.level = log_level
-        end
+        @logger = args[:logger]
+        # @logger   = args[:logger] ||= begin
+        #   _log_to = MultiIO.new(STDOUT)
+        #   log_to = args[:log_to]
+        #   log_age = args[:log_age] || 'daily'
+        #   _log_to.add_target(File.open(log_to, 'a')) if log_to
+        #   _logger = Logger.new(_log_to, log_age)
+        # end
+        #
+        # log_level = args[:log_level] ||= Logger::INFO
+        # if log_level
+        #   if log_level.is_a?(String)
+        #     log_level.downcase!
+        #     _log_level = %w(fatal error warn info debug).find { |v| v == log_level }
+        #     log_level = _log_level ? Logger::Severity.const_get(_log_level.to_sym.upcase) : Logger::INFO
+        #   end
+        #   @logger.level = log_level
+        # end
         @logger
       end
 
@@ -309,7 +299,7 @@ module Envoi
         watch_folder_def.delete('import_options')
 
         watch_folder_def['upload_args'] = upload_args
-        logger.debug { "Parameter 'upload/import arguments' initialized." }
+        logger.debug { "Parameter 'upload/import arguments' initialized. #{upload_args}" }
 
         maximum_active_processors = watch_folder_def['maximum_active_processors']
         if maximum_active_processors.nil?
@@ -356,12 +346,11 @@ module Envoi
 
       def process_stable_files
         active_processors         = @processors_by_watch_folder[wf]
-        maximum_active_processors = wf.definition['maximum_active_processors']
+        maximum_active_processors = definition['maximum_active_processors']
 
-        includes = wf.definition['includes']
-        excludes = wf.definition['excludes']
+        includes = definition['includes']
+        excludes = definition['excludes']
 
-        ignored_file_paths = wf.ignored_file_paths if wf.respond_to?(:ingored_file_paths)
         unless ignored_file_paths
           ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cantemo-portal-agent
 version: !ruby/object:Gem::Version
-  version: 1.2.7
+  version: 1.2.8
 platform: ruby
 authors:
 - John Whitson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-09 00:00:00.000000000 Z
+date: 2019-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asperalm