cantemo-portal-agent 1.0.9 → 1.1.0

data/lib/envoi/mam/agent/transfer_client/aspera.rb

@@ -1,4 +1,59 @@
 require 'envoi/mam/agent/transfer_client'
+require 'asperalm/fasp/local'
+
+
+
+module Asperalm
+  module Fasp
+    module InstallationExtensions
+
+      PRODUCTS = {
+        unix: {
+          :aspera_cli_local => {
+            :expected=>'Aspera CLI',
+            :exe_ext=>'',
+            :app_root=>File.join(Dir.home,'.aspera','cli'),
+            :run_root=>File.join(Dir.home,'.aspera','cli'),
+            :sub_bin=>'bin',
+            :sub_keys=>'etc',
+            :dsa=>'asperaweb_id_dsa.openssh'
+          }
+        }
+      }
+
+      def find_missing_products(os_type, common_places)
+        missing_products = []
+        case os_type
+        when :unix
+          extended_products = PRODUCTS[os_type] || { }
+          extended_products.each do |product_name, product|
+            product_found = common_places.find do |cp|
+              cp[:expected] == product[:expected] && cp[:app_root] == product[:app_root]
+            end
+            missing_products.push product unless product_found
+          end
+        end
+        missing_products
+      end
+
+      def add_missing_products(os_type, common_places)
+        missing_products = find_missing_products(os_type, common_places)
+        common_places.concat missing_products unless missing_products.empty?
+      end
+
+      def product_locations
+        common_places = super
+        add_missing_products(OpenApplication.current_os_type, common_places)
+        common_places
+      end
+
+    end
+
+    class Installation
+      include InstallationExtensions
+    end
+  end
+end
 
 module Envoi
   module Mam
@@ -32,10 +87,68 @@ module Envoi
             File.executable?(@ascp_path)
           end
 
+          def build_asperala_transfer_args(config, mode, source_path, target_path)
+
+            aspera_host_address = config['host_address'] || config['host']
+            aspera_username = config['username']
+            aspera_password = config['password']
+            aspera_ssh_port = config['ssh_port']
+
+            aspera_url = config['url'] || config['uri']
+            if aspera_url
+              aspera_uri = URI(aspera_url)
+              aspera_host_address ||= aspera_uri.host if aspera_uri
+              un, pw = (aspera_uri.userinfo || '').split(':')
+              aspera_username ||= un if un
+              aspera_password ||= pw if pw
+              aspera_ssh_port = aspera_uri.port
+            end
+
+            aspera_token = config['aspera_transfer_token'] || config['aspera_token'] || config['token'] || config['transfer_token']
+
+            # @ascp_path = config['ascp_path'] || default_ascp_path
+            # @ascp_path = File.expand_path(@ascp_path)
+            # aspera_ascp_path = @ascp_path
+
+            env_vars = { }
+            env_vars['ASPERA_SCP_PASS'] = aspera_password if aspera_password
+
+            ascp_args = config['ascp_args'] || default_ascp_args || agent.default_ascp_args
+
+            tags = config['tags'] || { }
+            aspera_tags = tags['aspera'] || { }
+            aspera_tags['xfer_id'] ||= SecureRandom.uuid
+
+            cmdline_args = [
+              '--mode', mode,
+              '--host', aspera_host_address,
+              '--user', aspera_username,
+              '--tags64', Base64.strict_encode64(JSON.generate(tags)),
+            ]
+            cmdline_args.concat [ '-P', aspera_ssh_port ] if aspera_ssh_port
+            cmdline_args.concat ascp_args.is_a?(Array) ? ascp_args : ascp_args.split(' ') if ascp_args && !ascp_args.empty?
+            cmdline_args.concat [ '-W', aspera_token ] if aspera_token && !aspera_token.empty?
+            cmdline_args.concat [ source_path.gsub('"', '\"'), target_path.gsub('"', '\"') ]
+
+            { env: env_vars, args: cmdline_args, ascp_version: :ascp }
+          end
+
           def build_ascp_command(config, mode, source_path, target_path)
             aspera_host_address = config['host_address'] || config['host']
             aspera_username = config['username']
             aspera_password = config['password']
+            aspera_ssh_port = config['ssh_port']
+
+            aspera_url = config['url'] || config['uri']
+            if aspera_url
+              aspera_uri = URI(aspera_url)
+              aspera_host_address ||= aspera_uri.host if aspera_uri
+              un, pw = (aspera_uri.userinfo || '').split(':')
+              aspera_username ||= un if un
+              aspera_password ||= pw if pw
+              aspera_ssh_port ||= aspera_uri.port
+            end
+
             aspera_token = config['aspera_transfer_token'] || config['aspera_token'] || config['token'] || config['transfer_token']
 
             @ascp_path = config['ascp_path'] || default_ascp_path
@@ -43,11 +156,19 @@ module Envoi
 
             aspera_ascp_path = @ascp_path
 
+            tags = config['tags'] || { }
+            aspera_tags = tags['aspera'] || { }
+            aspera_tags['xfer_id'] ||= SecureRandom.uuid
+
+
             ascp_args = config['ascp_args'] || default_ascp_args || agent.default_ascp_args
+
             # command = %(export ASPERA_SCP_PASS="#{aspera_password}"; "#{aspera_ascp_path}" #{ascp_args} --host="#{aspera_host_address}" --user=#{aspera_username} --mode=#{mode} "#{source_path}" "#{target_path}" )
             command = ''
             command << %(export ASPERA_SCP_PASS="#{aspera_password}"; ) if aspera_password
             command << %("#{aspera_ascp_path}" --mode=#{mode} --host="#{aspera_host_address}" --user="#{aspera_username}")
+            command << %( -P #{aspera_ssh_port}) if aspera_ssh_port
+            command << %(--tags64 #{Base64.strict_encode64(JSON.generate(tags))}) if tags && !tags.empty?
             command << %( #{ascp_args}) if ascp_args && !ascp_args.empty?
             command << %( -W "#{aspera_token}") if aspera_token && !aspera_token.empty?
             command << %( "#{source_path.gsub('"', '\"')}" "#{target_path.gsub('"', '\"')}")
@@ -56,20 +177,37 @@ module Envoi
           def download(config, path, destination_path = DEFAULT_DESTINATION_PATH)
             aspera_base_path = config['base_path'] || ''
             source_path = File.join(aspera_base_path, path)
-            command = build_ascp_command(config, 'recv', source_path, destination_path)
 
-            unless ascp_path_exists?
-              warn "ASCP not found. '#{ascp_path}'"
-              return false
-            end
-            agent.shell_execute(command)
+            mode = 'recv'
+            transfer(config, mode, source_path, destination_path)
           end
 
           def upload(config, path, destination_path = DEFAULT_DESTINATION_PATH)
             aspera_base_path = config['base_path'] || ''
             source_path = path
             destination_path = aspera_base_path.empty? ? destination_path : File.join(aspera_base_path, destination_path)
-            command = build_ascp_command(config, 'send', source_path, destination_path)
+
+            mode = 'send'
+            transfer(config, mode, source_path, destination_path)
+          end
+
+          def transfer(config, mode, source_path, destination_path)
+            if false
+              transfer_using_asperala(config, mode, source_path, destination_path)
+            else
+              transfer_using_shell_execute(config, mode, source_path, destination_path)
+            end
+          end
+
+          def transfer_using_asperala(config, mode, source_path, destination_path)
+            args_out = build_asperala_transfer_args(config, mode, source_path, destination_path)
+            fasp = Asperalm::Fasp::Local.instance
+            Asperalm::Log.instance.level = :debug
+            fasp.start_transfer_with_args_env(args_out, {})
+          end
+
+          def transfer_using_shell_execute(config, mode, source_path, destination_path)
+            command = build_ascp_command(config, mode, source_path, destination_path)
 
             unless ascp_path_exists?
               warn "ASCP not found. '#{ascp_path}'"

data/lib/envoi/mam/agent/version.rb

@@ -1,7 +1,7 @@
 module Envoi
   module Mam
     class Agent
-      VERSION = '1.0.2'.freeze
+      VERSION = '1.1.0'.freeze
     end
   end
 end

data/lib/envoi/mam/agent/watch_folder_utility/foreman.rb

@@ -0,0 +1,76 @@
+module Envoi
+  module Mam
+    class Agent
+      class WatchFolderUtility
+        class Foreman
+
+          AWF = Envoi::Aspera::WatchService::WatchFolder
+          LWF = Envoi::WatchFolderUtility::WatchFolder::Listen
+
+          attr_accessor :logger, :agent, :config, :watch_folders
+
+          def initialize(args = { })
+            initialize_logger(args)
+
+            @agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
+            @config = agent.config
+            cantemo_config = config[:cantemo] || config['cantemo']
+            watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
+
+            @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
+
+
+
+            @watch_folders = AWF.process_watch_folder_defs(watch_folder_defs)
+          end
+
+          def self.initialize_watch_folder(watch_folder_def)
+            case  (watch_folder_def['handler'] || '').downcase.gsub(/[-_ ]/, '')
+              when 'asperawatch'
+                wf_class = Envoi::Aspera::WatchService::WatchFolder
+            end
+            wf_class.new(watch_folder_def)
+          end
+
+          def self.process_watch_folder_def(watch_folder_def)
+            initialize_watch_folder(watch_folder_def)
+          end
+
+          def self.process_watch_folder_defs(watch_folder_defs)
+            if watch_folder_defs.is_a?(Array)
+              watch_folder_defs.map { |watch_folder_def| process_watch_folder_def(watch_folder_def) }
+            elsif watch_folder_defs.is_a?(Hash)
+              watch_folder_defs.map do |name, watch_folder_def|
+                watch_folder_def['path'] ||= name; process_watch_folder_def(watch_folder_def)
+              end
+            else
+              raise "Unhandled format: #{watch_folder_defs.class.name}"
+            end
+          end
+
+          def self.poll_watch_folder(watch_folder, &block)
+            watch_folder.poll(&block)
+          end
+
+          def self.poll_watch_folders(watch_folders, &block)
+            watch_folders.each { |watch_folder| poll_watch_folder(watch_folder, &block) }
+          end
+
+          def self.run_once(watch_folders, &block)
+            poll_watch_folders(watch_folders, &block)
+          end
+
+          # Foreman
+        end
+
+        # WatchFolderUtility
+      end
+
+      # Agent
+    end
+
+    # Mam
+  end
+
+  # Envoi
+end

data/lib/envoi/mam/agent.rb

@@ -55,6 +55,7 @@ module Envoi
         end
         logger.debug { "Executing Command: '#{command}'" }
 
+        success = false
         Open3.popen3(command) do |stdin, stdout, stderr, thread|
           # stdin.sync = true
           # stdout.sync = true
@@ -65,11 +66,15 @@ module Envoi
             output << stdout.read #rescue nil
             output << stderr.read # rescue nil
             unless output.empty?
-              print output
+              logger.debug { output }
               output.clear
             end
           end
+
+          success = thread.value == 0 ? true : false
         end
+
+        success
       end
 
       def self.load_from_config_file(args)

data/lib/envoi/mam/cantemo/agent/watch_folder_handler-working.rb

@@ -0,0 +1,111 @@
+require 'envoi/mam/cantemo/agent'
+require 'envoi/aspera/watch_service/watch_folder'
+
+module Envoi::Mam::Cantemo
+
+  class Agent
+
+    class WatchFolderHandler
+
+      AWF = Envoi::Aspera::WatchService::WatchFolder
+
+      attr_accessor :logger, :agent, :config, :watch_folders
+
+      def initialize(args = { })
+        initialize_logger(args)
+
+        @agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
+        @config = agent.config
+        cantemo_config = config[:cantemo] || config['cantemo']
+        watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
+
+        @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
+
+        @watch_folders = AWF.process_watch_folder_defs(watch_folder_defs)
+      end
+
+      def initialize_logger(args = { })
+        @logger = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
+        log_level = args[:log_level]
+        if log_level
+          @logger.level = log_level
+          args[:logger] = @logger
+        end
+        @logger
+      end
+
+      def add_to_ignore(wf, file)
+        @ignored_file_paths_by_watch_folder[wf] << file.path
+      end
+
+      def process_file(watch_folder, file, storage_id = nil, quarantine_directory_path = nil)
+        full_file_path = File.join(watch_folder.path, file.path)
+        if storage_id && agent.upload(file_path: full_file_path, storage_id: storage_id)
+          FileUtils.rm full_file_path
+        else
+          if Dir.exist?(quarantine_directory_path)
+            FileUtils.mv full_file_path, quarantine_directory_path
+          else
+            add_to_ignore(watch_folder, file)
+          end
+        end
+      end
+
+      def process_watch_folder(wf)
+        storage_id = wf.definition['upload_to_storage_id'] || wf.definition['storage_id']
+        quarantine_directory_path = wf.definition['quarantine_path']
+        exclude = wf.definition['exclude']
+        min_stable_poll_count = wf.definition['stable_poll_count'] || 3
+
+        maps = wf.state.details[:maps]
+        stable_paths = maps[:stable]
+        ignored_files = @ignored_file_paths_by_watch_folder[wf]
+        stable_paths.each do |fp, file|
+          if exclude
+            next if ignored_files.include?(file.path)
+            if [*exclude].find { |ep| File.fnmatch(ep, file.path) }
+              logger.debug { "Adding File to Ignore Cache: '#{file.path}'"}
+              ignored_files << file.path
+              next
+            end
+          end
+
+          # full_file_path = File.join(wf.path, file.path)
+
+          # pp file
+          # puts file_path
+          stable_poll_count = file[:stable_poll_count]
+
+          if stable_poll_count && stable_poll_count > min_stable_poll_count
+            process_file(wf, file, storage_id, quarantine_directory_path)
+          end
+        end
+
+        # process_watch_folder
+      end
+
+      def run
+        # AWF.run_once(watch_folders) { |wf| pp wf }
+        AWF.run(watch_folders) { |wf| process_watch_folder(wf) }
+      end
+
+      def run_once
+        AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
+      end
+
+      def self.run(args)
+        w = self.new(args)
+        w.run
+      end
+
+      def self.run_as_daemon(args)
+        # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
+        require 'daemons'
+        Daemons.run_proc('cantemo-portal-watch-folders') { self.run(args) }
+      end
+
+    end
+
+  end
+
+end

data/lib/envoi/mam/cantemo/agent/watch_folder_handler.rb

@@ -0,0 +1,176 @@
+require 'envoi/mam/cantemo/agent'
+require 'envoi/aspera/watch_service/watch_folder'
+require 'envoi/watch_folder_utility/watch_folder/handler/listen'
+
+module Envoi::Mam::Cantemo
+
+  class Agent
+
+    class WatchFolderHandler
+
+      AWF = Envoi::Aspera::WatchService::WatchFolder
+      LWF = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen
+
+      attr_accessor :logger, :agent, :config, :watch_folder_defs, :watch_folders
+
+      def initialize(args = { })
+        initialize_logger(args)
+
+        @agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
+        @config = agent.config
+        cantemo_config = config[:cantemo] || config['cantemo']
+        @watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
+
+        @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
+        process_watch_folder_defs
+      end
+
+      def process_watch_folder_def(watch_folder_def)
+        name = watch_folder_def['name']
+
+        path = watch_folder_def['path']
+        path ||= name
+        (watch_folder_def['paths'] ||= []).concat [*path]
+        watch_folder_def['paths'].uniq!
+        paths = watch_folder_def['paths']
+        watch_folder_def['path'] ||= paths.first if path.length == 1
+
+        exclude = watch_folder_def['exclude']
+        exclude ||= '**/.*'
+        (watch_folder_def['excludes'] ||= []).concat [*exclude]
+        watch_folder_def['excludes'].uniq!
+
+        args_out = { }
+        logging = watch_folder_def['logging'] || watch_folder_def
+        log_to = logging['log_to']
+        log_level = logging['log_level']
+        args_out[:log_to] ||= log_to if log_to && !log_to.empty?
+        args_out[:log_level] ||= log_level if log_level && !log_level.empty?
+        args_out[:logger] ||= logger.dup unless log_to
+        args_out[:definition] = watch_folder_def
+
+        LWF.new(args_out)
+      end
+
+      def process_watch_folder_defs
+        if watch_folder_defs.is_a?(Array)
+          @watch_folders = watch_folder_defs.map { |watch_folder_def| process_watch_folder_def(watch_folder_def) }
+        elsif watch_folder_defs.is_a?(Hash)
+          @watch_folders = watch_folder_defs.map do |name, watch_folder_def|
+            watch_folder_def['name'] ||= name
+
+            process_watch_folder_def(watch_folder_def)
+          end
+        else
+          raise "Unhandled format: #{watch_folder_defs.class.name}"
+        end
+      end
+
+      def initialize_logger(args = { })
+        @logger = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
+        log_level = args[:log_level]
+        if log_level
+          @logger.level = log_level
+          args[:logger] = @logger
+        end
+        @logger
+      end
+
+      def add_to_ignore(watch_folder, file)
+        logger.debug { "Adding File to Ignore Cache: '#{file.path}'"}
+        @ignored_file_paths_by_watch_folder[watch_folder] << file.path
+      end
+
+      def process_file(watch_folder, file, storage_id = nil, quarantine_directory_path = nil)
+        # full_file_path = File.join(watch_folder.path, file.path)
+        full_file_path = file.path
+        if storage_id && agent.upload(file_path: full_file_path, storage_id: storage_id)
+          FileUtils.rm full_file_path
+        else
+          if Dir.exist?(quarantine_directory_path)
+            FileUtils.mv full_file_path, quarantine_directory_path
+          else
+            add_to_ignore(watch_folder, file)
+          end
+        end
+      end
+
+      def process_watch_folder_stable_files(wf, stable_files)
+        storage_id = wf.definition['upload_to_storage_id'] || wf.definition['storage_id']
+        quarantine_directory_path = wf.definition['quarantine_path']
+        exclude = wf.definition['excludes']
+
+        ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
+        stable_files.each do |file|
+          next if ignored_file_paths.include?(file.path)
+
+          if exclude && [*exclude].find do |ep|
+            excluded = (r = Regexp.try_convert(ep) ? r.match(ep) : File.fnmatch(ep, file.path))
+            logger.debug { "#{ep} #{excluded ? 'matched' : "didn't match"} #{file.path}" }
+            excluded
+          end
+            add_to_ignore(wf, file)
+            next
+          end
+
+          process_file(wf, file, storage_id, quarantine_directory_path)
+        end
+
+      end
+
+      def poll
+        stable_files_by_watch_folder = {} # Hash.new { |h, k| h[k] = [] }
+        watch_folders.each do |watch_folder|
+          if !watch_folder.last_poll_time || (Time.now - watch_folder.last_poll_time) >= watch_folder.poll_interval
+            stable_files = watch_folder.poll
+            stable_files_by_watch_folder[watch_folder] = stable_files
+          end
+        end
+        stable_files_by_watch_folder.each do |watch_folder, stable_files|
+          process_watch_folder_stable_files(watch_folder, stable_files)
+        end
+      end
+
+      def run
+        # AWF.run_once(watch_folders) { |wf| pp wf }
+        # AWF.run(watch_folders) { |wf| process_watch_folder(wf) }
+
+        logger.info { 'Running...' }
+        watch_folders.map { |wf| wf.run }
+        loop do
+          begin
+            poll
+            sleep 1
+          rescue SystemExit, Interrupt
+            break
+          end
+        end
+        logger.info { 'Exiting...' }
+      rescue => e
+        logger.error { "An error occurred.\n#{e.message}\n#{e.backtrace.join("\n")}\n#{e.message}" }
+        raise e
+      end
+
+      def run_once
+        # AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
+      end
+
+      def self.run(args)
+        w = self.new(args)
+        w.run
+      rescue => e
+        abort("An error occurred.\n#{e.message}\n#{e.message}")
+      end
+
+      def self.run_as_daemon(args)
+        # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
+        require 'daemons'
+        # Daemons.run_proc('cantemo-portal-watch-folders', { force: true }) { self.run(args) }
+        Daemons.run_proc('cantemo-portal-watch-folders') { self.run(args) }
+      end
+
+    end
+
+  end
+
+end