aspera-cli 4.14.0 → 4.16.0

data/lib/aspera/cli/plugins/orchestrator.rb

@@ -1,17 +1,55 @@
 # frozen_string_literal: true
 
-require 'aspera/cli/plugins/node'
+require 'aspera/cli/basic_auth_plugin'
+require 'aspera/nagios'
+require 'aspera/log'
+require 'aspera/assert'
 require 'xmlsimple'
 
 module Aspera
   module Cli
     module Plugins
       class Orchestrator < Aspera::Cli::BasicAuthPlugin
+        class << self
+          STANDARD_PATH = '/aspera/orchestrator'
+          def detect(address_or_url)
+            address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
+            urls = [address_or_url]
+            urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
+            urls.each do |base_url|
+              next unless base_url.match?('https?://')
+              api = Rest.new(base_url: base_url)
+              test_endpoint = 'api/remote_node_ping'
+              result = api.read(test_endpoint, {format: :json})
+              next unless result[:data]['remote_orchestrator_info']
+              url = result[:http].uri.to_s
+              return {
+                version: result[:data]['remote_orchestrator_info']['orchestrator-version'],
+                url:     url[0..url.index(test_endpoint) - 2]
+              }
+            rescue StandardError => e
+              Log.log.debug{"detect error: #{e}"}
+            end
+            return nil
+          end
+
+          def wizard(object:, private_key_path: nil, pub_key_pem: nil)
+            options = object.options
+            return {
+              preset_value: {
+                url:      options.get_option(:url, mandatory: true),
+                username: options.get_option(:username, mandatory: true),
+                password: options.get_option(:password, mandatory: true)
+              },
+              test_args:    'workflow list'
+            }
+          end
+        end
+
         def initialize(env)
           super(env)
-          options.declare(:params, 'Start parameters', types: Hash, default: {})
-          options.declare(:result, "Specify result value as: 'work step:parameter'")
-          options.declare(:synchronous, 'Work step:parameter expected as result', values: :bool, default: :no)
+          options.declare(:result, "Specify result value as: 'work_step:parameter'")
+          options.declare(:synchronous, 'Wait for completion', values: :bool, default: :no)
           options.declare(:ret_style, 'How return type is requested in api', values: %i[header arg ext], default: :arg)
           options.declare(:auth_style, 'Authentication type', values: %i[arg_pass head_basic apikey], default: :head_basic)
           options.parse_options!
@@ -59,12 +97,12 @@ module Aspera
               call_args[:url_params][:format] = format
             when :ext
               call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
-            else raise 'unexpected'
+            else error_unexpected_value(call_type)
             end
           end
           result = @api_orch.call(call_args)
           result[:data] = XmlSimple.xml_in(result[:http].body, opt[:xml_opt] || {'ForceArray' => true}) if format.eql?('xml')
-          Log.dump(:data, result[:data])
+          Log.log.debug{Log.dump(:data, result[:data])}
           return result
         end
 
@@ -74,7 +112,7 @@ module Aspera
           when :arg_pass
             rest_params[:auth] = {
               type:      :url,
-              url_creds: {
+              url_query: {
                 'login'    => options.get_option(:username, mandatory: true),
                 'password' => options.get_option(:password, mandatory: true) }}
           when :head_basic
@@ -118,9 +156,9 @@ module Aspera
             end
             case command
             when :status
-              options = {}
-              options[:id] = wf_id unless wf_id.eql?(VAL_ALL)
-              result = call_ao('workflows_status', options)[:data]
+              call_opts = {}
+              call_opts[:id] = wf_id unless wf_id.eql?(ExtendedValue::ALL)
+              result = call_ao('workflows_status', call_opts)[:data]
               return {type: :object_list, data: result['workflows']['workflow']}
             when :list
               result = call_ao('workflows_list', id: 0)[:data]
@@ -145,19 +183,18 @@ module Aspera
               }
               call_params = {format: :json}
               override_accept = nil
-              # set external parameters if any
-              # TODO: make not an option, but a parameter
-              self.options.get_option(:params, mandatory: true).each do |name, value|
+              # get external parameters if any
+              options.get_next_argument('external_parameters', mandatory: false, type: Hash, default: {}).each do |name, value|
                 call_params["external_parameters[#{name}]"] = value
               end
               # synchronous call ?
-              call_params['synchronous'] = true if self.options.get_option(:synchronous, mandatory: true)
+              call_params['synchronous'] = true if options.get_option(:synchronous, mandatory: true)
               # expected result for synchro call ?
-              expected = self.options.get_option(:result)
-              unless expected.nil?
+              result_location = options.get_option(:result)
+              unless result_location.nil?
                 result[:type] = :status
-                fields = expected.split(':')
-                raise "Expects: work_step:result_name format, but got #{expected}" if fields.length != 2
+                fields = result_location.split(':')
+                raise Cli::BadArgument, "Expects: work_step:result_name : #{result_location}" if fields.length != 2
                 call_params['explicit_output_step'] = fields[0]
                 call_params['explicit_output_variable'] = fields[1]
                 # implicitly, call is synchronous
@@ -170,7 +207,7 @@ module Aspera
               result[:data] = call_ao('initiate', id: wf_id, args: call_params, accept: override_accept)[:data]
               return result
             end # wf command
-          else raise "ERROR, unknown command: [#{command}]"
+          else error_unexpected_value(command)
           end # case command
         end # execute_action
       end # Orchestrator

data/lib/aspera/cli/plugins/preview.rb

@@ -1,17 +1,20 @@
 # frozen_string_literal: true
 
+# cspell:ignore trevents
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/preview/generator'
 require 'aspera/preview/options'
 require 'aspera/preview/utils'
 require 'aspera/preview/file_types'
+require 'aspera/preview/terminal'
 require 'aspera/fasp/transfer_spec'
 require 'aspera/persistency_action_once'
 require 'aspera/node'
 require 'aspera/hash_ext'
 require 'aspera/timer_limiter'
 require 'aspera/id_generator'
-require 'date'
+require 'aspera/log'
+require 'aspera/assert'
 require 'securerandom'
 
 module Aspera
@@ -26,8 +29,11 @@ module Aspera
         PREVIEW_BASENAME = 'preview'
         # subfolder in system tmp folder
         TMP_DIR_PREFIX = 'prev_tmp'
+        # same value as in aspera.conf
         DEFAULT_PREVIEWS_FOLDER = 'previews'
+        # mark that this is used by a particular access key
         AK_MARKER_FILE = '.aspera_access_key'
+        # URL prefix for local storage
         PVCL_LOCAL_STORAGE = 'file:///'
         LOG_LIMITER_SEC = 30.0
         private_constant :PREV_GEN_TAG,
@@ -53,6 +59,8 @@ module Aspera
           @gen_options = Aspera::Preview::Options.new
           # used to trigger periodic processing
           @periodic = TimerLimiter.new(LOG_LIMITER_SEC)
+          # Proc
+          @filter_block = nil
           # link CLI options to gen_info attributes
           options.declare(
             :skip_format, 'Skip this preview format (multiple possible)', values: Aspera::Preview::Generator::PREVIEW_FORMATS,
@@ -66,7 +74,7 @@ module Aspera
           options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
           options.declare(:temp_folder, 'Path to temp folder', default: Dir.tmpdir)
           options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, default: [])
-          options.declare(:case, 'Basename of output for for test')
+          options.declare(:base, 'Basename of output for for test')
           options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
           options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
           options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', values: :bool, default: false)
@@ -88,7 +96,7 @@ module Aspera
           end
 
           options.parse_options!
-          raise 'skip_folder shall be an Array, use @json:[...]' unless @option_skip_folders.is_a?(Array)
+          assert_type(@option_skip_folders, Array){'skip_folder'}
           @tmp_folder = File.join(options.get_option(:temp_folder, mandatory: true), "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
           FileUtils.mkdir_p(@tmp_folder)
           Log.log.debug{"tmpdir: #{@tmp_folder}"}
@@ -98,7 +106,7 @@ module Aspera
           @skip_types = []
           value.split(',').each do |v|
             s = v.to_sym
-            raise "not supported: #{v}" unless Aspera::Preview::FileTypes::CONVERSION_TYPES.include?(s)
+            assert_values(s, Aspera::Preview::FileTypes::CONVERSION_TYPES){'skip_types'}
             @skip_types.push(s)
           end
         end
@@ -205,7 +213,7 @@ module Aspera
         end
 
         def do_transfer(direction, folder_id, source_filename, destination='/')
-          raise 'Internal ERROR' if destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)
+          assert(!(destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)))
           t_spec = @api_node.transfer_spec_gen4(folder_id, direction, {
             'paths' => [{'source' => source_filename}],
             'tags'  => {Fasp::TransferSpec::TAG_RESERVED => {PREV_GEN_TAG => true}}
@@ -232,6 +240,7 @@ module Aspera
         def get_infos_remote(gen_infos, entry)
           # store source directly here
           local_original_filepath = File.join(@tmp_folder, entry['name'])
+          # require 'date'
           # original_mtime=DateTime.parse(entry['modified_time'])
           # out: where previews are generated
           local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
@@ -255,9 +264,10 @@ module Aspera
           "#{entry['id']}#{PREVIEW_FOLDER_SUFFIX}"
         end
 
-        def preview_filename(preview_format, filename=nil)
-          filename ||= PREVIEW_BASENAME
-          return "#{filename}.#{preview_format}"
+        # Generate a file name based on basename and format (extension)
+        def preview_filename(preview_format, base_name=nil)
+          base_name ||= PREVIEW_BASENAME
+          return "#{base_name}.#{preview_format}"
         end
 
         # generate preview files for one folder entry (file) if necessary
@@ -290,10 +300,13 @@ module Aspera
                 next false if gen_info[:preview_newer_than_original]
               end
             end
-            # need generator for further checks
-            gen_info[:generator] = Aspera::Preview::Generator.new(@gen_options, gen_info[:src], gen_info[:dst], @tmp_folder, entry['content_type'])
-            # get conversion_type (if known) and check if supported
-            next false unless gen_info[:generator].supported?
+            begin
+              # need generator for further checks
+              gen_info[:generator] = Aspera::Preview::Generator.new(gen_info[:src], gen_info[:dst], @gen_options, @tmp_folder, entry['content_type'])
+            rescue
+              # no conversion supported
+              next false
+            end
             # shall we skip it ?
             next false if @skip_types.include?(gen_info[:generator].conversion_type)
             # ok we need to generate
@@ -307,7 +320,7 @@ module Aspera
             #  download original file to temp folder
             do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
           end
-          Log.log.info{"source: #{entry['id']}: #{entry['path']})"}
+          Log.log.info{"source: #{entry['id']}: #{entry['path']}"}
           gen_infos.each do |gen_info|
             gen_info[:generator].generate rescue nil
           end
@@ -328,24 +341,22 @@ module Aspera
         end # generate_preview
 
         # scan all files in provided folder entry
-        # @param scan_start subpath to start folder scan inside
-        def scan_folder_files(top_entry, scan_start=nil)
-          if !scan_start.nil?
+        # @param top_path subpath to start folder scan inside
+        def scan_folder_files(top_entry, top_path=nil)
+          unless top_path.nil?
             # canonical path: start with / and ends with /
-            scan_start = '/' + scan_start.split('/').reject(&:empty?).join('/')
-            scan_start = "#{scan_start}/" # unless scan_start.end_with?('/')
+            top_path = '/' + top_path.split('/').reject(&:empty?).join('/') + '/'
           end
-          filter_block = Aspera::Node.file_matcher(value_or_query(allowed_types: String))
-          Log.log.debug{"scan: #{top_entry} : #{scan_start}".green}
+          Log.log.debug{"scan: #{top_entry} : #{top_path}".green}
           # don't use recursive call, use list instead
           entries_to_process = [top_entry]
           until entries_to_process.empty?
             entry = entries_to_process.shift
-            # process this entry only if it is within the scan_start
+            # process this entry only if it is within the top_path
             entry_path_with_slash = entry['path']
             Log.log.info{"processing entry #{entry_path_with_slash}"} if @periodic.trigger?
             entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
-            if !scan_start.nil? && !scan_start.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(scan_start)
+            if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
               Log.log.debug{"#{entry['path']} folder (skip start)".bg_red}
               next
             end
@@ -353,7 +364,7 @@ module Aspera
             begin
               case entry['type']
               when 'file'
-                if filter_block.call(entry)
+                if @filter_block.call(entry)
                   generate_preview(entry)
                 else
                   Log.log.debug('skip by filter')
@@ -386,11 +397,11 @@ module Aspera
           end
         end
 
-        ACTIONS = %i[scan events trevents check test].freeze
+        ACTIONS = %i[scan events trevents check test show].freeze
 
         def execute_action
           command = options.get_next_command(ACTIONS)
-          unless %i[check test].include?(command)
+          unless %i[check test show].include?(command)
             # this will use node api
             @api_node = Aspera::Node.new(params: basic_auth_params)
             @transfer_server_address = URI.parse(@api_node.params[:base_url]).host
@@ -408,25 +419,25 @@ module Aspera
             if @access_remote
               # NOTE: the filter "name", it's why we take the first one
               @previews_folder_entry = get_folder_entries(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
-              raise CliError, "Folder #{@option_previews_folder} does not exist on node. "\
+              raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
                 'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
             else
-              raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
+              assert(@access_key_self['storage']['type'].eql?('local')){'only local storage allowed in this mode'}
               @local_storage_root = @access_key_self['storage']['path']
               # TODO: option to override @local_storage_root='xxx'
               @local_storage_root = @local_storage_root[PVCL_LOCAL_STORAGE.length..-1] if @local_storage_root.start_with?(PVCL_LOCAL_STORAGE)
               # TODO: windows could have "C:" ?
-              raise "not local storage: #{@local_storage_root}" unless @local_storage_root.start_with?('/')
-              raise CliError, "Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
+              assert(@local_storage_root.start_with?('/')){"not local storage: #{@local_storage_root}"}
+              assert(File.directory?(@local_storage_root), exception_class: Cli::Error){"Local storage root folder #{@local_storage_root} does not exist."}
               @local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
-              raise CliError, "Folder #{@local_preview_folder} does not exist locally. "\
+              raise Cli::Error, "Folder #{@local_preview_folder} does not exist locally. " \
                 'Please create it, or specify an alternate name.' unless File.directory?(@local_preview_folder)
               # protection to avoid clash of file id for two different access keys
               marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
               Log.log.debug{"marker file: #{marker_file}"}
               if File.exist?(marker_file)
                 ak = File.read(marker_file).chomp
-                raise "mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}" unless @access_key_self['id'].eql?(ak)
+                assert(@access_key_self['id'].eql?(ak)){"mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}"}
               else
                 File.write(marker_file, @access_key_self['id'])
               end
@@ -450,9 +461,11 @@ module Aspera
               else
                 @api_node.read("files/#{scan_id}")[:data]
               end
+            @filter_block = Aspera::Node.file_matcher_from_argument(options)
             scan_folder_files(folder_info, scan_path)
             return Main.result_status('scan finished')
           when :events, :trevents
+            @filter_block = Aspera::Node.file_matcher_from_argument(options)
             iteration_persistency = nil
             if options.get_option(:once_only, mandatory: true)
               iteration_persistency = PersistencyActionOnce.new(
@@ -470,15 +483,18 @@ module Aspera
             return Main.result_status("#{command} finished")
           when :check
             return Main.result_status('Tools validated')
-          when :test
-            format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS)
+          when :test, :show
             source = options.get_next_argument('source file')
-            dest = preview_filename(format, options.get_option(:case))
-            g = Aspera::Preview::Generator.new(@gen_options, source, dest, @tmp_folder, nil)
-            raise "cannot find file type for #{source}" if g.conversion_type.nil?
-            raise "out format #{format} not supported" unless g.supported?
+            format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS, default: :png)
+            generated_file_path = preview_filename(format, options.get_option(:base))
+            g = Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder, nil)
             g.generate
-            return Main.result_status("generated: #{dest}")
+            if command.eql?(:show)
+              terminal_options = options.get_option(:query, default: {}).symbolize_keys
+              Log.log.debug{"preview: #{generated_file_path}"}
+              formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
+            end
+            return Main.result_status("generated: #{generated_file_path}")
           else
             raise 'error'
           end

data/lib/aspera/cli/plugins/server.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
+# cspell:ignore ascmd zmode zuid zgid fasping
 require 'aspera/cli/basic_auth_plugin'
-require 'aspera/cli/plugins/sync'
-require 'aspera/ascmd'
+require 'aspera/cli/sync_actions'
 require 'aspera/fasp/transfer_spec'
+require 'aspera/ascmd'
 require 'aspera/ssh'
 require 'aspera/nagios'
+require 'aspera/log'
+require 'aspera/assert'
 require 'tempfile'
 require 'open3'
 
@@ -13,17 +16,8 @@ module Aspera
   module Cli
     module Plugins
       # implement basic remote access with FASP/SSH
-      class SyncSpecServer
-        def initialize(transfer_spec)
-          @transfer_spec = transfer_spec
-        end
-
-        def transfer_spec(direction, local_path, remote_path)
-          return @transfer_spec
-        end
-      end
-
       class Server < Aspera::Cli::BasicAuthPlugin
+        include SyncActions
         SSH_SCHEME = 'ssh'
         LOCAL_SCHEME = 'local'
         HTTPS_SCHEME = 'https'
@@ -49,11 +43,59 @@ module Aspera
           end
         end
 
+        class << self
+          def application_name
+            'HSTS Fasp/SSH'
+          end
+
+          def detect(address_or_url)
+            urls = if address_or_url.match?(%r{^[a-z]{1,6}://})
+              [address_or_url]
+            else
+              [
+                "ssh://#{address_or_url}:33001",
+                "ssh://#{address_or_url}:22"
+              ]
+              # wss not practical as it requires a token
+            end
+
+            urls.each do |base_url|
+              server_uri = URI.parse(base_url)
+              Log.log.debug{"URI=#{server_uri}, host=#{server_uri.hostname}, port=#{server_uri.port}, scheme=#{server_uri.scheme}"}
+              next unless server_uri.scheme.eql?(SSH_SCHEME)
+              begin
+                socket = TCPSocket.new(server_uri.hostname, server_uri.port)
+                socket.puts('SSH-2.0-Ascli_0.0')
+                version = socket.gets.chomp
+                if version.match?(/^SSH-2.0-/)
+                  return {version: version.gsub(/^SSH-2.0-/, ''), url: base_url}
+                end
+              rescue StandardError => e
+                Log.log.debug{"detect error: #{e}"}
+              end
+            end
+            return nil
+          end
+
+          def wizard(object:, private_key_path: nil, pub_key_pem: nil)
+            options = object.options
+            return {
+              preset_value: {
+                url:      options.get_option(:url, mandatory: true),
+                username: options.get_option(:username, mandatory: true),
+                password: options.get_option(:password, mandatory: true)
+              },
+              test_args:    'files br /'
+            }
+          end
+        end
+
         def initialize(env)
           super(env)
           options.declare(:ssh_keys, 'SSH key path list (Array or single)')
           options.declare(:passphrase, 'SSH private key passphrase')
           options.declare(:ssh_options, 'SSH options', types: Hash, default: {})
+          SyncActions.declare_options(options)
           options.parse_options!
           @ssh_opts = options.get_option(:ssh_options).symbolize_keys
         end
@@ -108,8 +150,9 @@ module Aspera
           end
           ssh_key_list = options.get_option(:ssh_keys)
           if !ssh_key_list.nil?
-            raise 'Expecting single value or array for ssh_keys' unless ssh_key_list.is_a?(Array) || ssh_key_list.is_a?(String)
             ssh_key_list = [ssh_key_list] if ssh_key_list.is_a?(String)
+            assert_type(ssh_key_list, Array){'ssh_keys'}
+            assert(ssh_key_list.all?(String))
             ssh_key_list.map!{|p|File.expand_path(p)}
             Log.log.debug{"SSH keys=#{ssh_key_list}"}
             if !ssh_key_list.empty?
@@ -138,8 +181,8 @@ module Aspera
             Fasp::TransferSpec.action_to_direction(transfer_spec, command)
             return Main.result_transfer(transfer.start(transfer_spec))
           when :sync
-            sync_plugin = Plugins::Sync.new(@agents, sync_spec: SyncSpecServer.new(transfer_spec))
-            return sync_plugin.execute_action
+            # lets ignore the arguments provided by execute_sync_action, we just give the transfer spec
+            return execute_sync_action {transfer_spec}
           end
         end
 
@@ -185,16 +228,16 @@ module Aspera
               else
                 nagios.add_critical('transfer', statuses.reject{|i|i.eql?(:success)}.first.to_s)
               end
-            else raise 'ERROR'
+            else error_unexpected_value(command_nagios)
             end
             return nagios.result
           when *TRANSFER_COMMANDS
             return execute_transfer(command, server_transfer_spec)
           when *Aspera::AsCmd::OPERATIONS
-            args = options.get_next_argument('ascmd command arguments', expected: :multiple, mandatory: false)
+            command_arguments = options.get_next_argument('ascmd command arguments', expected: :multiple, mandatory: false)
             ascmd = Aspera::AsCmd.new(ascmd_executor)
             begin
-              result = ascmd.send(:execute_single, command, args)
+              result = ascmd.send(:execute_single, command, command_arguments)
               case command
               when :mkdir, :mv, :cp, :rm
                 return Main.result_success
@@ -206,9 +249,9 @@ module Aspera
                 return {type: :single_object, data: result.stringify_keys}
               end
             rescue Aspera::AsCmd::Error => e
-              raise CliBadArgument, e.extended_message
+              raise Cli::BadArgument, e.extended_message
             end
-          else raise 'internal error: unexpected action'
+          else error_unreachable_line
           end
         end # execute_action
       end # Server