aspera-cli 4.13.0 → 4.15.0

data/lib/aspera/cli/plugins/orchestrator.rb

@@ -1,23 +1,55 @@
 # frozen_string_literal: true
 
-require 'aspera/cli/plugins/node'
+require 'aspera/cli/basic_auth_plugin'
+require 'aspera/nagios'
 require 'xmlsimple'
 
 module Aspera
   module Cli
     module Plugins
       class Orchestrator < Aspera::Cli::BasicAuthPlugin
+        class << self
+          STANDARD_PATH = '/aspera/orchestrator'
+          def detect(address_or_url)
+            address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
+            urls = [address_or_url]
+            urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
+            urls.each do |base_url|
+              next unless base_url.match?('https?://')
+              api = Rest.new(base_url: base_url)
+              test_endpoint = 'api/remote_node_ping'
+              result = api.read(test_endpoint, {format: :json})
+              next unless result[:data]['remote_orchestrator_info']
+              url = result[:http].uri.to_s
+              return {
+                version: result[:data]['remote_orchestrator_info']['orchestrator-version'],
+                url:     url[0..url.index(test_endpoint) - 2]
+              }
+            rescue StandardError => e
+              Log.log.debug{"detect error: #{e}"}
+            end
+            return nil
+          end
+
+          def wizard(object:, private_key_path: nil, pub_key_pem: nil)
+            options = object.options
+            return {
+              preset_value: {
+                url:      options.get_option(:url, mandatory: true),
+                username: options.get_option(:username, mandatory: true),
+                password: options.get_option(:password, mandatory: true)
+              },
+              test_args:    'workflow list'
+            }
+          end
+        end
+
         def initialize(env)
           super(env)
-          options.add_opt_simple(:params, 'parameters hash table, use @json:{"param":"value"}')
-          options.add_opt_simple(:result, "specify result value as: 'work step:parameter'")
-          options.add_opt_boolean(:synchronous, 'work step:parameter expected as result')
-          options.add_opt_list(:ret_style, %i[header arg ext], 'how return type is requested in api')
-          options.add_opt_list(:auth_style, %i[arg_pass head_basic apikey], 'authentication type')
-          options.set_option(:params, {})
-          options.set_option(:synchronous, :no)
-          options.set_option(:ret_style, :arg)
-          options.set_option(:auth_style, :head_basic)
+          options.declare(:result, "Specify result value as: 'work_step:parameter'")
+          options.declare(:synchronous, 'Wait for completion', values: :bool, default: :no)
+          options.declare(:ret_style, 'How return type is requested in api', values: %i[header arg ext], default: :arg)
+          options.declare(:auth_style, 'Authentication type', values: %i[arg_pass head_basic apikey], default: :head_basic)
           options.parse_options!
         end
 
@@ -49,7 +81,7 @@ module Aspera
           call_args[:subpath] = "#{opt[:prefix]}/#{call_args[:subpath]}" unless opt[:prefix].nil?
           # specify id if necessary
           call_args[:subpath] = "#{call_args[:subpath]}/#{opt[:id]}" if opt.key?(:id)
-          call_type = options.get_option(:ret_style, is_type: :mandatory)
+          call_type = options.get_option(:ret_style, mandatory: true)
           call_type = opt[:ret_style] if opt.key?(:ret_style)
           format = 'json'
           format = opt[:format] if opt.key?(:format)
@@ -68,24 +100,24 @@ module Aspera
           end
           result = @api_orch.call(call_args)
           result[:data] = XmlSimple.xml_in(result[:http].body, opt[:xml_opt] || {'ForceArray' => true}) if format.eql?('xml')
-          Log.dump(:data, result[:data])
+          Log.log.debug{Log.dump(:data, result[:data])}
           return result
         end
 
         def execute_action
-          rest_params = {base_url: options.get_option(:url, is_type: :mandatory)}
-          case options.get_option(:auth_style, is_type: :mandatory)
+          rest_params = {base_url: options.get_option(:url, mandatory: true)}
+          case options.get_option(:auth_style, mandatory: true)
           when :arg_pass
             rest_params[:auth] = {
               type:      :url,
-              url_creds: {
-                'login'    => options.get_option(:username, is_type: :mandatory),
-                'password' => options.get_option(:password, is_type: :mandatory) }}
+              url_query: {
+                'login'    => options.get_option(:username, mandatory: true),
+                'password' => options.get_option(:password, mandatory: true) }}
           when :head_basic
             rest_params[:auth] = {
               type:     :basic,
-              username: options.get_option(:username, is_type: :mandatory),
-              password: options.get_option(:password, is_type: :mandatory) }
+              username: options.get_option(:username, mandatory: true),
+              password: options.get_option(:password, mandatory: true) }
           when :apikey
             raise 'Not implemented'
           end
@@ -122,9 +154,9 @@ module Aspera
             end
             case command
             when :status
-              options = {}
-              options[:id] = wf_id unless wf_id.eql?(VAL_ALL)
-              result = call_ao('workflows_status', options)[:data]
+              call_opts = {}
+              call_opts[:id] = wf_id unless wf_id.eql?(ExtendedValue::ALL)
+              result = call_ao('workflows_status', call_opts)[:data]
               return {type: :object_list, data: result['workflows']['workflow']}
             when :list
               result = call_ao('workflows_list', id: 0)[:data]
@@ -149,18 +181,18 @@ module Aspera
               }
               call_params = {format: :json}
               override_accept = nil
-              # set external parameters if any
-              self.options.get_option(:params, is_type: :mandatory).each do |name, value|
+              # get external parameters if any
+              options.get_next_argument('external_parameters', mandatory: false, type: Hash, default: {}).each do |name, value|
                 call_params["external_parameters[#{name}]"] = value
               end
               # synchronous call ?
-              call_params['synchronous'] = true if self.options.get_option(:synchronous, is_type: :mandatory)
+              call_params['synchronous'] = true if options.get_option(:synchronous, mandatory: true)
               # expected result for synchro call ?
-              expected = self.options.get_option(:result)
-              unless expected.nil?
+              result_location = options.get_option(:result)
+              unless result_location.nil?
                 result[:type] = :status
-                fields = expected.split(':')
-                raise "Expects: work_step:result_name format, but got #{expected}" if fields.length != 2
+                fields = result_location.split(':')
+                raise Cli::BadArgument, "Expects: work_step:result_name : #{result_location}" if fields.length != 2
                 call_params['explicit_output_step'] = fields[0]
                 call_params['explicit_output_variable'] = fields[1]
                 # implicitly, call is synchronous

data/lib/aspera/cli/plugins/preview.rb

@@ -1,17 +1,18 @@
 # frozen_string_literal: true
 
+# cspell:ignore trevents
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/preview/generator'
 require 'aspera/preview/options'
 require 'aspera/preview/utils'
 require 'aspera/preview/file_types'
+require 'aspera/preview/terminal'
 require 'aspera/fasp/transfer_spec'
 require 'aspera/persistency_action_once'
 require 'aspera/node'
 require 'aspera/hash_ext'
 require 'aspera/timer_limiter'
 require 'aspera/id_generator'
-require 'date'
 require 'securerandom'
 
 module Aspera
@@ -26,8 +27,11 @@ module Aspera
         PREVIEW_BASENAME = 'preview'
         # subfolder in system tmp folder
         TMP_DIR_PREFIX = 'prev_tmp'
+        # same value as in aspera.conf
         DEFAULT_PREVIEWS_FOLDER = 'previews'
+        # mark that this is used by a particular access key
         AK_MARKER_FILE = '.aspera_access_key'
+        # URL prefix for local storage
         PVCL_LOCAL_STORAGE = 'file:///'
         LOG_LIMITER_SEC = 30.0
         private_constant :PREV_GEN_TAG,
@@ -53,44 +57,45 @@ module Aspera
           @gen_options = Aspera::Preview::Options.new
           # used to trigger periodic processing
           @periodic = TimerLimiter.new(LOG_LIMITER_SEC)
+          # Proc
+          @filter_block = nil
           # link CLI options to gen_info attributes
-          options.set_obj_attr(:skip_format, self, :option_skip_format, []) # no skip
-          options.set_obj_attr(:folder_reset_cache, self, :option_folder_reset_cache, :no)
-          options.set_obj_attr(:skip_types, self, :option_skip_types)
-          options.set_obj_attr(:previews_folder, self, :option_previews_folder, DEFAULT_PREVIEWS_FOLDER)
-          options.set_obj_attr(:skip_folders, self, :option_skip_folders, []) # no skip
-          options.set_obj_attr(:overwrite, self, :option_overwrite, :mtime)
-          options.set_obj_attr(:file_access, self, :option_file_access, :local)
-          options.add_opt_list(:skip_format, Aspera::Preview::Generator::PREVIEW_FORMATS, 'skip this preview format (multiple possible)')
-          options.add_opt_list(:folder_reset_cache, %i[no header read], 'force detection of generated preview by refresh cache')
-          options.add_opt_simple(:skip_types, 'skip types in comma separated list')
-          options.add_opt_simple(:previews_folder, 'preview folder in storage root')
-          options.add_opt_simple(:temp_folder, 'path to temp folder')
-          options.add_opt_simple(:skip_folders, 'list of folder to skip')
-          options.add_opt_simple(:case, 'basename of output for for test')
-          options.add_opt_simple(:scan_path, 'subpath in folder id to start scan in (default=/)')
-          options.add_opt_simple(:scan_id, 'folder id in storage to start scan in, default is access key main folder id')
-          options.add_opt_boolean(:mimemagic, 'use Mime type detection of gem mimemagic')
-          options.add_opt_list(:overwrite, %i[always never mtime], 'when to overwrite result file')
-          options.add_opt_list(:file_access, %i[local remote], 'how to read and write files in repository')
-          options.set_option(:temp_folder, Dir.tmpdir)
-          options.set_option(:mimemagic, false)
+          options.declare(
+            :skip_format, 'Skip this preview format (multiple possible)', values: Aspera::Preview::Generator::PREVIEW_FORMATS,
+            handler: {o: self, m: :option_skip_format}, default: [])
+          options.declare(
+            :folder_reset_cache, 'Force detection of generated preview by refresh cache',
+            values: %i[no header read],
+            handler: {o: self, m: :option_folder_reset_cache},
+            default: :no)
+          options.declare(:skip_types, 'Skip types in comma separated list', handler: {o: self, m: :option_skip_types})
+          options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
+          options.declare(:temp_folder, 'Path to temp folder', default: Dir.tmpdir)
+          options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, default: [])
+          options.declare(:base, 'Basename of output for for test')
+          options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
+          options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
+          options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', values: :bool, default: false)
+          options.declare(:overwrite, 'When to overwrite result file', values: %i[always never mtime], handler: {o: self, m: :option_overwrite}, default: :mtime)
+          options.declare(
+            :file_access, 'How to read and write files in repository',
+            values: %i[local remote],
+            handler: {o: self, m: :option_file_access},
+            default: :local)
 
           # add other options for generator (and set default values)
           Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
-            options.set_obj_attr(opt[:name], @gen_options, opt[:name], opt[:default])
-            if opt.key?(:values)
-              options.add_opt_list(opt[:name], opt[:values], opt[:description])
+            values = if opt.key?(:values)
+              opt[:values]
             elsif Cli::Manager::BOOLEAN_SIMPLE.include?(opt[:default])
-              options.add_opt_boolean(opt[:name], opt[:description])
-            else
-              options.add_opt_simple(opt[:name], opt[:description])
+              :bool
             end
+            options.declare(opt[:name], opt[:description].capitalize, values: values, handler: {o: @gen_options, m: opt[:name]}, default: opt[:default])
           end
 
           options.parse_options!
           raise 'skip_folder shall be an Array, use @json:[...]' unless @option_skip_folders.is_a?(Array)
-          @tmp_folder = File.join(options.get_option(:temp_folder, is_type: :mandatory), "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
+          @tmp_folder = File.join(options.get_option(:temp_folder, mandatory: true), "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
           FileUtils.mkdir_p(@tmp_folder)
           Log.log.debug{"tmpdir: #{@tmp_folder}"}
         end
@@ -206,34 +211,12 @@ module Aspera
         end
 
         def do_transfer(direction, folder_id, source_filename, destination='/')
-          raise 'error' if destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)
-          if @default_transfer_spec.nil?
-            # make a dummy call to get some default transfer parameters
-            res = @api_node.create('files/upload_setup', {'transfer_requests' => [{'transfer_request' => {'paths' => [{}], 'destination_root' => '/'}}]})
-            template_ts = res[:data]['transfer_specs'].first['transfer_spec']
-            # get ports, anyway that should be 33001 for both. add remote_user ?
-            @default_transfer_spec = %w[ssh_port fasp_port].each_with_object({}){|e, h|h[e] = template_ts[e]; }
-            if !@default_transfer_spec['remote_user'].eql?(Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
-              Log.log.warn('remote_user shall be xfer')
-              @default_transfer_spec['remote_user'] = Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER
-            end
-            @api_node.ts_basic_token(@default_transfer_spec)
-            # NOTE: we use the same address for ascp than for node api instead of the one from upload_setup
-            # TODO: configurable ? useful ?
-            @default_transfer_spec['remote_host'] = @transfer_server_address
-          end
-          t_spec = @default_transfer_spec.merge({
-            'direction' => direction,
-            'paths'     => [{'source' => source_filename}],
-            'tags'      => {
-              Fasp::TransferSpec::TAG_RESERVED => {
-                PREV_GEN_TAG => true,
-                'node'       => {
-                  'access_key' => @access_key_self['id'],
-                  'file_id'    => folder_id }}}
+          raise 'Internal ERROR' if destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)
+          t_spec = @api_node.transfer_spec_gen4(folder_id, direction, {
+            'paths' => [{'source' => source_filename}],
+            'tags'  => {Fasp::TransferSpec::TAG_RESERVED => {PREV_GEN_TAG => true}}
           })
-          # force destination
-          # t_spec['destination_root']=destination
+          # force destination, need to set this in transfer agent else it gets overwritten, not do: t_spec['destination_root']=destination
           transfer.option_transfer_spec_deep_merge({'destination_root' => destination})
           Main.result_transfer(transfer.start(t_spec))
         end
@@ -255,6 +238,7 @@ module Aspera
         def get_infos_remote(gen_infos, entry)
           # store source directly here
           local_original_filepath = File.join(@tmp_folder, entry['name'])
+          # require 'date'
           # original_mtime=DateTime.parse(entry['modified_time'])
           # out: where previews are generated
           local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
@@ -278,9 +262,10 @@ module Aspera
           "#{entry['id']}#{PREVIEW_FOLDER_SUFFIX}"
         end
 
-        def preview_filename(preview_format, filename=nil)
-          filename ||= PREVIEW_BASENAME
-          return "#{filename}.#{preview_format}"
+        # Generate a file name based on basename and format (extension)
+        def preview_filename(preview_format, base_name=nil)
+          base_name ||= PREVIEW_BASENAME
+          return "#{base_name}.#{preview_format}"
         end
 
         # generate preview files for one folder entry (file) if necessary
@@ -313,10 +298,13 @@ module Aspera
                 next false if gen_info[:preview_newer_than_original]
               end
             end
-            # need generator for further checks
-            gen_info[:generator] = Aspera::Preview::Generator.new(@gen_options, gen_info[:src], gen_info[:dst], @tmp_folder, entry['content_type'])
-            # get conversion_type (if known) and check if supported
-            next false unless gen_info[:generator].supported?
+            begin
+              # need generator for further checks
+              gen_info[:generator] = Aspera::Preview::Generator.new(gen_info[:src], gen_info[:dst], @gen_options, @tmp_folder, entry['content_type'])
+            rescue
+              # no conversion supported
+              next false
+            end
             # shall we skip it ?
             next false if @skip_types.include?(gen_info[:generator].conversion_type)
             # ok we need to generate
@@ -330,7 +318,7 @@ module Aspera
             #  download original file to temp folder
             do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
           end
-          Log.log.info{"source: #{entry['id']}: #{entry['path']})"}
+          Log.log.info{"source: #{entry['id']}: #{entry['path']}"}
           gen_infos.each do |gen_info|
             gen_info[:generator].generate rescue nil
           end
@@ -351,24 +339,22 @@ module Aspera
         end # generate_preview
 
         # scan all files in provided folder entry
-        # @param scan_start subpath to start folder scan inside
-        def scan_folder_files(top_entry, scan_start=nil)
-          if !scan_start.nil?
+        # @param top_path subpath to start folder scan inside
+        def scan_folder_files(top_entry, top_path=nil)
+          unless top_path.nil?
             # canonical path: start with / and ends with /
-            scan_start = '/' + scan_start.split('/').reject(&:empty?).join('/')
-            scan_start = "#{scan_start}/" # unless scan_start.end_with?('/')
+            top_path = '/' + top_path.split('/').reject(&:empty?).join('/') + '/'
           end
-          filter_block = Aspera::Node.file_matcher(options.get_option(:value))
-          Log.log.debug{"scan: #{top_entry} : #{scan_start}".green}
+          Log.log.debug{"scan: #{top_entry} : #{top_path}".green}
           # don't use recursive call, use list instead
           entries_to_process = [top_entry]
           until entries_to_process.empty?
             entry = entries_to_process.shift
-            # process this entry only if it is within the scan_start
+            # process this entry only if it is within the top_path
             entry_path_with_slash = entry['path']
             Log.log.info{"processing entry #{entry_path_with_slash}"} if @periodic.trigger?
             entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
-            if !scan_start.nil? && !scan_start.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(scan_start)
+            if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
               Log.log.debug{"#{entry['path']} folder (skip start)".bg_red}
               next
             end
@@ -376,7 +362,7 @@ module Aspera
             begin
               case entry['type']
               when 'file'
-                if filter_block.call(entry)
+                if @filter_block.call(entry)
                   generate_preview(entry)
                 else
                   Log.log.debug('skip by filter')
@@ -409,11 +395,11 @@ module Aspera
           end
         end
 
-        ACTIONS = %i[scan events trevents check test].freeze
+        ACTIONS = %i[scan events trevents check test show].freeze
 
         def execute_action
           command = options.get_next_command(ACTIONS)
-          unless %i[check test].include?(command)
+          unless %i[check test show].include?(command)
             # this will use node api
             @api_node = Aspera::Node.new(params: basic_auth_params)
             @transfer_server_address = URI.parse(@api_node.params[:base_url]).host
@@ -431,7 +417,7 @@ module Aspera
             if @access_remote
               # NOTE: the filter "name", it's why we take the first one
               @previews_folder_entry = get_folder_entries(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
-              raise CliError, "Folder #{@option_previews_folder} does not exist on node. "\
+              raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
                 'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
             else
               raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
@@ -440,9 +426,9 @@ module Aspera
               @local_storage_root = @local_storage_root[PVCL_LOCAL_STORAGE.length..-1] if @local_storage_root.start_with?(PVCL_LOCAL_STORAGE)
               # TODO: windows could have "C:" ?
               raise "not local storage: #{@local_storage_root}" unless @local_storage_root.start_with?('/')
-              raise CliError, "Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
+              raise Cli::Error, "Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
               @local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
-              raise CliError, "Folder #{@local_preview_folder} does not exist locally. "\
+              raise Cli::Error, "Folder #{@local_preview_folder} does not exist locally. " \
                 'Please create it, or specify an alternate name.' unless File.directory?(@local_preview_folder)
               # protection to avoid clash of file id for two different access keys
               marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
@@ -455,7 +441,7 @@ module Aspera
               end
             end
           end
-          Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, is_type: :mandatory)
+          Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, mandatory: true)
           # check tools that are anyway required for all cases
           Aspera::Preview::Utils.check_tools(@skip_types)
           case command
@@ -473,19 +459,21 @@ module Aspera
               else
                 @api_node.read("files/#{scan_id}")[:data]
               end
+            @filter_block = Aspera::Node.file_matcher_from_argument(options)
             scan_folder_files(folder_info, scan_path)
             return Main.result_status('scan finished')
           when :events, :trevents
+            @filter_block = Aspera::Node.file_matcher_from_argument(options)
             iteration_persistency = nil
-            if options.get_option(:once_only, is_type: :mandatory)
+            if options.get_option(:once_only, mandatory: true)
               iteration_persistency = PersistencyActionOnce.new(
                 manager: @agents[:persistency],
                 data:    [],
                 id:      IdGenerator.from_list([
                   'preview_iteration',
                   command.to_s,
-                  options.get_option(:url, is_type: :mandatory),
-                  options.get_option(:username, is_type: :mandatory)
+                  options.get_option(:url, mandatory: true),
+                  options.get_option(:username, mandatory: true)
                 ]))
             end
             # call processing method specified by command line command
@@ -493,19 +481,23 @@ module Aspera
             return Main.result_status("#{command} finished")
           when :check
             return Main.result_status('Tools validated')
-          when :test
-            format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS)
+          when :test, :show
             source = options.get_next_argument('source file')
-            dest = preview_filename(format, options.get_option(:case))
-            g = Aspera::Preview::Generator.new(@gen_options, source, dest, @tmp_folder, nil)
-            raise "cannot find file type for #{source}" if g.conversion_type.nil?
-            raise "out format #{format} not supported" unless g.supported?
+            format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS, default: :png)
+            generated_file_path = preview_filename(format, options.get_option(:base))
+            g = Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder, nil)
             g.generate
-            return Main.result_status("generated: #{dest}")
+            if command.eql?(:show)
+              terminal_options = options.get_option(:query, default: {}).symbolize_keys
+              Log.log.debug{"preview: #{generated_file_path}"}
+              formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
+            end
+            return Main.result_status("generated: #{generated_file_path}")
           else
             raise 'error'
           end
         ensure
+          Log.log.debug{"cleaning up temp folder #{@tmp_folder}"}
           FileUtils.rm_rf(@tmp_folder)
         end # execute_action
       end # Preview