aspera-cli 4.24.1 → 4.25.0.pre

data/lib/aspera/cli/plugins/orchestrator.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'aspera/cli/basic_auth_plugin'
+require 'aspera/cli/plugins/basic_auth'
 require 'aspera/cli/special_values'
 require 'aspera/nagios'
 require 'aspera/log'
@@ -10,7 +10,8 @@ require 'xmlsimple'
 module Aspera
   module Cli
     module Plugins
-      class Orchestrator < Cli::BasicAuthPlugin
+      # Aspera Orchestrator
+      class Orchestrator < BasicAuth
         STANDARD_PATH = '/aspera/orchestrator'
         TEST_ENDPOINT = 'api/remote_node_ping'
         private_constant :STANDARD_PATH, :TEST_ENDPOINT
@@ -23,11 +24,11 @@ module Aspera
             urls.each do |base_url|
               next unless base_url.match?('https?://')
               api = Rest.new(base_url: base_url)
-              result = api.call(operation: 'GET', subpath: TEST_ENDPOINT, headers: {'Accept' => Rest::MIME_JSON}, query: {format: :json})
-              next unless result[:data]['remote_orchestrator_info']
-              url = result[:http].uri.to_s
+              data, http = api.read(TEST_ENDPOINT, query: {format: :json}, ret: :both)
+              next unless data['remote_orchestrator_info']
+              url = http.uri.to_s
               return {
-                version: result[:data]['remote_orchestrator_info']['orchestrator-version'],
+                version: data['remote_orchestrator_info']['orchestrator-version'],
                 url:     url[0..url.index(TEST_ENDPOINT) - 2]
               }
             rescue StandardError => e
@@ -37,32 +38,32 @@ module Aspera
             raise error if error
             return
           end
+        end
 
-          def wizard(object:)
-            options = object.options
-            return {
-              preset_value: {
-                url:      options.get_option(:url, mandatory: true),
-                username: options.get_option(:username, mandatory: true),
-                password: options.get_option(:password, mandatory: true)
-              },
-              test_args:    'workflow list'
-            }
-          end
+        # @param wizard  [Wizard] The wizard object
+        # @param app_url [Wizard] The wizard object
+        # @return [Hash] :preset_value, :test_args
+        def wizard(wizard, app_url)
+          return {
+            preset_value: {
+              url:      app_url,
+              username: options.get_option(:username, mandatory: true),
+              password: options.get_option(:password, mandatory: true)
+            },
+            test_args:    'workflow list'
+          }
         end
 
         def initialize(**_)
           super
           @api_orch = nil
           options.declare(:result, "Specify result value as: 'work_step:parameter'")
-          options.declare(:synchronous, 'Wait for completion', values: :bool, default: :no)
-          options.declare(:ret_style, 'How return type is requested in api', values: %i[header arg ext], default: :arg)
-          options.declare(:auth_style, 'Authentication type', values: %i[arg_pass head_basic apikey], default: :head_basic)
+          options.declare(:synchronous, 'Wait for completion', allowed: Allowed::TYPES_BOOLEAN, default: false)
+          options.declare(:ret_style, 'How return type is requested in api', allowed: %i[header arg ext], default: :arg)
+          options.declare(:auth_style, 'Authentication type', allowed: %i[arg_pass head_basic apikey], default: :head_basic)
           options.parse_options!
         end
 
-        ACTIONS = %i[health info workflow plugins processes].freeze
-
         # Call orchestrator API, it's a bit special
         # @param endpoint   [String]  the endpoint to call
         # @param ret_style  [Symbol]  the return style, :header, :arg, :ext(extension)
@@ -72,28 +73,31 @@ module Aspera
         # @param http       [Boolean] if true, returns the HttpResponse, else
         def call_ao(endpoint, ret_style: nil, format: 'json', args: nil, xml_arrays: true, http: false)
           # calls are all GET
-          call_args = {operation: 'GET', subpath: "api/#{endpoint}"}
+          call_args = {operation: 'GET', subpath: "api/#{endpoint}", ret: :both, query: {}}
           ret_style = options.get_option(:ret_style, mandatory: true) if ret_style.nil?
-          call_args[:query] = args unless args.nil?
+          call_args[:query].merge!(args) unless args.nil?
           unless format.nil?
             case ret_style
             when :header
               call_args[:headers] = {'Accept' => "application/#{format}"}
             when :arg
-              call_args[:query] ||= {}
               call_args[:query][:format] = format
             when :ext
               call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
             else Aspera.error_unexpected_value(ret_style)
             end
           end
-          result = @api_orch.call(**call_args)
-          return result[:http] if http
-          result = format.eql?('xml') ? XmlSimple.xml_in(result[:http].body, {'ForceArray' => xml_arrays}) : result[:data]
+          add_query = options.get_option(:query)
+          call_args[:query].merge!(add_query.symbolize_keys) unless add_query.nil?
+          data, resp = @api_orch.call(**call_args)
+          return resp if http
+          result = format.eql?('xml') ? XmlSimple.xml_in(resp.body, {'ForceArray' => xml_arrays}) : data
           Log.dump(:data, result)
           return result
         end
 
+        ACTIONS = %i[health info workflows workorders workstep plugins processes monitors].freeze
+
         def execute_action
           auth_params =
             case options.get_option(:auth_style, mandatory: true)
@@ -131,37 +135,33 @@ module Aspera
             rescue StandardError => e
               nagios.add_critical('node api', e.to_s)
             end
-            return nagios.result
+            Main.result_object_list(nagios.status_list)
+          # 14. Ping the remote Instance
           when :info
             result = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
             return Main.result_single_object(result)
+          # 12. Orchestrator Background Process status
           when :processes
             # TODO: Bug ? API has only XML format
             result = call_ao('processes_status', format: 'xml')
             return Main.result_object_list(result['process'])
+          # 13. Orchestrator Monitor
+          when :monitors
+            result = call_ao('monitor_snapshot')
+            return Main.result_single_object(result['monitor'])
           when :plugins
             # TODO: Bug ? only json format on url
             result = call_ao('plugin_version')
             return Main.result_object_list(result['Plugin'])
-          when :workflow
-            command = options.get_next_command(%i[list status inputs details start export])
+          when :workflows
+            command = options.get_next_command(%i[list status inputs details start export workorders outputs])
             case command
-            when :status
-              wf_id = instance_identifier
-              result = call_ao(wf_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{wf_id}")
-              return Main.result_object_list(result['workflows']['workflow'])
+            # 1. List all available workflows on the system
             when :list
-              result = call_ao('workflows_list/0')
+              result = call_ao('workflows_list')
               return Main.result_object_list(result['workflows']['workflow'], fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification])
-            when :details
-              result = call_ao("workflow_details/#{instance_identifier}")
-              return Main.result_object_list(result['workflows']['workflow']['statuses'])
-            when :inputs
-              result = call_ao("workflow_inputs_spec/#{instance_identifier}")
-              return Main.result_single_object(result['workflow_inputs_spec'])
-            when :export
-              result = call_ao("export_workflow/#{instance_identifier}", format: nil, http: true)
-              return Main.result_text(result.body)
+            # 2.1 Initiate a workorder - Asynchronous
+            # 2.2 Initiate a workorder - Synchronous
             when :start
               result = {
                 type: :single_object,
@@ -189,6 +189,69 @@ module Aspera
               result[:type] = :text if call_params['synchronous']
               result[:data] = call_ao("initiate/#{wf_id}", args: call_params)
               return result
+            # 3. Fetch input specification for a workflow
+            when :inputs
+              result = call_ao("workflow_inputs_spec/#{instance_identifier}")
+              return Main.result_single_object(result['workflow_inputs_spec'])
+            # 4. Check the running status for all workflows
+            # 5. Check the running status for a particular workflow
+            when :status
+              wf_id = instance_identifier
+              result = call_ao(wf_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{wf_id}")
+              return Main.result_object_list(result['workflows']['workflow'])
+            # 6. Check the detailed running status for a particular workflow
+            when :details
+              result = call_ao("workflow_details/#{instance_identifier}")
+              return Main.result_object_list(result['workflows']['workflow']['statuses'])
+            # 15. Fetch output specification for a particular work flow
+            when :outputs
+              result = call_ao("workflow_outputs_spec/#{instance_identifier}")
+              return Main.result_object_list(result['workflow_outputs_spec']['output'])
+            # 19.Fetch all workorders from a workflow
+            when :workorders
+              result = call_ao("work_orders_list/#{instance_identifier}")
+              return Main.result_object_list(result['work_orders'])
+            when :export
+              result = call_ao("export_workflow/#{instance_identifier}", format: nil, http: true)
+              return Main.result_text(result.body)
+            end
+          when :workorders
+            command = options.get_next_command(%i[status cancel reset output])
+            case command
+            # 7. Check the status for a particular work order
+            when :status
+              wo_id = instance_identifier
+              result = call_ao("work_order_status/#{wo_id}")
+              return Main.result_single_object(result['work_order'])
+            # 9. Cancel a Work Order
+            when :cancel
+              wo_id = instance_identifier
+              result = call_ao("work_order_cancel/#{wo_id}")
+              return Main.result_single_object(result['work_order'])
+            # 11. Reset a Work order
+            when :reset
+              wo_id = instance_identifier
+              result = call_ao("work_order_reset/#{wo_id}")
+              return Main.result_single_object(result['work_order'])
+            # 16. Fetch output of a work order
+            when :output
+              wo_id = instance_identifier
+              result = call_ao("work_order_output/#{wo_id}", format: 'xml')
+              return Main.result_object_list(result['variable'])
+            end
+          when :workstep
+            command = options.get_next_command(%i[status cancel])
+            case command
+            # 8. Check the status of a Step
+            when :status
+              ws_id = instance_identifier
+              result = call_ao("work_step_status/#{ws_id}")
+              return Main.result_single_object(result)
+            # 10. Cancel a Work Step
+            when :cancel
+              ws_id = instance_identifier
+              result = call_ao("work_step_cancel/#{ws_id}")
+              return Main.result_single_object(result)
             end
           else Aspera.error_unexpected_value(command)
           end
@@ -198,3 +261,24 @@ module Aspera
     end
   end
 end
+
+# 17.Persist custom data
+# 18.Fetch queued items from queue
+# 20.List Task for a User
+# 21. Fetch Task details
+# 22. Submit Task
+# 23. Control Process
+# engine monitor worker
+# 24. Lookup Queued Item
+# 25. Reorder Queued Items
+# 26. Bulk Reorder Queued Items
+# 27. Queue Item (Add an item to a Queue)
+#
+# Required Input:
+# Optional Input:
+# 28.List all queues
+# 29. Portlet Version
+# 30. Plugin Version
+# 31. Restart Work Order from a Step
+# 32. Delete element from a Managed Queue
+#

data/lib/aspera/cli/plugins/preview.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # cspell:ignore trevents
-require 'aspera/cli/basic_auth_plugin'
+require 'aspera/cli/plugins/basic_auth'
 require 'aspera/preview/generator'
 require 'aspera/preview/options'
 require 'aspera/preview/utils'
@@ -9,6 +9,7 @@ require 'aspera/preview/file_types'
 require 'aspera/preview/terminal'
 require 'aspera/transfer/spec'
 require 'aspera/persistency_action_once'
+require 'aspera/temp_file_manager'
 require 'aspera/api/node'
 require 'aspera/hash_ext'
 require 'aspera/timer_limiter'
@@ -20,7 +21,7 @@ require 'securerandom'
 module Aspera
   module Cli
     module Plugins
-      class Preview < Cli::BasicAuthPlugin
+      class Preview < BasicAuth
         # special tag to identify transfers related to generator
         PREV_GEN_TAG = 'preview_generator'
         # defined by node API: suffix for folder containing previews
@@ -45,16 +46,15 @@ module Aspera
           :AK_MARKER_FILE,
           :LOG_LIMITER_SEC
 
-        # option_skip_format has special accessors
-        attr_accessor :option_previews_folder
-        attr_accessor :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
+        attr_accessor :option_skip_types, :option_previews_folder, :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
 
         def initialize(**_)
           super
-          @skip_types = []
-          @default_transfer_spec = nil
-          # by default generate all supported formats (clone, as altered by options)
-          @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
+          @option_skip_types = []
+          @option_skip_folders = []
+          @option_previews_folder = nil
+          @option_overwrite = nil
+          @option_folder_reset_cache = nil
           # options for generation
           @gen_options = Aspera::Preview::Options.new
           # used to trigger periodic processing
@@ -63,69 +63,49 @@ module Aspera
           @filter_block = nil
           # link CLI options to gen_info attributes
           options.declare(
-            :skip_format, 'Skip this preview format (multiple possible)', values: Aspera::Preview::Generator::PREVIEW_FORMATS,
-            handler: {o: self, m: :option_skip_format}, default: []
+            :skip_format, 'Skip this preview format',
+            allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
           )
           options.declare(
             :folder_reset_cache, 'Force detection of generated preview by refresh cache',
-            values: %i[no header read],
+            allowed: %i[no header read],
             handler: {o: self, m: :option_folder_reset_cache},
             default: :no
           )
-          options.declare(:skip_types, 'Skip types in comma separated list', handler: {o: self, m: :option_skip_types})
+          options.declare(:skip_types, 'Skip generation for those types of files', handler: {o: self, m: :option_skip_types}, allowed: Allowed::TYPES_SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES)
           options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
-          options.declare(:temp_folder, 'Path to temp folder', default: Dir.tmpdir)
-          options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, default: [])
+          options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, allowed: Allowed::TYPES_STRING_ARRAY)
           options.declare(:base, 'Basename of output for for test')
           options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
           options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
-          options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', values: :bool, default: false)
-          options.declare(:overwrite, 'When to overwrite result file', values: %i[always never mtime], handler: {o: self, m: :option_overwrite}, default: :mtime)
+          options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', allowed: Allowed::TYPES_BOOLEAN, default: false)
+          options.declare(:overwrite, 'When to overwrite result file', handler: {o: self, m: :option_overwrite}, allowed: %i[always never mtime], default: :mtime)
           options.declare(
             :file_access, 'How to read and write files in repository',
-            values: %i[local remote],
+            allowed: %i[local remote],
             handler: {o: self, m: :option_file_access},
             default: :local
           )
-
           # add other options for generator (and set default values)
           Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
             values = if opt.key?(:values)
               opt[:values]
             elsif Cli::Manager::BOOLEAN_SIMPLE.include?(opt[:default])
-              :bool
+              Allowed::TYPES_BOOLEAN
             end
-            options.declare(opt[:name], opt[:description].capitalize, values: values, handler: {o: @gen_options, m: opt[:name]}, default: opt[:default])
+            options.declare(opt[:name], opt[:description].capitalize, allowed: values, handler: {o: @gen_options, m: opt[:name]}, default: opt[:default])
           end
 
           options.parse_options!
-          Aspera.assert_type(@option_skip_folders, Array){'skip_folder'}
-          @tmp_folder = File.join(options.get_option(:temp_folder, mandatory: true), "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
+          # by default generate all supported formats (clone, as altered by options)
+          @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
+          skip = options.get_option(:skip_format)
+          @preview_formats_to_generate.delete(skip) if skip
+          @tmp_folder = File.join(TempFileManager.instance.global_temp, "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
           FileUtils.mkdir_p(@tmp_folder)
           Log.log.debug{"tmpdir: #{@tmp_folder}"}
         end
 
-        def option_skip_types=(value)
-          @skip_types = []
-          value.split(',').each do |v|
-            s = v.to_sym
-            Aspera.assert_values(s, Aspera::Preview::FileTypes::CONVERSION_TYPES){'skip_types'}
-            @skip_types.push(s)
-          end
-        end
-
-        def option_skip_types
-          return @skip_types.map(&:to_s).join(',')
-        end
-
-        def option_skip_format=(value)
-          @preview_formats_to_generate.delete(value)
-        end
-
-        def option_skip_format
-          return @preview_formats_to_generate.map(&:to_s).join(',')
-        end
-
         # /files/id/files is normally cached in redis, but we can discard the cache
         # but /files/id is not cached
         def get_folder_entries(file_id, request_args = nil)
@@ -136,7 +116,7 @@ module Aspera
             subpath:   "files/#{file_id}/files",
             headers:   headers,
             query:     request_args
-          )[:data]
+          )
         end
 
         # old version based on folders
@@ -221,8 +201,8 @@ module Aspera
             'paths' => [{'source' => source_filename}],
             'tags'  => {Transfer::Spec::TAG_RESERVED => {PREV_GEN_TAG => true}}
           })
-          # force destination, need to set this in transfer agent else it gets overwritten, not do: t_spec['destination_root']=destination
-          transfer.option_transfer_spec_deep_merge({'destination_root' => destination})
+          # force destination, need to set this in transfer agent else it gets overwritten, do not do: t_spec['destination_root']=destination
+          transfer.user_transfer_spec['destination_root'] = destination
           Main.result_transfer(transfer.start(t_spec))
         end
 
@@ -311,7 +291,7 @@ module Aspera
               next false
             end
             # shall we skip it ?
-            next false if @skip_types.include?(gen_info[:generator].conversion_type)
+            next false if @option_skip_types.include?(gen_info[:generator].conversion_type)
             # ok we need to generate
             true
           end
@@ -444,7 +424,7 @@ module Aspera
           end
           Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, mandatory: true)
           # check tools that are anyway required for all cases
-          Aspera::Preview::Utils.check_tools(@skip_types)
+          Aspera::Preview::Utils.check_tools(@option_skip_types)
           case command
           when :scan
             scan_path = options.get_option(:scan_path)
@@ -491,7 +471,7 @@ module Aspera
             g = Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder, nil)
             g.generate
             if command.eql?(:show)
-              terminal_options = options.get_option(:query, default: {}).symbolize_keys
+              terminal_options = (options.get_option(:query) || {}).symbolize_keys
               Log.log.debug{"preview: #{generated_file_path}"}
               formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
             end

data/lib/aspera/cli/plugins/server.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # cspell:ignore ascmd zmode zuid zgid fasping
-require 'aspera/cli/basic_auth_plugin'
+require 'aspera/cli/plugins/basic_auth'
 require 'aspera/cli/sync_actions'
 require 'aspera/transfer/spec'
 require 'aspera/ascmd'
@@ -14,7 +14,7 @@ module Aspera
   module Cli
     module Plugins
       # Operations on HSTS with SSH/FASP (ascmd/ascp)
-      class Server < Cli::BasicAuthPlugin
+      class Server < BasicAuth
         include SyncActions
 
         SSH_SCHEME = 'ssh'
@@ -67,27 +67,29 @@ module Aspera
             raise error if error
             return
           end
+        end
 
-          def wizard(object:)
-            options = object.options
-            return {
-              preset_value: {
-                url:      options.get_option(:url, mandatory: true),
-                username: options.get_option(:username, mandatory: true),
-                password: options.get_option(:password, mandatory: true)
-              },
-              test_args:    'files br /'
-            }
-          end
+        # @param wizard  [Wizard] The wizard object
+        # @param app_url [Wizard] The wizard object
+        # @return [Hash] :preset_value, :test_args
+        def wizard(wizard, app_url)
+          return {
+            preset_value: {
+              url:      app_url,
+              username: options.get_option(:username, mandatory: true),
+              password: options.get_option(:password, mandatory: true)
+            },
+            test_args:    'files browse /'
+          }
         end
 
         def initialize(**_)
           super
           @ssh_opts = {}
           @connection_type = :ssh
-          options.declare(:ssh_keys, 'SSH key path list (Array or single)')
+          options.declare(:ssh_keys, 'SSH key path list', allowed: Allowed::TYPES_STRING_ARRAY)
           options.declare(:passphrase, 'SSH private key passphrase')
-          options.declare(:ssh_options, 'SSH options', types: Hash, handler: {o: self, m: :option_ssh_opts})
+          options.declare(:ssh_options, 'SSH options', allowed: Hash, handler: {o: self, m: :option_ssh_opts})
           SyncActions.declare_options(options)
           options.parse_options!
         end
@@ -119,7 +121,7 @@ module Aspera
             ENV['SSH_CLIENT'] = 'local 0 0'
             @connection_type = :local
             return server_transfer_spec
-          elsif transfer.option_transfer_spec['token'].is_a?(String) && server_uri.scheme.eql?(HTTPS_SCHEME)
+          elsif transfer.user_transfer_spec['token'].is_a?(String) && server_uri.scheme.eql?(HTTPS_SCHEME)
             server_transfer_spec['wss_enabled'] = true
             server_transfer_spec['wss_port'] = server_uri.port
             @connection_type = :wss
@@ -152,9 +154,7 @@ module Aspera
           end
           ssh_key_list = options.get_option(:ssh_keys)
           if !ssh_key_list.nil?
-            ssh_key_list = [ssh_key_list] if ssh_key_list.is_a?(String)
-            Aspera.assert_type(ssh_key_list, Array){'ssh_keys'}
-            Aspera.assert(ssh_key_list.all?(String))
+            Aspera.assert_array_all(ssh_key_list, String){'ssh_keys'}
             ssh_key_list.map!{ |p| File.expand_path(p)}
             Log.log.debug{"SSH keys=#{ssh_key_list}"}
             if !ssh_key_list.empty?
@@ -171,7 +171,7 @@ module Aspera
             server_transfer_spec['ssh_private_key_passphrase'] = ssh_passphrase
           end
           # if user provided transfer spec has a token, we will use bypass keys
-          cred_set = true if transfer.option_transfer_spec['token'].is_a?(String)
+          cred_set = true if transfer.user_transfer_spec['token'].is_a?(String)
           Aspera.assert(cred_set, type: BadArgument){'Either password, key , or transfer spec token must be provided'}
           return server_transfer_spec
         end
@@ -225,7 +225,7 @@ module Aspera
               end
             else Aspera.error_unexpected_value(command_nagios)
             end
-            return nagios.result
+            Main.result_object_list(nagios.status_list)
           when *TRANSFER_COMMANDS
             return execute_transfer(command, server_transfer_spec)
           when *AsCmd::OPERATIONS