aspera-cli 4.22.0 → 4.24.0

data/lib/aspera/cli/plugins/orchestrator.rb

@@ -35,10 +35,10 @@ module Aspera
               Log.log.debug{"detect error: #{e}"}
             end
             raise error if error
-            return nil
+            return
           end
 
-          def wizard(object:, private_key_path: nil, pub_key_pem: nil)
+          def wizard(object:)
             options = object.options
             return {
               preset_value: {
@@ -51,8 +51,9 @@ module Aspera
           end
         end
 
-        def initialize(**env)
+        def initialize(**_)
           super
+          @api_orch = nil
           options.declare(:result, "Specify result value as: 'work_step:parameter'")
           options.declare(:synchronous, 'Wait for completion', values: :bool, default: :no)
           options.declare(:ret_style, 'How return type is requested in api', values: %i[header arg ext], default: :arg)
@@ -89,7 +90,7 @@ module Aspera
           result = @api_orch.call(**call_args)
           return result[:http] if http
           result = format.eql?('xml') ? XmlSimple.xml_in(result[:http].body, {'ForceArray' => xml_arrays}) : result[:data]
-          Log.log.debug{Log.dump(:data, result)}
+          Log.dump(:data, result)
           return result
         end
 
@@ -101,7 +102,8 @@ module Aspera
                 type:      :url,
                 url_query: {
                   'login'    => options.get_option(:username, mandatory: true),
-                  'password' => options.get_option(:password, mandatory: true)}
+                  'password' => options.get_option(:password, mandatory: true)
+                }
               }
             when :head_basic
               {
@@ -110,7 +112,7 @@ module Aspera
                 password: options.get_option(:password, mandatory: true)
               }
             when :apikey
-              raise 'Not implemented'
+              Aspera.error_not_implemented
             end
 
           @api_orch = Rest.new(
@@ -150,11 +152,7 @@ module Aspera
               return Main.result_object_list(result['workflows']['workflow'])
             when :list
               result = call_ao('workflows_list/0')
-              return {
-                type:   :object_list,
-                data:   result['workflows']['workflow'],
-                fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification]
-              }
+              return Main.result_object_list(result['workflows']['workflow'], fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification])
             when :details
               result = call_ao("workflow_details/#{instance_identifier}")
               return Main.result_object_list(result['workflows']['workflow']['statuses'])
@@ -188,9 +186,7 @@ module Aspera
                 # implicitly, call is synchronous
                 call_params['synchronous'] = true
               end
-              if call_params['synchronous']
-                result[:type] = :text
-              end
+              result[:type] = :text if call_params['synchronous']
               result[:data] = call_ao("initiate/#{wf_id}", args: call_params)
               return result
             end

data/lib/aspera/cli/plugins/preview.rb

@@ -49,7 +49,7 @@ module Aspera
         attr_accessor :option_previews_folder
         attr_accessor :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
 
-        def initialize(**env)
+        def initialize(**_)
           super
           @skip_types = []
           @default_transfer_spec = nil
@@ -64,12 +64,14 @@ module Aspera
           # link CLI options to gen_info attributes
           options.declare(
             :skip_format, 'Skip this preview format (multiple possible)', values: Aspera::Preview::Generator::PREVIEW_FORMATS,
-            handler: {o: self, m: :option_skip_format}, default: [])
+            handler: {o: self, m: :option_skip_format}, default: []
+          )
           options.declare(
             :folder_reset_cache, 'Force detection of generated preview by refresh cache',
             values: %i[no header read],
             handler: {o: self, m: :option_folder_reset_cache},
-            default: :no)
+            default: :no
+          )
           options.declare(:skip_types, 'Skip types in comma separated list', handler: {o: self, m: :option_skip_types})
           options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
           options.declare(:temp_folder, 'Path to temp folder', default: Dir.tmpdir)
@@ -83,7 +85,8 @@ module Aspera
             :file_access, 'How to read and write files in repository',
             values: %i[local remote],
             handler: {o: self, m: :option_file_access},
-            default: :local)
+            default: :local
+          )
 
           # add other options for generator (and set default values)
           Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
@@ -125,14 +128,15 @@ module Aspera
 
         # /files/id/files is normally cached in redis, but we can discard the cache
         # but /files/id is not cached
-        def get_folder_entries(file_id, request_args=nil)
+        def get_folder_entries(file_id, request_args = nil)
           headers = {'Accept' => Rest::MIME_JSON}
           headers['X-Aspera-Cache-Control'] = 'no-cache' if @option_folder_reset_cache.eql?(:header)
           return @api_node.call(
             operation: 'GET',
             subpath:   "files/#{file_id}/files",
             headers:   headers,
-            query:     request_args)[:data]
+            query:     request_args
+          )[:data]
         end
 
         # old version based on folders
@@ -196,12 +200,8 @@ module Aspera
               if !file_entry.nil? &&
                   @option_skip_folders.none?{ |d| file_entry['path'].start_with?(d)}
                 file_entry['parent_file_id'] = event['data']['parent_file_id']
-                if event['types'].include?('file.deleted')
-                  Log.log.error('TODO'.red)
-                end
-                if event['types'].include?('file.deleted')
-                  generate_preview(file_entry)
-                end
+                Log.log.error('TODO'.red) if event['types'].include?('file.deleted')
+                generate_preview(file_entry) if event['types'].include?('file.deleted')
               end
             end
             # log/persist periodically or last one
@@ -215,7 +215,7 @@ module Aspera
           end
         end
 
-        def do_transfer(direction, folder_id, source_filename, destination='/')
+        def do_transfer(direction, folder_id, source_filename, destination = '/')
           Aspera.assert(!(destination.nil? && direction.eql?(Transfer::Spec::DIRECTION_RECEIVE)))
           t_spec = @api_node.transfer_spec_gen4(folder_id, direction, {
             'paths' => [{'source' => source_filename}],
@@ -268,7 +268,7 @@ module Aspera
         end
 
         # Generate a file name based on basename and format (extension)
-        def preview_filename(preview_format, base_name=nil)
+        def preview_filename(preview_format, base_name = nil)
           base_name ||= PREVIEW_BASENAME
           return "#{base_name}.#{preview_format}"
         end
@@ -319,7 +319,7 @@ module Aspera
           # create folder if needed
           FileUtils.mkdir_p(local_entry_preview_dir)
           if @access_remote
-            raise 'missing parent_file_id in entry' if entry['parent_file_id'].nil?
+            Aspera.assert(!entry['parent_file_id'].nil?){'missing parent_file_id in entry'}
             #  download original file to temp folder
             do_transfer(Transfer::Spec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
           end
@@ -335,9 +335,7 @@ module Aspera
             File.delete(File.join(@tmp_folder, entry['name']))
           end
           # force read file updated previews
-          if @option_folder_reset_cache.eql?(:read)
-            @api_node.read("files/#{entry['id']}")
-          end
+          @api_node.read("files/#{entry['id']}") if @option_folder_reset_cache.eql?(:read)
         rescue StandardError => e
           Log.log.error{"Ignore: #{e.message}"}
           Log.log.debug(e.backtrace.join("\n").red)
@@ -345,10 +343,10 @@ module Aspera
 
         # scan all files in provided folder entry
         # @param top_path subpath to start folder scan inside
-        def scan_folder_files(top_entry, top_path=nil)
+        def scan_folder_files(top_entry, top_path = nil)
           unless top_path.nil?
             # canonical path: start with / and ends with /
-            top_path = '/' + top_path.split('/').reject(&:empty?).join('/') + '/'
+            top_path = "/#{top_path.split('/').reject(&:empty?).join('/')}/"
           end
           Log.log.debug{"scan: #{top_entry} : #{top_path}".green}
           # don't use recursive call, use list instead
@@ -384,9 +382,7 @@ module Aspera
                   # process all items in current folder
                   folder_entries.each do |folder_entry|
                     # add path for older versions of ES
-                    if !folder_entry.key?('path')
-                      folder_entry['path'] = entry_path_with_slash + folder_entry['name']
-                    end
+                    folder_entry['path'] = entry_path_with_slash + folder_entry['name'] if !folder_entry.key?('path')
                     folder_entry['parent_file_id'] = entry['id']
                     entries_to_process.push(folder_entry)
                   end
@@ -418,7 +414,7 @@ module Aspera
             Log.log.debug{"remote: #{@access_remote}"}
             Log.log.debug{"access key info: #{@access_key_self}"}
             # TODO: can the previews folder parameter be read from node api ?
-            @option_skip_folders.push('/' + @option_previews_folder)
+            @option_skip_folders.push("/#{@option_previews_folder}")
             if @access_remote
               # NOTE: the filter "name", it's why we take the first one
               @previews_folder_entry = get_folder_entries(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
@@ -431,7 +427,7 @@ module Aspera
               @local_storage_root = @local_storage_root[PVCL_LOCAL_STORAGE.length..-1] if @local_storage_root.start_with?(PVCL_LOCAL_STORAGE)
               # TODO: windows could have "C:" ?
               Aspera.assert(@local_storage_root.start_with?('/')){"not local storage: #{@local_storage_root}"}
-              Aspera.assert(File.directory?(@local_storage_root), exception_class: Cli::Error){"Local storage root folder #{@local_storage_root} does not exist."}
+              Aspera.assert(File.directory?(@local_storage_root), type: Cli::Error){"Local storage root folder #{@local_storage_root} does not exist."}
               @local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
               raise Cli::Error, "Folder #{@local_preview_folder} does not exist locally. " \
                 'Please create it, or specify an alternate name.' unless File.directory?(@local_preview_folder)
@@ -460,7 +456,8 @@ module Aspera
                   'id'   => @access_key_self['root_file_id'],
                   'name' => '/',
                   'type' => 'folder',
-                  'path' => '/'}
+                  'path' => '/'
+                }
               else
                 @api_node.read("files/#{scan_id}")
               end
@@ -479,7 +476,8 @@ module Aspera
                   command.to_s,
                   options.get_option(:url, mandatory: true),
                   options.get_option(:username, mandatory: true)
-                ]))
+                ])
+              )
             end
             # call processing method specified by command line command
             send(:"process_#{command}", iteration_persistency)
@@ -498,8 +496,7 @@ module Aspera
               formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
             end
             return Main.result_status("generated: #{generated_file_path}")
-          else
-            raise 'error'
+          else Aspera.error_unexpected_value(command)
           end
         ensure
           Log.log.debug{"cleaning up temp folder #{@tmp_folder}"}

data/lib/aspera/cli/plugins/server.rb

@@ -10,15 +10,13 @@ require 'aspera/nagios'
 require 'aspera/log'
 require 'aspera/assert'
 require 'aspera/environment'
-require 'tempfile'
-require 'open3'
-
 module Aspera
   module Cli
     module Plugins
-      # implement basic remote access with FASP/SSH
+      # Operations on HSTS with SSH/FASP (ascmd/ascp)
       class Server < Cli::BasicAuthPlugin
         include SyncActions
+
         SSH_SCHEME = 'ssh'
         LOCAL_SCHEME = 'local'
         HTTPS_SCHEME = 'https'
@@ -33,8 +31,8 @@ module Aspera
         private_constant :SSH_SCHEME, :URI_SCHEMES, :ASCMD_ALIASES, :TRANSFER_COMMANDS
 
         class LocalExecutor
-          def execute(ascmd_path, line)
-            return Environment.secure_capture(exec: ascmd_path, stdin_data: line, binmode: true)
+          def execute(ascmd_path, input:)
+            return Environment.secure_capture(exec: ascmd_path, stdin_data: input, binmode: true, exception: false)
           end
         end
 
@@ -61,18 +59,16 @@ module Aspera
               socket = TCPSocket.new(server_uri.hostname, server_uri.port)
               socket.puts('SSH-2.0-Ascli_0.0')
               version = socket.gets.chomp
-              if version.match?(/^SSH-2.0-/)
-                return {version: version.gsub(/^SSH-2.0-/, ''), url: base_url}
-              end
+              return {version: version.gsub(/^SSH-2.0-/, ''), url: base_url} if version.match?(/^SSH-2.0-/)
             rescue StandardError => e
               error = e
               Log.log.debug{"detect error: #{e}"}
             end
             raise error if error
-            return nil
+            return
           end
 
-          def wizard(object:, private_key_path: nil, pub_key_pem: nil)
+          def wizard(object:)
             options = object.options
             return {
               preset_value: {
@@ -85,14 +81,23 @@ module Aspera
           end
         end
 
-        def initialize(**env)
+        def initialize(**_)
           super
+          @ssh_opts = {}
+          @connection_type = :ssh
           options.declare(:ssh_keys, 'SSH key path list (Array or single)')
           options.declare(:passphrase, 'SSH private key passphrase')
-          options.declare(:ssh_options, 'SSH options', types: Hash, default: {})
+          options.declare(:ssh_options, 'SSH options', types: Hash, handler: {o: self, m: :option_ssh_opts})
           SyncActions.declare_options(options)
           options.parse_options!
-          @ssh_opts = options.get_option(:ssh_options).symbolize_keys
+        end
+
+        def option_ssh_opts; @ssh_opts; end
+
+        # multiple option are merged
+        def option_ssh_opts=(value)
+          Aspera.assert_type(value, Hash)
+          @ssh_opts.deep_merge!(value.compact.symbolize_keys)
         end
 
         # Read command line options
@@ -110,12 +115,14 @@ module Aspera
           if server_uri.scheme.eql?(LOCAL_SCHEME)
             # Using local execution (mostly for testing)
             server_transfer_spec['remote_host'] = 'localhost'
-            # simulate SSH environment, else ascp will fail
+            # simulate SSH environment, else ascmd will fail
             ENV['SSH_CLIENT'] = 'local 0 0'
+            @connection_type = :local
             return server_transfer_spec
           elsif transfer.option_transfer_spec['token'].is_a?(String) && server_uri.scheme.eql?(HTTPS_SCHEME)
             server_transfer_spec['wss_enabled'] = true
             server_transfer_spec['wss_port'] = server_uri.port
+            @connection_type = :wss
             # Using WSS
             return server_transfer_spec
           end
@@ -165,7 +172,7 @@ module Aspera
           end
           # if user provided transfer spec has a token, we will use bypass keys
           cred_set = true if transfer.option_transfer_spec['token'].is_a?(String)
-          raise 'Either password, key , or transfer spec token must be provided' if !cred_set
+          Aspera.assert(cred_set, type: BadArgument){'Either password, key , or transfer spec token must be provided'}
           return server_transfer_spec
         end
 
@@ -187,12 +194,11 @@ module Aspera
 
         def execute_action
           server_transfer_spec = options_to_base_transfer_spec
-          ascmd_executor = if !@ssh_opts.empty?
-            Ssh.new(server_transfer_spec['remote_host'], server_transfer_spec['remote_user'], @ssh_opts)
-          elsif server_transfer_spec.key?('wss_enabled')
-            nil
-          else
-            LocalExecutor.new
+          ascmd_executor = case @connection_type
+          when :local then LocalExecutor.new
+          when :wss then nil
+          when :ssh then Ssh.new(server_transfer_spec['remote_host'], server_transfer_spec['remote_user'], @ssh_opts)
+          else Aspera.error_unexpected_value(@connection_type){'connection type'}
           end
           # the set of available commands depends on SSH executor availability (i.e. no WSS)
           available_commands = ascmd_executor.nil? ? BASE_ACTIONS : ACTIONS
@@ -205,18 +211,13 @@ module Aspera
             command_nagios = options.get_next_command(%i[transfer])
             case command_nagios
             when :transfer
-              file = Tempfile.new('transfer_test')
-              filepath = file.path
-              file.write('This is a test file for transfer test')
-              file.close
               probe_ts = server_transfer_spec.merge({
                 'direction'     => 'send',
                 'cookie'        => 'aspera.sync', # hide in console
                 'resume_policy' => 'none',
-                'paths'         => [{'source' => filepath, 'destination' => '.fasping'}]
+                'paths'         => [{'source' => 'faux:///pingfile?1k', 'destination' => '.fasping'}]
               })
               statuses = transfer.start(probe_ts)
-              file.unlink
               if TransferAgent.session_status(statuses).eql?(:success)
                 nagios.add_ok('transfer', 'ok')
               else

data/lib/aspera/cli/plugins/shares.rb

@@ -21,11 +21,9 @@ module Aspera
               # TODO: use ping instead ?
               api.read("#{NODE_API_PATH}/app")
             rescue RestCallError => e
-              if e.response.code.to_s.eql?('401') && e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
-                found = true
-              end
+              found = true if e.response.code.to_s.eql?('401') && e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
             end
-            return nil unless found
+            return unless found
             version = 'unknown'
             test_page = api.call(operation: 'GET', subpath: 'login')
             if (m = test_page[:http].body.match(/\(v(1\..*)\)/))
@@ -37,7 +35,7 @@ module Aspera
             }
           end
 
-          def wizard(object:, private_key_path: nil, pub_key_pem: nil)
+          def wizard(object:)
             options = object.options
             return {
               preset_value: {
@@ -72,8 +70,9 @@ module Aspera
                 .call(
                   operation: 'GET',
                   subpath: 'ping',
-                  headers: {'content-type': Rest::MIME_JSON})
-              raise 'Shares not detected' unless res[:http].body.eql?(' ')
+                  headers: {'content-type': Rest::MIME_JSON}
+                )
+              raise Error, 'Shares not detected' unless res[:http].body.eql?(' ')
               nagios.add_ok('shares api', 'accessible')
             rescue StandardError => e
               nagios.add_critical('API', e.to_s)
@@ -83,59 +82,72 @@ module Aspera
             api_shares_node = basic_auth_api(NODE_API_PATH)
             repo_command = options.get_next_command(Node::COMMANDS_SHARES)
             return Node
-                .new(**init_params, api: api_shares_node)
+                .new(context: context, api: api_shares_node)
                 .execute_action(repo_command)
           when :admin
             api_shares_admin = basic_auth_api(ADMIN_API_PATH)
             admin_command = options.get_next_command(%i[node share transfer_settings user group].freeze)
             case admin_command
             when :node
-              return entity_action(api_shares_admin, 'data/nodes')
+              return entity_execute(api: api_shares_admin, entity: 'data/nodes')
             when :share
               share_command = options.get_next_command(%i[user_permissions group_permissions].concat(Plugin::ALL_OPS))
               case share_command
               when *Plugin::ALL_OPS
-                return entity_command(
-                  share_command, api_shares_admin, 'data/shares',
-                  display_fields: %w[id name node_id directory percent_free])
+                return entity_execute(
+                  api: api_shares_admin,
+                  entity: 'data/shares',
+                  command: share_command,
+                  display_fields: %w[id name node_id directory percent_free]
+                )
               when :user_permissions, :group_permissions
                 share_id = instance_identifier
-                return entity_action(api_shares_admin, "data/shares/#{share_id}/#{share_command}")
+                return entity_execute(api: api_shares_admin, entity: "data/shares/#{share_id}/#{share_command}")
               end
             when :transfer_settings
               xfer_settings_command = options.get_next_command(%i[show modify])
-              return entity_command(xfer_settings_command, api_shares_admin, 'data/transfer_settings', is_singleton: true)
+              return entity_execute(
+                api: api_shares_admin,
+                entity: 'data/transfer_settings',
+                command: xfer_settings_command,
+                is_singleton: true
+              )
             when :user, :group
               entity_type = admin_command
               entities_location = options.get_next_command(%i[all local ldap saml])
               entities_prefix = entities_location.eql?(:all) ? '' : "#{entities_location}_"
               entities_path = "data/#{entities_prefix}#{entity_type}s"
-              entity_action = nil
+              entity_commands = nil
               case entities_location
               when :all
-                entity_action = %i[list show delete]
-                entity_action.concat(USR_GRP_SETTINGS)
-                entity_action.push(:users) if entity_type.eql?(:group)
-                entity_action.freeze
+                entity_commands = %i[list show delete]
+                entity_commands.concat(USR_GRP_SETTINGS)
+                entity_commands.push(:users) if entity_type.eql?(:group)
+                entity_commands.freeze
               when :local
-                entity_action = %i[list show delete create modify]
-                entity_action.push(:users) if entity_type.eql?(:group)
-                entity_action.freeze
+                entity_commands = %i[list show delete create modify]
+                entity_commands.push(:users) if entity_type.eql?(:group)
+                entity_commands.freeze
               when :ldap
-                entity_action = %i[add].freeze
+                entity_commands = %i[add].freeze
               when :saml
-                entity_action = %i[import].freeze
+                entity_commands = %i[import].freeze
               end
-              entity_verb = options.get_next_command(entity_action)
+              entity_verb = options.get_next_command(entity_commands)
               case entity_verb
               when *Plugin::ALL_OPS # list, show, delete, create, modify
                 display_fields = entity_type.eql?(:user) ? %w[id username first_name last_name email] : nil
                 display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:all)
-                return entity_command(entity_verb, api_shares_admin, entities_path, display_fields: display_fields)
+                return entity_execute(
+                  api: api_shares_admin,
+                  entity: entities_path,
+                  command: entity_verb,
+                  display_fields: display_fields
+                )
               when *USR_GRP_SETTINGS # transfer_settings, app_authorizations, share_permissions
                 group_id = instance_identifier
                 entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
-                return entity_action(api_shares_admin, entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
+                return entity_execute(api: api_shares_admin, entity: entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
               when :import # saml
                 return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
                   entity_parameters = entity_parameters.transform_keys{ |k| k.gsub(/\s+/, '_').downcase}
@@ -151,7 +163,7 @@ module Aspera
                   api_shares_admin.create(entities_path, {entity_type=>entity_name})
                 end
               when :users # group
-                return entity_action(api_shares_admin, "#{entities_path}/#{instance_identifier}/#{entities_prefix}users")
+                return entity_execute(api: api_shares_admin, entity: "#{entities_path}/#{instance_identifier}/#{entities_prefix}users")
               else Aspera.error_unexpected_value(entity_verb)
               end
             end