aspera-cli 4.20.0 → 4.21.2

data/lib/aspera/cli/plugins/node.rb

@@ -127,10 +127,13 @@ module Aspera
         # commands for execute_command_gen4
         COMMANDS_GEN4 = %i[mkdir rename delete upload download sync http_node_download show modify permission thumbnail v3].concat(NODE4_READ_ACTIONS).freeze
 
+        # commands supported in ATS for COS
         COMMANDS_COS = %i[upload download info access_keys api_details transfer].freeze
         COMMANDS_SHARES = (BASE_ACTIONS - %i[search]).freeze
         COMMANDS_FASPEX = COMMON_ACTIONS
 
+        GEN4_LS_FIELDS = %w[name type recursive_size size modified_time access_level].freeze
+
         def initialize(api: nil, **env)
           super(**env, basic_options: api.nil?)
           Node.declare_options(options) if api.nil?
@@ -474,7 +477,7 @@ module Aspera
               result[:password] = apifid[:api].auth_params[:password]
             when :oauth2
               result[:username] = apifid[:api].params[:headers][Api::Node::HEADER_X_ASPERA_ACCESS_KEY]
-              result[:password] = apifid[:api].oauth.token
+              result[:password] = apifid[:api].oauth.authorization
             else Aspera.error_unreachable_line
             end
             return {type: :single_object, data: result} if command_repo.eql?(:node_info)
@@ -484,22 +487,15 @@ module Aspera
           when :browse
             apifid = apifid_from_next_arg(top_file_id)
             file_info = apifid[:api].read_with_cache("files/#{apifid[:file_id]}")
-            if file_info['type'].eql?('folder')
-              result = apifid[:api].call(
-                operation: 'GET',
-                subpath:   "files/#{apifid[:file_id]}/files",
-                headers:   Api::Node.cache_control_headers,
-                query:     query_read_delete)
-              items = result[:data]
-              formatter.display_item_count(result[:data].length, result[:http]['X-Total-Count'])
-            else
-              items = [file_info]
+            unless file_info['type'].eql?('folder')
+              # a single file
+              return {type: :object_list, data: [file_info], fields: GEN4_LS_FIELDS}
             end
-            return {type: :object_list, data: items, fields: %w[name type recursive_size size modified_time access_level]}
+            return {type: :object_list, data: apifid[:api].list_files(apifid[:file_id]), fields: GEN4_LS_FIELDS}
           when :find
             apifid = apifid_from_next_arg(top_file_id)
-            test_block = Api::Node.file_matcher_from_argument(options)
-            return {type: :object_list, data: @api_node.find_files(apifid[:file_id], test_block), fields: ['path']}
+            find_lambda = Api::Node.file_matcher_from_argument(options)
+            return {type: :object_list, data: @api_node.find_files(apifid[:file_id], find_lambda), fields: ['path']}
           when :mkdir
             containing_folder_path = options.get_next_argument('path').split(Api::Node::PATH_SEPARATOR)
             new_folder = containing_folder_path.pop
@@ -605,16 +601,19 @@ module Aspera
             return Main.result_image(result[:http].body, formatter: formatter)
           when :permission
             apifid = apifid_from_next_arg(top_file_id)
-            command_perm = options.get_next_command(%i[list create delete])
+            command_perm = options.get_next_command(%i[list show create delete])
             case command_perm
             when :list
-              # generic options : TODO: as arg ? query_read_delete
-              list_options ||= {'include' => Rest.array_params(%w[access_level permission_count])}
-              # add which one to get
-              list_options['file_id'] = apifid[:file_id]
-              list_options['inherited'] ||= false
-              items = apifid[:api].read('permissions', list_options)
+              list_query = query_read_delete(default: {'include' => Rest.array_params(%w[access_level permission_count])})
+              # specify file to get permissions for unless not specified
+              list_query['file_id'] = apifid[:file_id] unless apifid[:file_id].to_s.empty?
+              list_query['inherited'] = false if list_query.key?('file_id') && !list_query.key?('inherited')
+              # NOTE: supports per_page and page and header X-Total-Count
+              items = apifid[:api].read('permissions', list_query)
               return {type: :object_list, data: items}
+            when :show
+              perm_id = instance_identifier
+              return Main.result_single_object(apifid[:api].read("permissions/#{perm_id}"))
             when :delete
               return do_bulk_operation(command: command_perm, descr: 'identifier', values: :identifier) do |one_id|
                 apifid[:api].delete("permissions/#{one_id}")
@@ -802,42 +801,53 @@ module Aspera
             end
             case command
             when :list
+              transfer_filter = query_read_delete(default: {})
+              last_iteration_token = nil
               iteration_persistency = nil
-              iteration_data = []
               if options.get_option(:once_only, mandatory: true)
                 iteration_persistency = PersistencyActionOnce.new(
                   manager: persistency,
-                  data:    iteration_data,
+                  data:    [],
                   id:      IdGenerator.from_list([
                     'node_transfers',
                     options.get_option(:url, mandatory: true),
                     options.get_option(:username, mandatory: true)
                   ]))
+                if transfer_filter.delete('reset')
+                  iteration_persistency.data.clear
+                  iteration_persistency.save
+                  return Main.result_status('Persistency reset')
+                end
+                last_iteration_token = iteration_persistency.data.first
               end
-              transfer_filter = query_read_delete(default: {})
-              if transfer_filter.delete('reset')
-                iteration_data.clear
-                iteration_persistency&.save
-                return Main.result_status('Persistency reset')
-              end
+              raise 'reset only with once_only' if transfer_filter.key?('reset') && iteration_persistency.nil?
               max_items = transfer_filter.delete(MAX_ITEMS)
-              transfer_filter['iteration_token'] = iteration_persistency.data[0] unless iteration_data.empty?
               transfers_data = []
               loop do
+                transfer_filter['iteration_token'] = last_iteration_token unless last_iteration_token.nil?
                 result = @api_node.call(operation: 'GET', subpath: res_class_path, query: transfer_filter)
-                data = result[:data]
-                transfers_data.concat(data)
-                if !max_items.nil? && (transfers_data.length >= max_items)
+                # no data
+                break if result[:data].empty?
+                # get next iteration token from link
+                next_iteration_token = nil
+                link_info = result[:http]['Link']
+                unless link_info.nil?
+                  m = link_info.match(/<([^>]+)>/)
+                  raise "Cannot parse iteration in Link: #{link_info}" if m.nil?
+                  next_iteration_token = CGI.parse(URI.parse(m[1]).query)['iteration_token']&.first
+                end
+                # same as last iteration: stop
+                break if next_iteration_token&.eql?(last_iteration_token)
+                last_iteration_token = next_iteration_token
+                transfers_data.concat(result[:data])
+                if max_items&.<=(transfers_data.length)
+                  # if !max_items.nil? && (transfers_data.length >= max_items)
                   transfers_data = transfers_data.slice(0, max_items)
                   break
                 end
-                link_info = result[:http]['Link']
-                break if iteration_persistency.nil? || data.empty? || link_info.nil?
-                m = link_info.match(/<([^>]+)>/)
-                raise "Problem with iteration: #{link_info}" if m.nil?
-                iteration_token = CGI.parse(URI.parse(m[1]).query)['iteration_token']&.first
-                iteration_data[0] = transfer_filter['iteration_token'] = iteration_token
+                break if last_iteration_token.nil?
               end
+              iteration_persistency&.data&.[]=(0, last_iteration_token)
               iteration_persistency&.save
               return {
                 type:   :object_list,
@@ -1006,7 +1016,7 @@ module Aspera
             Environment.instance.open_uri("#{options.get_option(:asperabrowserurl)}?goto=#{encoded_params}")
             return Main.result_status('done')
           when :basic_token
-            return Main.result_status(Rest.basic_token(options.get_option(:username, mandatory: true), options.get_option(:password, mandatory: true)))
+            return Main.result_status(Rest.basic_authorization(options.get_option(:username, mandatory: true), options.get_option(:password, mandatory: true)))
           when :bearer_token
             private_key = OpenSSL::PKey::RSA.new(options.get_next_argument('private RSA key PEM value', validation: String))
             token_info = options.get_next_argument('user and group identification', validation: Hash)
@@ -1019,7 +1029,7 @@ module Aspera
             raise 'Missing key: url' unless parameters.key?(:url)
             uri = URI.parse(parameters[:url])
             server = WebServerSimple.new(uri, certificate: parameters[:certificate])
-            server.mount(uri.path, NodeSimulatorServlet, parameters[:credentials], transfer)
+            server.mount(uri.path, NodeSimulatorServlet, parameters[:credentials], NodeSimulator.new)
             server.start
             return Main.result_status('Simulator terminated')
           end

data/lib/aspera/cli/transfer_progress.rb

@@ -79,6 +79,8 @@ module Aspera
         new_title = @sessions.length < 2 ? @title.to_s : "[#{@sessions.length}] #{@title}"
         @progress_bar.title = new_title unless @progress_bar.title.eql?(new_title)
         @progress_bar.increment if !progress_provided && !@completed
+      rescue ProgressBar::InvalidProgressError => e
+        Log.log.error{"Progress error: #{e}"}
       end
 
       private

data/lib/aspera/cli/version.rb

@@ -4,6 +4,6 @@ module Aspera
   module Cli
     # for beta add extension : .beta1
     # for dev version add extension : .pre
-    VERSION = '4.20.0'
+    VERSION = '4.21.2'
   end
 end

data/lib/aspera/command_line_builder.rb

@@ -77,7 +77,7 @@ module Aspera
       @param_hash.each_pair{|key, val|Log.log.warn{"unrecognized parameter: #{key} = \"#{val}\""} if !@used_param_names.include?(key)}
       # set result
       env_args[:env].merge!(@result[:env])
-      env_args[:args].push(*@result[:args])
+      env_args[:args].concat(@result[:args])
       return nil
     end
 

data/lib/aspera/coverage.rb

@@ -20,14 +20,16 @@ if ENV.key?('ENABLE_COVERAGE')
   # lines with those words are ignored from coverage
   no_cov_functions = %w[error_unreachable_line error_unexpected_value Log.log.trace].freeze
   SimpleCov.start do
-    add_filter 'lib/aspera/cli/plugins/faspex.rb'
-    add_filter 'lib/aspera/node_simulator.rb'
-    add_filter 'lib/aspera/keychain/macos_security.rb'
+    # assert usually do not trigger
     add_filter do |source_file|
       source_file.lines.each do |line|
         line.skipped! if no_cov_functions.any?{|i|line.src.include?(i)}
       end
       false
     end
+    # no coverage test in those
+    add_filter 'lib/aspera/cli/plugins/faspex.rb'
+    add_filter 'lib/aspera/node_simulator.rb'
+    add_filter 'lib/aspera/keychain/macos_security.rb'
   end
 end

data/lib/aspera/environment.rb

@@ -46,8 +46,7 @@ module Aspera
           return OS_LINUX
         when /aix/
           return OS_AIX
-        else
-          raise "Unknown OS: #{RbConfig::CONFIG['host_os']}"
+        else Aspera.error_unexpected_value(RbConfig::CONFIG['host_os']){'host_os'}
         end
       end
 
@@ -62,8 +61,8 @@ module Aspera
           return CPU_S390
         when /arm/, /aarch64/
           return CPU_ARM64
+        else Aspera.error_unexpected_value(RbConfig::CONFIG['host_cpu']){'host_cpu'}
         end
-        raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
       end
 
       # normalized architecture name
@@ -73,9 +72,9 @@ module Aspera
       end
 
       # executable file extension for current OS
-      def exe_extension
-        return '.exe' if os.eql?(OS_WINDOWS)
-        return ''
+      def exe_file(name='')
+        return "#{name}.exe" if os.eql?(OS_WINDOWS)
+        return name
       end
 
       # on Windows, the env var %USERPROFILE% provides the path to user's home more reliably than %HOMEDRIVE%%HOMEPATH%
@@ -96,34 +95,78 @@ module Aspera
         Kernel.send('lave'.reverse, code, empty_binding, file, line)
       end
 
-      def log_spawn(env:, exec:, args:)
+      # Generate log line for external program with arguments
+      # @param env [Hash, nil]   environment variables
+      # @param exec [String]     path to executable
+      # @param args [Array, nil] arguments
+      # @return [String] log line with environment, program and arguments
+      def log_spawn(exec:, args: nil, env: nil)
         [
           'execute:'.red,
-          env.map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
+          env&.map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
           Shellwords.shellescape(exec),
-          args.map{|a|Shellwords.shellescape(a)}
-        ].flatten.join(' ')
+          args&.map{|a|Shellwords.shellescape(a)}
+        ].compact.flatten.join(' ')
       end
 
-      # start process in background, or raise exception
+      # Start process in background
       # caller can call Process.wait on returned value
-      def secure_spawn(exec:, args: [], env: [])
-        Log.log.debug {log_spawn(env: env, exec: exec, args: args)}
+      # @param exec    [String]     path to executable
+      # @param args    [Array, nil] arguments for executable
+      # @param env     [Hash, nil]  environment variables
+      # @param options [Hash, nil]  spawn options
+      # @return [String]            PID of process
+      # @raise  [Exception]         if problem
+      def secure_spawn(exec:, args: nil, env: nil, **options)
+        Aspera.assert_type(exec, String)
+        Aspera.assert_type(args, Array) unless args.nil?
+        Aspera.assert_type(env, Hash) unless env.nil?
+        Aspera.assert_type(options, Hash) unless options.nil?
+        Log.log.debug {log_spawn(exec: exec, args: args, env: env)}
         # start ascp in separate process
-        ascp_pid = Process.spawn(env, [exec, exec], *args, close_others: true)
+        spawn_args = []
+        spawn_args.push(env) unless env.nil?
+        spawn_args.push([exec, exec])
+        spawn_args.concat(args) unless args.nil?
+        opts = {close_others: true}
+        opts.merge!(options) unless options.nil?
+        ascp_pid = Process.spawn(*spawn_args, **opts)
         Log.log.debug{"pid: #{ascp_pid}"}
         return ascp_pid
       end
 
+      # start process and wait for completion
+      # @param env [Hash, nil]   environment variables
+      # @param exec [String]     path to executable
+      # @param args [Array, nil] arguments
+      # @return [String] PID of process
+      def secure_execute(exec:, args: nil, env: nil, **system_args)
+        Aspera.assert_type(exec, String)
+        Aspera.assert_type(args, Array) unless args.nil?
+        Aspera.assert_type(env, Hash) unless env.nil?
+        Log.log.debug {log_spawn(exec: exec, args: args, env: env)}
+        # start in separate process
+        spawn_args = []
+        spawn_args.push(env) unless env.nil?
+        # ensure no shell expansion
+        spawn_args.push([exec, exec])
+        spawn_args.concat(args) unless args.nil?
+        kwargs = {exception: true}
+        kwargs.merge!(system_args)
+        Kernel.system(*spawn_args, **kwargs)
+        nil
+      end
+
+      # Execute process and capture stdout
       # @param exec [String] path to executable
       # @param args [Array] arguments to executable
       # @param opts [Hash] options to capture3
-      # @return stdout of executable or raise expcetion
+      # @return stdout of executable or raise exception
       def secure_capture(exec:, args: [], **opts)
         Aspera.assert_type(exec, String)
         Aspera.assert_type(args, Array)
         Aspera.assert_type(opts, Hash)
-        Log.log.debug {log_spawn(env: {}, exec: exec, args: args)}
+        Log.log.debug {log_spawn(exec: exec, args: args)}
         stdout, stderr, status = Open3.capture3(exec, *args, **opts)
         Log.log.debug{"status=#{status}, stderr=#{stderr}"}
         Log.log.trace1{"stdout=#{stdout}"}

data/lib/aspera/faspex_postproc.rb

@@ -39,7 +39,7 @@ module Aspera
           response.body = {status: 'error', message: 'Empty request'}.to_json
           return
         end
-        # build script path by removing domain, and adding script folder
+        # build script path by removing domain and adding script folder
         script_file = request.path[@parameters[:root].size..]
         Log.log.debug{"script file=#{script_file}"}
         script_path = File.join(@parameters[:script_folder], script_file)
@@ -48,11 +48,9 @@ module Aspera
         Log.log.debug{Log.dump(:webhook_parameters, webhook_parameters)}
         # env expects only strings
         environment = webhook_parameters.each_with_object({}) { |(k, v), h| h[k] = v.to_s }
-        post_proc_pid = Process.spawn(environment, [script_path, script_path])
-        Log.log.debug{"pid=#{post_proc_pid}"}
-        raise 'no pid' if post_proc_pid.nil?
-        # "wait" for process to avoid zombie
+        post_proc_pid = Environment.secure_spawn(env: environment, exec: script_path)
         Timeout.timeout(@parameters[:timeout_seconds]) do
+          # "wait" for process to avoid zombie
           Process.wait(post_proc_pid)
           post_proc_pid = nil
         end

data/lib/aspera/hash_ext.rb

@@ -20,17 +20,7 @@ class ::Hash
   end
 end
 
-# in 2.5
-unless Hash.method_defined?(:transform_keys)
-  class Hash
-    def transform_keys
-      raise 'missing block' unless block_given?
-      return each_with_object({}){|(k, v), memo|memo[yield(k)] = v}
-    end
-  end
-end
-
-# rails
+# Exists in Rails
 unless Hash.method_defined?(:symbolize_keys)
   class Hash
     def symbolize_keys
@@ -39,7 +29,7 @@ unless Hash.method_defined?(:symbolize_keys)
   end
 end
 
-# rails
+# Exists in Rails
 unless Hash.method_defined?(:stringify_keys)
   class Hash
     def stringify_keys