aspera-cli 4.22.0 → 4.24.0

data/lib/aspera/rest.rb

@@ -6,12 +6,14 @@ require 'aspera/log'
 require 'aspera/assert'
 require 'aspera/oauth'
 require 'aspera/hash_ext'
+require 'aspera/timer_limiter'
 require 'net/http'
 require 'net/https'
 require 'json'
 require 'base64'
 require 'singleton'
 require 'securerandom'
+require 'fileutils'
 
 # Cancel method for HTTP
 class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
@@ -86,11 +88,11 @@ module Aspera
       # Build URI from URL and parameters and check it is http or https
       # encode array [] parameters
       # @param query [Hash,Array]
-      def build_uri(url, query=nil)
+      def build_uri(url, query = nil)
         uri = URI.parse(url)
         Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
         return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
-        Log.log.debug{Log.dump('query', query)}
+        Log.dump(:query, query)
         query_array = []
         case query
         when Hash
@@ -172,6 +174,10 @@ module Aspera
         return result
       end
 
+      # Parse a header string as returned by HTTP
+      # @param header [String] header string, e.g. "application/json; charset=utf-8"
+      # @return [Hash] parsed header with type and parameters
+      #   {type: 'application/json', parameters: {charset: 'utf-8'}}
       def parse_header(header)
         type, *params = header.split(/;\s*/)
         parameters = params.map do |param|
@@ -188,9 +194,7 @@ module Aspera
 
     # create and start keep alive connection on demand
     def http_session
-      if @http_session.nil?
-        @http_session = self.class.start_http_session(@base_url)
-      end
+      @http_session = self.class.start_http_session(@base_url) if @http_session.nil?
       return @http_session
     end
 
@@ -233,7 +237,7 @@ module Aspera
     )
       Aspera.assert_type(base_url, String)
       # base url with no trailing slashes (note: string may be frozen)
-      @base_url = base_url.gsub(%r{/+$}, '')
+      @base_url = base_url.chomp('/')
       # remove trailing port if it is 443 and scheme is https
       @base_url = @base_url.gsub(/:443$/, '') if @base_url.start_with?('https://')
       @base_url = @base_url.gsub(/:80$/, '') if @base_url.start_with?('http://')
@@ -260,7 +264,7 @@ module Aspera
       if @oauth.nil?
         Aspera.assert(@auth_params[:type].eql?(:oauth2)){'no OAuth defined'}
         oauth_parameters = @auth_params.reject{ |k, _v| k.eql?(:type)}
-        Log.log.debug{Log.dump('oauth parameters', oauth_parameters)}
+        Log.dump(:oauth_parameters, oauth_parameters)
         @oauth = OAuth::Factory.instance.create(**oauth_parameters)
       end
       return @oauth
@@ -287,8 +291,8 @@ module Aspera
     )
       subpath = subpath.to_s if subpath.is_a?(Symbol)
       subpath = '' if subpath.nil?
-      Log.log.debug{"#{operation} [#{subpath}]".red.bold.bg_green}
-      Log.log.debug{Log.dump(:body, body)}
+      Log.log.debug{"call #{operation} [#{subpath}]".red.bold.bg_green}
+      Log.dump(:body, body)
       Aspera.assert_type(subpath, String)
       if headers.nil?
         headers = @headers.clone
@@ -346,7 +350,7 @@ module Aspera
         end
         # :type = :basic
         req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
-        Log.log.trace1{Log.dump(:req_body, req.body)}
+        Log.dump(:req_body, req.body, level: :trace1)
         # we try the call, and will retry on some error types
         error_tries ||= 1 + RestParameters.instance.retry_max
         # initialize with number of initial retries allowed, nil gives zero
@@ -358,20 +362,18 @@ module Aspera
         http_session.request(req) do |response|
           result[:http] = response
           result_mime = self.class.parse_header(result[:http]['Content-Type'] || MIME_TEXT)[:type]
+          Log.log.debug{"response: code=#{result[:http].code}, mime=#{result_mime}, mime2= #{response['Content-Type']}"}
           # JSON data needs to be parsed, in case it contains an error code
           if !save_to_file.nil? &&
               result[:http].code.to_s.start_with?('2') &&
-              !result[:http]['Content-Length'].nil? &&
               !JSON_DECODE.include?(result_mime)
-            total_size = result[:http]['Content-Length'].to_i
+            total_size = result[:http]['Content-Length']&.to_i
             Log.log.debug('before write file')
             target_file = save_to_file
             # override user's path to path in header
             if !response['Content-Disposition'].nil?
               disposition = self.class.parse_header(response['Content-Disposition'])
-              if disposition[:parameters].key?(:filename)
-                target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename])
-              end
+              target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename]) if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
             end
             # download with temp filename
             target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
@@ -379,15 +381,18 @@ module Aspera
             written_size = 0
             session_id = SecureRandom.uuid.freeze
             RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
-            RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size)
+            RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
+            FileUtils.mkdir_p(File.dirname(target_file_tmp))
+            limiter = TimerLimiter.new(0.5)
             File.open(target_file_tmp, 'wb') do |file|
               result[:http].read_body do |fragment|
                 file.write(fragment)
                 written_size += fragment.length
-                RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size)
+                RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
               end
             end
-            RestParameters.instance.progress_bar&.event(:end, session_id: session_id)
+            RestParameters.instance.progress_bar&.event(:session_end, session_id: session_id)
+            RestParameters.instance.progress_bar&.event(:end)
             # rename at the end
             File.rename(target_file_tmp, target_file)
             file_saved = true
@@ -400,12 +405,15 @@ module Aspera
         case result_mime
         when *JSON_DECODE
           result[:data] = JSON.parse(result[:http].body) rescue result[:http].body
-          Log.log.debug{Log.dump('result_data', result[:data])}
+          Log.dump(:result_data, result[:data])
         else # when MIME_TEXT
           result[:data] = result[:http].body
         end
         RestErrorAnalyzer.instance.raise_on_error(req, result)
-        File.write(save_to_file, result[:http].body, binmode: true) unless file_saved || save_to_file.nil?
+        unless file_saved || save_to_file.nil?
+          FileUtils.mkdir_p(File.dirname(save_to_file))
+          File.write(save_to_file, result[:http].body, binmode: true)
+        end
       rescue RestCallError => e
         do_retry = false
         # AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
@@ -436,7 +444,7 @@ module Aspera
         if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
           tries_remain_redirect -= 1
           current_uri = URI.parse(@base_url)
-          new_url = e.response['location']
+          new_url = e.response['Location']
           # special case: relative redirect
           if URI.parse(new_url).host.nil?
             # we don't manage relative redirects with non-absolute path
@@ -444,14 +452,24 @@ module Aspera
             new_url = "#{current_uri.scheme}://#{current_uri.host}#{new_url}"
           end
           # forwards the request to the new location
-          return self.class.new(base_url: new_url, redirect_max: tries_remain_redirect).call(
-            operation: operation, query: query, body: body, content_type: content_type,
-            save_to_file: save_to_file, return_error: return_error, headers: headers)
+          return self.class.new(
+            base_url: new_url,
+            redirect_max: tries_remain_redirect
+          ).call(
+            operation: operation,
+            subpath: new_url.end_with?('/') ? '/' : nil,
+            query: query,
+            body: body,
+            content_type: content_type,
+            save_to_file: save_to_file,
+            return_error: return_error,
+            headers: headers
+          )
         end
         # raise exception if could not retry and not return error in result
         raise e unless return_error
       end
-      Log.log.debug{"result=#{result}"}
+      Log.log.debug{"result=http:#{result[:http]}, data:#{result[:data].class}"}
       return result
     end
 
@@ -464,7 +482,7 @@ module Aspera
       return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
     end
 
-    def read(subpath, query=nil)
+    def read(subpath, query = nil)
       return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query)[:data]
     end
 
@@ -472,7 +490,7 @@ module Aspera
       return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
     end
 
-    def delete(subpath, params=nil)
+    def delete(subpath, params = nil)
       return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params)[:data]
     end
 
@@ -502,7 +520,7 @@ module Aspera
         name_matches = matching_items.select{ |i| i['name'].casecmp?(search_name)}
         case name_matches.length
         when 1 then return name_matches.first
-        when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
+        when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
         else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{ |i| i['name']}}"
         end
       end

data/lib/aspera/rest_call_error.rb

@@ -8,7 +8,7 @@ module Aspera
     # @param req HTTP Request object
     # @param resp HTTP Response object
     # @param msg Error message
-    def initialize(msg, req=nil, resp=nil)
+    def initialize(msg, req = nil, resp = nil)
       @request = req
       @response = resp
       super(msg)

data/lib/aspera/rest_error_analyzer.rb

@@ -8,6 +8,7 @@ module Aspera
   # analyze error codes returned by REST calls and raise ruby exception
   class RestErrorAnalyzer
     include Singleton
+
     attr_accessor :log_file
 
     # the singleton object is registered with application specific handlers

data/lib/aspera/resumer.rb

@@ -16,7 +16,7 @@ module Aspera
     }.freeze
 
     # @param params see DEFAULTS
-    def initialize(params=nil)
+    def initialize(params = nil)
       @parameters = DEFAULTS.dup
       if !params.nil?
         Aspera.assert_type(params, Hash)

data/lib/aspera/secret_hider.rb

@@ -2,10 +2,13 @@
 
 # cspell:ignore FILEPASS
 require 'logger'
+require 'singleton'
 
 module Aspera
   # remove secret from logs and output
   class SecretHider
+    include Singleton
+
     # configurable:
     ADDITIONAL_KEYS_TO_HIDE = []
     # display string for hidden secrets
@@ -34,56 +37,59 @@ module Aspera
       /(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
     ].freeze
     private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :KEY_FALSE_POSITIVES, :REGEX_LOG_REPLACES
-    @log_secrets = false
-    class << self
-      attr_accessor :log_secrets
+    attr_accessor :log_secrets
 
-      # @return new log formatter that hides secrets
-      def log_formatter(original_formatter)
-        original_formatter ||= Logger::Formatter.new
-        # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
-        return lambda do |severity, date_time, program_name, msg|
-          if msg.is_a?(String) && !@log_secrets
-            REGEX_LOG_REPLACES.each do |reg_ex|
-              msg = msg.gsub(reg_ex){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
-            end
+    # @return new log formatter that hides secrets
+    def log_formatter(original_formatter)
+      original_formatter ||= Logger::Formatter.new
+      # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
+      return lambda do |severity, date_time, program_name, msg|
+        if msg.is_a?(String) && !@log_secrets
+          REGEX_LOG_REPLACES.each do |reg_ex|
+            msg = msg.gsub(reg_ex){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
           end
-          original_formatter.call(severity, date_time, program_name, msg)
         end
+        original_formatter.call(severity, date_time, program_name, msg)
       end
+    end
 
-      def hide_secrets_in_string(value)
-        return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
-      end
+    def hide_secrets_in_string(value)
+      return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
+    end
 
-      # @return true if the key denotes a secret
-      def secret?(keyword, value)
-        keyword = keyword.to_s if keyword.is_a?(Symbol)
-        # only Strings can be secrets, not booleans, or hash, arrays
-        return false unless keyword.is_a?(String) && value.is_a?(String)
-        # those are not secrets
-        return false if KEY_FALSE_POSITIVES.any?{ |f| f.match?(keyword)}
-        return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
-        # check if keyword (name) contains an element that designate it as a secret
-        ALL_SECRETS.any?{ |kw| keyword.include?(kw)}
-      end
+    # @return true if the key denotes a secret
+    def secret?(keyword, value)
+      keyword = keyword.to_s if keyword.is_a?(Symbol)
+      # only Strings can be secrets, not booleans, or hash, arrays
+      return false unless keyword.is_a?(String) && value.is_a?(String)
+      # those are not secrets
+      return false if KEY_FALSE_POSITIVES.any?{ |f| f.match?(keyword)}
+      return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
+      # check if keyword (name) contains an element that designate it as a secret
+      ALL_SECRETS.any?{ |kw| keyword.include?(kw)}
+    end
 
-      # Hides recursively secrets in Hash or Array of Hash
-      def deep_remove_secret(obj)
-        case obj
-        when Array
-          obj.each{ |i| deep_remove_secret(i)}
-        when Hash
-          obj.each do |k, v|
-            if secret?(k, v)
-              obj[k] = HIDDEN_PASSWORD
-            elsif obj[k].is_a?(Hash)
-              deep_remove_secret(obj[k])
-            end
+    # Hides recursively secrets in Hash or Array of Hash
+    def deep_remove_secret(obj)
+      case obj
+      when Array
+        obj.each{ |i| deep_remove_secret(i)}
+      when Hash
+        obj.each do |k, v|
+          if secret?(k, v)
+            obj[k] = HIDDEN_PASSWORD
+          elsif obj[k].is_a?(Hash)
+            deep_remove_secret(obj[k])
           end
         end
-        return obj
       end
+      return obj
+    end
+
+    private
+
+    def initialize
+      @log_secrets = false
     end
   end
 end

data/lib/aspera/ssh.rb

@@ -8,6 +8,8 @@ module Aspera
   # A simple wrapper around Net::SSH
   # executes one command and get its result from stdout
   class Ssh
+    class Error < Aspera::Error
+    end
     class << self
       def disable_ed25519_keys
         Log.log.debug('Disabling SSH ed25519 user keys')
@@ -42,27 +44,35 @@ module Aspera
       @ssh_options[:logger] = Log.log
     end
 
-    def execute(cmd, input=nil)
+    # Anything on stderr raises an exception
+    def execute(cmd, input: nil, exception: false)
       Aspera.assert_type(cmd, String)
       Log.log.debug{"cmd=#{cmd}"}
       response = []
+      error = []
       Net::SSH.start(@host, @username, @ssh_options) do |session|
         ssh_channel = session.open_channel do |channel|
           # prepare stdout processing
           channel.on_data{ |_chan, data| response.push(data)}
           # prepare stderr processing, stderr if type = 1
           channel.on_extended_data do |_chan, _type, data|
-            error_message = "#{cmd}: [#{data.chomp}]"
+            error.push(data)
+          end
+          channel.on_request('exit-status') do |_ch, data|
+            exit_code = data.read_long
+            next if exit_code.zero?
+            error_message = "#{cmd}: exit #{exit_code}, #{error.join.chomp}"
+            raise Error, error_message if  exception
             # Happens when windows user hasn't logged in and created home account.
             error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
-            raise error_message
+            Log.log.debug(error_message)
           end
           # send command to SSH channel (execute) cspell: disable-next-line
           channel.send('cexe'.reverse, cmd){ |_ch, _success| channel.send_data(input) unless input.nil?}
         end
         # wait for channel to finish (command exit)
         ssh_channel.wait
-        # main ssh session loop
+        # main SSH session loop
         session.loop
       end
       # response as single string

data/lib/aspera/sync/args.schema.yaml

@@ -0,0 +1,102 @@
+$schema: https://json-schema.org/draft/2020-12/schema
+$id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/transfer/sync_instance.schema.yaml
+$comment: >-
+  `x-` fields documented in `command_line_builder.rb`
+  This spec is specific to `ascli`.
+  The native async `conf` format is now preferred.
+title: SyncInstanceSpec
+description: Instance (global) and Session (per-sync) parameters for async.
+type: object
+properties:
+  alt_logdir:
+    type: string
+  watchd:
+    type: string
+  apply_local_docroot:
+    x-cli-switch: true
+  quiet:
+    x-cli-switch: true
+  ws_connect:
+    x-cli-switch: true
+  sessions:
+    type: array
+    items:
+      description: Session parameters for async.
+      type: object
+      properties:
+        name:
+          type: string
+        local_dir:
+          type: string
+        remote_dir:
+          type: string
+        local_db_dir:
+          type: string
+        remote_db_dir:
+          type: string
+        host:
+          type: string
+          x-ts-name: remote_host
+        user:
+          type: string
+          x-ts-name: remote_user
+        private_key_paths:
+          type: array
+          x-cli-option: "--private-key-path"
+        direction:
+          type: string
+        checksum:
+          type: string
+        tags:
+          type: object
+          x-cli-option: "--tags64"
+          x-cli-convert: json64
+          x-ts-name: tags
+        tcp_port:
+          type: integer
+          x-ts-name: ssh_port
+        rate_policy:
+          type: string
+        target_rate:
+          type: string
+        cooloff:
+          type: integer
+        pending_max:
+          type: integer
+        scan_intensity:
+          type: string
+        cipher:
+          type: string
+          x-cli-convert: remove_hyphen
+          x-ts-name: cipher
+        transfer_threads:
+          type: integer
+        preserve_time:
+          x-cli-switch: true
+          x-ts-name: preserve_times
+        preserve_access_time:
+          x-cli-switch: true
+        preserve_modification_time:
+          x-cli-switch: true
+        preserve_uid:
+          x-cli-switch: true
+          x-ts-name: preserve_file_owner_uid
+        preserve_gid:
+          x-cli-switch: true
+          x-ts-name: preserve_file_owner_gid
+        create_dir:
+          x-cli-switch: true
+          x-ts-name: create_dir
+        reset:
+          x-cli-switch: true
+        remote_password:
+          x-cli-envvar: ASPERA_SCP_PASS
+          x-ts-name: remote_password
+        cookie:
+          x-cli-envvar: ASPERA_SCP_COOKIE
+          x-ts-name: cookie
+        token:
+          x-cli-envvar: ASPERA_SCP_TOKEN
+          x-ts-name: token
+        license:
+          x-cli-envvar: ASPERA_SCP_LICENSE