aspera-cli 4.25.0.pre2 → 4.25.0

data/lib/aspera/oauth/base.rb

@@ -14,50 +14,38 @@ module Aspera
     # Bearer Token Usage: https://tools.ietf.org/html/rfc6750
     class Base
       Aspera.require_method!(:create_token)
-      # @param **             Parameters for REST
-      # @param client_id      [String, nil]
-      # @param client_secret  [String, nil]
-      # @param scope          [String, nil]
-      # @param use_query      [bool]        Provide parameters in query instead of body
-      # @param path_token     [String]      API end point to create a token from base URL
-      # @param token_field    [String]      Field in result that contains the token
-      # @param cache_ids      [Array, nil]  List of unique identifiers for cache id generation
+      # @param params         [Hash]    Parameters for token creation (client_id, client_secret, scope, etc...)
+      # @param use_query      [Boolean] Provide parameters in query instead of body
+      # @param path_token     [String]  API end point to create a token from base URL
+      # @param token_field    [String]  Field in result that contains the token
+      # @param cache_ids      [Array]   List of unique identifiers for cache id generation
+      # @param **rest_params  [Hash]    Parameters for REST
       def initialize(
-        client_id: nil,
-        client_secret: nil,
-        scope: nil,
+        params: {},
         use_query: false,
         path_token:  'token',
         token_field: Factory::TOKEN_FIELD,
-        cache_ids: nil,
+        cache_ids: [],
         **rest_params
       )
+        Aspera.assert_type(params, Hash)
+        Aspera.assert_type(cache_ids, Array)
         # This is the OAuth API
         @api = Rest.new(**rest_params)
-        @scope = nil
-        @token_cache_id = nil
+        @params = params.dup.freeze
         @path_token = path_token
         @token_field = token_field
-        @client_id = client_id
-        @client_secret = client_secret
         @use_query = use_query
-        @base_cache_ids = cache_ids.nil? ? [] : cache_ids.clone
-        Aspera.assert_type(@base_cache_ids, Array)
-        # TODO: this shall be done in class, using cache_ids
-        @base_cache_ids.push(@api.auth_params[:username]) if @api.auth_params.key?(:username)
-        @base_cache_ids.compact!
-        @base_cache_ids.freeze
-        self.scope = scope
+        # TODO: :username and :scope shall be done in class, using cache_ids
+        @token_cache_id = Factory.cache_id(@api.base_url, self.class, cache_ids, rest_params[:username], @params[:scope])
       end
 
-      # Scope can be modified after creation, then update identifier for cache
-      def scope=(scope)
-        @scope = scope
-        # generate token unique identifier for persistency (memory/disk cache)
-        @token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
-      end
-
-      attr_reader :scope, :api, :path_token, :client_id
+      # The OAuth API Object
+      attr_reader :api
+      # Sub path to generate token
+      attr_reader :path_token
+      # Parameters to generate token
+      attr_reader :params
 
       # Helper method to create token as per RFC
       # @return [HTTPResponse]
@@ -68,17 +56,16 @@ module Aspera
         return @api.create(@path_token, creation_params, content_type: Rest::MIME_WWW, ret: :resp)
       end
 
+      # Create base parameters for token creation calls
       # @param add_secret [Boolean] Add secret in default call parameters
       # @return [Hash] Optional general parameters
-      def optional_scope_client_id(add_secret: false)
-        call_params = {}
-        call_params[:scope] = @scope unless @scope.nil?
-        call_params[:client_id] = @client_id unless @client_id.nil?
-        call_params[:client_secret] = @client_secret unless !add_secret || @client_id.nil? || @client_secret.nil?
+      def base_params(add_secret: false)
+        call_params = @params.dup
+        call_params.delete(:client_secret) unless add_secret
         return call_params
       end
 
-      # @return value suitable for Authorization header
+      # @return [String] value suitable for Authorization header
       def authorization(**kwargs)
         return OAuth::Factory.bearer_authorization(token(**kwargs))
       end
@@ -114,7 +101,7 @@ module Aspera
             if !refresh_token.nil?
               Log.log.info{"refresh token=[#{refresh_token}]".bg_green}
               begin
-                http = create_token_call(optional_scope_client_id(add_secret: true).merge(grant_type: 'refresh_token', refresh_token: refresh_token))
+                http = create_token_call(base_params(add_secret: true).merge(grant_type: 'refresh_token', refresh_token: refresh_token))
                 # Save only if success
                 json_data = http.body
                 token_data = JSON.parse(json_data)

data/lib/aspera/oauth/factory.rb

@@ -36,14 +36,13 @@ module Aspera
           return authorization[SPACE_BEARER_AUTH_SCHEME.length..-1]
         end
 
+        # Generate a unique cache id for a token creator
+        # @param url           [String] Base URL of the OAuth server
+        # @param creator_class [Class]  Class of the token creator
+        # @param params        [Array]  List of parameters (can be nested) to uniquely identify the token
         # @return a unique cache identifier
         def cache_id(url, creator_class, *params)
-          return IdGenerator.from_list([
-            PERSIST_CATEGORY_TOKEN,
-            url,
-            Factory.class_to_id(creator_class)
-          ] +
-            params)
+          return IdGenerator.from_list(PERSIST_CATEGORY_TOKEN, url, Factory.class_to_id(creator_class), params)
         end
 
         # @return snake version of class name

data/lib/aspera/oauth/generic.rb

@@ -23,7 +23,7 @@ module Aspera
       end
 
       def create_token
-        return create_token_call(optional_scope_client_id.merge(@create_params))
+        return create_token_call(base_params.merge(@create_params))
       end
     end
     Factory.instance.register_token_creator(Generic)

data/lib/aspera/oauth/jwt.rb

@@ -61,7 +61,7 @@ module Aspera
         Log.log.debug{"private=[#{@private_key_obj}]"}
         assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
         Log.log.debug{"assertion=[#{assertion}]"}
-        return create_token_call(optional_scope_client_id.merge(grant_type: GRANT_TYPE, assertion: assertion))
+        return create_token_call(base_params.merge(grant_type: GRANT_TYPE, assertion: assertion))
       end
     end
     Factory.instance.register_token_creator(Jwt)

data/lib/aspera/oauth/url_json.rb

@@ -6,8 +6,9 @@ module Aspera
   module OAuth
     # This class is used to create a token using a JSON body and a URL
     class UrlJson < Base
-      # @param url  URL to send the JSON body
-      # @param json JSON body to send
+      # @param url  [Hash] Query parameters to send
+      # @param json [Hash] Body parameters to send as JSON
+      # @param generic_params [Hash] Generic parameters for OAuth::Base
       def initialize(
         url:,
         json:,
@@ -22,7 +23,7 @@ module Aspera
         api.call(
           operation:    'POST',
           subpath:      path_token,
-          query:        @query.merge(scope: scope), # scope is here because it may change over time (node)
+          query:        @query.merge(scope: params[:scope]), # scope is here because it may change over time (node)
           content_type: Rest::MIME_JSON,
           body:         @body,
           headers:      {'Accept' => Rest::MIME_JSON},

data/lib/aspera/oauth/web.rb

@@ -32,7 +32,7 @@ module Aspera
         random_state = SecureRandom.uuid
         login_page_url = Rest.build_uri(
           "#{api.base_url}/#{@path_authorize}",
-          optional_scope_client_id.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
+          base_params.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
         )
         # here, we need a human to authorize on a web page
         Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
@@ -44,7 +44,7 @@ module Aspera
         received_params = web_server.received_request
         Aspera.assert(random_state.eql?(received_params['state'])){'wrong received state'}
         # exchange code for token
-        return create_token_call(optional_scope_client_id(add_secret: true).merge(
+        return create_token_call(base_params(add_secret: true).merge(
           grant_type:   'authorization_code',
           code:         received_params['code'],
           redirect_uri: @redirect_uri

data/lib/aspera/preview/file_types.rb

@@ -53,7 +53,7 @@ module Aspera
 
       private_constant :SUPPORTED_MIME_TYPES
 
-      # @attr use_mimemagic [bool] true to use mimemagic to determine real mime type based on file content
+      # @attr use_mimemagic [Boolean] `true` to use mimemagic to determine real mime type based on file content
       attr_accessor :use_mimemagic
 
       def initialize

data/lib/aspera/rest.rb

@@ -307,9 +307,12 @@ module Aspera
     # @param body       [Hash, String] body parameters
     # @param headers      [Hash] additional headers (override Content-Type)
     # @param save_to_file [String, nil](filepath)
-    # @param exception    [Bool] `true`, error raise exception
+    # @param exception    [Boolean] `true`, error raise exception
     # @param ret [:data, :resp, :both] Tell to return only data, only http response, or both
-    # @return [Object, Array] only data, only http response, or both
+    # @return [(HTTPResponse,Hash)] If ret is :both
+    # @return [HTTPResponse] If ret is :resp
+    # @return [Hash] If ret is :data
+    # @raise [RestCallError] on error if `exception` is true
     def call(
       operation:,
       subpath: nil,

data/lib/aspera/ssh.rb

@@ -35,16 +35,17 @@ module Aspera
     # ssh_options: same as Net::SSH.start
     # see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
     def initialize(host, username, ssh_options)
-      Log.log.debug{"ssh:#{username}@#{host}"}
-      Log.log.debug{"ssh_options:#{ssh_options}"}
       Aspera.assert_type(host, String)
       Aspera.assert_type(username, String)
       Aspera.assert_type(ssh_options, Hash)
-      ssh_options[:use_agent] = false unless ssh_options.key?(:use_agent)
       @host = host
       @username = username
-      @ssh_options = ssh_options
-      @ssh_options[:logger] = Log.log
+      @ssh_options = ssh_options.dup
+      @ssh_options[:logger] = Log.log unless @ssh_options.key?(:logger)
+      @ssh_options[:verbose] = :warn unless @ssh_options.key?(:verbose)
+      @ssh_options[:use_agent] = false unless @ssh_options.key?(:use_agent)
+      Log.log.debug{"ssh:#{@username}@#{@host}"}
+      Log.dump(:ssh_options, @ssh_options)
     end
 
     # Anything on stderr raises an exception

data/lib/aspera/sync/conf.schema.yaml

@@ -4,8 +4,8 @@ description: Async session description, identical to `asyncs` Node API.
 $comment: >-
   This is a YAML version of the original JSON schema from asyncs API.
   It is modified with special `x-` fields.
-  x-ts-name      [Bool,String]  `true` if same name in transfer spec, else real name in transfer spec
-  x-ts-convert   [String]       Name of methods to convert value from transfer spec to `conf`
+  x-ts-name      [Boolean,String] `true` if same name in transfer spec, else real name in transfer spec
+  x-ts-convert   [String]         Name of methods to convert value from transfer spec to `conf`
   `x-` fields are also documented in `command_line_builder.rb`
 type: object
 required:

data/lib/aspera/transfer/parameters.rb

@@ -45,12 +45,12 @@ module Aspera
         end
       end
 
-      # @param job_spec        [Hash]   Transfer spec
-      # @param ascp_args       [Array]  Other ascp args
-      # @param quiet           [Bool]   Remove ascp output
-      # @param trusted_certs   [Array]  Trusted certificates
-      # @param client_ssh_key  [Symbol] :rsa or :dsa
-      # @param check_ignore_cb [Proc]   Callback
+      # @param job_spec        [Hash]    Transfer spec
+      # @param ascp_args       [Array]   Other ascp args
+      # @param quiet           [Boolean] Remove ascp output
+      # @param trusted_certs   [Array]   Trusted certificates
+      # @param client_ssh_key  [:rsa,:dsa] :rsa or :dsa
+      # @param check_ignore_cb [Proc]    Callback
       def initialize(
         job_spec,
         ascp_args:       nil,

data/lib/aspera/transfer/spec.schema.yaml

@@ -7,8 +7,8 @@ $comment: >-
   The following extensions are used for mapping to ascp arguments.
   x-cli-envvar   [String]       Name of env var
   x-cli-option   [String]       Command line option (starts with "-"), or `true`: same as ts, or `false`: not an option
-  x-cli-switch   [Bool]         `true` if option has no arg, else by default option has a value
-  x-cli-special  [Bool]         `true` if not anaged by command line generator (special handling: option or argument)
+  x-cli-switch   [Boolean]      `true` if option has no arg, else by default option has a value
+  x-cli-special  [Boolean]      `true` if not anaged by command line generator (special handling: option or argument)
   x-cli-convert  [String,Hash]  Method name for Convert object or Conversion Hash for enum: ts to arg
   x-agents       [Array]        Supported agents (for doc only), if not specified: all
   x-deprecation  [String]       Deprecation message for doc
@@ -106,7 +106,7 @@ properties:
   delete_source:
     $comment: "TODO: implement"
     description: >-
-      Remove transfered source files after transfer success.
+      Remove transferred source files after transfer success.
       Equivalent to `remove_after_transfer` + `remove_empty_directories` + `remove_empty_source_directory`.
       Take precedence over those.
     type: boolean

data/lib/aspera/transfer/spec_doc.rb

@@ -8,20 +8,14 @@ module Aspera
     # Generate documentation from Schema, for Transfer Spec, or async Conf spec
     class SpecDoc
       class << self
-        # First letter of agent name symbol
-        def agent_to_short(agent_sym)
-          agent_sym.to_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
-        end
-
-        # @param formatter      [Cli::Formatter] Formatter to use, methods: markdown, tick, check_row
+        # @param formatter      [Cli::Formatter] Formatter to use, methods: markdown_text, tick, check_row
         # @param include_option [Boolean]        `true` : include CLI options
         # @param agent_columns  [Boolean]        `true` : include agents columns
         # @param schema         [Hash]           The JSON spec
         # @return [Array] a table suitable to display in manual
         def man_table(formatter, include_option: false, agent_columns: true, schema: Spec::SCHEMA)
-          col_local = agent_to_short(:direct)
           cols = %i[name type description]
-          cols.insert(-2, *AGENT_LIST.map(&:last)) if agent_columns
+          cols.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short]}.sort) if agent_columns
           rows = []
           schema['properties'].each do |name, info|
             rows.concat(man_table(formatter, include_option: include_option, agent_columns: agent_columns, schema: info).last.map{ |h| h.merge(name: "#{name}.#{h[:name]}")}) if info['type'].eql?('object') && info['properties']
@@ -35,25 +29,25 @@ module Aspera
             # Render Markdown formatting and split lines
             columns[:description] =
               info['description']
-                .gsub(Markdown::FORMATS){formatter.markdown(Regexp.last_match)}
+                .gsub(Markdown::FORMATS){formatter.markdown_text(Regexp.last_match)}
                 .split("\n") if info.key?('description')
             columns[:description].unshift("DEPRECATED: #{info['x-deprecation']}") if info.key?('x-deprecation')
             # Add flags for supported agents in doc
             agents = []
-            AGENT_LIST.each do |agent_info|
-              agents.push(agent_info.last) if info['x-agents'].nil? || info['x-agents'].include?(agent_info.first.to_s)
+            Agent::Factory::ALL.each_key do |sym|
+              agents.push(sym) if info['x-agents'].nil? || info['x-agents'].include?(sym.to_s)
             end
-            Aspera.assert(agents.include?(col_local)){"#{name}: x-cli-option requires agent direct (or nil)"} if info['x-cli-option']
+            Aspera.assert(agents.include?(:direct)){"#{name}: x-cli-option requires agent direct (or nil)"} if info['x-cli-option']
             if agent_columns
-              AGENT_LIST.each do |agent_info|
-                columns[agent_info.last] = formatter.tick(agents.include?(agent_info.last))
+              Agent::Factory::ALL.each do |sym, names|
+                columns[names[:short]] = formatter.tick(agents.include?(sym))
               end
             else
-              columns[:description].push("(#{agents.map(&:upcase).join(', ')})") unless agents.length.eql?(AGENT_LIST.length)
+              columns[:description].push("(#{agents.map{ |i| Agent::Factory::ALL[i][:short].to_s.upcase}.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
             end
             # Only keep lines that are usable in supported agents
             next false if agents.empty?
-            columns[:description].push("Allowed values: #{info['enum'].map{ |v| formatter.markdown("`#{v}`")}.join(', ')}") if info.key?('enum')
+            columns[:description].push("Allowed values: #{info['enum'].map{ |v| formatter.markdown_text("`#{v}`")}.join(', ')}") if info.key?('enum')
             if include_option
               envvar_prefix = ''
               cli_option =
@@ -70,17 +64,13 @@ module Aspera
                   "#{info['x-cli-option']}#{sep}#{"(#{conversion_tag})" if conversion_tag}#{arg_type}"
                 end
               short = info.key?('x-cli-short') ? "(#{info['x-cli-short']})" : nil
-              columns[:description].push("(#{'special:' if info['x-cli-special']}#{envvar_prefix}#{formatter.markdown("`#{cli_option}`")})#{short}") if cli_option
+              columns[:description].push("(#{'special:' if info['x-cli-special']}#{envvar_prefix}#{formatter.markdown_text("`#{cli_option}`")})#{short}") if cli_option
             end
             rows.push(formatter.check_row(columns))
           end
           [cols, rows.sort_by{ |i| i[:name]}]
         end
       end
-      # Agents shown in manual for parameters (sub list)
-      AGENT_LIST = Agent::Factory.instance.list.map do |agent_sym|
-        [agent_sym, agent_sym.to_s.capitalize, agent_to_short(agent_sym)]
-      end.sort_by(&:last).freeze
     end
   end
 end

data/lib/aspera/uri_reader.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'uri'
+require 'base64'
 require 'aspera/assert'
 require 'aspera/rest'
 require 'aspera/temp_file_manager'
@@ -19,6 +20,13 @@ module Aspera
         case uri.scheme
         when 'http', 'https'
           return Rest.new(base_url: uri_to_read, redirect_max: 5).read(nil, headers: {'Accept' => '*/*'})
+        when 'data'
+          metadata, encoded_data = uri.opaque.split(',', 2)
+          if metadata.end_with?(';base64')
+            Base64.decode64(encoded_data)
+          else
+            URI.decode_www_form_component(encoded_data)
+          end
         when SCHEME_FILE, NilClass
           local_file_path = uri.path
           raise Error, 'URL shall have a path, check syntax' if local_file_path.nil?
@@ -35,12 +43,18 @@ module Aspera
           # require specific file scheme: the path part is "relative", or absolute if there are 4 slash
           raise "use format: #{SCHEME_FILE_PFX2}<path>" unless url.start_with?(SCHEME_FILE_PFX2)
           return File.expand_path(url[SCHEME_FILE_PFX2.length..-1])
+        elsif url.start_with?('data:')
+          # download to temp file
+          # auto-delete on exit
+          temp_file = TempFileManager.instance.new_file_path_global('uri_reader')
+          File.write(temp_file, read(url), binmode: true)
+          return temp_file
         else
           # download to temp file
           # auto-delete on exit
-          sdk_archive_path = TempFileManager.instance.new_file_path_global(suffix: File.basename(url))
-          Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to_file: sdk_archive_path)
-          return sdk_archive_path
+          temp_file = TempFileManager.instance.new_file_path_global(suffix: File.basename(url))
+          Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to_file: temp_file)
+          return temp_file
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.25.0.pre2
+  version: 4.25.0
 platform: ruby
 authors:
 - Laurent Martin
@@ -367,6 +367,7 @@ files:
 - lib/aspera/data/5
 - lib/aspera/data/6
 - lib/aspera/data_repository.rb
+- lib/aspera/dot_container.rb
 - lib/aspera/environment.rb
 - lib/aspera/faspex_gw.rb
 - lib/aspera/faspex_postproc.rb