aspera-cli 4.29.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +2 -1
  3. data/CHANGELOG.md +76 -2
  4. data/README.md +20 -1
  5. data/SECURITY.md +14 -0
  6. data/docs/README.md +277 -38
  7. data/lib/aspera/api/node.rb +9 -2
  8. data/lib/aspera/ascp/installation.rb +26 -4
  9. data/lib/aspera/ascp/log_analyzer/lines.rb +2 -2
  10. data/lib/aspera/ascp/log_analyzer.rb +19 -3
  11. data/lib/aspera/ascp/log_locations.rb +163 -0
  12. data/lib/aspera/cli/ascp_logs_actions.rb +12 -0
  13. data/lib/aspera/cli/bootstrapper.rb +1 -0
  14. data/lib/aspera/cli/command_line.rb +39 -2
  15. data/lib/aspera/cli/command_registry.rb +3 -0
  16. data/lib/aspera/cli/command_spec.rb +13 -0
  17. data/lib/aspera/cli/error.rb +3 -0
  18. data/lib/aspera/cli/extended_value.rb +5 -4
  19. data/lib/aspera/cli/formatter.rb +14 -8
  20. data/lib/aspera/cli/http.rb +26 -1
  21. data/lib/aspera/cli/info.rb +1 -1
  22. data/lib/aspera/cli/mcp/tool.rb +5 -3
  23. data/lib/aspera/cli/option_value.rb +14 -8
  24. data/lib/aspera/cli/options.schema.yaml +156 -19
  25. data/lib/aspera/cli/parser.rb +15 -3
  26. data/lib/aspera/cli/plugins/aoc.rb +4 -2
  27. data/lib/aspera/cli/plugins/ats.rb +1 -1
  28. data/lib/aspera/cli/plugins/base.rb +33 -6
  29. data/lib/aspera/cli/plugins/config.rb +12 -1
  30. data/lib/aspera/cli/plugins/faspex5.rb +3 -1
  31. data/lib/aspera/cli/plugins/mcp.rb +14 -0
  32. data/lib/aspera/cli/plugins/node.rb +8 -8
  33. data/lib/aspera/cli/plugins/oauth.rb +18 -5
  34. data/lib/aspera/cli/plugins/orchestrator.rb +28 -24
  35. data/lib/aspera/cli/plugins/preview.rb +3 -1
  36. data/lib/aspera/cli/plugins/server.rb +3 -3
  37. data/lib/aspera/cli/plugins/shares.rb +1 -2
  38. data/lib/aspera/cli/plugins/transferd.rb +7 -2
  39. data/lib/aspera/cli/poll.rb +236 -0
  40. data/lib/aspera/cli/preset_manager.rb +63 -19
  41. data/lib/aspera/cli/prompt.rb +153 -8
  42. data/lib/aspera/cli/runner.rb +25 -7
  43. data/lib/aspera/cli/security.rb +10 -0
  44. data/lib/aspera/cli/sync_actions.rb +2 -0
  45. data/lib/aspera/cli/transfer_agent.rb +14 -0
  46. data/lib/aspera/cli/transfer_progress.rb +6 -5
  47. data/lib/aspera/cli/version.rb +1 -1
  48. data/lib/aspera/environment.rb +3 -1
  49. data/lib/aspera/id_generator.rb +16 -3
  50. data/lib/aspera/inflate.rb +43 -0
  51. data/lib/aspera/oauth/client/base.rb +38 -8
  52. data/lib/aspera/oauth/client/factory.rb +35 -15
  53. data/lib/aspera/oauth/client/json_credentials.rb +3 -2
  54. data/lib/aspera/oauth/client/shares_web.rb +13 -4
  55. data/lib/aspera/oauth/client/url_json.rb +3 -2
  56. data/lib/aspera/oauth/client/web.rb +2 -2
  57. data/lib/aspera/orchestrator_workflow_checker.rb +10 -1
  58. data/lib/aspera/preview/terminal.rb +8 -2
  59. data/lib/aspera/products/desktop.rb +13 -1
  60. data/lib/aspera/rest/client.rb +23 -5
  61. data/lib/aspera/schema/registry.rb +1 -0
  62. data/lib/aspera/sealed_hash.rb +2 -18
  63. data/lib/aspera/secret/hider.rb +13 -1
  64. data/lib/aspera/secret/store/one_password_cli.rb +13 -11
  65. data/lib/aspera/sync/operations.rb +4 -4
  66. data/lib/aspera/transfer/result.rb +20 -0
  67. data/lib/aspera/transfer/spec.rb +2 -2
  68. data/lib/aspera/{transfer/units.rb → units/decimal.rb} +4 -12
  69. data/lib/aspera/units/duration.rb +109 -0
  70. data/lib/aspera/uri_reader.rb +11 -6
  71. data/lib/aspera/web_auth/reply.html.erb +1 -1
  72. data/llms-install.md +88 -0
  73. data/llms.txt +6 -0
  74. data/skills/ascli/SKILL.md +6 -0
  75. data.tar.gz.sig +0 -0
  76. metadata +7 -3
  77. metadata.gz.sig +0 -0
  78. data/lib/aspera/duration.rb +0 -45
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require 'zlib'
8
+
9
+ module Aspera
10
+ # Decompression of zlib data that comes from outside (e.g. a code pasted by a user, a token given by a client or a server).
11
+ # A small input can inflate to gigabytes (decompression bomb): the size of the result is checked while decompressing, not after.
12
+ module Inflate
13
+ # Size of the slices of compressed data given to the inflater.
14
+ # Deflate expands at most about 1032 times, so one slice gives at most about 4 MiB.
15
+ # The input is sliced because `Zlib::Inflate#inflate` with a block does not yield chunks on JRuby:
16
+ # it yields the whole result once decompressed.
17
+ INPUT_SLICE = 4096
18
+ private_constant :INPUT_SLICE
19
+ class << self
20
+ # @param data [String] Zlib data
21
+ # @param max_size [Integer] Maximum size of the result, in bytes
22
+ # @param what [String] What is decompressed, for error messages, e.g. `Code`
23
+ # @return [String] Inflated data
24
+ # @raise [ParameterError] If the result is larger than `max_size`, or the data is incomplete
25
+ # @raise [Zlib::Error] If the data is not zlib data
26
+ def limited(data, max_size:, what:)
27
+ output = +''
28
+ inflater = Zlib::Inflate.new
29
+ begin
30
+ 0.step(data.bytesize - 1, INPUT_SLICE) do |offset|
31
+ output << inflater.inflate(data.byteslice(offset, INPUT_SLICE))
32
+ raise ParameterError, "#{what} is too large when decompressed: more than #{max_size} bytes" if output.bytesize > max_size
33
+ break if inflater.finished?
34
+ end
35
+ raise ParameterError, "#{what} is incomplete" unless inflater.finished?
36
+ ensure
37
+ inflater.close
38
+ end
39
+ output
40
+ end
41
+ end
42
+ end
43
+ end
@@ -10,6 +10,7 @@ require_relative '../../secret/hider'
10
10
  require_relative '../../mime'
11
11
  require_relative '../../rainbow'
12
12
  require 'date'
13
+ require 'time'
13
14
  using Rainbow
14
15
 
15
16
  module Aspera
@@ -21,10 +22,15 @@ module Aspera
21
22
  # Bearer Token Usage: https://tools.ietf.org/html/rfc6750
22
23
  class Base
23
24
  Aspera.require_method!(:create_token)
25
+ class << self
26
+ # @param path [String] Path of an end point
27
+ # @return [Boolean] `true` if `path` is an absolute URL (`http` or `https`)
28
+ def absolute_url?(path) = path.match?(%r{\Ahttps?://}i)
29
+ end
24
30
  # @param params [Hash] Parameters for token creation (client_id, client_secret, scope, etc...)
25
- # @param path_token [String] API end point to create a token from base URL
31
+ # @param path_token [String] API end point to create a token: path from base URL, or absolute URL (`http` or `https`), e.g. on another host
26
32
  # @param use_query [Boolean] Provide parameters in query instead of body
27
- # @param token_field [String] Field in result that contains the token
33
+ # @param token_field [String] Field in result that contains the token, or dot-separated path of a nested field (e.g. `authed_user.access_token`)
28
34
  # @param cache_ids [Array] List of unique identifiers for cache id generation
29
35
  # @param rest_params [Hash] Parameters for REST
30
36
  def initialize(
@@ -53,6 +59,23 @@ module Aspera
53
59
  # Sub path to generate token
54
60
  attr_reader :path_token
55
61
 
62
+ # URL of an end point of the OAuth server
63
+ # @param path [String] Path from the base URL, or absolute URL (`http` or `https`), used as is
64
+ # @return [String] URL of the end point
65
+ def endpoint_url(path)
66
+ Base.absolute_url?(path) ? path : "#{@api.base_url}/#{path}"
67
+ end
68
+
69
+ # API and subpath to call an end point of the OAuth server
70
+ # An absolute URL (e.g. token on another host than authorization) gets its own API, same parameters.
71
+ # The token cache id does not depend on it: it uses the base URL.
72
+ # @param path [String] Path from the base URL, or absolute URL (`http` or `https`)
73
+ # @return [Array(Rest::Client, String)] API and subpath
74
+ def endpoint_api(path)
75
+ return [@api, path] unless Base.absolute_url?(path)
76
+ [Rest::Client.new(**@api.params.merge(base_url: path)), '']
77
+ end
78
+
56
79
  # Parameters for token creation call
57
80
  # @param include_secret [Boolean] Add secret in default call parameters
58
81
  # @param other_params [Hash] Other parameters to merge
@@ -69,8 +92,9 @@ module Aspera
69
92
  def create_token_base(include_secret: false, **other_params)
70
93
  Log.log.debug { 'Generating a new token'.bg(:green) }
71
94
  creation_params = base_parameters(include_secret: include_secret, **other_params)
72
- return @api.create(@path_token, nil, query: creation_params, ret: :resp) if @use_query
73
- @api.create(@path_token, creation_params, content_type: Mime::WWW, ret: :resp)
95
+ token_api, subpath = endpoint_api(@path_token)
96
+ return token_api.create(subpath, nil, query: creation_params, ret: :resp) if @use_query
97
+ token_api.create(subpath, creation_params, content_type: Mime::WWW, ret: :resp)
74
98
  end
75
99
 
76
100
  # Get the value of the HTTP `Authorization` header for this OAuth client, i.e. `Bearer <token>`.
@@ -141,11 +165,16 @@ module Aspera
141
165
  end
142
166
 
143
167
  # Cache the token creation response
168
+ # Adds `obtained_at` and, if the server gives `expires_in`, `expires_at` (expiration of an opaque token)
144
169
  # @param http [Net::HTTPResponse] Response of the token creation (see {#create_token_base})
145
170
  # @return [Hash{String => Object}] Token creation response: `@token_field`, optional `refresh_token`, `expires_in`, ...
146
171
  def save_token_data(http)
147
172
  token_data = JSON.parse(http.body)
148
- Factory.instance.persist_mgr.put(@token_cache_id, http.body)
173
+ now = Time.now
174
+ token_data[Factory::OBTAINED_AT_FIELD] = now.utc.iso8601
175
+ lifetime = Integer(token_data['expires_in'], exception: false)
176
+ token_data[Factory::EXPIRES_AT_FIELD] = (now + lifetime).utc.iso8601 if lifetime && !token_data.key?(Factory::EXPIRES_AT_FIELD)
177
+ Factory.instance.persist_mgr.put(@token_cache_id, JSON.generate(token_data))
149
178
  # Logs decoded token information
150
179
  cached_token_info
151
180
  token_data
@@ -154,9 +183,10 @@ module Aspera
154
183
  # @param token_data [Hash{String => Object}] Token creation response
155
184
  # @return [String] The bearer token, registered as secret: hidden wherever it appears (logs, results)
156
185
  def bearer_token(token_data)
157
- Aspera.assert(token_data.key?(@token_field)) { "API error: No such field in answer: #{@token_field}" }
158
- Secret::Hider.instance.add_secret_values(token_data[@token_field], token_data['refresh_token'])
159
- token_data[@token_field]
186
+ token_value = Factory.token_value(token_data, @token_field)
187
+ Aspera.assert(!token_value.nil?) { "API error: No such field in answer: #{@token_field}" }
188
+ Secret::Hider.instance.add_secret_values(token_value, token_data['refresh_token'])
189
+ token_value
160
190
  end
161
191
  end
162
192
  end
@@ -8,6 +8,8 @@ require_relative '../../assert'
8
8
  require_relative '../../string_ext'
9
9
  require 'singleton'
10
10
  require 'base64'
11
+ require 'json'
12
+ require 'time'
11
13
  module Aspera
12
14
  module OAuth
13
15
  # Factory to create tokens and manage their cache
@@ -23,8 +25,13 @@ module Aspera
23
25
  # prefix for bearer authorization when in header
24
26
  SPACE_BEARER_AUTH_SCHEME = 'Bearer '
25
27
  TOKEN_FIELD = 'access_token'
28
+ # Separator of the path of a nested token field, e.g. `authed_user.access_token`
29
+ FIELD_PATH_SEPARATOR = '.'
30
+ # fields added to the cached token creation response
31
+ OBTAINED_AT_FIELD = 'obtained_at'
32
+ EXPIRES_AT_FIELD = 'expires_at'
26
33
 
27
- private_constant :PERSIST_CATEGORY_TOKEN, :SPACE_BEARER_AUTH_SCHEME
34
+ private_constant :PERSIST_CATEGORY_TOKEN, :SPACE_BEARER_AUTH_SCHEME, :FIELD_PATH_SEPARATOR
28
35
 
29
36
  class << self
30
37
  # Format a token for use in Authorization header
@@ -58,6 +65,14 @@ module Aspera
58
65
  return IdGenerator.from_list(PERSIST_CATEGORY_TOKEN, url, Factory.class_to_id(creator_class), params)
59
66
  end
60
67
 
68
+ # Value of the token in a token creation response
69
+ # @param token_data [Hash{String => Object}] Token creation response
70
+ # @param token_field [String] Field that contains the token, or dot-separated path of a nested field, e.g. `authed_user.access_token`
71
+ # @return [Object, nil] Value of the field, `nil` if absent (or an intermediate value is not a Hash)
72
+ def token_value(token_data, token_field)
73
+ token_field.split(FIELD_PATH_SEPARATOR).reduce(token_data) { |data, key| data.is_a?(Hash) ? data[key] : nil }
74
+ end
75
+
61
76
  # Convert a class name to snake_case symbol
62
77
  # @param creator_class [Class] The class to convert
63
78
  # @return [Symbol] snake_case version of class name
@@ -81,7 +96,9 @@ module Aspera
81
96
  # tokens older than this duration in sec. will be discarded from cache
82
97
  token_cache_max_age: 1800,
83
98
  # tokens valid for less than this duration in sec. will be regenerated
84
- token_refresh_threshold: 120
99
+ token_refresh_threshold: 120,
100
+ # tokens obtained since more than this duration in sec. will be regenerated (0: no limit)
101
+ token_refresh_max_age: 0
85
102
  }
86
103
  end
87
104
 
@@ -133,29 +150,32 @@ module Aspera
133
150
 
134
151
  # Get token information from cache
135
152
  # @param id [String] identifier of token
136
- # @param token_field [String, nil] Field containing the token, or nil to auto-detect
153
+ # @param token_field [String, nil] Field containing the token (or dot-separated path), or nil to auto-detect
137
154
  # @return [Hash] token internal information , including Date object for `expiration_date`
138
155
  def get_token_info(id, token_field: nil)
139
156
  token_raw_string = persist_mgr.get(id)
140
157
  return if token_raw_string.nil?
141
158
  token_data = JSON.parse(token_raw_string)
142
159
  Aspera.assert_type(token_data, Hash)
143
- token_value = token_data[token_field] || token_data[TOKEN_FIELD] || token_data['token']
160
+ token_value = (self.class.token_value(token_data, token_field) unless token_field.nil?) || token_data[TOKEN_FIELD] || token_data['token']
144
161
  decoded_token = decode_token(token_value)
145
162
  info = {data: token_data}
146
- if decoded_token.is_a?(Hash)
147
- info[:decoded] = decoded_token
148
- # TODO: move date decoding to token decoder ?
149
- expiration_date =
150
- if decoded_token['expires_at'].is_a?(String) then Time.parse(decoded_token['expires_at']).to_time
151
- elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
152
- end
153
- unless expiration_date.nil?
154
- info[:expiration] = expiration_date
155
- info[:ttl_sec] = expiration_date - Time.now
156
- info[:expired] = info[:ttl_sec] < @parameters[:token_refresh_threshold]
163
+ info[:decoded] = decoded_token if decoded_token.is_a?(Hash)
164
+ # TODO: move date decoding to token decoder ?
165
+ expiration_date =
166
+ if decoded_token.is_a?(Hash) && decoded_token['expires_at'].is_a?(String) then Time.parse(decoded_token['expires_at'])
167
+ elsif decoded_token.is_a?(Hash) && decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
168
+ elsif token_data[EXPIRES_AT_FIELD].is_a?(String) then Time.parse(token_data[EXPIRES_AT_FIELD])
157
169
  end
170
+ unless expiration_date.nil?
171
+ info[:expiration] = expiration_date
172
+ info[:ttl_sec] = expiration_date - Time.now
173
+ info[:expired] = info[:ttl_sec] < @parameters[:token_refresh_threshold]
158
174
  end
175
+ # Force refresh of an old token, e.g. before its refresh token expires
176
+ max_age = @parameters[:token_refresh_max_age].to_i
177
+ obtained_at = token_data[OBTAINED_AT_FIELD]
178
+ info[:expired] = true if max_age.positive? && obtained_at.is_a?(String) && Time.now - Time.parse(obtained_at) > max_age
159
179
  Log.dump(:token_info, info)
160
180
  return info
161
181
  end
@@ -22,9 +22,10 @@ module Aspera
22
22
  end
23
23
 
24
24
  def create_token
25
- api.call(
25
+ token_api, subpath = endpoint_api(path_token)
26
+ token_api.call(
26
27
  operation: 'POST',
27
- subpath: path_token,
28
+ subpath: subpath,
28
29
  content_type: Mime::JSON,
29
30
  body: @body,
30
31
  headers: {'Accept' => Mime::JSON},
@@ -62,14 +62,23 @@ module Aspera
62
62
  end
63
63
 
64
64
  # `paste`: Shares shows a code, to paste in the terminal
65
- # The state is part of the key of the code: the code is not readable without it
65
+ # The prompt shows if the code is valid, and asks again if not.
66
66
  # @return [Hash{String=>String}] Content of the code
67
67
  def receive_code(state)
68
68
  Environment.instance.open_uri(login_page("#{DEVICE_CODE_RELAY_STATE}#{state}"))
69
69
  Log.log.warn { 'USER ACTION: Log in with the web browser, then copy the code displayed by Shares and paste it in the prompt below.' }
70
- input = Cli::Prompt.prompt_user_input('Code', sensitive: true)
71
- Aspera.assert(SealedHash.sealed?(input), type: ParameterError) { 'Not a code: copy the code displayed by Shares after login' }
72
- SealedHash.unseal(input, secret: state, base_key: Cli::Info::CMD_NAME)
70
+ code = Cli::Prompt.prompt_user_input('Code', sensitive: true, validate: ->(text) { unseal_code(text, state) })
71
+ unseal_code(code, state)
72
+ end
73
+
74
+ # The state is part of the key of the code: the code is not readable without it
75
+ # @param text [String] Pasted by the user
76
+ # @param state [String] State of this login
77
+ # @return [Hash{String=>String}] Content of the code
78
+ # @raise [ParameterError] if the text is not a valid code of this login (or it is expired)
79
+ def unseal_code(text, state)
80
+ Aspera.assert(SealedHash.sealed?(text), type: ParameterError) { 'Not a code: copy the code displayed by Shares after login' }
81
+ SealedHash.unseal(text, secret: state, base_key: Cli::Info::CMD_NAME)
73
82
  end
74
83
 
75
84
  # @param relay_state [String] Where Shares sends the result
@@ -23,9 +23,10 @@ module Aspera
23
23
  end
24
24
 
25
25
  def create_token
26
- api.call(
26
+ token_api, subpath = endpoint_api(path_token)
27
+ token_api.call(
27
28
  operation: 'POST',
28
- subpath: path_token,
29
+ subpath: subpath,
29
30
  query: @query.merge(scope: params[:scope]), # scope is here because it may change over time (node)
30
31
  content_type: Mime::JSON,
31
32
  body: @body,
@@ -26,7 +26,7 @@ module Aspera
26
26
  end
27
27
  end
28
28
  # @param redirect_uri [String] url to receive the code after auth (to be exchanged for token)
29
- # @param path_authorize [String] path to login page on web app
29
+ # @param path_authorize [String] path to login page on web app, or absolute URL (`http` or `https`), e.g. on another host than the token
30
30
  # @param pkce [Boolean] use PKCE (RFC 7636): the code can be exchanged only by this process
31
31
  def initialize(
32
32
  redirect_uri:,
@@ -57,7 +57,7 @@ module Aspera
57
57
  token_params[:code_verifier] = code_verifier
58
58
  end
59
59
  login_page_url = Rest.build_uri(
60
- "#{api.base_url}/#{@path_authorize}",
60
+ endpoint_url(@path_authorize),
61
61
  base_parameters(**authorize_params)
62
62
  )
63
63
  # here, we need a human to authorize on a web page
@@ -70,6 +70,9 @@ module Aspera
70
70
  TABLE_SEPARATOR = ':::'
71
71
  # Elements of a step that a `Map_to` of type `output` gives as attributes (in lower case)
72
72
  STEP_IDENTITY = %w[Step_name Step_type Action_id].freeze
73
+ # Maximum size of the entries of a package once decompressed: a small package can decompress to gigabytes (decompression bomb)
74
+ PACKAGE_MAX_SIZE = 256 * 1024 * 1024
75
+ private_constant :PACKAGE_MAX_SIZE
73
76
  private_constant :WORKFLOW_ATTRIBUTES, :WHITEBOARD_ATTRIBUTES, :VALUE_TYPES, :ELEMENT_COUNTS, :TRUE_VALUES, :UNNAMED, :SUCCESS_STATUSES,
74
77
  :BOOLEAN_TEXT, :STEP_CHILDREN, :PARAMETER_MAPPINGS, :TEMPLATE_COLUMNS, :FALSE_VALUES, :INTEGER_TYPES, :SPEC_CODE, :PARTLY_DEFINED, :PLACEHOLDER,
75
78
  :STEP_IDENTITY, :SINGLE_NAME, :TABLE_SEPARATOR, :LOOKUP_DELIMITER, :LOOKUP_INDEX_INPUT
@@ -122,10 +125,16 @@ module Aspera
122
125
  # @param content [String] Bytes of a `.wkf` file: gzipped tar with a manifest (list of files, the first one is the main workflow)
123
126
  def check_package(content)
124
127
  files = {}
128
+ size = 0
125
129
  begin
126
130
  Zlib::GzipReader.wrap(StringIO.new(content)) do |gzip|
127
131
  Gem::Package::TarReader.new(gzip) do |tar|
128
- tar.each { |entry| files[entry.full_name] = entry.read if entry.file? }
132
+ tar.each do |entry|
133
+ # Size declared in the header: checked before the entry is decompressed (read, or skipped)
134
+ size += entry.header.size
135
+ return add(ERROR, 'package.too_large', 'file', "package is too large when decompressed: more than #{PACKAGE_MAX_SIZE} bytes") if size > PACKAGE_MAX_SIZE
136
+ files[entry.full_name] = entry.read if entry.file?
137
+ end
129
138
  end
130
139
  end
131
140
  rescue Zlib::Error, Gem::Package::Error => e
@@ -18,6 +18,9 @@ module Aspera
18
18
  module Backend
19
19
  # Base decoder that rescales image data to the current terminal geometry.
20
20
  class Base
21
+ # Rows and columns used when the size of the terminal is unknown
22
+ DEFAULT_TERMINAL_SIZE = [24, 80].freeze
23
+ private_constant :DEFAULT_TERMINAL_SIZE
21
24
  # @param reserve [Integer] number of terminal rows reserved for non-image output
22
25
  # @param double [Boolean] when `true`, render two image rows in one terminal row
23
26
  # @param font_ratio [Float] terminal font aspect ratio: height divided by width
@@ -33,7 +36,9 @@ module Aspera
33
36
  # @param columns [Integer] source image width in pixels
34
37
  # @return [Array<Integer>] scaled width and height for terminal rendering
35
38
  def terminal_scaling(rows, columns)
36
- (term_rows, term_columns) = IO.console.winsize || [24, 80]
39
+ (term_rows, term_columns) = IO.console&.winsize
40
+ # A terminal that does not know its size (e.g. a pseudo-terminal created without size) reports 0 x 0
41
+ (term_rows, term_columns) = DEFAULT_TERMINAL_SIZE unless term_rows&.positive? && term_columns&.positive?
37
42
  term_rows = [term_rows - @reserve, 2].max
38
43
  fit_term_ratio = [term_rows.to_f * @font_ratio / rows.to_f, term_columns.to_f / columns.to_f].min
39
44
  [(columns * fit_term_ratio).to_i, (rows * fit_term_ratio * @height_ratio / @font_ratio).to_i]
@@ -145,7 +150,8 @@ module Aspera
145
150
  begin
146
151
  Log.log.debug('Trying rmagick')
147
152
  Backend::RMagick.new(blob, reserve: reserve, double: double, font_ratio: font_ratio).terminal_pixels
148
- rescue => e
153
+ rescue StandardError, LoadError => e
154
+ # LoadError: optional gem rmagick not installed
149
155
  Log.log.debug(e.message)
150
156
  nil
151
157
  end
@@ -18,12 +18,24 @@ module Aspera
18
18
  case Aspera::Environment.instance.os
19
19
  when Aspera::Environment::OS_MACOS then [{
20
20
  app_root: File.join('', 'Applications', 'IBM Aspera.app'),
21
- log_root: File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER),
21
+ log_root: log_folder,
22
22
  sub_bin: File.join('Contents', 'Resources', 'transferd', 'bin')
23
23
  }]
24
24
  else []
25
25
  end.map { |i| i.merge({expected: APP_NAME}) }
26
26
  end
27
+
28
+ # Folder of the logs of the application, of `transferd` and of `ascp` (`log_directory` and `fasp_runtime.log.dir` of `transferd`)
29
+ # @return [String, nil] nil if unknown
30
+ def log_folder
31
+ case Aspera::Environment.instance.os
32
+ when Aspera::Environment::OS_MACOS then File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER)
33
+ when Aspera::Environment::OS_WINDOWS
34
+ appdata = ENV.fetch('APPDATA', nil)
35
+ File.join(appdata, APP_IDENTIFIER, 'logs') unless appdata.nil?
36
+ else File.join(ENV.fetch('XDG_CONFIG_HOME', File.join(Dir.home, '.config')), APP_IDENTIFIER, 'logs')
37
+ end
38
+ end
27
39
  end
28
40
  end
29
41
  end
@@ -127,8 +127,8 @@ module Aspera
127
127
  # @param operation [String] HTTP operation (GET, POST, PUT, DELETE)
128
128
  # @param subpath [String] subpath of REST API
129
129
  # @param query [Hash{String,Symbol => Object}] URL parameters
130
- # @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, else use headers
131
- # @param body [Hash, Array, String, nil] Body parameters (see `build_request`)
130
+ # @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, `nil` for a raw body (type in `headers`)
131
+ # @param body [Hash, Array, String, IO, nil] Body parameters (see `build_request`)
132
132
  # @param headers [Hash{String => String}] Additional headers (override Content-Type)
133
133
  # @param save_to [String, Pathname, IO, nil] File path or IO object to save response body; progress bar is used when set
134
134
  # @param exception [Boolean] Whether to raise an exception on HTTP error
@@ -164,6 +164,8 @@ module Aspera
164
164
  # We must have a way to check return code
165
165
  Aspera.assert(exception || !ret.eql?(:data), 'ret: :data requires exception handler')
166
166
  req_headers, req_query = prepare_call(headers, query)
167
+ # A raw body stream is sent again from the same position on retry
168
+ stream_start = body.pos if content_type.nil? && body.respond_to?(:read) && body.respond_to?(:seek)
167
169
  result_http = nil
168
170
  result_data = nil
169
171
  # number of tries on error (first call included)
@@ -173,6 +175,7 @@ module Aspera
173
175
  # start a block to be able to retry the actual HTTP request in case of OAuth token expiration
174
176
  begin
175
177
  Log.log.debug("send request (redirects=#{@redirect_max})")
178
+ body.seek(stream_start) unless stream_start.nil?
176
179
  req = build_request(operation, subpath, req_query, content_type, body, req_headers)
177
180
  result_mime = nil
178
181
  file_saved = false
@@ -334,9 +337,11 @@ module Aspera
334
337
  # @param operation [String] HTTP operation (GET, POST, ...)
335
338
  # @param subpath [String] Subpath of REST API
336
339
  # @param query [Hash, String, Array, nil] Query of request
337
- # @param content_type [String, nil] One of Mime::JSON, Mime::WWW, Mime::TEXT, Mime::MULTIPART, or `nil` for no body
338
- # @param body [Hash, Array, String, nil] Body of request, serialized according to `content_type`
340
+ # @param content_type [String, nil] One of Mime::JSON, Mime::WWW, Mime::TEXT, Mime::MULTIPART, or `nil` for a raw body
341
+ # @param body [Hash, Array, String, IO, nil] Body of request, serialized according to `content_type`
339
342
  # For Mime::MULTIPART: Hash `name => value`, or Array of `[name, value, opts]` with opts `filename`, `content_type` for a file part
343
+ # For `nil`: no body, or raw body sent as is, `Content-Type` given in `headers`: String (e.g. binary),
344
+ # or IO (responds to `read`) streamed from its current position, e.g. a large file not loaded in memory
340
345
  # @param headers [Hash] Headers of request
341
346
  # @return [Net::HTTPRequest] The request
342
347
  def build_request(operation, subpath, query, content_type, body, headers)
@@ -352,7 +357,20 @@ module Aspera
352
357
  raise "unsupported operation : #{operation}"
353
358
  end
354
359
  case content_type
355
- when nil # ignore
360
+ when nil
361
+ case body
362
+ when nil # no body
363
+ when String then req.body = body
364
+ else
365
+ Aspera.assert(body.respond_to?(:read)) { "raw body: String or IO expected, not #{body.class}" }
366
+ req.body_stream = body
367
+ # Net::HTTP needs a length or chunked encoding: length of the rest of the stream, if known
368
+ if body.respond_to?(:size) && body.respond_to?(:pos)
369
+ req['Content-Length'] = (body.size - body.pos).to_s
370
+ else
371
+ req['Transfer-Encoding'] = 'chunked'
372
+ end
373
+ end
356
374
  when Mime::JSON
357
375
  req.body = JSON.generate(body) # , ascii_only: true
358
376
  req['Content-Type'] = Mime::JSON
@@ -118,6 +118,7 @@ module Aspera
118
118
  IMAGE_OPTIONS = "#{OPTIONS}:components.schemas.ImageOptions".freeze
119
119
  ASCP_LOGS_OPTIONS = "#{OPTIONS}:components.schemas.AscpLogsOptions".freeze
120
120
  OUT_OPTIONS = "#{OPTIONS}:components.schemas.OutOptions".freeze
121
+ POLL_OPTIONS = "#{OPTIONS}:components.schemas.PollOptions".freeze
121
122
  PACKAGE_FOLDER_OPTIONS = "#{OPTIONS}:components.schemas.PackageFolderOptions".freeze
122
123
  WEB_SERVER_OPTIONS = "#{OPTIONS}:components.schemas.WebServerOptions".freeze
123
124
  GATEWAY_OPTIONS = "#{OPTIONS}:components.schemas.Faspex4GatewayOptions".freeze
@@ -4,6 +4,7 @@
4
4
  # SPDX-License-Identifier: Apache-2.0
5
5
 
6
6
  require_relative 'assert'
7
+ require_relative 'inflate'
7
8
  require 'json'
8
9
  require 'zlib'
9
10
  require 'openssl'
@@ -113,7 +114,7 @@ module Aspera
113
114
  else
114
115
  deflated = raw
115
116
  end
116
- data = JSON.parse(inflate(deflated))
117
+ data = JSON.parse(Inflate.limited(deflated, max_size: MAX_INFLATED_SIZE, what: 'Code'))
117
118
  raise ParameterError, 'Invalid code: unexpected content' unless data.is_a?(Hash)
118
119
  data
119
120
  rescue JSON::ParserError, Zlib::Error
@@ -144,23 +145,6 @@ module Aspera
144
145
  rescue OpenSSL::Cipher::CipherError
145
146
  nil
146
147
  end
147
-
148
- # @param body [String] Deflated data
149
- # @return [String] Inflated data, limited in size
150
- def inflate(body)
151
- output = +''
152
- inflater = Zlib::Inflate.new
153
- begin
154
- inflater.inflate(body) do |chunk|
155
- output << chunk
156
- raise ParameterError, 'Invalid code: content is too large' if output.bytesize > MAX_INFLATED_SIZE
157
- end
158
- raise ParameterError, 'Code is incomplete' unless inflater.finished?
159
- ensure
160
- inflater.close
161
- end
162
- output
163
- end
164
148
  end
165
149
  end
166
150
  end
@@ -34,6 +34,8 @@ module Aspera
34
34
  NON_ENV_SECRETS = (KEY_SECRETS + HTTP_SECRETS).freeze
35
35
  # not secrets: `*_file` is the path of a file (e.g. `ca_file`)
36
36
  KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/, /public_key$/, /^token_type$/, /_file$/].freeze
37
+ # boolean values as text: not secrets, even for a name that denotes a secret (e.g. `show_secrets`)
38
+ BOOLEAN_TEXTS = %w[yes no true false].freeze
37
39
  # min length of hidden secrets, use `+` or `{n,}`
38
40
  SECRET_LENGTH = '{5,}'
39
41
  # min length of a declared secret value to be hidden inside a string: shorter ones are hidden only when they are the whole string,
@@ -60,7 +62,7 @@ module Aspera
60
62
  # cred in http dump
61
63
  /(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
62
64
  ].freeze
63
- private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :SECRET_LENGTH, :VALUE_IN_STRING_LENGTH, :REGEX_LOG_REPLACES
65
+ private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :BOOLEAN_TEXTS, :SECRET_LENGTH, :VALUE_IN_STRING_LENGTH, :REGEX_LOG_REPLACES
64
66
  attr_accessor :log_secrets
65
67
 
66
68
  # @return [Proc] new log formatter that hides secrets
@@ -156,6 +158,16 @@ module Aspera
156
158
  end
157
159
  end
158
160
 
161
+ # Declare the secret values of a named value (e.g. an option): the whole value if the name denotes a secret,
162
+ # else the secret values found by key name in it.
163
+ # A boolean value given as text is not a secret, e.g. `no` for option `cache_tokens`, else it would be hidden everywhere.
164
+ # @param name [String, Symbol] Name, e.g. option name
165
+ # @param value [Object] Value
166
+ def add_named_secrets(name, value)
167
+ return add_secrets_in(value) unless secret?(name, '')
168
+ add_secret_values(value) unless BOOLEAN_TEXTS.include?(value)
169
+ end
170
+
159
171
  # Declare additional exact key names whose values are secrets
160
172
  # @param keys [Array<String, Symbol>] Key names
161
173
  def add_secret_keys(keys)
@@ -50,11 +50,14 @@ module Aspera
50
50
  validate_set(options)
51
51
  assert_new_label(options[:label])
52
52
  args = ['item', 'create', '--category', ITEM_CATEGORY, "--title=#{options[:label]}"]
53
- args << "--url=#{options[:url]}" if options[:url]
54
- args << "#{FIELD_USERNAME}=#{options[:username]}" if options[:username]
55
- args << "#{FIELD_PASSWORD}[password]=#{options[:password]}" if options[:password]
56
- args << "#{FIELD_NOTES_ID}=#{options[:description]}" if options[:description]
57
- op_run(*args)
53
+ args << "--url=#{options[:url]}" if options[:url]
54
+ # `op` reads an item template from its standard input when it is not a terminal (always the case here):
55
+ # fields are given in the template, which also keeps the secret out of the command line
56
+ fields = []
57
+ fields << {id: FIELD_USERNAME, type: 'STRING', purpose: 'USERNAME', label: FIELD_USERNAME, value: options[:username]} if options[:username]
58
+ fields << {id: FIELD_PASSWORD, type: 'CONCEALED', purpose: 'PASSWORD', label: FIELD_PASSWORD, value: options[:password]} if options[:password]
59
+ fields << {id: FIELD_NOTES_ID, type: 'STRING', purpose: 'NOTES', label: FIELD_NOTES_ID, value: options[:description]} if options[:description]
60
+ op_capture(*args, stdin_data: JSON.generate({fields: fields}), exception: true)
58
61
  nil
59
62
  end
60
63
 
@@ -97,12 +100,11 @@ module Aspera
97
100
  nil
98
101
  end
99
102
 
100
- def op_capture(*args, exception: true)
101
- Environment.instance.class.secure_execute(
102
- OP_EXECUTABLE, *args, *common_flags,
103
- mode: :capture,
104
- exception: exception
105
- )
103
+ # @param stdin_data [String, nil] Data sent to standard input of `op` (e.g. an item template)
104
+ def op_capture(*args, exception: true, stdin_data: nil)
105
+ kwargs = {mode: :capture, exception: exception}
106
+ kwargs[:stdin_data] = stdin_data unless stdin_data.nil?
107
+ Environment.instance.class.secure_execute(OP_EXECUTABLE, *args, *common_flags, **kwargs)
106
108
  end
107
109
  end
108
110
  end
@@ -6,13 +6,13 @@
6
6
  require_relative '../ascp/installation'
7
7
  require_relative '../agent/direct'
8
8
  require_relative '../transfer/spec'
9
- require_relative '../transfer/units'
9
+ require_relative '../units/decimal'
10
10
  require_relative '../exec_spec'
11
11
  require_relative '../command_line_converter'
12
12
  require_relative '../command_line_builder'
13
13
  require_relative '../log'
14
14
  require_relative '../assert'
15
- require_relative '../duration'
15
+ require_relative '../units/duration'
16
16
  require_relative '../environment'
17
17
  require_relative '../schema/registry'
18
18
  require 'json'
@@ -268,7 +268,7 @@ module Aspera
268
268
  # Convert `target_rate` given as a string with optional suffix (e.g. `100m`) into integer bps
269
269
  # @param params [Hash, nil] `transport` (`conf` format) or session (`args` format), modified in place
270
270
  def normalize_target_rate(params)
271
- params['target_rate'] = Transfer::Units.rate_string_to_bps(params['target_rate']) if params.is_a?(Hash) && params['target_rate'].is_a?(String)
271
+ params['target_rate'] = Units::Decimal.rate_string_to_bps(params['target_rate']) if params.is_a?(Hash) && params['target_rate'].is_a?(String)
272
272
  end
273
273
 
274
274
  # Transfer specification to synchronization information
@@ -349,7 +349,7 @@ module Aspera
349
349
  # a special switch sets the non-default enum value, e.g. `--continuous` sets `mode` to `continuous`
350
350
  props.key?('enum') ? (props['enum'] - [props['default']]).first : true
351
351
  elsif props.key?('x-unit')
352
- Duration.to_unit(args.shift, props['x-unit'])
352
+ Units::Duration.new(args.shift.to_s).to_unit(props['x-unit'])
353
353
  else
354
354
  args.shift
355
355
  end