aspera-cli 4.29.0 → 4.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -1
- data/CHANGELOG.md +76 -2
- data/README.md +20 -1
- data/SECURITY.md +14 -0
- data/docs/README.md +277 -38
- data/lib/aspera/api/node.rb +9 -2
- data/lib/aspera/ascp/installation.rb +26 -4
- data/lib/aspera/ascp/log_analyzer/lines.rb +2 -2
- data/lib/aspera/ascp/log_analyzer.rb +19 -3
- data/lib/aspera/ascp/log_locations.rb +163 -0
- data/lib/aspera/cli/ascp_logs_actions.rb +12 -0
- data/lib/aspera/cli/bootstrapper.rb +1 -0
- data/lib/aspera/cli/command_line.rb +39 -2
- data/lib/aspera/cli/command_registry.rb +3 -0
- data/lib/aspera/cli/command_spec.rb +13 -0
- data/lib/aspera/cli/error.rb +3 -0
- data/lib/aspera/cli/extended_value.rb +5 -4
- data/lib/aspera/cli/formatter.rb +14 -8
- data/lib/aspera/cli/http.rb +26 -1
- data/lib/aspera/cli/info.rb +1 -1
- data/lib/aspera/cli/mcp/tool.rb +5 -3
- data/lib/aspera/cli/option_value.rb +14 -8
- data/lib/aspera/cli/options.schema.yaml +156 -19
- data/lib/aspera/cli/parser.rb +15 -3
- data/lib/aspera/cli/plugins/aoc.rb +4 -2
- data/lib/aspera/cli/plugins/ats.rb +1 -1
- data/lib/aspera/cli/plugins/base.rb +33 -6
- data/lib/aspera/cli/plugins/config.rb +12 -1
- data/lib/aspera/cli/plugins/faspex5.rb +3 -1
- data/lib/aspera/cli/plugins/mcp.rb +14 -0
- data/lib/aspera/cli/plugins/node.rb +8 -8
- data/lib/aspera/cli/plugins/oauth.rb +18 -5
- data/lib/aspera/cli/plugins/orchestrator.rb +28 -24
- data/lib/aspera/cli/plugins/preview.rb +3 -1
- data/lib/aspera/cli/plugins/server.rb +3 -3
- data/lib/aspera/cli/plugins/shares.rb +1 -2
- data/lib/aspera/cli/plugins/transferd.rb +7 -2
- data/lib/aspera/cli/poll.rb +236 -0
- data/lib/aspera/cli/preset_manager.rb +63 -19
- data/lib/aspera/cli/prompt.rb +153 -8
- data/lib/aspera/cli/runner.rb +25 -7
- data/lib/aspera/cli/security.rb +10 -0
- data/lib/aspera/cli/sync_actions.rb +2 -0
- data/lib/aspera/cli/transfer_agent.rb +14 -0
- data/lib/aspera/cli/transfer_progress.rb +6 -5
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/environment.rb +3 -1
- data/lib/aspera/id_generator.rb +16 -3
- data/lib/aspera/inflate.rb +43 -0
- data/lib/aspera/oauth/client/base.rb +38 -8
- data/lib/aspera/oauth/client/factory.rb +35 -15
- data/lib/aspera/oauth/client/json_credentials.rb +3 -2
- data/lib/aspera/oauth/client/shares_web.rb +13 -4
- data/lib/aspera/oauth/client/url_json.rb +3 -2
- data/lib/aspera/oauth/client/web.rb +2 -2
- data/lib/aspera/orchestrator_workflow_checker.rb +10 -1
- data/lib/aspera/preview/terminal.rb +8 -2
- data/lib/aspera/products/desktop.rb +13 -1
- data/lib/aspera/rest/client.rb +23 -5
- data/lib/aspera/schema/registry.rb +1 -0
- data/lib/aspera/sealed_hash.rb +2 -18
- data/lib/aspera/secret/hider.rb +13 -1
- data/lib/aspera/secret/store/one_password_cli.rb +13 -11
- data/lib/aspera/sync/operations.rb +4 -4
- data/lib/aspera/transfer/result.rb +20 -0
- data/lib/aspera/transfer/spec.rb +2 -2
- data/lib/aspera/{transfer/units.rb → units/decimal.rb} +4 -12
- data/lib/aspera/units/duration.rb +109 -0
- data/lib/aspera/uri_reader.rb +11 -6
- data/lib/aspera/web_auth/reply.html.erb +1 -1
- data/llms-install.md +88 -0
- data/llms.txt +6 -0
- data/skills/ascli/SKILL.md +6 -0
- data.tar.gz.sig +0 -0
- metadata +7 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/duration.rb +0 -45
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
7
|
+
require 'zlib'
|
|
8
|
+
|
|
9
|
+
module Aspera
|
|
10
|
+
# Decompression of zlib data that comes from outside (e.g. a code pasted by a user, a token given by a client or a server).
|
|
11
|
+
# A small input can inflate to gigabytes (decompression bomb): the size of the result is checked while decompressing, not after.
|
|
12
|
+
module Inflate
|
|
13
|
+
# Size of the slices of compressed data given to the inflater.
|
|
14
|
+
# Deflate expands at most about 1032 times, so one slice gives at most about 4 MiB.
|
|
15
|
+
# The input is sliced because `Zlib::Inflate#inflate` with a block does not yield chunks on JRuby:
|
|
16
|
+
# it yields the whole result once decompressed.
|
|
17
|
+
INPUT_SLICE = 4096
|
|
18
|
+
private_constant :INPUT_SLICE
|
|
19
|
+
class << self
|
|
20
|
+
# @param data [String] Zlib data
|
|
21
|
+
# @param max_size [Integer] Maximum size of the result, in bytes
|
|
22
|
+
# @param what [String] What is decompressed, for error messages, e.g. `Code`
|
|
23
|
+
# @return [String] Inflated data
|
|
24
|
+
# @raise [ParameterError] If the result is larger than `max_size`, or the data is incomplete
|
|
25
|
+
# @raise [Zlib::Error] If the data is not zlib data
|
|
26
|
+
def limited(data, max_size:, what:)
|
|
27
|
+
output = +''
|
|
28
|
+
inflater = Zlib::Inflate.new
|
|
29
|
+
begin
|
|
30
|
+
0.step(data.bytesize - 1, INPUT_SLICE) do |offset|
|
|
31
|
+
output << inflater.inflate(data.byteslice(offset, INPUT_SLICE))
|
|
32
|
+
raise ParameterError, "#{what} is too large when decompressed: more than #{max_size} bytes" if output.bytesize > max_size
|
|
33
|
+
break if inflater.finished?
|
|
34
|
+
end
|
|
35
|
+
raise ParameterError, "#{what} is incomplete" unless inflater.finished?
|
|
36
|
+
ensure
|
|
37
|
+
inflater.close
|
|
38
|
+
end
|
|
39
|
+
output
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -10,6 +10,7 @@ require_relative '../../secret/hider'
|
|
|
10
10
|
require_relative '../../mime'
|
|
11
11
|
require_relative '../../rainbow'
|
|
12
12
|
require 'date'
|
|
13
|
+
require 'time'
|
|
13
14
|
using Rainbow
|
|
14
15
|
|
|
15
16
|
module Aspera
|
|
@@ -21,10 +22,15 @@ module Aspera
|
|
|
21
22
|
# Bearer Token Usage: https://tools.ietf.org/html/rfc6750
|
|
22
23
|
class Base
|
|
23
24
|
Aspera.require_method!(:create_token)
|
|
25
|
+
class << self
|
|
26
|
+
# @param path [String] Path of an end point
|
|
27
|
+
# @return [Boolean] `true` if `path` is an absolute URL (`http` or `https`)
|
|
28
|
+
def absolute_url?(path) = path.match?(%r{\Ahttps?://}i)
|
|
29
|
+
end
|
|
24
30
|
# @param params [Hash] Parameters for token creation (client_id, client_secret, scope, etc...)
|
|
25
|
-
# @param path_token [String] API end point to create a token from base URL
|
|
31
|
+
# @param path_token [String] API end point to create a token: path from base URL, or absolute URL (`http` or `https`), e.g. on another host
|
|
26
32
|
# @param use_query [Boolean] Provide parameters in query instead of body
|
|
27
|
-
# @param token_field [String] Field in result that contains the token
|
|
33
|
+
# @param token_field [String] Field in result that contains the token, or dot-separated path of a nested field (e.g. `authed_user.access_token`)
|
|
28
34
|
# @param cache_ids [Array] List of unique identifiers for cache id generation
|
|
29
35
|
# @param rest_params [Hash] Parameters for REST
|
|
30
36
|
def initialize(
|
|
@@ -53,6 +59,23 @@ module Aspera
|
|
|
53
59
|
# Sub path to generate token
|
|
54
60
|
attr_reader :path_token
|
|
55
61
|
|
|
62
|
+
# URL of an end point of the OAuth server
|
|
63
|
+
# @param path [String] Path from the base URL, or absolute URL (`http` or `https`), used as is
|
|
64
|
+
# @return [String] URL of the end point
|
|
65
|
+
def endpoint_url(path)
|
|
66
|
+
Base.absolute_url?(path) ? path : "#{@api.base_url}/#{path}"
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# API and subpath to call an end point of the OAuth server
|
|
70
|
+
# An absolute URL (e.g. token on another host than authorization) gets its own API, same parameters.
|
|
71
|
+
# The token cache id does not depend on it: it uses the base URL.
|
|
72
|
+
# @param path [String] Path from the base URL, or absolute URL (`http` or `https`)
|
|
73
|
+
# @return [Array(Rest::Client, String)] API and subpath
|
|
74
|
+
def endpoint_api(path)
|
|
75
|
+
return [@api, path] unless Base.absolute_url?(path)
|
|
76
|
+
[Rest::Client.new(**@api.params.merge(base_url: path)), '']
|
|
77
|
+
end
|
|
78
|
+
|
|
56
79
|
# Parameters for token creation call
|
|
57
80
|
# @param include_secret [Boolean] Add secret in default call parameters
|
|
58
81
|
# @param other_params [Hash] Other parameters to merge
|
|
@@ -69,8 +92,9 @@ module Aspera
|
|
|
69
92
|
def create_token_base(include_secret: false, **other_params)
|
|
70
93
|
Log.log.debug { 'Generating a new token'.bg(:green) }
|
|
71
94
|
creation_params = base_parameters(include_secret: include_secret, **other_params)
|
|
72
|
-
|
|
73
|
-
|
|
95
|
+
token_api, subpath = endpoint_api(@path_token)
|
|
96
|
+
return token_api.create(subpath, nil, query: creation_params, ret: :resp) if @use_query
|
|
97
|
+
token_api.create(subpath, creation_params, content_type: Mime::WWW, ret: :resp)
|
|
74
98
|
end
|
|
75
99
|
|
|
76
100
|
# Get the value of the HTTP `Authorization` header for this OAuth client, i.e. `Bearer <token>`.
|
|
@@ -141,11 +165,16 @@ module Aspera
|
|
|
141
165
|
end
|
|
142
166
|
|
|
143
167
|
# Cache the token creation response
|
|
168
|
+
# Adds `obtained_at` and, if the server gives `expires_in`, `expires_at` (expiration of an opaque token)
|
|
144
169
|
# @param http [Net::HTTPResponse] Response of the token creation (see {#create_token_base})
|
|
145
170
|
# @return [Hash{String => Object}] Token creation response: `@token_field`, optional `refresh_token`, `expires_in`, ...
|
|
146
171
|
def save_token_data(http)
|
|
147
172
|
token_data = JSON.parse(http.body)
|
|
148
|
-
|
|
173
|
+
now = Time.now
|
|
174
|
+
token_data[Factory::OBTAINED_AT_FIELD] = now.utc.iso8601
|
|
175
|
+
lifetime = Integer(token_data['expires_in'], exception: false)
|
|
176
|
+
token_data[Factory::EXPIRES_AT_FIELD] = (now + lifetime).utc.iso8601 if lifetime && !token_data.key?(Factory::EXPIRES_AT_FIELD)
|
|
177
|
+
Factory.instance.persist_mgr.put(@token_cache_id, JSON.generate(token_data))
|
|
149
178
|
# Logs decoded token information
|
|
150
179
|
cached_token_info
|
|
151
180
|
token_data
|
|
@@ -154,9 +183,10 @@ module Aspera
|
|
|
154
183
|
# @param token_data [Hash{String => Object}] Token creation response
|
|
155
184
|
# @return [String] The bearer token, registered as secret: hidden wherever it appears (logs, results)
|
|
156
185
|
def bearer_token(token_data)
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
token_data[
|
|
186
|
+
token_value = Factory.token_value(token_data, @token_field)
|
|
187
|
+
Aspera.assert(!token_value.nil?) { "API error: No such field in answer: #{@token_field}" }
|
|
188
|
+
Secret::Hider.instance.add_secret_values(token_value, token_data['refresh_token'])
|
|
189
|
+
token_value
|
|
160
190
|
end
|
|
161
191
|
end
|
|
162
192
|
end
|
|
@@ -8,6 +8,8 @@ require_relative '../../assert'
|
|
|
8
8
|
require_relative '../../string_ext'
|
|
9
9
|
require 'singleton'
|
|
10
10
|
require 'base64'
|
|
11
|
+
require 'json'
|
|
12
|
+
require 'time'
|
|
11
13
|
module Aspera
|
|
12
14
|
module OAuth
|
|
13
15
|
# Factory to create tokens and manage their cache
|
|
@@ -23,8 +25,13 @@ module Aspera
|
|
|
23
25
|
# prefix for bearer authorization when in header
|
|
24
26
|
SPACE_BEARER_AUTH_SCHEME = 'Bearer '
|
|
25
27
|
TOKEN_FIELD = 'access_token'
|
|
28
|
+
# Separator of the path of a nested token field, e.g. `authed_user.access_token`
|
|
29
|
+
FIELD_PATH_SEPARATOR = '.'
|
|
30
|
+
# fields added to the cached token creation response
|
|
31
|
+
OBTAINED_AT_FIELD = 'obtained_at'
|
|
32
|
+
EXPIRES_AT_FIELD = 'expires_at'
|
|
26
33
|
|
|
27
|
-
private_constant :PERSIST_CATEGORY_TOKEN, :SPACE_BEARER_AUTH_SCHEME
|
|
34
|
+
private_constant :PERSIST_CATEGORY_TOKEN, :SPACE_BEARER_AUTH_SCHEME, :FIELD_PATH_SEPARATOR
|
|
28
35
|
|
|
29
36
|
class << self
|
|
30
37
|
# Format a token for use in Authorization header
|
|
@@ -58,6 +65,14 @@ module Aspera
|
|
|
58
65
|
return IdGenerator.from_list(PERSIST_CATEGORY_TOKEN, url, Factory.class_to_id(creator_class), params)
|
|
59
66
|
end
|
|
60
67
|
|
|
68
|
+
# Value of the token in a token creation response
|
|
69
|
+
# @param token_data [Hash{String => Object}] Token creation response
|
|
70
|
+
# @param token_field [String] Field that contains the token, or dot-separated path of a nested field, e.g. `authed_user.access_token`
|
|
71
|
+
# @return [Object, nil] Value of the field, `nil` if absent (or an intermediate value is not a Hash)
|
|
72
|
+
def token_value(token_data, token_field)
|
|
73
|
+
token_field.split(FIELD_PATH_SEPARATOR).reduce(token_data) { |data, key| data.is_a?(Hash) ? data[key] : nil }
|
|
74
|
+
end
|
|
75
|
+
|
|
61
76
|
# Convert a class name to snake_case symbol
|
|
62
77
|
# @param creator_class [Class] The class to convert
|
|
63
78
|
# @return [Symbol] snake_case version of class name
|
|
@@ -81,7 +96,9 @@ module Aspera
|
|
|
81
96
|
# tokens older than this duration in sec. will be discarded from cache
|
|
82
97
|
token_cache_max_age: 1800,
|
|
83
98
|
# tokens valid for less than this duration in sec. will be regenerated
|
|
84
|
-
token_refresh_threshold: 120
|
|
99
|
+
token_refresh_threshold: 120,
|
|
100
|
+
# tokens obtained since more than this duration in sec. will be regenerated (0: no limit)
|
|
101
|
+
token_refresh_max_age: 0
|
|
85
102
|
}
|
|
86
103
|
end
|
|
87
104
|
|
|
@@ -133,29 +150,32 @@ module Aspera
|
|
|
133
150
|
|
|
134
151
|
# Get token information from cache
|
|
135
152
|
# @param id [String] identifier of token
|
|
136
|
-
# @param token_field [String, nil] Field containing the token, or nil to auto-detect
|
|
153
|
+
# @param token_field [String, nil] Field containing the token (or dot-separated path), or nil to auto-detect
|
|
137
154
|
# @return [Hash] token internal information , including Date object for `expiration_date`
|
|
138
155
|
def get_token_info(id, token_field: nil)
|
|
139
156
|
token_raw_string = persist_mgr.get(id)
|
|
140
157
|
return if token_raw_string.nil?
|
|
141
158
|
token_data = JSON.parse(token_raw_string)
|
|
142
159
|
Aspera.assert_type(token_data, Hash)
|
|
143
|
-
token_value = token_data
|
|
160
|
+
token_value = (self.class.token_value(token_data, token_field) unless token_field.nil?) || token_data[TOKEN_FIELD] || token_data['token']
|
|
144
161
|
decoded_token = decode_token(token_value)
|
|
145
162
|
info = {data: token_data}
|
|
146
|
-
if decoded_token.is_a?(Hash)
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
end
|
|
153
|
-
unless expiration_date.nil?
|
|
154
|
-
info[:expiration] = expiration_date
|
|
155
|
-
info[:ttl_sec] = expiration_date - Time.now
|
|
156
|
-
info[:expired] = info[:ttl_sec] < @parameters[:token_refresh_threshold]
|
|
163
|
+
info[:decoded] = decoded_token if decoded_token.is_a?(Hash)
|
|
164
|
+
# TODO: move date decoding to token decoder ?
|
|
165
|
+
expiration_date =
|
|
166
|
+
if decoded_token.is_a?(Hash) && decoded_token['expires_at'].is_a?(String) then Time.parse(decoded_token['expires_at'])
|
|
167
|
+
elsif decoded_token.is_a?(Hash) && decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
|
|
168
|
+
elsif token_data[EXPIRES_AT_FIELD].is_a?(String) then Time.parse(token_data[EXPIRES_AT_FIELD])
|
|
157
169
|
end
|
|
170
|
+
unless expiration_date.nil?
|
|
171
|
+
info[:expiration] = expiration_date
|
|
172
|
+
info[:ttl_sec] = expiration_date - Time.now
|
|
173
|
+
info[:expired] = info[:ttl_sec] < @parameters[:token_refresh_threshold]
|
|
158
174
|
end
|
|
175
|
+
# Force refresh of an old token, e.g. before its refresh token expires
|
|
176
|
+
max_age = @parameters[:token_refresh_max_age].to_i
|
|
177
|
+
obtained_at = token_data[OBTAINED_AT_FIELD]
|
|
178
|
+
info[:expired] = true if max_age.positive? && obtained_at.is_a?(String) && Time.now - Time.parse(obtained_at) > max_age
|
|
159
179
|
Log.dump(:token_info, info)
|
|
160
180
|
return info
|
|
161
181
|
end
|
|
@@ -22,9 +22,10 @@ module Aspera
|
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
def create_token
|
|
25
|
-
|
|
25
|
+
token_api, subpath = endpoint_api(path_token)
|
|
26
|
+
token_api.call(
|
|
26
27
|
operation: 'POST',
|
|
27
|
-
subpath:
|
|
28
|
+
subpath: subpath,
|
|
28
29
|
content_type: Mime::JSON,
|
|
29
30
|
body: @body,
|
|
30
31
|
headers: {'Accept' => Mime::JSON},
|
|
@@ -62,14 +62,23 @@ module Aspera
|
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
# `paste`: Shares shows a code, to paste in the terminal
|
|
65
|
-
# The
|
|
65
|
+
# The prompt shows if the code is valid, and asks again if not.
|
|
66
66
|
# @return [Hash{String=>String}] Content of the code
|
|
67
67
|
def receive_code(state)
|
|
68
68
|
Environment.instance.open_uri(login_page("#{DEVICE_CODE_RELAY_STATE}#{state}"))
|
|
69
69
|
Log.log.warn { 'USER ACTION: Log in with the web browser, then copy the code displayed by Shares and paste it in the prompt below.' }
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
code = Cli::Prompt.prompt_user_input('Code', sensitive: true, validate: ->(text) { unseal_code(text, state) })
|
|
71
|
+
unseal_code(code, state)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The state is part of the key of the code: the code is not readable without it
|
|
75
|
+
# @param text [String] Pasted by the user
|
|
76
|
+
# @param state [String] State of this login
|
|
77
|
+
# @return [Hash{String=>String}] Content of the code
|
|
78
|
+
# @raise [ParameterError] if the text is not a valid code of this login (or it is expired)
|
|
79
|
+
def unseal_code(text, state)
|
|
80
|
+
Aspera.assert(SealedHash.sealed?(text), type: ParameterError) { 'Not a code: copy the code displayed by Shares after login' }
|
|
81
|
+
SealedHash.unseal(text, secret: state, base_key: Cli::Info::CMD_NAME)
|
|
73
82
|
end
|
|
74
83
|
|
|
75
84
|
# @param relay_state [String] Where Shares sends the result
|
|
@@ -23,9 +23,10 @@ module Aspera
|
|
|
23
23
|
end
|
|
24
24
|
|
|
25
25
|
def create_token
|
|
26
|
-
|
|
26
|
+
token_api, subpath = endpoint_api(path_token)
|
|
27
|
+
token_api.call(
|
|
27
28
|
operation: 'POST',
|
|
28
|
-
subpath:
|
|
29
|
+
subpath: subpath,
|
|
29
30
|
query: @query.merge(scope: params[:scope]), # scope is here because it may change over time (node)
|
|
30
31
|
content_type: Mime::JSON,
|
|
31
32
|
body: @body,
|
|
@@ -26,7 +26,7 @@ module Aspera
|
|
|
26
26
|
end
|
|
27
27
|
end
|
|
28
28
|
# @param redirect_uri [String] url to receive the code after auth (to be exchanged for token)
|
|
29
|
-
# @param path_authorize [String] path to login page on web app
|
|
29
|
+
# @param path_authorize [String] path to login page on web app, or absolute URL (`http` or `https`), e.g. on another host than the token
|
|
30
30
|
# @param pkce [Boolean] use PKCE (RFC 7636): the code can be exchanged only by this process
|
|
31
31
|
def initialize(
|
|
32
32
|
redirect_uri:,
|
|
@@ -57,7 +57,7 @@ module Aspera
|
|
|
57
57
|
token_params[:code_verifier] = code_verifier
|
|
58
58
|
end
|
|
59
59
|
login_page_url = Rest.build_uri(
|
|
60
|
-
|
|
60
|
+
endpoint_url(@path_authorize),
|
|
61
61
|
base_parameters(**authorize_params)
|
|
62
62
|
)
|
|
63
63
|
# here, we need a human to authorize on a web page
|
|
@@ -70,6 +70,9 @@ module Aspera
|
|
|
70
70
|
TABLE_SEPARATOR = ':::'
|
|
71
71
|
# Elements of a step that a `Map_to` of type `output` gives as attributes (in lower case)
|
|
72
72
|
STEP_IDENTITY = %w[Step_name Step_type Action_id].freeze
|
|
73
|
+
# Maximum size of the entries of a package once decompressed: a small package can decompress to gigabytes (decompression bomb)
|
|
74
|
+
PACKAGE_MAX_SIZE = 256 * 1024 * 1024
|
|
75
|
+
private_constant :PACKAGE_MAX_SIZE
|
|
73
76
|
private_constant :WORKFLOW_ATTRIBUTES, :WHITEBOARD_ATTRIBUTES, :VALUE_TYPES, :ELEMENT_COUNTS, :TRUE_VALUES, :UNNAMED, :SUCCESS_STATUSES,
|
|
74
77
|
:BOOLEAN_TEXT, :STEP_CHILDREN, :PARAMETER_MAPPINGS, :TEMPLATE_COLUMNS, :FALSE_VALUES, :INTEGER_TYPES, :SPEC_CODE, :PARTLY_DEFINED, :PLACEHOLDER,
|
|
75
78
|
:STEP_IDENTITY, :SINGLE_NAME, :TABLE_SEPARATOR, :LOOKUP_DELIMITER, :LOOKUP_INDEX_INPUT
|
|
@@ -122,10 +125,16 @@ module Aspera
|
|
|
122
125
|
# @param content [String] Bytes of a `.wkf` file: gzipped tar with a manifest (list of files, the first one is the main workflow)
|
|
123
126
|
def check_package(content)
|
|
124
127
|
files = {}
|
|
128
|
+
size = 0
|
|
125
129
|
begin
|
|
126
130
|
Zlib::GzipReader.wrap(StringIO.new(content)) do |gzip|
|
|
127
131
|
Gem::Package::TarReader.new(gzip) do |tar|
|
|
128
|
-
tar.each
|
|
132
|
+
tar.each do |entry|
|
|
133
|
+
# Size declared in the header: checked before the entry is decompressed (read, or skipped)
|
|
134
|
+
size += entry.header.size
|
|
135
|
+
return add(ERROR, 'package.too_large', 'file', "package is too large when decompressed: more than #{PACKAGE_MAX_SIZE} bytes") if size > PACKAGE_MAX_SIZE
|
|
136
|
+
files[entry.full_name] = entry.read if entry.file?
|
|
137
|
+
end
|
|
129
138
|
end
|
|
130
139
|
end
|
|
131
140
|
rescue Zlib::Error, Gem::Package::Error => e
|
|
@@ -18,6 +18,9 @@ module Aspera
|
|
|
18
18
|
module Backend
|
|
19
19
|
# Base decoder that rescales image data to the current terminal geometry.
|
|
20
20
|
class Base
|
|
21
|
+
# Rows and columns used when the size of the terminal is unknown
|
|
22
|
+
DEFAULT_TERMINAL_SIZE = [24, 80].freeze
|
|
23
|
+
private_constant :DEFAULT_TERMINAL_SIZE
|
|
21
24
|
# @param reserve [Integer] number of terminal rows reserved for non-image output
|
|
22
25
|
# @param double [Boolean] when `true`, render two image rows in one terminal row
|
|
23
26
|
# @param font_ratio [Float] terminal font aspect ratio: height divided by width
|
|
@@ -33,7 +36,9 @@ module Aspera
|
|
|
33
36
|
# @param columns [Integer] source image width in pixels
|
|
34
37
|
# @return [Array<Integer>] scaled width and height for terminal rendering
|
|
35
38
|
def terminal_scaling(rows, columns)
|
|
36
|
-
(term_rows, term_columns) = IO.console
|
|
39
|
+
(term_rows, term_columns) = IO.console&.winsize
|
|
40
|
+
# A terminal that does not know its size (e.g. a pseudo-terminal created without size) reports 0 x 0
|
|
41
|
+
(term_rows, term_columns) = DEFAULT_TERMINAL_SIZE unless term_rows&.positive? && term_columns&.positive?
|
|
37
42
|
term_rows = [term_rows - @reserve, 2].max
|
|
38
43
|
fit_term_ratio = [term_rows.to_f * @font_ratio / rows.to_f, term_columns.to_f / columns.to_f].min
|
|
39
44
|
[(columns * fit_term_ratio).to_i, (rows * fit_term_ratio * @height_ratio / @font_ratio).to_i]
|
|
@@ -145,7 +150,8 @@ module Aspera
|
|
|
145
150
|
begin
|
|
146
151
|
Log.log.debug('Trying rmagick')
|
|
147
152
|
Backend::RMagick.new(blob, reserve: reserve, double: double, font_ratio: font_ratio).terminal_pixels
|
|
148
|
-
rescue => e
|
|
153
|
+
rescue StandardError, LoadError => e
|
|
154
|
+
# LoadError: optional gem rmagick not installed
|
|
149
155
|
Log.log.debug(e.message)
|
|
150
156
|
nil
|
|
151
157
|
end
|
|
@@ -18,12 +18,24 @@ module Aspera
|
|
|
18
18
|
case Aspera::Environment.instance.os
|
|
19
19
|
when Aspera::Environment::OS_MACOS then [{
|
|
20
20
|
app_root: File.join('', 'Applications', 'IBM Aspera.app'),
|
|
21
|
-
log_root:
|
|
21
|
+
log_root: log_folder,
|
|
22
22
|
sub_bin: File.join('Contents', 'Resources', 'transferd', 'bin')
|
|
23
23
|
}]
|
|
24
24
|
else []
|
|
25
25
|
end.map { |i| i.merge({expected: APP_NAME}) }
|
|
26
26
|
end
|
|
27
|
+
|
|
28
|
+
# Folder of the logs of the application, of `transferd` and of `ascp` (`log_directory` and `fasp_runtime.log.dir` of `transferd`)
|
|
29
|
+
# @return [String, nil] nil if unknown
|
|
30
|
+
def log_folder
|
|
31
|
+
case Aspera::Environment.instance.os
|
|
32
|
+
when Aspera::Environment::OS_MACOS then File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER)
|
|
33
|
+
when Aspera::Environment::OS_WINDOWS
|
|
34
|
+
appdata = ENV.fetch('APPDATA', nil)
|
|
35
|
+
File.join(appdata, APP_IDENTIFIER, 'logs') unless appdata.nil?
|
|
36
|
+
else File.join(ENV.fetch('XDG_CONFIG_HOME', File.join(Dir.home, '.config')), APP_IDENTIFIER, 'logs')
|
|
37
|
+
end
|
|
38
|
+
end
|
|
27
39
|
end
|
|
28
40
|
end
|
|
29
41
|
end
|
data/lib/aspera/rest/client.rb
CHANGED
|
@@ -127,8 +127,8 @@ module Aspera
|
|
|
127
127
|
# @param operation [String] HTTP operation (GET, POST, PUT, DELETE)
|
|
128
128
|
# @param subpath [String] subpath of REST API
|
|
129
129
|
# @param query [Hash{String,Symbol => Object}] URL parameters
|
|
130
|
-
# @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization,
|
|
131
|
-
# @param body [Hash, Array, String, nil] Body parameters (see `build_request`)
|
|
130
|
+
# @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, `nil` for a raw body (type in `headers`)
|
|
131
|
+
# @param body [Hash, Array, String, IO, nil] Body parameters (see `build_request`)
|
|
132
132
|
# @param headers [Hash{String => String}] Additional headers (override Content-Type)
|
|
133
133
|
# @param save_to [String, Pathname, IO, nil] File path or IO object to save response body; progress bar is used when set
|
|
134
134
|
# @param exception [Boolean] Whether to raise an exception on HTTP error
|
|
@@ -164,6 +164,8 @@ module Aspera
|
|
|
164
164
|
# We must have a way to check return code
|
|
165
165
|
Aspera.assert(exception || !ret.eql?(:data), 'ret: :data requires exception handler')
|
|
166
166
|
req_headers, req_query = prepare_call(headers, query)
|
|
167
|
+
# A raw body stream is sent again from the same position on retry
|
|
168
|
+
stream_start = body.pos if content_type.nil? && body.respond_to?(:read) && body.respond_to?(:seek)
|
|
167
169
|
result_http = nil
|
|
168
170
|
result_data = nil
|
|
169
171
|
# number of tries on error (first call included)
|
|
@@ -173,6 +175,7 @@ module Aspera
|
|
|
173
175
|
# start a block to be able to retry the actual HTTP request in case of OAuth token expiration
|
|
174
176
|
begin
|
|
175
177
|
Log.log.debug("send request (redirects=#{@redirect_max})")
|
|
178
|
+
body.seek(stream_start) unless stream_start.nil?
|
|
176
179
|
req = build_request(operation, subpath, req_query, content_type, body, req_headers)
|
|
177
180
|
result_mime = nil
|
|
178
181
|
file_saved = false
|
|
@@ -334,9 +337,11 @@ module Aspera
|
|
|
334
337
|
# @param operation [String] HTTP operation (GET, POST, ...)
|
|
335
338
|
# @param subpath [String] Subpath of REST API
|
|
336
339
|
# @param query [Hash, String, Array, nil] Query of request
|
|
337
|
-
# @param content_type [String, nil] One of Mime::JSON, Mime::WWW, Mime::TEXT, Mime::MULTIPART, or `nil` for
|
|
338
|
-
# @param body [Hash, Array, String, nil] Body of request, serialized according to `content_type`
|
|
340
|
+
# @param content_type [String, nil] One of Mime::JSON, Mime::WWW, Mime::TEXT, Mime::MULTIPART, or `nil` for a raw body
|
|
341
|
+
# @param body [Hash, Array, String, IO, nil] Body of request, serialized according to `content_type`
|
|
339
342
|
# For Mime::MULTIPART: Hash `name => value`, or Array of `[name, value, opts]` with opts `filename`, `content_type` for a file part
|
|
343
|
+
# For `nil`: no body, or raw body sent as is, `Content-Type` given in `headers`: String (e.g. binary),
|
|
344
|
+
# or IO (responds to `read`) streamed from its current position, e.g. a large file not loaded in memory
|
|
340
345
|
# @param headers [Hash] Headers of request
|
|
341
346
|
# @return [Net::HTTPRequest] The request
|
|
342
347
|
def build_request(operation, subpath, query, content_type, body, headers)
|
|
@@ -352,7 +357,20 @@ module Aspera
|
|
|
352
357
|
raise "unsupported operation : #{operation}"
|
|
353
358
|
end
|
|
354
359
|
case content_type
|
|
355
|
-
when nil
|
|
360
|
+
when nil
|
|
361
|
+
case body
|
|
362
|
+
when nil # no body
|
|
363
|
+
when String then req.body = body
|
|
364
|
+
else
|
|
365
|
+
Aspera.assert(body.respond_to?(:read)) { "raw body: String or IO expected, not #{body.class}" }
|
|
366
|
+
req.body_stream = body
|
|
367
|
+
# Net::HTTP needs a length or chunked encoding: length of the rest of the stream, if known
|
|
368
|
+
if body.respond_to?(:size) && body.respond_to?(:pos)
|
|
369
|
+
req['Content-Length'] = (body.size - body.pos).to_s
|
|
370
|
+
else
|
|
371
|
+
req['Transfer-Encoding'] = 'chunked'
|
|
372
|
+
end
|
|
373
|
+
end
|
|
356
374
|
when Mime::JSON
|
|
357
375
|
req.body = JSON.generate(body) # , ascii_only: true
|
|
358
376
|
req['Content-Type'] = Mime::JSON
|
|
@@ -118,6 +118,7 @@ module Aspera
|
|
|
118
118
|
IMAGE_OPTIONS = "#{OPTIONS}:components.schemas.ImageOptions".freeze
|
|
119
119
|
ASCP_LOGS_OPTIONS = "#{OPTIONS}:components.schemas.AscpLogsOptions".freeze
|
|
120
120
|
OUT_OPTIONS = "#{OPTIONS}:components.schemas.OutOptions".freeze
|
|
121
|
+
POLL_OPTIONS = "#{OPTIONS}:components.schemas.PollOptions".freeze
|
|
121
122
|
PACKAGE_FOLDER_OPTIONS = "#{OPTIONS}:components.schemas.PackageFolderOptions".freeze
|
|
122
123
|
WEB_SERVER_OPTIONS = "#{OPTIONS}:components.schemas.WebServerOptions".freeze
|
|
123
124
|
GATEWAY_OPTIONS = "#{OPTIONS}:components.schemas.Faspex4GatewayOptions".freeze
|
data/lib/aspera/sealed_hash.rb
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
# SPDX-License-Identifier: Apache-2.0
|
|
5
5
|
|
|
6
6
|
require_relative 'assert'
|
|
7
|
+
require_relative 'inflate'
|
|
7
8
|
require 'json'
|
|
8
9
|
require 'zlib'
|
|
9
10
|
require 'openssl'
|
|
@@ -113,7 +114,7 @@ module Aspera
|
|
|
113
114
|
else
|
|
114
115
|
deflated = raw
|
|
115
116
|
end
|
|
116
|
-
data = JSON.parse(
|
|
117
|
+
data = JSON.parse(Inflate.limited(deflated, max_size: MAX_INFLATED_SIZE, what: 'Code'))
|
|
117
118
|
raise ParameterError, 'Invalid code: unexpected content' unless data.is_a?(Hash)
|
|
118
119
|
data
|
|
119
120
|
rescue JSON::ParserError, Zlib::Error
|
|
@@ -144,23 +145,6 @@ module Aspera
|
|
|
144
145
|
rescue OpenSSL::Cipher::CipherError
|
|
145
146
|
nil
|
|
146
147
|
end
|
|
147
|
-
|
|
148
|
-
# @param body [String] Deflated data
|
|
149
|
-
# @return [String] Inflated data, limited in size
|
|
150
|
-
def inflate(body)
|
|
151
|
-
output = +''
|
|
152
|
-
inflater = Zlib::Inflate.new
|
|
153
|
-
begin
|
|
154
|
-
inflater.inflate(body) do |chunk|
|
|
155
|
-
output << chunk
|
|
156
|
-
raise ParameterError, 'Invalid code: content is too large' if output.bytesize > MAX_INFLATED_SIZE
|
|
157
|
-
end
|
|
158
|
-
raise ParameterError, 'Code is incomplete' unless inflater.finished?
|
|
159
|
-
ensure
|
|
160
|
-
inflater.close
|
|
161
|
-
end
|
|
162
|
-
output
|
|
163
|
-
end
|
|
164
148
|
end
|
|
165
149
|
end
|
|
166
150
|
end
|
data/lib/aspera/secret/hider.rb
CHANGED
|
@@ -34,6 +34,8 @@ module Aspera
|
|
|
34
34
|
NON_ENV_SECRETS = (KEY_SECRETS + HTTP_SECRETS).freeze
|
|
35
35
|
# not secrets: `*_file` is the path of a file (e.g. `ca_file`)
|
|
36
36
|
KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/, /public_key$/, /^token_type$/, /_file$/].freeze
|
|
37
|
+
# boolean values as text: not secrets, even for a name that denotes a secret (e.g. `show_secrets`)
|
|
38
|
+
BOOLEAN_TEXTS = %w[yes no true false].freeze
|
|
37
39
|
# min length of hidden secrets, use `+` or `{n,}`
|
|
38
40
|
SECRET_LENGTH = '{5,}'
|
|
39
41
|
# min length of a declared secret value to be hidden inside a string: shorter ones are hidden only when they are the whole string,
|
|
@@ -60,7 +62,7 @@ module Aspera
|
|
|
60
62
|
# cred in http dump
|
|
61
63
|
/(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
|
|
62
64
|
].freeze
|
|
63
|
-
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :SECRET_LENGTH, :VALUE_IN_STRING_LENGTH, :REGEX_LOG_REPLACES
|
|
65
|
+
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :BOOLEAN_TEXTS, :SECRET_LENGTH, :VALUE_IN_STRING_LENGTH, :REGEX_LOG_REPLACES
|
|
64
66
|
attr_accessor :log_secrets
|
|
65
67
|
|
|
66
68
|
# @return [Proc] new log formatter that hides secrets
|
|
@@ -156,6 +158,16 @@ module Aspera
|
|
|
156
158
|
end
|
|
157
159
|
end
|
|
158
160
|
|
|
161
|
+
# Declare the secret values of a named value (e.g. an option): the whole value if the name denotes a secret,
|
|
162
|
+
# else the secret values found by key name in it.
|
|
163
|
+
# A boolean value given as text is not a secret, e.g. `no` for option `cache_tokens`, else it would be hidden everywhere.
|
|
164
|
+
# @param name [String, Symbol] Name, e.g. option name
|
|
165
|
+
# @param value [Object] Value
|
|
166
|
+
def add_named_secrets(name, value)
|
|
167
|
+
return add_secrets_in(value) unless secret?(name, '')
|
|
168
|
+
add_secret_values(value) unless BOOLEAN_TEXTS.include?(value)
|
|
169
|
+
end
|
|
170
|
+
|
|
159
171
|
# Declare additional exact key names whose values are secrets
|
|
160
172
|
# @param keys [Array<String, Symbol>] Key names
|
|
161
173
|
def add_secret_keys(keys)
|
|
@@ -50,11 +50,14 @@ module Aspera
|
|
|
50
50
|
validate_set(options)
|
|
51
51
|
assert_new_label(options[:label])
|
|
52
52
|
args = ['item', 'create', '--category', ITEM_CATEGORY, "--title=#{options[:label]}"]
|
|
53
|
-
args << "--url=#{options[:url]}"
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
53
|
+
args << "--url=#{options[:url]}" if options[:url]
|
|
54
|
+
# `op` reads an item template from its standard input when it is not a terminal (always the case here):
|
|
55
|
+
# fields are given in the template, which also keeps the secret out of the command line
|
|
56
|
+
fields = []
|
|
57
|
+
fields << {id: FIELD_USERNAME, type: 'STRING', purpose: 'USERNAME', label: FIELD_USERNAME, value: options[:username]} if options[:username]
|
|
58
|
+
fields << {id: FIELD_PASSWORD, type: 'CONCEALED', purpose: 'PASSWORD', label: FIELD_PASSWORD, value: options[:password]} if options[:password]
|
|
59
|
+
fields << {id: FIELD_NOTES_ID, type: 'STRING', purpose: 'NOTES', label: FIELD_NOTES_ID, value: options[:description]} if options[:description]
|
|
60
|
+
op_capture(*args, stdin_data: JSON.generate({fields: fields}), exception: true)
|
|
58
61
|
nil
|
|
59
62
|
end
|
|
60
63
|
|
|
@@ -97,12 +100,11 @@ module Aspera
|
|
|
97
100
|
nil
|
|
98
101
|
end
|
|
99
102
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
)
|
|
103
|
+
# @param stdin_data [String, nil] Data sent to standard input of `op` (e.g. an item template)
|
|
104
|
+
def op_capture(*args, exception: true, stdin_data: nil)
|
|
105
|
+
kwargs = {mode: :capture, exception: exception}
|
|
106
|
+
kwargs[:stdin_data] = stdin_data unless stdin_data.nil?
|
|
107
|
+
Environment.instance.class.secure_execute(OP_EXECUTABLE, *args, *common_flags, **kwargs)
|
|
106
108
|
end
|
|
107
109
|
end
|
|
108
110
|
end
|
|
@@ -6,13 +6,13 @@
|
|
|
6
6
|
require_relative '../ascp/installation'
|
|
7
7
|
require_relative '../agent/direct'
|
|
8
8
|
require_relative '../transfer/spec'
|
|
9
|
-
require_relative '../
|
|
9
|
+
require_relative '../units/decimal'
|
|
10
10
|
require_relative '../exec_spec'
|
|
11
11
|
require_relative '../command_line_converter'
|
|
12
12
|
require_relative '../command_line_builder'
|
|
13
13
|
require_relative '../log'
|
|
14
14
|
require_relative '../assert'
|
|
15
|
-
require_relative '../duration'
|
|
15
|
+
require_relative '../units/duration'
|
|
16
16
|
require_relative '../environment'
|
|
17
17
|
require_relative '../schema/registry'
|
|
18
18
|
require 'json'
|
|
@@ -268,7 +268,7 @@ module Aspera
|
|
|
268
268
|
# Convert `target_rate` given as a string with optional suffix (e.g. `100m`) into integer bps
|
|
269
269
|
# @param params [Hash, nil] `transport` (`conf` format) or session (`args` format), modified in place
|
|
270
270
|
def normalize_target_rate(params)
|
|
271
|
-
params['target_rate'] =
|
|
271
|
+
params['target_rate'] = Units::Decimal.rate_string_to_bps(params['target_rate']) if params.is_a?(Hash) && params['target_rate'].is_a?(String)
|
|
272
272
|
end
|
|
273
273
|
|
|
274
274
|
# Transfer specification to synchronization information
|
|
@@ -349,7 +349,7 @@ module Aspera
|
|
|
349
349
|
# a special switch sets the non-default enum value, e.g. `--continuous` sets `mode` to `continuous`
|
|
350
350
|
props.key?('enum') ? (props['enum'] - [props['default']]).first : true
|
|
351
351
|
elsif props.key?('x-unit')
|
|
352
|
-
Duration.
|
|
352
|
+
Units::Duration.new(args.shift.to_s).to_unit(props['x-unit'])
|
|
353
353
|
else
|
|
354
354
|
args.shift
|
|
355
355
|
end
|