aspera-cli 4.28.0 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +143 -2
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +135 -56
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +103 -26
- data/SECURITY.md +70 -0
- data/bin/ascli +8 -5
- data/bin/asession +9 -6
- data/docs/README.md +1633 -609
- data/docs/test-mcp-with-ai.md +8 -8
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +20 -19
- data/lib/aspera/agent/factory.rb +6 -3
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +7 -4
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +20 -20
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +71 -65
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +354 -0
- data/lib/aspera/ascp/management.rb +16 -8
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +12 -19
- data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +37 -33
- data/lib/aspera/cli/command_line.rb +24 -6
- data/lib/aspera/cli/command_registry.rb +6 -1
- data/lib/aspera/cli/command_spec.rb +34 -7
- data/lib/aspera/cli/completion/ascli.bash +19 -5
- data/lib/aspera/cli/completion/ascli.fish +13 -2
- data/lib/aspera/cli/completion/ascli.ps1 +31 -9
- data/lib/aspera/cli/completion/ascli.zsh +22 -6
- data/lib/aspera/cli/completion_actions.rb +303 -69
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +3 -0
- data/lib/aspera/cli/extended_value.rb +61 -27
- data/lib/aspera/cli/formatter.rb +29 -22
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -10
- data/lib/aspera/cli/http.rb +12 -9
- data/lib/aspera/cli/info.rb +12 -7
- data/lib/aspera/cli/mailer.rb +6 -3
- data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
- data/lib/aspera/cli/mcp/secrets.rb +8 -5
- data/lib/aspera/cli/mcp/tool.rb +15 -14
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +65 -12
- data/lib/aspera/cli/options.schema.yaml +487 -63
- data/lib/aspera/cli/parser.rb +56 -32
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +244 -35
- data/lib/aspera/cli/plugins/ats.rb +9 -8
- data/lib/aspera/cli/plugins/base.rb +17 -12
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +198 -103
- data/lib/aspera/cli/plugins/console.rb +6 -3
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +21 -7
- data/lib/aspera/cli/plugins/faspex5.rb +28 -25
- data/lib/aspera/cli/plugins/faspio.rb +7 -5
- data/lib/aspera/cli/plugins/httpgw.rb +7 -4
- data/lib/aspera/cli/plugins/mcp.rb +28 -22
- data/lib/aspera/cli/plugins/node.rb +31 -30
- data/lib/aspera/cli/plugins/oauth.rb +5 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
- data/lib/aspera/cli/plugins/preview.rb +69 -44
- data/lib/aspera/cli/plugins/server.rb +63 -11
- data/lib/aspera/cli/plugins/shares.rb +86 -16
- data/lib/aspera/cli/plugins/transferd.rb +223 -0
- data/lib/aspera/cli/preset_actions.rb +144 -30
- data/lib/aspera/cli/preset_manager.rb +52 -18
- data/lib/aspera/cli/prompt.rb +4 -1
- data/lib/aspera/cli/result.rb +12 -8
- data/lib/aspera/cli/runner.rb +69 -46
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +79 -9
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +13 -10
- data/lib/aspera/cli/table_fitter.rb +3 -0
- data/lib/aspera/cli/terminal_formatter.rb +9 -6
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +29 -20
- data/lib/aspera/cli/transfer_progress.rb +13 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +188 -42
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.rb +12 -10
- data/lib/aspera/cli/wizard.rb +42 -39
- data/lib/aspera/command_line_builder.rb +9 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +4 -1
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/duration.rb +5 -2
- data/lib/aspera/environment.rb +21 -12
- data/lib/aspera/exec_spec.rb +3 -0
- data/lib/aspera/faspex_gw.rb +13 -9
- data/lib/aspera/faspex_postproc.rb +15 -12
- data/lib/aspera/formatter_interface.rb +4 -1
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +4 -2
- data/lib/aspera/json_rpc/client.rb +6 -5
- data/lib/aspera/json_rpc/version.rb +3 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +18 -5
- data/lib/aspera/markdown.rb +5 -2
- data/lib/aspera/mime.rb +3 -0
- data/lib/aspera/nagios.rb +6 -4
- data/lib/aspera/node_emulator.rb +15 -13
- data/lib/aspera/oauth/client/base.rb +91 -70
- data/lib/aspera/oauth/client/boot.rb +7 -4
- data/lib/aspera/oauth/client/factory.rb +6 -3
- data/lib/aspera/oauth/client/generic.rb +5 -1
- data/lib/aspera/oauth/client/json_credentials.rb +4 -1
- data/lib/aspera/oauth/client/jwt.rb +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +85 -0
- data/lib/aspera/oauth/client/url_json.rb +4 -1
- data/lib/aspera/oauth/client/web.rb +39 -12
- data/lib/aspera/oauth/server/pkce.rb +6 -5
- data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +8 -4
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +4 -1
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +3 -0
- data/lib/aspera/rest/aspera_errors.rb +4 -1
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +22 -29
- data/lib/aspera/rest/error_analyzer.rb +5 -2
- data/lib/aspera/rest/list.rb +8 -4
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +8 -5
- data/lib/aspera/rest.rb +9 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +8 -2
- data/lib/aspera/schema/reader.rb +7 -0
- data/lib/aspera/schema/registry.rb +43 -28
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +166 -0
- data/lib/aspera/secret/hider.rb +201 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +110 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +5 -2
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +3 -0
- data/lib/aspera/sync/conf.schema.yaml +9 -4
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +16 -14
- data/lib/aspera/temp_file_manager.rb +8 -4
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +17 -14
- data/lib/aspera/transfer/result.rb +3 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +9 -22
- data/lib/aspera/transfer/units.rb +75 -0
- data/lib/aspera/uri_reader.rb +57 -7
- data/lib/aspera/web_auth/index.html.erb +6 -2
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +47 -10
- data/lib/aspera/web_page.rb +7 -2
- data/lib/aspera/web_server_simple.rb +6 -3
- data/lib/aspera/yaml.rb +5 -1
- data/llms.txt +81 -0
- data/skills/ascli/SKILL.md +88 -0
- data.tar.gz.sig +0 -0
- metadata +72 -19
- metadata.gz.sig +0 -0
- data/lib/aspera/keychain/base.rb +0 -30
- data/lib/aspera/keychain/encrypted_hash.rb +0 -192
- data/lib/aspera/keychain/factory.rb +0 -58
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
- data/lib/aspera/keychain/macos_security.rb +0 -277
- data/lib/aspera/keychain/one_password_api.rb +0 -91
- data/lib/aspera/keychain/one_password_base.rb +0 -39
- data/lib/aspera/keychain/one_password_cli.rb +0 -100
- data/lib/aspera/secret_hider.rb +0 -196
- data/lib/aspera/secret_strength.rb +0 -114
data/lib/aspera/api/aoc.rb
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../log'
|
|
7
|
+
require_relative '../assert'
|
|
8
|
+
require_relative '../rest'
|
|
9
|
+
require_relative '../hash_ext'
|
|
10
|
+
require_relative '../data_repository'
|
|
11
|
+
require_relative '../transfer/spec'
|
|
12
|
+
require_relative 'node'
|
|
13
|
+
require_relative '../rest/list'
|
|
14
|
+
require_relative '../rainbow'
|
|
11
15
|
require 'base64'
|
|
12
|
-
require 'aspera/rainbow'
|
|
13
16
|
using Rainbow
|
|
14
17
|
|
|
15
18
|
module Aspera
|
|
@@ -62,7 +65,7 @@ module Aspera
|
|
|
62
65
|
# `''` Use default workspace if it is set, else none
|
|
63
66
|
DEFAULT_WORKSPACE = ''
|
|
64
67
|
# Production domain of AoC
|
|
65
|
-
SAAS_DOMAIN_PROD = 'ibmaspera.com'
|
|
68
|
+
SAAS_DOMAIN_PROD = 'ibmaspera.com'
|
|
66
69
|
# To avoid infinite loop in pub link redirection
|
|
67
70
|
MAX_AOC_URL_REDIRECT = 10
|
|
68
71
|
CLIENT_ID_PREFIX = 'aspera.'
|
|
@@ -103,7 +106,7 @@ module Aspera
|
|
|
103
106
|
# Scope `admin:all`
|
|
104
107
|
ADMIN = 'admin:all'
|
|
105
108
|
ADMIN_USER = 'admin-user:all'
|
|
106
|
-
ADMIN_USER_USER = "#{ADMIN_USER}+#{USER}"
|
|
109
|
+
ADMIN_USER_USER = "#{ADMIN_USER}+#{USER}".freeze
|
|
107
110
|
end
|
|
108
111
|
API_V1 = 'api/v1'
|
|
109
112
|
|
data/lib/aspera/api/ats.rb
CHANGED
data/lib/aspera/api/cos_node.rb
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../log'
|
|
7
|
+
require_relative '../assert'
|
|
8
|
+
require_relative 'node'
|
|
6
9
|
require 'xmlsimple'
|
|
7
10
|
|
|
8
11
|
module Aspera
|
|
@@ -12,7 +15,7 @@ module Aspera
|
|
|
12
15
|
IBM_CLOUD_TOKEN_URL = 'https://iam.cloud.ibm.com/identity'
|
|
13
16
|
TOKEN_FIELD = 'delegated_refresh_token'
|
|
14
17
|
FASP_INFO_KEYS = %w[ATSEndpoint AccessKey].freeze
|
|
15
|
-
SERVICE_CREDS_PARAMS = %w[apikey resource_instance_id endpoints]
|
|
18
|
+
SERVICE_CREDS_PARAMS = %w[apikey resource_instance_id endpoints].freeze
|
|
16
19
|
class << self
|
|
17
20
|
def parameters_from_svc_credentials(service_credentials, bucket_region)
|
|
18
21
|
# check necessary contents
|
data/lib/aspera/api/faspex.rb
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../assert'
|
|
7
|
+
require_relative '../rest'
|
|
8
|
+
require_relative '../oauth/client/base'
|
|
9
|
+
require_relative '../rest/list'
|
|
7
10
|
require 'digest'
|
|
8
11
|
|
|
9
12
|
module Aspera
|
|
@@ -94,7 +97,7 @@ module Aspera
|
|
|
94
97
|
# states for jobs not in final state
|
|
95
98
|
JOB_RUNNING = %w[queued working].freeze
|
|
96
99
|
PATH_STANDARD_ROOT = '/aspera/faspex'
|
|
97
|
-
PATH_API_DETECT = "#{PATH_API_V5}/#{PATH_HEALTH}"
|
|
100
|
+
PATH_API_DETECT = "#{PATH_API_V5}/#{PATH_HEALTH}".freeze
|
|
98
101
|
HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
|
|
99
102
|
HEADER_FASPEX_VERSION = 'X-IBM-Aspera'
|
|
100
103
|
EMAIL_NOTIF_LIST = %w[
|
|
@@ -125,7 +128,7 @@ module Aspera
|
|
|
125
128
|
sync_failed
|
|
126
129
|
account_exist
|
|
127
130
|
mfa_code
|
|
128
|
-
]
|
|
131
|
+
].freeze
|
|
129
132
|
class << self
|
|
130
133
|
# @return [Boolean] true if the URL is a public link
|
|
131
134
|
def public_link?(url)
|
|
@@ -146,7 +149,7 @@ module Aspera
|
|
|
146
149
|
# @param root [String] Root path for API
|
|
147
150
|
# @param password [String, nil] For :boot auth, the token copied directly from browser in developer mode
|
|
148
151
|
# @param client_id [String, nil] For :web and :jwt auth, the client_id of web UI application
|
|
149
|
-
# @param client_secret [String, nil] For :web auth, the client_secret of
|
|
152
|
+
# @param client_secret [String, nil] For :web auth, the client_secret of the API client, required if it has HTTP basic auth enabled (not needed for :jwt)
|
|
150
153
|
# @param redirect_uri [String, nil] For :web auth, the redirect_uri of web UI application (must be the same as in application configuration)
|
|
151
154
|
# @param username [String, nil] For :jwt auth, the username of the user to impersonate
|
|
152
155
|
# @param private_key [String, nil] For :jwt auth, the private key to sign JWT token
|
|
@@ -215,9 +218,13 @@ module Aspera
|
|
|
215
218
|
type: :oauth2,
|
|
216
219
|
base_url: "#{url}/#{PATH_AUTH}",
|
|
217
220
|
grant_method: :web,
|
|
221
|
+
# Faspex 5.0.16+ checks the code verifier, earlier versions ignore it
|
|
222
|
+
pkce: true,
|
|
218
223
|
params: {
|
|
219
|
-
client_id:
|
|
220
|
-
|
|
224
|
+
client_id: client_id,
|
|
225
|
+
# required when the API client has HTTP basic auth enabled, else ignored by Faspex
|
|
226
|
+
client_secret: client_secret
|
|
227
|
+
}.compact,
|
|
221
228
|
redirect_uri: redirect_uri
|
|
222
229
|
}
|
|
223
230
|
}
|
data/lib/aspera/api/httpgw.rb
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../log'
|
|
7
|
+
require_relative '../rest'
|
|
8
|
+
require_relative '../transfer/faux_file'
|
|
9
|
+
require_relative '../assert'
|
|
10
|
+
require_relative '../rainbow'
|
|
7
11
|
require 'net/protocol'
|
|
8
12
|
require 'securerandom'
|
|
9
13
|
require 'websocket'
|
|
10
14
|
require 'base64'
|
|
11
15
|
require 'json'
|
|
12
|
-
require 'aspera/rainbow'
|
|
13
16
|
using Rainbow
|
|
14
17
|
|
|
15
18
|
# Throw exception on error, instead of error code
|
|
@@ -209,7 +212,7 @@ module Aspera
|
|
|
209
212
|
fileIndex: file_index
|
|
210
213
|
}
|
|
211
214
|
file = file_to_send[:file]
|
|
212
|
-
file = File.open(file) unless file.is_a?(Transfer::FauxFile)
|
|
215
|
+
file = File.open(file) unless file.is_a?(Transfer::FauxFile)
|
|
213
216
|
begin
|
|
214
217
|
until file.eof?
|
|
215
218
|
slice_bin_data = file.read(@upload_chunk_size)
|
data/lib/aspera/api/node.rb
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../link_header'
|
|
7
|
+
require_relative '../transfer/spec'
|
|
8
|
+
require_relative '../rest'
|
|
9
|
+
require_relative '../oauth/client/factory'
|
|
10
|
+
require_relative '../log'
|
|
11
|
+
require_relative '../assert'
|
|
12
|
+
require_relative '../secret/hider'
|
|
13
|
+
require_relative '../environment'
|
|
14
|
+
require_relative '../rainbow'
|
|
11
15
|
require 'base64'
|
|
12
16
|
require 'openssl'
|
|
13
17
|
require 'pathname'
|
|
14
18
|
require 'zlib'
|
|
15
19
|
require 'net/ssh/buffer'
|
|
16
|
-
require 'aspera/rainbow'
|
|
17
20
|
using Rainbow
|
|
18
21
|
|
|
19
22
|
module Aspera
|
|
@@ -181,7 +184,7 @@ module Aspera
|
|
|
181
184
|
''
|
|
182
185
|
].join("\n")))
|
|
183
186
|
# Hidden wherever it appears (logs, results)
|
|
184
|
-
|
|
187
|
+
Secret::Hider.instance.add_secret_values(token)
|
|
185
188
|
return token
|
|
186
189
|
end
|
|
187
190
|
|
|
@@ -292,8 +295,10 @@ module Aspera
|
|
|
292
295
|
if use_v4
|
|
293
296
|
headers[HEADER_ACCEPT_VERSION] = '4.0'
|
|
294
297
|
query['per_page'] = 1000 unless query.key?('per_page')
|
|
295
|
-
|
|
296
|
-
|
|
298
|
+
else
|
|
299
|
+
# API v3 requires both `page` (first is 1) and `per_page`
|
|
300
|
+
query['page'] = 1 unless query.key?('page')
|
|
301
|
+
query['per_page'] = 1000 unless query.key?('per_page')
|
|
297
302
|
end
|
|
298
303
|
loop do
|
|
299
304
|
Rest::Parameters.instance.spinner_cb.call(folder_items.count)
|
|
@@ -304,8 +309,9 @@ module Aspera
|
|
|
304
309
|
break if iteration_token.nil? || iteration_token.empty?
|
|
305
310
|
query['iteration_token'] = iteration_token
|
|
306
311
|
else
|
|
307
|
-
|
|
308
|
-
query
|
|
312
|
+
# Last page: empty, or shorter than requested
|
|
313
|
+
break if data.empty? || (query.key?('per_page') && data.length < query['per_page'].to_i)
|
|
314
|
+
query['page'] = query['page'].to_i + 1
|
|
309
315
|
end
|
|
310
316
|
end
|
|
311
317
|
rescue StandardError => e
|
|
@@ -450,11 +456,6 @@ module Aspera
|
|
|
450
456
|
return find_state[:found]
|
|
451
457
|
end
|
|
452
458
|
|
|
453
|
-
# Generate a refreshed auth token
|
|
454
|
-
def refreshed_transfer_token
|
|
455
|
-
return oauth.authorization(refresh: true)
|
|
456
|
-
end
|
|
457
|
-
|
|
458
459
|
# Get a base download transfer spec (gen3)
|
|
459
460
|
# @return [Hash] Base transfer spec
|
|
460
461
|
def base_spec
|
|
@@ -484,7 +485,6 @@ module Aspera
|
|
|
484
485
|
ak_token = Rest.basic_authorization(auth_params[:username], auth_params[:password])
|
|
485
486
|
when :oauth2
|
|
486
487
|
ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
|
|
487
|
-
# TODO: token_generation_lambda = lambda{|do_refresh|oauth.authorization(refresh: do_refresh)}
|
|
488
488
|
# Get bearer token, possibly use cache
|
|
489
489
|
ak_token = oauth.authorization
|
|
490
490
|
when :none
|
data/lib/aspera/ascmd.rb
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
# cspell:ignore smode errstr zstr zmode zuid zgid zctime zatime zmtime fcount dcount btype blist ascmdtypes
|
|
7
|
+
require_relative 'log'
|
|
8
|
+
require_relative 'assert'
|
|
9
|
+
require_relative 'rainbow'
|
|
7
10
|
using Rainbow
|
|
8
11
|
|
|
9
12
|
module Aspera
|
|
@@ -193,7 +196,7 @@ module Aspera
|
|
|
193
196
|
num_bytes = type_name.eql?(:zstr) ? buffer.length : type_descr[:size]
|
|
194
197
|
Aspera.assert(buffer.length >= num_bytes, 'not enough bytes')
|
|
195
198
|
byte_array = buffer.shift(num_bytes)
|
|
196
|
-
byte_array =
|
|
199
|
+
byte_array = Array(byte_array)
|
|
197
200
|
result = byte_array.pack('C*').unpack1(type_descr[:unpack])
|
|
198
201
|
result.force_encoding('UTF-8') if type_name.eql?(:zstr)
|
|
199
202
|
Log.log.trace1 { "#{' .' * indent_level}-> base:#{byte_array} -> #{result}" }
|
|
@@ -1,27 +1,26 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
require 'xmlsimple'
|
|
21
|
-
require 'base64'
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
# cspell:ignore ckpt
|
|
7
|
+
require_relative '../yaml'
|
|
8
|
+
require_relative '../environment'
|
|
9
|
+
require_relative '../data_repository'
|
|
10
|
+
require_relative '../log'
|
|
11
|
+
require_relative '../rest'
|
|
12
|
+
require_relative '../uri_reader'
|
|
13
|
+
require_relative '../assert'
|
|
14
|
+
require_relative '../web_server_simple'
|
|
15
|
+
require_relative '../hash_ext'
|
|
16
|
+
require_relative '../products/desktop'
|
|
17
|
+
require_relative '../products/connect'
|
|
18
|
+
require_relative '../products/transferd'
|
|
19
|
+
require_relative '../products/other'
|
|
22
20
|
require 'fileutils'
|
|
23
21
|
require 'openssl'
|
|
24
|
-
require '
|
|
22
|
+
require 'singleton'
|
|
23
|
+
require 'stringio'
|
|
25
24
|
module Aspera
|
|
26
25
|
module Ascp
|
|
27
26
|
# Singleton that tells where to find ascp and other local resources (keys..) , using the "path(:name)" method.
|
|
@@ -57,18 +56,10 @@ module Aspera
|
|
|
57
56
|
def sdk_locations
|
|
58
57
|
Aspera.assert_type(@transferd_urls, Array) { 'locations_url' }
|
|
59
58
|
location_urls = @transferd_urls.empty? ? TRANSFERD_ARCHIVE_LOCATION_URLS : @transferd_urls
|
|
60
|
-
location_urls
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
transferd_locations = last ? UriReader.read(location_url) : read_fail_fast(location_url)
|
|
65
|
-
locations = Yaml.safe_load(transferd_locations)
|
|
66
|
-
Aspera.assert_type(locations, Array) { "SDK locations from #{location_url}" }
|
|
67
|
-
return locations
|
|
68
|
-
rescue StandardError => e
|
|
69
|
-
raise if last
|
|
70
|
-
Log.log.warn { "Failed to retrieve SDK locations from #{location_url}: #{e.message}, trying next" }
|
|
71
|
-
end
|
|
59
|
+
return UriReader.read_first(location_urls, what: 'SDK locations') do |content|
|
|
60
|
+
locations = Yaml.safe_load(content)
|
|
61
|
+
Aspera.assert_type(locations, Array) { 'SDK locations' }
|
|
62
|
+
locations
|
|
72
63
|
end
|
|
73
64
|
end
|
|
74
65
|
|
|
@@ -141,7 +132,6 @@ module Aspera
|
|
|
141
132
|
file_key = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert_private_key.pem')
|
|
142
133
|
file_cert = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert.pem')
|
|
143
134
|
if !File.exist?(file_key) || !File.exist?(file_cert)
|
|
144
|
-
require 'openssl'
|
|
145
135
|
# create new self signed certificate for http fallback
|
|
146
136
|
private_key = OpenSSL::PKey::RSA.new(4096)
|
|
147
137
|
cert = WebServerSimple.self_signed_cert(private_key)
|
|
@@ -247,8 +237,10 @@ module Aspera
|
|
|
247
237
|
return ascp_data
|
|
248
238
|
end
|
|
249
239
|
|
|
250
|
-
# @
|
|
251
|
-
|
|
240
|
+
# @param platform [nil, String] Platform, if nil: current one
|
|
241
|
+
# @param version [nil, String] Version, if nil: latest
|
|
242
|
+
# @return [Hash] the entry of SDK locations for the given platform and version
|
|
243
|
+
def sdk_location_for_platform(platform: nil, version: nil)
|
|
252
244
|
all_locations = sdk_locations
|
|
253
245
|
platform = Environment.instance.architecture if platform.nil?
|
|
254
246
|
locations = all_locations.select { |l| l['platform'].eql?(platform) }
|
|
@@ -256,7 +248,20 @@ module Aspera
|
|
|
256
248
|
version = locations.max_by { |entry| Gem::Version.new(entry['version']) }['version'] if version.nil?
|
|
257
249
|
info = locations.select { |entry| entry['version'].eql?(version) }
|
|
258
250
|
Aspera.assert(!info.empty?) { "No such version: #{version} for #{platform}" }
|
|
259
|
-
return info.first
|
|
251
|
+
return info.first
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
# @return [String] the url for download of SDK archive for the given platform and version
|
|
255
|
+
def sdk_url_for_platform(platform: nil, version: nil)
|
|
256
|
+
return sdk_location_for_platform(platform: platform, version: version)['url']
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
# @return [String, nil] Version of the SDK installed in the SDK folder (from its meta data file), or nil if not installed
|
|
260
|
+
def installed_sdk_version
|
|
261
|
+
return unless File.exist?(sdk_exe_path(:transferd))
|
|
262
|
+
meta_data_file = File.join(Products::Transferd.sdk_directory, Products::Other::INFO_META_FILE)
|
|
263
|
+
return unless File.exist?(meta_data_file)
|
|
264
|
+
return File.read(meta_data_file).scan(%r{<version>(.*?)</version>}).flatten.first
|
|
260
265
|
end
|
|
261
266
|
|
|
262
267
|
# @param url [String] URL or filename used to detect archive format
|
|
@@ -346,8 +351,8 @@ module Aspera
|
|
|
346
351
|
extracted_files[file_basename] = entry_name
|
|
347
352
|
FileUtils.mkdir_p(dest_folder)
|
|
348
353
|
if link_target.nil?
|
|
349
|
-
# Security: Check if destination already exists
|
|
350
|
-
if File.exist?(dest_file)
|
|
354
|
+
# Security: Check if destination already exists, including a dangling symlink (`exist?` follows links)
|
|
355
|
+
if File.symlink?(dest_file) || File.exist?(dest_file)
|
|
351
356
|
Log.log.warn { "Rejecting write to existing file or link: #{dest_file}" }
|
|
352
357
|
next
|
|
353
358
|
end
|
|
@@ -357,13 +362,13 @@ module Aspera
|
|
|
357
362
|
FileUtils.mkdir_p(File.dirname(dest_file))
|
|
358
363
|
# Check where the file would resolve to (handles existing symlinks in path)
|
|
359
364
|
resolved_dest = File.realpath(File.dirname(dest_file))
|
|
360
|
-
unless
|
|
365
|
+
unless inside_folder?(resolved_dest, install_boundary)
|
|
361
366
|
Log.log.warn { "Rejecting file outside installation directory: #{dest_file} resolves to #{resolved_dest}" }
|
|
362
367
|
next
|
|
363
368
|
end
|
|
364
369
|
rescue Errno::ENOENT
|
|
365
370
|
# Directory doesn't exist yet, verify the intended path
|
|
366
|
-
unless dest_file.
|
|
371
|
+
unless inside_folder?(File.expand_path(dest_file), File.expand_path(folder))
|
|
367
372
|
Log.log.warn { "Rejecting file with path outside installation directory: #{dest_file}" }
|
|
368
373
|
next
|
|
369
374
|
end
|
|
@@ -372,7 +377,8 @@ module Aspera
|
|
|
372
377
|
else
|
|
373
378
|
# Security: Validate symlink target stays within installation boundary
|
|
374
379
|
# Resolve the symlink target relative to its location
|
|
375
|
-
|
|
380
|
+
# (real path of the folder, so that links already on disk are resolved)
|
|
381
|
+
link_dir = File.realpath(File.dirname(dest_file))
|
|
376
382
|
resolved_target = if link_target.start_with?('/')
|
|
377
383
|
# Absolute symlink target
|
|
378
384
|
link_target
|
|
@@ -381,7 +387,7 @@ module Aspera
|
|
|
381
387
|
File.expand_path(link_target, link_dir)
|
|
382
388
|
end
|
|
383
389
|
# Check if resolved target would be outside installation directory
|
|
384
|
-
unless
|
|
390
|
+
unless inside_folder?(File.expand_path(resolved_target), install_boundary)
|
|
385
391
|
Log.log.warn { "Rejecting symlink pointing outside installation directory: #{entry_name} -> #{link_target} (resolves to #{resolved_target})" }
|
|
386
392
|
next
|
|
387
393
|
end
|
|
@@ -393,8 +399,10 @@ module Aspera
|
|
|
393
399
|
# Downloads, installs and configures the SDK, returns name, version and install folder
|
|
394
400
|
# @param url [nil, String] URL to SDK archive, if nil: default url for version
|
|
395
401
|
# @param version [nil, String] Specific version, if nil: latest version
|
|
402
|
+
# @param folder [nil, String] Install folder, if nil: the SDK folder (option `sdk_folder`), which is unchanged
|
|
396
403
|
# @return [Array<String>] sdk name, sdk version, install folder
|
|
397
|
-
def install_sdk(url: nil, version: nil)
|
|
404
|
+
def install_sdk(url: nil, version: nil, folder: nil)
|
|
405
|
+
return with_sdk_directory(folder) { install_sdk(url: url, version: version) } unless folder.nil?
|
|
398
406
|
folder = Products::Transferd.sdk_directory
|
|
399
407
|
download_sdk(folder: folder, url: url, version: version)
|
|
400
408
|
# Ensure necessary files are there, or generate them, restrict file access on SDK executables
|
|
@@ -429,6 +437,25 @@ module Aspera
|
|
|
429
437
|
|
|
430
438
|
private
|
|
431
439
|
|
|
440
|
+
# Run the block with another SDK folder (files of the SDK are located from it), then set the previous one back
|
|
441
|
+
# @param folder [String] Temporary SDK folder
|
|
442
|
+
def with_sdk_directory(folder)
|
|
443
|
+
previous = Products::Transferd.sdk_directory
|
|
444
|
+
begin
|
|
445
|
+
Products::Transferd.sdk_directory = folder
|
|
446
|
+
return yield
|
|
447
|
+
ensure
|
|
448
|
+
Products::Transferd.sdk_directory = previous
|
|
449
|
+
end
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
# @param path [String] Path to check
|
|
453
|
+
# @param boundary [String] Folder that must contain `path`
|
|
454
|
+
# @return [Boolean] `true` if `path` is `boundary` or is below it (a sibling sharing the name prefix is not)
|
|
455
|
+
def inside_folder?(path, boundary)
|
|
456
|
+
path.eql?(boundary) || path.start_with?(boundary.end_with?(File::SEPARATOR) ? boundary : "#{boundary}#{File::SEPARATOR}")
|
|
457
|
+
end
|
|
458
|
+
|
|
432
459
|
DEFAULT_ASPERA_CONF = <<~END_OF_CONFIG_FILE
|
|
433
460
|
<?xml version='1.0' encoding='UTF-8'?>
|
|
434
461
|
<CONF version="2">
|
|
@@ -449,10 +476,8 @@ module Aspera
|
|
|
449
476
|
'https://ibm.biz/sdk_location',
|
|
450
477
|
'https://raw.githubusercontent.com/IBM/aspera-cli/refs/heads/main/docs/sdk_location.yaml'
|
|
451
478
|
].freeze
|
|
452
|
-
# Timeout in seconds when another location URL remains to be tried
|
|
453
|
-
LOCATION_FAIL_FAST_TIMEOUT = 10
|
|
454
479
|
# filename for ascp with optional extension (Windows)
|
|
455
|
-
private_constant :DEFAULT_ASPERA_CONF, :EXE_FILES, :SDK_FILES, :TRANSFERD_ARCHIVE_LOCATION_URLS
|
|
480
|
+
private_constant :DEFAULT_ASPERA_CONF, :EXE_FILES, :SDK_FILES, :TRANSFERD_ARCHIVE_LOCATION_URLS
|
|
456
481
|
|
|
457
482
|
def initialize
|
|
458
483
|
# cache for installed products found
|
|
@@ -465,25 +490,6 @@ module Aspera
|
|
|
465
490
|
@ascp_folder = nil
|
|
466
491
|
end
|
|
467
492
|
|
|
468
|
-
# Read URL with a short timeout and no retry (temporarily changes global HTTP parameters)
|
|
469
|
-
# @param url [String] URL to read
|
|
470
|
-
# @return [String] content
|
|
471
|
-
def read_fail_fast(url)
|
|
472
|
-
parameters = Rest::Parameters.instance
|
|
473
|
-
session_cb = parameters.session_cb
|
|
474
|
-
retry_max = parameters.retry_max
|
|
475
|
-
parameters.retry_max = 0
|
|
476
|
-
parameters.session_cb = lambda do |http_session|
|
|
477
|
-
session_cb&.call(http_session)
|
|
478
|
-
http_session.open_timeout = [http_session.open_timeout, LOCATION_FAIL_FAST_TIMEOUT].compact.min
|
|
479
|
-
http_session.read_timeout = [http_session.read_timeout, LOCATION_FAIL_FAST_TIMEOUT].compact.min
|
|
480
|
-
end
|
|
481
|
-
UriReader.read(url)
|
|
482
|
-
ensure
|
|
483
|
-
parameters.session_cb = session_cb
|
|
484
|
-
parameters.retry_max = retry_max
|
|
485
|
-
end
|
|
486
|
-
|
|
487
493
|
# @param file_type [Symbol] one of EXE_FILES
|
|
488
494
|
# @return [String] path to executable in SDK folder
|
|
489
495
|
def sdk_exe_path(file_type)
|