aspera-cli 4.27.5 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +301 -4
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +139 -58
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +103 -26
- data/SECURITY.md +70 -0
- data/bin/ascli +8 -5
- data/bin/asession +10 -7
- data/docs/README.md +2551 -772
- data/docs/test-mcp-with-ai.md +13 -11
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +34 -22
- data/lib/aspera/agent/factory.rb +8 -4
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +8 -5
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +23 -19
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +84 -42
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +354 -0
- data/lib/aspera/ascp/management.rb +102 -48
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +16 -37
- data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +37 -34
- data/lib/aspera/cli/command_line.rb +24 -6
- data/lib/aspera/cli/command_registry.rb +49 -4
- data/lib/aspera/cli/command_spec.rb +34 -4
- data/lib/aspera/cli/completion/ascli.bash +53 -5
- data/lib/aspera/cli/completion/ascli.fish +27 -8
- data/lib/aspera/cli/completion/ascli.ps1 +63 -0
- data/lib/aspera/cli/completion/ascli.zsh +36 -4
- data/lib/aspera/cli/completion_actions.rb +594 -0
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +45 -0
- data/lib/aspera/cli/extended_value.rb +83 -24
- data/lib/aspera/cli/formatter.rb +142 -73
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -11
- data/lib/aspera/cli/http.rb +17 -13
- data/lib/aspera/cli/info.rb +17 -6
- data/lib/aspera/cli/mailer.rb +7 -4
- data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
- data/lib/aspera/cli/mcp/secrets.rb +146 -0
- data/lib/aspera/cli/mcp/tool.rb +232 -0
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +72 -11
- data/lib/aspera/cli/options.schema.yaml +856 -81
- data/lib/aspera/cli/parser.rb +64 -31
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +250 -44
- data/lib/aspera/cli/plugins/ats.rb +9 -8
- data/lib/aspera/cli/plugins/base.rb +51 -25
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +207 -141
- data/lib/aspera/cli/plugins/console.rb +9 -4
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +22 -8
- data/lib/aspera/cli/plugins/faspex5.rb +37 -44
- data/lib/aspera/cli/plugins/faspio.rb +41 -8
- data/lib/aspera/cli/plugins/httpgw.rb +8 -4
- data/lib/aspera/cli/plugins/mcp.rb +186 -121
- data/lib/aspera/cli/plugins/node.rb +41 -34
- data/lib/aspera/cli/plugins/oauth.rb +5 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
- data/lib/aspera/cli/plugins/preview.rb +69 -44
- data/lib/aspera/cli/plugins/server.rb +66 -12
- data/lib/aspera/cli/plugins/shares.rb +288 -48
- data/lib/aspera/cli/plugins/transferd.rb +223 -0
- data/lib/aspera/cli/preset_actions.rb +155 -33
- data/lib/aspera/cli/preset_manager.rb +72 -16
- data/lib/aspera/cli/prompt.rb +7 -2
- data/lib/aspera/cli/result.rb +28 -11
- data/lib/aspera/cli/runner.rb +193 -80
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +204 -0
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +13 -10
- data/lib/aspera/cli/table_fitter.rb +119 -0
- data/lib/aspera/cli/terminal_formatter.rb +32 -40
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +31 -19
- data/lib/aspera/cli/transfer_progress.rb +13 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +197 -50
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.html.erb +61 -0
- data/lib/aspera/cli/web_secret.rb +222 -0
- data/lib/aspera/cli/wizard.rb +58 -45
- data/lib/aspera/command_line_builder.rb +35 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +42 -25
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/duration.rb +45 -0
- data/lib/aspera/environment.rb +106 -33
- data/lib/aspera/exec_spec.rb +9 -5
- data/lib/aspera/faspex_gw.rb +105 -35
- data/lib/aspera/faspex_postproc.rb +193 -47
- data/lib/aspera/formatter_interface.rb +4 -2
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +6 -3
- data/lib/aspera/json_rpc/client.rb +55 -35
- data/lib/aspera/json_rpc/version.rb +4 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +20 -10
- data/lib/aspera/markdown.rb +155 -115
- data/lib/aspera/mime.rb +9 -0
- data/lib/aspera/nagios.rb +9 -5
- data/lib/aspera/node_emulator.rb +111 -112
- data/lib/aspera/oauth/client/base.rb +163 -0
- data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
- data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
- data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
- data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
- data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +85 -0
- data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
- data/lib/aspera/oauth/client/web.rb +84 -0
- data/lib/aspera/oauth/server/authorize.html.erb +34 -0
- data/lib/aspera/oauth/server/pkce.rb +427 -0
- data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +11 -6
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +4 -1
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +30 -1
- data/lib/aspera/rest/aspera_errors.rb +55 -60
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +42 -42
- data/lib/aspera/rest/error_analyzer.rb +14 -2
- data/lib/aspera/rest/list.rb +14 -10
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +13 -4
- data/lib/aspera/rest.rb +9 -7
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +56 -78
- data/lib/aspera/schema/reader.rb +46 -11
- data/lib/aspera/schema/registry.rb +45 -24
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +166 -0
- data/lib/aspera/secret/hider.rb +201 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +110 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +114 -0
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +6 -1
- data/lib/aspera/sync/conf.schema.yaml +25 -10
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +34 -12
- data/lib/aspera/temp_file_manager.rb +85 -22
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +22 -14
- data/lib/aspera/transfer/result.rb +3 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +45 -18
- data/lib/aspera/transfer/spec.schema.yaml +8 -5
- data/lib/aspera/transfer/units.rb +75 -0
- data/lib/aspera/uri_reader.rb +57 -7
- data/lib/aspera/web_auth/index.html.erb +20 -0
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +59 -14
- data/lib/aspera/web_page.html.erb +262 -0
- data/lib/aspera/web_page.rb +66 -0
- data/lib/aspera/web_server_simple.rb +82 -13
- data/lib/aspera/yaml.rb +5 -1
- data/llms.txt +81 -0
- data/skills/ascli/SKILL.md +88 -0
- data.tar.gz.sig +0 -0
- metadata +116 -48
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/mcp_tool.rb +0 -189
- data/lib/aspera/keychain/base.rb +0 -25
- data/lib/aspera/keychain/encrypted_hash.rb +0 -188
- data/lib/aspera/keychain/factory.rb +0 -59
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
- data/lib/aspera/keychain/macos_security.rb +0 -171
- data/lib/aspera/keychain/one_password_api.rb +0 -86
- data/lib/aspera/keychain/one_password_base.rb +0 -34
- data/lib/aspera/keychain/one_password_cli.rb +0 -98
- data/lib/aspera/oauth/base.rb +0 -141
- data/lib/aspera/oauth/web.rb +0 -57
- data/lib/aspera/oauth.rb +0 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
- data/lib/aspera/secret_hider.rb +0 -117
- data/lib/aspera/web_auth/index.html +0 -147
|
@@ -1,71 +1,66 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'error_analyzer'
|
|
5
7
|
require 'xmlsimple'
|
|
6
8
|
|
|
7
9
|
module Aspera
|
|
8
10
|
module Rest
|
|
9
|
-
# REST error handlers for various Aspera REST APIs
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
ErrorAnalyzer.add_error(context, type, parsed['content'].gsub(/\A"|"\z/, ''))
|
|
53
|
-
parsed.each do |k, v|
|
|
54
|
-
next if k.eql?('content')
|
|
55
|
-
ErrorAnalyzer.add_error(context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
next
|
|
59
|
-
end
|
|
60
|
-
next unless data.is_a?(Hash)
|
|
61
|
-
work_order = data['work_order']
|
|
62
|
-
next unless work_order.is_a?(Hash)
|
|
63
|
-
ErrorAnalyzer.add_error(context, type, work_order['statusDetails'])
|
|
64
|
-
data['missing_parameters']&.each do |param|
|
|
65
|
-
ErrorAnalyzer.add_error(context, type, "missing parameter: #{param}")
|
|
66
|
-
end
|
|
11
|
+
# REST error handlers for various Aspera REST APIs, registered on require
|
|
12
|
+
# handlers should probably be defined by plugins for modularity
|
|
13
|
+
# Faspex 4: both user_message and internal_message, and code 200
|
|
14
|
+
# example: missing meta data on package creation
|
|
15
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 1: error:user_message', path: %w[error user_message], always: true)
|
|
16
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 2: error:description', path: %w[error description])
|
|
17
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 3: error:internal_message', path: %w[error internal_message])
|
|
18
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 5', path: ['error_description'])
|
|
19
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['message'])
|
|
20
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['failure'], always: true)
|
|
21
|
+
# AoC Automation
|
|
22
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
|
|
23
|
+
ErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |type, context|
|
|
24
|
+
next unless context[:data].is_a?(Hash) && context[:data]['errors'].is_a?(Hash)
|
|
25
|
+
# special for Shares: false positive ? (update global transfer_settings)
|
|
26
|
+
next if context[:data].key?('min_connect_version')
|
|
27
|
+
context[:data]['errors'].each do |k, v|
|
|
28
|
+
ErrorAnalyzer.add_error(context, type, "#{k}: #{v}")
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
# call to upload_setup and download_setup of node api
|
|
32
|
+
ErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, context|
|
|
33
|
+
next unless context[:data].is_a?(Hash)
|
|
34
|
+
d_t_s = context[:data]['transfer_specs']
|
|
35
|
+
next unless d_t_s.is_a?(Array)
|
|
36
|
+
d_t_s.each do |res|
|
|
37
|
+
r_err = res.dig(*%w[transfer_spec error]) || res['error']
|
|
38
|
+
next unless r_err.is_a?(Hash)
|
|
39
|
+
ErrorAnalyzer.add_error(context, type, r_err.values.join(': '))
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
|
|
43
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
|
|
44
|
+
ErrorAnalyzer.instance.add_handler('Orchestrator') do |type, context|
|
|
45
|
+
next if context[:response].code.start_with?('2')
|
|
46
|
+
data = context[:data]
|
|
47
|
+
if data.is_a?(String) && data.start_with?('<?xml')
|
|
48
|
+
parsed = XmlSimple.xml_in(data, {'ForceArray' => false}) rescue nil
|
|
49
|
+
if parsed.is_a?(Hash) && parsed['content'].is_a?(String)
|
|
50
|
+
ErrorAnalyzer.add_error(context, type, parsed['content'].gsub(/\A"|"\z/, ''))
|
|
51
|
+
parsed.each do |k, v|
|
|
52
|
+
next if k.eql?('content')
|
|
53
|
+
ErrorAnalyzer.add_error(context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
|
|
67
54
|
end
|
|
68
55
|
end
|
|
56
|
+
next
|
|
57
|
+
end
|
|
58
|
+
next unless data.is_a?(Hash)
|
|
59
|
+
work_order = data['work_order']
|
|
60
|
+
next unless work_order.is_a?(Hash)
|
|
61
|
+
ErrorAnalyzer.add_error(context, type, work_order['statusDetails'])
|
|
62
|
+
data['missing_parameters']&.each do |param|
|
|
63
|
+
ErrorAnalyzer.add_error(context, type, "missing parameter: #{param}")
|
|
69
64
|
end
|
|
70
65
|
end
|
|
71
66
|
end
|
data/lib/aspera/rest/client.rb
CHANGED
|
@@ -1,23 +1,25 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../mime'
|
|
7
|
+
require_relative 'parameters'
|
|
8
|
+
require_relative 'util'
|
|
9
|
+
require_relative 'call_error'
|
|
10
|
+
require_relative 'error_analyzer'
|
|
11
|
+
require_relative '../log'
|
|
12
|
+
require_relative '../assert'
|
|
13
|
+
require_relative '../oauth/client/factory'
|
|
14
|
+
require_relative '../hash_ext'
|
|
15
|
+
require_relative '../timer_limiter'
|
|
16
|
+
require_relative '../rainbow'
|
|
14
17
|
require 'net/http'
|
|
15
18
|
require 'net/https'
|
|
16
19
|
require 'json'
|
|
17
20
|
require 'securerandom'
|
|
18
21
|
require 'fileutils'
|
|
19
22
|
require 'pathname'
|
|
20
|
-
require 'aspera/rainbow'
|
|
21
23
|
using Rainbow
|
|
22
24
|
|
|
23
25
|
# Cancel method for HTTP
|
|
@@ -126,11 +128,12 @@ module Aspera
|
|
|
126
128
|
# @param subpath [String] subpath of REST API
|
|
127
129
|
# @param query [Hash{String,Symbol => Object}] URL parameters
|
|
128
130
|
# @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, else use headers
|
|
129
|
-
# @param body [Hash, String, nil] Body parameters
|
|
131
|
+
# @param body [Hash, Array, String, nil] Body parameters (see `build_request`)
|
|
130
132
|
# @param headers [Hash{String => String}] Additional headers (override Content-Type)
|
|
131
133
|
# @param save_to [String, Pathname, IO, nil] File path or IO object to save response body; progress bar is used when set
|
|
132
134
|
# @param exception [Boolean] Whether to raise an exception on HTTP error
|
|
133
135
|
# @param ret [Symbol] One of :data, :resp, :both - controls return value
|
|
136
|
+
# @param json_opts [Hash] Options of `JSON.parse` for a JSON response, e.g. `{allow_duplicate_key: true}`
|
|
134
137
|
# @return [Array(Hash, Net::HTTPResponse)] When `ret` is :both
|
|
135
138
|
# @return [Net::HTTPResponse] When `ret` is :resp
|
|
136
139
|
# @return [Hash] When `ret` is :data
|
|
@@ -144,7 +147,8 @@ module Aspera
|
|
|
144
147
|
headers: nil,
|
|
145
148
|
save_to: nil,
|
|
146
149
|
exception: true,
|
|
147
|
-
ret: :data
|
|
150
|
+
ret: :data,
|
|
151
|
+
json_opts: {}
|
|
148
152
|
)
|
|
149
153
|
subpath = subpath.to_s if subpath.is_a?(Symbol)
|
|
150
154
|
subpath = '' if subpath.nil?
|
|
@@ -185,7 +189,7 @@ module Aspera
|
|
|
185
189
|
# TODO : related to mime type encoding ?
|
|
186
190
|
# result_http.body.force_encoding('UTF-8') if result_http.body.is_a?(String)
|
|
187
191
|
# Log.log.debug{"result: body=#{result_http.body}"}
|
|
188
|
-
result_data = parse_response(result_http, result_mime)
|
|
192
|
+
result_data = parse_response(result_http, mime_type: result_mime, json_opts: json_opts)
|
|
189
193
|
ErrorAnalyzer.instance.raise_on_error(req, result_data, result_http)
|
|
190
194
|
unless file_saved || save_to.nil?
|
|
191
195
|
raise 'save_to: IO object requires a streaming response' if save_to.respond_to?(:write)
|
|
@@ -201,11 +205,15 @@ module Aspera
|
|
|
201
205
|
# not authorized: OAuth token expired
|
|
202
206
|
if !token_renewed && @not_auth_codes.include?(result_http.code.to_s) && @auth_params[:type].eql?(:oauth2)
|
|
203
207
|
token_renewed = true
|
|
204
|
-
|
|
205
|
-
|
|
208
|
+
begin
|
|
209
|
+
req_headers['Authorization'] = oauth.renewed_authorization
|
|
206
210
|
Log.log.debug('using new token')
|
|
207
|
-
req_headers['Authorization'] = new_authorization
|
|
208
211
|
retry
|
|
212
|
+
rescue StandardError => e_renew
|
|
213
|
+
# The reason why no token could be obtained is more useful than the original error
|
|
214
|
+
raise e_renew if exception
|
|
215
|
+
# No exception requested: the result is the original response
|
|
216
|
+
Log.log.error { "Token renewal failed: #{e_renew.message}" }
|
|
209
217
|
end
|
|
210
218
|
end
|
|
211
219
|
if retry_error?(e) && (error_tries -= 1).positive?
|
|
@@ -223,7 +231,8 @@ module Aspera
|
|
|
223
231
|
save_to: save_to,
|
|
224
232
|
exception: exception,
|
|
225
233
|
headers: headers,
|
|
226
|
-
ret: ret
|
|
234
|
+
ret: ret,
|
|
235
|
+
json_opts: json_opts
|
|
227
236
|
)
|
|
228
237
|
end
|
|
229
238
|
# raise exception if could not retry and not return error in result
|
|
@@ -264,20 +273,6 @@ module Aspera
|
|
|
264
273
|
sleep(Parameters.instance.retry_sleep) unless Parameters.instance.retry_sleep.eql?(0)
|
|
265
274
|
end
|
|
266
275
|
|
|
267
|
-
# Renew OAuth token: use refresh token, or generate a new one
|
|
268
|
-
# @return [String, nil] New value for header `Authorization`, or `nil` if no new token could be obtained
|
|
269
|
-
def renew_oauth_authorization
|
|
270
|
-
oauth.authorization(refresh: true)
|
|
271
|
-
rescue StandardError => e
|
|
272
|
-
Log.log.error("refresh failed: #{e.message}".bg(:red))
|
|
273
|
-
begin
|
|
274
|
-
oauth.authorization(cache: false)
|
|
275
|
-
rescue StandardError => e
|
|
276
|
-
Log.log.error("new token failed: #{e.message}".bg(:red))
|
|
277
|
-
nil
|
|
278
|
-
end
|
|
279
|
-
end
|
|
280
|
-
|
|
281
276
|
# Forward the call to the location of a redirect response.
|
|
282
277
|
# Same server: same API parameters (auth, headers). Other server: credentials are not forwarded.
|
|
283
278
|
# @param request_uri [URI] URI of the redirected request
|
|
@@ -339,8 +334,9 @@ module Aspera
|
|
|
339
334
|
# @param operation [String] HTTP operation (GET, POST, ...)
|
|
340
335
|
# @param subpath [String] Subpath of REST API
|
|
341
336
|
# @param query [Hash, String, Array, nil] Query of request
|
|
342
|
-
# @param content_type [String, nil] One of Mime::JSON, Mime::WWW, Mime::TEXT, or `nil` for no body
|
|
343
|
-
# @param body [Hash, String, nil]
|
|
337
|
+
# @param content_type [String, nil] One of Mime::JSON, Mime::WWW, Mime::TEXT, Mime::MULTIPART, or `nil` for no body
|
|
338
|
+
# @param body [Hash, Array, String, nil] Body of request, serialized according to `content_type`
|
|
339
|
+
# For Mime::MULTIPART: Hash `name => value`, or Array of `[name, value, opts]` with opts `filename`, `content_type` for a file part
|
|
344
340
|
# @param headers [Hash] Headers of request
|
|
345
341
|
# @return [Net::HTTPRequest] The request
|
|
346
342
|
def build_request(operation, subpath, query, content_type, body, headers)
|
|
@@ -366,6 +362,9 @@ module Aspera
|
|
|
366
362
|
when Mime::TEXT
|
|
367
363
|
req.body = body
|
|
368
364
|
req['Content-Type'] = Mime::TEXT
|
|
365
|
+
when Mime::MULTIPART
|
|
366
|
+
# values are kept in the request: re-sent on retry
|
|
367
|
+
req.set_form(body.to_a, Mime::MULTIPART)
|
|
369
368
|
else Aspera.error_unexpected_value(content_type) { 'body type' }
|
|
370
369
|
end
|
|
371
370
|
# set headers
|
|
@@ -438,17 +437,18 @@ module Aspera
|
|
|
438
437
|
end
|
|
439
438
|
|
|
440
439
|
# Parse response body, according to media type
|
|
441
|
-
# @param
|
|
442
|
-
# @param
|
|
440
|
+
# @param http_res [Net::HTTPResponse] Response
|
|
441
|
+
# @param mime_type [String] Media type of response
|
|
442
|
+
# @param json_opts [Hash] Options of `JSON.parse`
|
|
443
443
|
# @return [Hash, Array, String, nil] Parsed JSON, or raw body
|
|
444
|
-
def parse_response(
|
|
445
|
-
result_data =
|
|
444
|
+
def parse_response(http_res, mime_type:, json_opts: {})
|
|
445
|
+
result_data = http_res.body
|
|
446
446
|
Log.dump(:result_data_raw, result_data, level: :trace1)
|
|
447
447
|
# TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
|
|
448
448
|
node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
|
|
449
449
|
result_data = result_data[0..node_api_bug] if node_api_bug
|
|
450
|
-
|
|
451
|
-
result_data = JSON.parse(result_data) if Mime.json?(
|
|
450
|
+
mime_type = 'application/xml' if result_data.is_a?(String) && result_data.start_with?('<?xml')
|
|
451
|
+
result_data = JSON.parse(result_data, **json_opts) if Mime.json?(mime_type) && !result_data.nil? && !result_data.empty?
|
|
452
452
|
Log.dump(:result_data, result_data)
|
|
453
453
|
result_data
|
|
454
454
|
end
|
|
@@ -508,7 +508,7 @@ module Aspera
|
|
|
508
508
|
end
|
|
509
509
|
|
|
510
510
|
# HTTP codes of service unavailable
|
|
511
|
-
UNAVAILABLE_CODES = ['503']
|
|
511
|
+
UNAVAILABLE_CODES = ['503'].freeze
|
|
512
512
|
# Network errors that can be retried
|
|
513
513
|
NETWORK_ERRORS = [
|
|
514
514
|
Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout,
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'call_error'
|
|
7
|
+
require_relative '../log'
|
|
5
8
|
require 'singleton'
|
|
6
9
|
require 'net/http'
|
|
7
10
|
|
|
@@ -15,6 +18,10 @@ module Aspera
|
|
|
15
18
|
class ErrorAnalyzer
|
|
16
19
|
include Singleton
|
|
17
20
|
|
|
21
|
+
# Line of the backtrace of a Ruby exception (Exception#full_message), e.g. `\tfrom /app/x.rb:12:in 'y'`
|
|
22
|
+
BACKTRACE_LINE = /^\s+from \S.*:\d+:in /
|
|
23
|
+
private_constant :BACKTRACE_LINE
|
|
24
|
+
|
|
18
25
|
attr_accessor :log_file
|
|
19
26
|
|
|
20
27
|
# the singleton object is registered with application specific handlers
|
|
@@ -97,6 +104,11 @@ module Aspera
|
|
|
97
104
|
# @param type [String] type of exception, for logging purpose
|
|
98
105
|
# @param message [String] one error message to add to list
|
|
99
106
|
def add_error(context, type, message)
|
|
107
|
+
# A server in Ruby (e.g. Orchestrator) can return the backtrace of the exception: in debug log only
|
|
108
|
+
if message.is_a?(String) && message.match?(BACKTRACE_LINE)
|
|
109
|
+
Log.log.debug { "Error with backtrace: #{message}" }
|
|
110
|
+
message = message.lines.grep_v(BACKTRACE_LINE).join.strip
|
|
111
|
+
end
|
|
100
112
|
context[:messages].push(message)
|
|
101
113
|
Log.log.trace1 { "Found error: #{type}: #{message}" }
|
|
102
114
|
log_file = instance.log_file
|
data/lib/aspera/rest/list.rb
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'parameters'
|
|
7
|
+
require_relative '../cli/error'
|
|
8
|
+
require_relative '../mime'
|
|
9
|
+
require_relative '../assert'
|
|
10
|
+
require_relative '../log'
|
|
7
11
|
|
|
8
12
|
module Aspera
|
|
9
13
|
module Rest
|
|
@@ -120,19 +124,19 @@ module Aspera
|
|
|
120
124
|
Aspera.assert_values(field, ['name']) { 'Default query field' }
|
|
121
125
|
query = {'q'=> value}
|
|
122
126
|
end
|
|
123
|
-
lookup_entity_generic(
|
|
127
|
+
lookup_entity_generic(field: field, value: value, entity: entity) { list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first }
|
|
124
128
|
end
|
|
125
129
|
|
|
126
130
|
# Lookup entity by field and value.
|
|
127
131
|
# Extracts a single result from the list returned by the block.
|
|
128
132
|
#
|
|
129
|
-
# @param
|
|
130
|
-
# @param value [String] Value to match against the field.
|
|
131
|
-
# @param
|
|
132
|
-
# @
|
|
133
|
+
# @param field [String] Field to match in the hashes.
|
|
134
|
+
# @param value [String] Value to match against the field (exact match with `eql?`: case and type sensitive).
|
|
135
|
+
# @param entity [String] Type of entity to lookup (typically, API path, only used for exception)
|
|
136
|
+
# @yieldreturn [Array<Hash>] The list of entities to search (block is mandatory).
|
|
133
137
|
# @return [Hash] The unique matching object.
|
|
134
138
|
# @raise [Cli::BadIdentifier] If 0 or >1 matches are found.
|
|
135
|
-
def lookup_entity_generic(
|
|
139
|
+
def lookup_entity_generic(field:, value:, entity:)
|
|
136
140
|
Aspera.assert(block_given?, 'block required for lookup_entity_generic')
|
|
137
141
|
found = yield
|
|
138
142
|
Aspera.assert_array_all(found, Hash)
|
data/lib/aspera/rest/util.rb
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'parameters'
|
|
7
|
+
require_relative '../log'
|
|
8
|
+
require_relative '../assert'
|
|
9
|
+
require_relative '../secret/hider'
|
|
6
10
|
require 'net/http'
|
|
7
11
|
require 'net/https'
|
|
8
12
|
require 'uri'
|
|
@@ -16,7 +20,12 @@ module Aspera
|
|
|
16
20
|
# @param user [String] Username
|
|
17
21
|
# @param pass [String] Password
|
|
18
22
|
# @return [String] Basic auth token
|
|
19
|
-
def basic_authorization(user, pass)
|
|
23
|
+
def basic_authorization(user, pass)
|
|
24
|
+
credentials = Base64.strict_encode64("#{user}:#{pass}")
|
|
25
|
+
# Hidden wherever it appears (logs, results)
|
|
26
|
+
Secret::Hider.instance.add_secret_values(credentials)
|
|
27
|
+
"Basic #{credentials}"
|
|
28
|
+
end
|
|
20
29
|
|
|
21
30
|
# Indicate that the given Hash query uses php style for array parameters
|
|
22
31
|
# @param query [Hash] A key can have Array value and result will use PHP format: a[]=1&a[]=2
|
data/lib/aspera/rest.rb
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'rest/parameters'
|
|
7
|
+
require_relative 'rest/util'
|
|
8
|
+
require_relative 'rest/call_error'
|
|
9
|
+
require_relative 'rest/error_analyzer'
|
|
10
|
+
require_relative 'rest/client'
|
|
11
|
+
require_relative 'rest/list'
|
|
@@ -32764,8 +32764,6 @@ components:
|
|
|
32764
32764
|
type: string
|
|
32765
32765
|
examples:
|
|
32766
32766
|
- eyJyZXNvdXJjZSI6InBhY2thZ2VzIiwidHlwZSI6ImV4dGVybmFsX3ZpZXdfcGFja2FnZSIsImlkIjoiMzAiLCJwYXNzY29kZSI6IjA2MzBkZDIzLTA3YTAtNGExYi04N2IzLWM3YjZkMGViNjE5NiIsInBhY2thZ2VfaWQiOiIzMCJ9
|
|
32767
|
-
maxLength: 255
|
|
32768
|
-
minLength: 1
|
|
32769
32767
|
relays_count:
|
|
32770
32768
|
description: Package relays count
|
|
32771
32769
|
type: object
|
|
@@ -39578,13 +39576,13 @@ components:
|
|
|
39578
39576
|
description: JWT access token used to authorize access to protected endpoints.
|
|
39579
39577
|
type: string
|
|
39580
39578
|
examples:
|
|
39581
|
-
-
|
|
39579
|
+
- <ACCESS_TOKEN>
|
|
39582
39580
|
minLength: 1
|
|
39583
39581
|
refresh_token:
|
|
39584
39582
|
description: The refresh token used to request a new access token. This is required when `grant_type = refresh_token`.
|
|
39585
39583
|
type: string
|
|
39586
39584
|
examples:
|
|
39587
|
-
-
|
|
39585
|
+
- <REFRESH_TOKEN>
|
|
39588
39586
|
maxLength: 255
|
|
39589
39587
|
minLength: 1
|
|
39590
39588
|
JwtGetTokenRequest:
|
|
@@ -39606,7 +39604,7 @@ components:
|
|
|
39606
39604
|
description: JWT access token used to authorize access to protected endpoints.
|
|
39607
39605
|
type: string
|
|
39608
39606
|
examples:
|
|
39609
|
-
-
|
|
39607
|
+
- <ACCESS_TOKEN>
|
|
39610
39608
|
minLength: 1
|
|
39611
39609
|
WebPostTokenRequest:
|
|
39612
39610
|
description: Parameters to authorize to an OAuth 2 client.
|
|
@@ -39639,7 +39637,7 @@ components:
|
|
|
39639
39637
|
description: JWT access token used to authorize access to protected endpoints.
|
|
39640
39638
|
type: string
|
|
39641
39639
|
examples:
|
|
39642
|
-
-
|
|
39640
|
+
- <ACCESS_TOKEN>
|
|
39643
39641
|
minLength: 1
|
|
39644
39642
|
redirect_uri:
|
|
39645
39643
|
description: The URL to redirect users to after successful authentication. Must use the HTTPS protocol.
|
|
@@ -39676,7 +39674,7 @@ components:
|
|
|
39676
39674
|
description: The refresh token used to request a new access token. This is required when `grant_type = refresh_token`.
|
|
39677
39675
|
type: string
|
|
39678
39676
|
examples:
|
|
39679
|
-
-
|
|
39677
|
+
- <REFRESH_TOKEN>
|
|
39680
39678
|
maxLength: 255
|
|
39681
39679
|
minLength: 1
|
|
39682
39680
|
client_secret:
|
|
@@ -39721,7 +39719,7 @@ components:
|
|
|
39721
39719
|
description: JWT access token used to authorize access to protected endpoints.
|
|
39722
39720
|
type: string
|
|
39723
39721
|
examples:
|
|
39724
|
-
-
|
|
39722
|
+
- <ACCESS_TOKEN>
|
|
39725
39723
|
minLength: 1
|
|
39726
39724
|
redirect_uri:
|
|
39727
39725
|
description: The URL to redirect users to after successful authentication. Must use the HTTPS protocol.
|
|
@@ -39758,7 +39756,7 @@ components:
|
|
|
39758
39756
|
description: The refresh token used to request a new access token. This is required when `grant_type = refresh_token`.
|
|
39759
39757
|
type: string
|
|
39760
39758
|
examples:
|
|
39761
|
-
-
|
|
39759
|
+
- <REFRESH_TOKEN>
|
|
39762
39760
|
maxLength: 255
|
|
39763
39761
|
minLength: 1
|
|
39764
39762
|
client_secret:
|
|
@@ -39797,7 +39795,7 @@ components:
|
|
|
39797
39795
|
description: The assertion for the urn:ietf:params:oauth:grant-type:jwt-bearer grant_type.
|
|
39798
39796
|
type: string
|
|
39799
39797
|
examples:
|
|
39800
|
-
-
|
|
39798
|
+
- <JWT>
|
|
39801
39799
|
maxLength: 255
|
|
39802
39800
|
minLength: 1
|
|
39803
39801
|
required:
|
|
@@ -39828,7 +39826,7 @@ components:
|
|
|
39828
39826
|
description: The refresh token used to request a new access token. This is required when `grant_type = refresh_token`.
|
|
39829
39827
|
type: string
|
|
39830
39828
|
examples:
|
|
39831
|
-
-
|
|
39829
|
+
- <REFRESH_TOKEN>
|
|
39832
39830
|
maxLength: 255
|
|
39833
39831
|
minLength: 1
|
|
39834
39832
|
required:
|