aspera-cli 4.28.0 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +143 -2
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +135 -56
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +103 -26
- data/SECURITY.md +70 -0
- data/bin/ascli +8 -5
- data/bin/asession +9 -6
- data/docs/README.md +1633 -609
- data/docs/test-mcp-with-ai.md +8 -8
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +20 -19
- data/lib/aspera/agent/factory.rb +6 -3
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +7 -4
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +20 -20
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +71 -65
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +354 -0
- data/lib/aspera/ascp/management.rb +16 -8
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +12 -19
- data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +37 -33
- data/lib/aspera/cli/command_line.rb +24 -6
- data/lib/aspera/cli/command_registry.rb +6 -1
- data/lib/aspera/cli/command_spec.rb +34 -7
- data/lib/aspera/cli/completion/ascli.bash +19 -5
- data/lib/aspera/cli/completion/ascli.fish +13 -2
- data/lib/aspera/cli/completion/ascli.ps1 +31 -9
- data/lib/aspera/cli/completion/ascli.zsh +22 -6
- data/lib/aspera/cli/completion_actions.rb +303 -69
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +3 -0
- data/lib/aspera/cli/extended_value.rb +61 -27
- data/lib/aspera/cli/formatter.rb +29 -22
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -10
- data/lib/aspera/cli/http.rb +12 -9
- data/lib/aspera/cli/info.rb +12 -7
- data/lib/aspera/cli/mailer.rb +6 -3
- data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
- data/lib/aspera/cli/mcp/secrets.rb +8 -5
- data/lib/aspera/cli/mcp/tool.rb +15 -14
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +65 -12
- data/lib/aspera/cli/options.schema.yaml +487 -63
- data/lib/aspera/cli/parser.rb +56 -32
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +244 -35
- data/lib/aspera/cli/plugins/ats.rb +9 -8
- data/lib/aspera/cli/plugins/base.rb +17 -12
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +198 -103
- data/lib/aspera/cli/plugins/console.rb +6 -3
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +21 -7
- data/lib/aspera/cli/plugins/faspex5.rb +28 -25
- data/lib/aspera/cli/plugins/faspio.rb +7 -5
- data/lib/aspera/cli/plugins/httpgw.rb +7 -4
- data/lib/aspera/cli/plugins/mcp.rb +28 -22
- data/lib/aspera/cli/plugins/node.rb +31 -30
- data/lib/aspera/cli/plugins/oauth.rb +5 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
- data/lib/aspera/cli/plugins/preview.rb +69 -44
- data/lib/aspera/cli/plugins/server.rb +63 -11
- data/lib/aspera/cli/plugins/shares.rb +86 -16
- data/lib/aspera/cli/plugins/transferd.rb +223 -0
- data/lib/aspera/cli/preset_actions.rb +144 -30
- data/lib/aspera/cli/preset_manager.rb +52 -18
- data/lib/aspera/cli/prompt.rb +4 -1
- data/lib/aspera/cli/result.rb +12 -8
- data/lib/aspera/cli/runner.rb +69 -46
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +79 -9
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +13 -10
- data/lib/aspera/cli/table_fitter.rb +3 -0
- data/lib/aspera/cli/terminal_formatter.rb +9 -6
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +29 -20
- data/lib/aspera/cli/transfer_progress.rb +13 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +188 -42
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.rb +12 -10
- data/lib/aspera/cli/wizard.rb +42 -39
- data/lib/aspera/command_line_builder.rb +9 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +4 -1
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/duration.rb +5 -2
- data/lib/aspera/environment.rb +21 -12
- data/lib/aspera/exec_spec.rb +3 -0
- data/lib/aspera/faspex_gw.rb +13 -9
- data/lib/aspera/faspex_postproc.rb +15 -12
- data/lib/aspera/formatter_interface.rb +4 -1
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +4 -2
- data/lib/aspera/json_rpc/client.rb +6 -5
- data/lib/aspera/json_rpc/version.rb +3 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +18 -5
- data/lib/aspera/markdown.rb +5 -2
- data/lib/aspera/mime.rb +3 -0
- data/lib/aspera/nagios.rb +6 -4
- data/lib/aspera/node_emulator.rb +15 -13
- data/lib/aspera/oauth/client/base.rb +91 -70
- data/lib/aspera/oauth/client/boot.rb +7 -4
- data/lib/aspera/oauth/client/factory.rb +6 -3
- data/lib/aspera/oauth/client/generic.rb +5 -1
- data/lib/aspera/oauth/client/json_credentials.rb +4 -1
- data/lib/aspera/oauth/client/jwt.rb +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +85 -0
- data/lib/aspera/oauth/client/url_json.rb +4 -1
- data/lib/aspera/oauth/client/web.rb +39 -12
- data/lib/aspera/oauth/server/pkce.rb +6 -5
- data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +8 -4
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +4 -1
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +3 -0
- data/lib/aspera/rest/aspera_errors.rb +4 -1
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +22 -29
- data/lib/aspera/rest/error_analyzer.rb +5 -2
- data/lib/aspera/rest/list.rb +8 -4
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +8 -5
- data/lib/aspera/rest.rb +9 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +8 -2
- data/lib/aspera/schema/reader.rb +7 -0
- data/lib/aspera/schema/registry.rb +43 -28
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +166 -0
- data/lib/aspera/secret/hider.rb +201 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +110 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +5 -2
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +3 -0
- data/lib/aspera/sync/conf.schema.yaml +9 -4
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +16 -14
- data/lib/aspera/temp_file_manager.rb +8 -4
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +17 -14
- data/lib/aspera/transfer/result.rb +3 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +9 -22
- data/lib/aspera/transfer/units.rb +75 -0
- data/lib/aspera/uri_reader.rb +57 -7
- data/lib/aspera/web_auth/index.html.erb +6 -2
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +47 -10
- data/lib/aspera/web_page.rb +7 -2
- data/lib/aspera/web_server_simple.rb +6 -3
- data/lib/aspera/yaml.rb +5 -1
- data/llms.txt +81 -0
- data/skills/ascli/SKILL.md +88 -0
- data.tar.gz.sig +0 -0
- metadata +72 -19
- metadata.gz.sig +0 -0
- data/lib/aspera/keychain/base.rb +0 -30
- data/lib/aspera/keychain/encrypted_hash.rb +0 -192
- data/lib/aspera/keychain/factory.rb +0 -58
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
- data/lib/aspera/keychain/macos_security.rb +0 -277
- data/lib/aspera/keychain/one_password_api.rb +0 -91
- data/lib/aspera/keychain/one_password_base.rb +0 -39
- data/lib/aspera/keychain/one_password_cli.rb +0 -100
- data/lib/aspera/secret_hider.rb +0 -196
- data/lib/aspera/secret_strength.rb +0 -114
data/lib/aspera/coverage.rb
CHANGED
|
@@ -1,39 +1,109 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
#
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
module Aspera
|
|
7
|
+
# Test coverage with SimpleCov, enabled with env var `ENABLE_COVERAGE`.
|
|
8
|
+
# Each tested process stores only its own result in `parts/<uuid>`: merging all results at each process exit is slow when there are many processes.
|
|
9
|
+
# The report is generated by `report`, which merges all parts (called by the test Rake tasks).
|
|
10
|
+
module Coverage
|
|
11
|
+
# Development top folder, computed from this source location
|
|
12
|
+
ROOT = File.dirname(File.realpath(__FILE__), 3)
|
|
13
|
+
# Report folder, relative to `ROOT`
|
|
14
|
+
DIR = 'tmp/coverage'
|
|
15
|
+
# One sub-folder per process, each with its own `.resultset.json`
|
|
16
|
+
PARTS = File.join(DIR, 'parts')
|
|
17
|
+
# Coverage badge (shields.io endpoint), relative to `ROOT`
|
|
18
|
+
BADGE = 'docs/coverage.json'
|
|
19
|
+
# Badge color: first one whose lower limit is reached
|
|
20
|
+
BADGE_COLORS = {90 => 'brightgreen', 80 => 'green', 70 => 'yellowgreen', 60 => 'yellow', 50 => 'orange'}.freeze
|
|
21
|
+
# Results are kept for 1 day (must be longer than time to run the whole test suite, possibly in several runs)
|
|
22
|
+
MERGE_TIMEOUT = 86_400
|
|
23
|
+
# Lines with those words are ignored from coverage (assertions usually do not trigger)
|
|
24
|
+
NO_COV_FUNCTIONS = %w[error_unreachable_line error_unexpected_value Log.log.trace].freeze
|
|
25
|
+
# No coverage in these files or folders
|
|
26
|
+
NO_COV_PATHS = %w[
|
|
27
|
+
build/
|
|
28
|
+
tmp/
|
|
29
|
+
lib/transferd_pb.rb
|
|
30
|
+
lib/transferd_services_pb.rb
|
|
31
|
+
lib/aspera/secret/store/macos_security.rb
|
|
32
|
+
lib/aspera/assert.rb
|
|
33
|
+
].freeze
|
|
34
|
+
private_constant :BADGE_COLORS, :MERGE_TIMEOUT, :NO_COV_FUNCTIONS, :NO_COV_PATHS
|
|
35
|
+
|
|
36
|
+
class << self
|
|
37
|
+
# Start coverage of current process.
|
|
38
|
+
# Must be called before any library is loaded.
|
|
39
|
+
def start
|
|
40
|
+
require 'simplecov'
|
|
41
|
+
require 'securerandom'
|
|
42
|
+
name = SecureRandom.uuid
|
|
43
|
+
SimpleCov.root(ROOT)
|
|
44
|
+
SimpleCov.coverage_dir(File.join(PARTS, name))
|
|
45
|
+
SimpleCov.command_name(name)
|
|
46
|
+
SimpleCov.merge_subprocesses(true)
|
|
47
|
+
SimpleCov.merge_timeout(MERGE_TIMEOUT)
|
|
48
|
+
# The trend of a single process is meaningless
|
|
49
|
+
SimpleCov.history_limit(0)
|
|
50
|
+
# Only store this process result (in its own folder): no report
|
|
51
|
+
SimpleCov.at_exit { SimpleCov.result }
|
|
52
|
+
SimpleCov.start { Aspera::Coverage.filters(self) }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Merge the results of all processes and generate the report in `DIR`.
|
|
56
|
+
# Results older than `MERGE_TIMEOUT` are deleted.
|
|
57
|
+
# @return [Float, nil] Percentage of covered lines, or `nil` if there is no result
|
|
58
|
+
def report
|
|
59
|
+
require 'simplecov'
|
|
60
|
+
require 'fileutils'
|
|
61
|
+
require 'etc'
|
|
62
|
+
parts = Dir.glob(File.join(ROOT, PARTS, '*'))
|
|
63
|
+
old, parts = parts.partition { |dir| Time.now - File.mtime(dir) > MERGE_TIMEOUT }
|
|
64
|
+
FileUtils.rm_rf(old)
|
|
65
|
+
results = parts.map { |dir| File.join(dir, '.resultset.json') }.select { |file| File.exist?(file) }
|
|
66
|
+
return if results.empty?
|
|
67
|
+
# Else the merged result would be added to the previous one
|
|
68
|
+
FileUtils.rm_f(File.join(ROOT, DIR, '.resultset.json'))
|
|
69
|
+
SimpleCov.root(ROOT)
|
|
70
|
+
SimpleCov.coverage_dir(DIR)
|
|
71
|
+
percent = nil
|
|
72
|
+
# The merged command name lists all process UUIDs: show their number instead
|
|
73
|
+
SimpleCov.at_exit do
|
|
74
|
+
result = SimpleCov.result
|
|
75
|
+
result.command_name = "#{result.command_names.size} processes"
|
|
76
|
+
result.format!
|
|
77
|
+
percent = result.covered_percent
|
|
78
|
+
end
|
|
79
|
+
SimpleCov.collate(results, processes: Etc.nprocessors) { Aspera::Coverage.filters(self) }
|
|
80
|
+
percent
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Write the coverage badge: a shields.io endpoint file, shown in the top `README.md`
|
|
84
|
+
# @param percent [Float] Percentage of covered lines
|
|
85
|
+
def write_badge(percent)
|
|
86
|
+
require 'json'
|
|
87
|
+
color = BADGE_COLORS.find { |limit, _| percent >= limit }&.last || 'red'
|
|
88
|
+
File.write(File.join(ROOT, BADGE), "#{JSON.pretty_generate(schemaVersion: 1, label: 'coverage', message: format('%.1f%%', percent), color: color)}\n")
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Configure coverage criteria and filters (in `SimpleCov.start` or `SimpleCov.collate` block)
|
|
92
|
+
# Branch coverage is measured in addition to line coverage (shown in the report, the badge shows lines).
|
|
93
|
+
# @param config [SimpleCov] Receiver of the configuration block
|
|
94
|
+
def filters(config)
|
|
95
|
+
config.enable_coverage(:branch)
|
|
96
|
+
config.skip do |source_file|
|
|
97
|
+
source_file.lines.each do |line|
|
|
98
|
+
line.skipped! if NO_COV_FUNCTIONS.any? { |i| line.src.include?(i) }
|
|
99
|
+
end
|
|
100
|
+
false
|
|
101
|
+
end
|
|
102
|
+
NO_COV_PATHS.each { |path| config.skip(path) }
|
|
32
103
|
end
|
|
33
|
-
false
|
|
34
104
|
end
|
|
35
|
-
# no coverage in these files
|
|
36
|
-
add_filter 'lib/aspera/keychain/macos_security.rb'
|
|
37
|
-
add_filter 'lib/aspera/assert.rb'
|
|
38
105
|
end
|
|
39
106
|
end
|
|
107
|
+
|
|
108
|
+
# Start coverage when this file is loaded, if enabled: must be required before any other library
|
|
109
|
+
Aspera::Coverage.start if ENV.key?('ENABLE_COVERAGE')
|
data/lib/aspera/dot_container.rb
CHANGED
data/lib/aspera/duration.rb
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
4
7
|
require 'strscan'
|
|
5
8
|
|
|
6
9
|
module Aspera
|
|
@@ -29,7 +32,7 @@ module Aspera
|
|
|
29
32
|
scanner = StringScanner.new(value)
|
|
30
33
|
units = []
|
|
31
34
|
until scanner.eos?
|
|
32
|
-
Aspera.assert(scanner.scan(/(\d+)(ms|us|[wdhms])/), type: ParameterError) { "Invalid duration: #{value}" }
|
|
35
|
+
Aspera.assert(scanner.scan(/(\d+)(ms|us|[wdhms])/), type: ParameterError) { "Invalid duration: #{value}" } # cspell:disable-line
|
|
33
36
|
units.push([scanner[1], scanner[2]])
|
|
34
37
|
end
|
|
35
38
|
Aspera.assert(!units.empty?, type: ParameterError) { "Invalid duration: #{value}" }
|
data/lib/aspera/environment.rb
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'log'
|
|
7
|
+
require_relative 'assert'
|
|
8
|
+
require_relative 'rainbow'
|
|
6
9
|
require 'rbconfig'
|
|
7
10
|
require 'singleton'
|
|
8
11
|
require 'openssl'
|
|
@@ -10,10 +13,9 @@ require 'open3'
|
|
|
10
13
|
require 'English'
|
|
11
14
|
require 'io/console'
|
|
12
15
|
require 'shellwords'
|
|
13
|
-
require 'aspera/rainbow'
|
|
14
16
|
using Rainbow
|
|
15
17
|
|
|
16
|
-
# cspell:words
|
|
18
|
+
# cspell:words bccwin conhost Cmder
|
|
17
19
|
|
|
18
20
|
module Aspera
|
|
19
21
|
# detect OS, architecture, and specific stuff
|
|
@@ -76,9 +78,10 @@ module Aspera
|
|
|
76
78
|
return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
|
|
77
79
|
end
|
|
78
80
|
|
|
79
|
-
#
|
|
81
|
+
# Fresh binding: no local variables, `self` is a plain `Object` (not a sandbox).
|
|
82
|
+
# Constants still resolve through the `Aspera` nesting.
|
|
80
83
|
def empty_binding
|
|
81
|
-
return
|
|
84
|
+
return Object.new.instance_eval { binding }
|
|
82
85
|
end
|
|
83
86
|
|
|
84
87
|
# Secure execution of Ruby code
|
|
@@ -109,8 +112,8 @@ module Aspera
|
|
|
109
112
|
return str if str.match?(%r{\A[A-Za-z0-9_.,:/@+=-]+\z})
|
|
110
113
|
# return str if Shellwords.shellescape(str) == str
|
|
111
114
|
|
|
112
|
-
# Otherwise use single quotes
|
|
113
|
-
"'#{str.gsub("'"
|
|
115
|
+
# Otherwise use single quotes (gsub with block: in a replacement string, \' is the post-match)
|
|
116
|
+
"'#{str.gsub("'") { %q('\'') }}'"
|
|
114
117
|
end
|
|
115
118
|
|
|
116
119
|
# Executes a command without invoking a shell.
|
|
@@ -158,7 +161,7 @@ module Aspera
|
|
|
158
161
|
# https://docs.ruby-lang.org/en/master/Process.html#method-c-spawn
|
|
159
162
|
# https://docs.ruby-lang.org/en/master/Process.html#module-Process-label-Execution+Options
|
|
160
163
|
kwargs[:close_others] = true unless kwargs.key?(:close_others)
|
|
161
|
-
pid = Process.spawn(*build_spawn_argv(cmd, kwargs), **kwargs)
|
|
164
|
+
pid = ::Process.spawn(*build_spawn_argv(cmd, kwargs), **kwargs)
|
|
162
165
|
Log.dump(:pid, pid)
|
|
163
166
|
pid
|
|
164
167
|
when :capture
|
|
@@ -288,7 +291,8 @@ module Aspera
|
|
|
288
291
|
# url_method: :text or :graphical, how URLs, secrets and files are presented to the user
|
|
289
292
|
# browser_command: [Array<String>, nil] command (and arguments) that opens a URL (URL is appended), instead of
|
|
290
293
|
# the system default
|
|
291
|
-
|
|
294
|
+
# web_auth: :port or :paste, how the result of a web login is received: local port, or URL pasted by the user
|
|
295
|
+
attr_accessor :url_method, :browser_command, :web_auth, :file_illegal_characters
|
|
292
296
|
attr_reader :os, :cpu, :default_gui_mode
|
|
293
297
|
|
|
294
298
|
def initialize
|
|
@@ -333,6 +337,7 @@ module Aspera
|
|
|
333
337
|
end
|
|
334
338
|
@url_method = @default_gui_mode
|
|
335
339
|
@browser_command = nil
|
|
340
|
+
@web_auth = :port
|
|
336
341
|
@file_illegal_characters = REPLACE_CHARACTER + WINDOWS_FILENAME_INVALID_CHARACTERS
|
|
337
342
|
nil
|
|
338
343
|
end
|
|
@@ -387,7 +392,11 @@ module Aspera
|
|
|
387
392
|
# @return [nil]
|
|
388
393
|
def open_uri_graphical(uri)
|
|
389
394
|
if @browser_command
|
|
390
|
-
|
|
395
|
+
begin
|
|
396
|
+
self.class.secure_execute(*@browser_command, uri.to_s)
|
|
397
|
+
rescue Errno::ENOENT
|
|
398
|
+
raise ParameterError, "Browser command not found (option ui.command): #{@browser_command.first}. Check that it is in PATH, or use its full path."
|
|
399
|
+
end
|
|
391
400
|
else
|
|
392
401
|
case @os
|
|
393
402
|
when Environment::OS_MACOS then self.class.secure_execute('open', uri.to_s)
|
data/lib/aspera/exec_spec.rb
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
3
6
|
module Aspera
|
|
4
7
|
# Holds the executable name, environment variables and command line arguments for a spawned process,
|
|
5
8
|
# and the temporary files it uses (for example a file list), which must exist until it ends.
|
data/lib/aspera/faspex_gw.rb
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'api/faspex'
|
|
7
|
+
require_relative 'log'
|
|
8
|
+
require_relative 'assert'
|
|
9
|
+
require_relative 'hash_ext'
|
|
10
|
+
require_relative 'mime'
|
|
11
|
+
require_relative 'secret/strength'
|
|
12
|
+
require_relative 'servlet_helpers'
|
|
13
|
+
require_relative 'web_server_simple'
|
|
10
14
|
require 'webrick'
|
|
11
15
|
require 'json'
|
|
12
16
|
|
|
@@ -38,8 +42,8 @@ module Aspera
|
|
|
38
42
|
not_allowed = parameters.keys - ALLOWED_PARAMETERS
|
|
39
43
|
Aspera.assert(not_allowed.empty?, type: ParameterError) { "unsupported parameters: #{not_allowed.join(', ')}" }
|
|
40
44
|
Aspera.assert(parameters[:username].nil? == parameters[:password].nil?, type: ParameterError) { 'Parameters username and password must be set together' }
|
|
41
|
-
parameters[:password_min_bits] ||=
|
|
42
|
-
|
|
45
|
+
parameters[:password_min_bits] ||= Secret::Strength::DEFAULT_MIN_BITS
|
|
46
|
+
Secret::Strength.assert_strong(parameters[:password], name: 'password', min_bits: parameters[:password_min_bits]) unless parameters[:password].nil?
|
|
43
47
|
parameters
|
|
44
48
|
end
|
|
45
49
|
|
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'log'
|
|
7
|
+
require_relative 'assert'
|
|
8
|
+
require_relative 'environment'
|
|
9
|
+
require_relative 'hash_ext'
|
|
10
|
+
require_relative 'secret/strength'
|
|
11
|
+
require_relative 'servlet_helpers'
|
|
12
|
+
require_relative 'web_server_simple'
|
|
3
13
|
require 'json'
|
|
4
14
|
require 'timeout'
|
|
5
15
|
require 'English'
|
|
6
16
|
require 'ipaddr'
|
|
7
17
|
require 'openssl'
|
|
8
18
|
require 'webrick'
|
|
9
|
-
require 'aspera/log'
|
|
10
|
-
require 'aspera/assert'
|
|
11
|
-
require 'aspera/environment'
|
|
12
|
-
require 'aspera/hash_ext'
|
|
13
|
-
require 'aspera/secret_strength'
|
|
14
|
-
require 'aspera/servlet_helpers'
|
|
15
|
-
require 'aspera/web_server_simple'
|
|
16
19
|
|
|
17
20
|
module Aspera
|
|
18
21
|
# Start a Faspex-4 style post-processing script using Faspex-5 webhook
|
|
@@ -49,9 +52,9 @@ module Aspera
|
|
|
49
52
|
parameters[:script_folder] ||= '.'
|
|
50
53
|
parameters[:fail_on_error] ||= false
|
|
51
54
|
parameters[:timeout_seconds] ||= 60
|
|
52
|
-
parameters[:secret_token_min_bits] ||=
|
|
55
|
+
parameters[:secret_token_min_bits] ||= Secret::Strength::DEFAULT_MIN_BITS
|
|
53
56
|
Aspera.assert(File.directory?(parameters[:script_folder])) { "script_folder not found: #{parameters[:script_folder]}" }
|
|
54
|
-
|
|
57
|
+
Secret::Strength.assert_strong(parameters[:secret_token], name: 'secret_token', min_bits: parameters[:secret_token_min_bits]) if parameters.key?(:secret_token)
|
|
55
58
|
# allowed_ips: optional IP addresses or networks (CIDR); nil means no restriction
|
|
56
59
|
parameters[:allowed_ips] = Array(parameters[:allowed_ips]).map { |ip| ip_address(ip) } if parameters.key?(:allowed_ips)
|
|
57
60
|
parameters
|
|
@@ -153,7 +156,7 @@ module Aspera
|
|
|
153
156
|
post_proc_pid = Environment.secure_execute(script_path, mode: :background, env: script_environment(webhook_parameters))
|
|
154
157
|
Timeout.timeout(@parameters[:timeout_seconds]) do
|
|
155
158
|
# "wait" for process to avoid zombie
|
|
156
|
-
Process.wait(post_proc_pid)
|
|
159
|
+
::Process.wait(post_proc_pid)
|
|
157
160
|
post_proc_pid = nil
|
|
158
161
|
end
|
|
159
162
|
process_status = $CHILD_STATUS
|
|
@@ -223,8 +226,8 @@ module Aspera
|
|
|
223
226
|
# Kill the script after a timeout or an error
|
|
224
227
|
# @param pid [Integer] Process id of the script
|
|
225
228
|
def kill_script(pid)
|
|
226
|
-
Process.kill('SIGKILL', pid)
|
|
227
|
-
Process.wait(pid)
|
|
229
|
+
::Process.kill('SIGKILL', pid)
|
|
230
|
+
::Process.wait(pid)
|
|
228
231
|
Log.log.error("Killed process: #{pid}")
|
|
229
232
|
rescue SystemCallError
|
|
230
233
|
# already terminated
|
data/lib/aspera/graphql.rb
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'rest/call_error'
|
|
7
|
+
require_relative 'assert'
|
|
4
8
|
|
|
5
9
|
module Aspera
|
|
6
10
|
# Helper to execute GraphQL queries via a Rest::Client object.
|
data/lib/aspera/hash_ext.rb
CHANGED
data/lib/aspera/id_generator.rb
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
5
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
require_relative '../rest/error_analyzer'
|
|
7
|
+
require_relative '../rest/call_error'
|
|
8
|
+
require_relative '../assert'
|
|
9
|
+
require_relative 'version'
|
|
9
10
|
|
|
10
11
|
# Error in non-2XX response. Error in 2XX response is raised by the client.
|
|
11
12
|
Aspera::Rest::ErrorAnalyzer.instance.add_simple_handler(name: 'JSON RPC', path: %w[error message])
|
data/lib/aspera/line_logger.rb
CHANGED
data/lib/aspera/link_header.rb
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
3
6
|
require 'strscan'
|
|
4
7
|
|
|
5
8
|
module Aspera
|
|
@@ -72,7 +75,7 @@ module Aspera
|
|
|
72
75
|
end
|
|
73
76
|
|
|
74
77
|
# RFC 2616 token: any char except separators
|
|
75
|
-
TOKEN_RE = /[^()
|
|
78
|
+
TOKEN_RE = /[^()<>@,;:"\[\]?={}\s]+/ # RFC 2616 token
|
|
76
79
|
QUOTED_RE = /"((?:[^"\\]|\\.)*)"/ # double-quoted string with backslash escapes
|
|
77
80
|
HREF_RE = /\s*<([^>]*)>\s*;?\s*/ # <URI> possibly followed by ;
|
|
78
81
|
ATTR_RE = /(#{TOKEN_RE})\s*=\s*(#{TOKEN_RE}|#{QUOTED_RE})\s*/ # key=value or key="value"
|
data/lib/aspera/log.rb
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
7
|
+
require_relative 'secret/hider'
|
|
8
|
+
require_relative 'rainbow'
|
|
5
9
|
require 'logger'
|
|
6
10
|
require 'pp'
|
|
7
11
|
require 'json'
|
|
8
12
|
require 'singleton'
|
|
9
13
|
require 'stringio'
|
|
10
|
-
require 'aspera/rainbow'
|
|
11
14
|
using Rainbow
|
|
12
15
|
|
|
13
16
|
# Ignore warnings
|
|
@@ -111,7 +114,7 @@ module Aspera
|
|
|
111
114
|
Aspera.assert(object.nil? || block.nil?, 'Use either object, or block, not both')
|
|
112
115
|
object = yield if block_given?
|
|
113
116
|
# Secrets hidden by key name in the structure, whatever the dump format (the log formatter then hides secrets in text)
|
|
114
|
-
object =
|
|
117
|
+
object = Secret::Hider.instance.hidden_copy(object) unless Secret::Hider.instance.log_secrets
|
|
115
118
|
dump_text =
|
|
116
119
|
case instance.dump_format
|
|
117
120
|
when :json
|
|
@@ -175,6 +178,16 @@ module Aspera
|
|
|
175
178
|
nil
|
|
176
179
|
end
|
|
177
180
|
|
|
181
|
+
# Send logs to the logger of a host application (e.g. `Rails.logger`), until `logger_type` is set.
|
|
182
|
+
# Its level and formatter are not changed: secrets are hidden in dumps (`Log.dump`), not in log lines.
|
|
183
|
+
# @param logger [Logger] Logger of the host application
|
|
184
|
+
# @return [nil]
|
|
185
|
+
def logger=(logger)
|
|
186
|
+
Aspera.assert(%i[add level].all? { |m| logger.respond_to?(m) }) { "not a logger: #{logger.class}" }
|
|
187
|
+
@logger = logger
|
|
188
|
+
nil
|
|
189
|
+
end
|
|
190
|
+
|
|
178
191
|
# Set log level of underlying logger given symbol level
|
|
179
192
|
# @param new_level [Symbol] One of LEVELS
|
|
180
193
|
def level=(new_level)
|
|
@@ -193,7 +206,7 @@ module Aspera
|
|
|
193
206
|
raise Error, 'Formatter must be a String, a Logger::Formatter or a Proc'
|
|
194
207
|
end
|
|
195
208
|
# Update formatter with password hiding
|
|
196
|
-
@logger.formatter =
|
|
209
|
+
@logger.formatter = Secret::Hider.instance.log_formatter(formatter)
|
|
197
210
|
nil
|
|
198
211
|
end
|
|
199
212
|
|
data/lib/aspera/markdown.rb
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
4
7
|
|
|
5
8
|
module Aspera
|
|
6
9
|
# Formatting for Markdown
|
|
@@ -170,8 +173,8 @@ module Aspera
|
|
|
170
173
|
headings = table.shift
|
|
171
174
|
table.unshift(col_widths.map { |col_width| '-' * col_width })
|
|
172
175
|
table.unshift(headings)
|
|
176
|
+
# Separator row with spaces around pipes, as other rows (markdownlint MD060, style `compact`)
|
|
173
177
|
lines = table.map { |line| "| #{line.map { |i| i.to_s.gsub('\\', '\\\\').gsub('|', '\|') }.join(' | ')} |\n" }
|
|
174
|
-
lines[1] = lines[1].tr(' ', '-')
|
|
175
178
|
return lines.join.chomp
|
|
176
179
|
end
|
|
177
180
|
end
|
data/lib/aspera/mime.rb
CHANGED
data/lib/aspera/nagios.rb
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'log'
|
|
7
|
+
require_relative 'assert'
|
|
6
8
|
|
|
7
9
|
module Aspera
|
|
8
10
|
class Nagios
|
|
@@ -46,7 +48,7 @@ module Aspera
|
|
|
46
48
|
# display status for nagios
|
|
47
49
|
puts("#{status} - [#{message}]\n")
|
|
48
50
|
# provide exit code to nagios
|
|
49
|
-
Process.exit(LEVELS.index(data.first['status'].to_sym))
|
|
51
|
+
::Process.exit(LEVELS.index(data.first['status'].to_sym))
|
|
50
52
|
end
|
|
51
53
|
end
|
|
52
54
|
|
data/lib/aspera/node_emulator.rb
CHANGED
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
7
|
+
require_relative 'ascp/installation'
|
|
8
|
+
require_relative 'agent/transferd'
|
|
9
|
+
require_relative 'transfer/spec'
|
|
10
|
+
require_relative 'transfer/error'
|
|
11
|
+
require_relative 'secret/hider'
|
|
12
|
+
require_relative 'string_ext'
|
|
13
|
+
require_relative 'log'
|
|
14
|
+
require_relative 'servlet_helpers'
|
|
13
15
|
require 'webrick'
|
|
14
16
|
require 'etc'
|
|
15
17
|
require 'securerandom'
|
|
@@ -295,7 +297,7 @@ module Aspera
|
|
|
295
297
|
@node_id = SecureRandom.uuid
|
|
296
298
|
@cluster_id = SecureRandom.uuid
|
|
297
299
|
# transferd runs as the user of the emulator
|
|
298
|
-
@transfer_user = Etc.getpwuid(Process.euid)&.name || Etc.getlogin.to_s
|
|
300
|
+
@transfer_user = Etc.getpwuid(::Process.euid)&.name || Etc.getlogin.to_s
|
|
299
301
|
end
|
|
300
302
|
|
|
301
303
|
# Get the node information
|
|
@@ -345,7 +347,7 @@ module Aspera
|
|
|
345
347
|
def start(transfer_spec)
|
|
346
348
|
Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
|
|
347
349
|
# copy without secrets, returned in transfer `start_spec`
|
|
348
|
-
start_spec =
|
|
350
|
+
start_spec = Secret::Hider.instance.hidden_copy(transfer_spec)
|
|
349
351
|
Log.dump(:start_spec, start_spec)
|
|
350
352
|
request = ::Transferd::Api::TransferRequest.new(
|
|
351
353
|
transferType: ::Transferd::Api::TransferType::FILE_REGULAR,
|
|
@@ -434,7 +436,7 @@ module Aspera
|
|
|
434
436
|
|
|
435
437
|
# @return [Float] current time in seconds, for durations
|
|
436
438
|
def monotonic_time
|
|
437
|
-
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
439
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
438
440
|
end
|
|
439
441
|
|
|
440
442
|
# Convert a Node API path to a real path
|