aspera-cli 4.28.0 → 4.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +217 -2
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +135 -56
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +122 -26
- data/SECURITY.md +84 -0
- data/bin/ascli +8 -5
- data/bin/asession +9 -6
- data/docs/README.md +1895 -632
- data/docs/test-mcp-with-ai.md +8 -8
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +20 -19
- data/lib/aspera/agent/factory.rb +6 -3
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +7 -4
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +29 -22
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +95 -67
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +370 -0
- data/lib/aspera/ascp/log_locations.rb +163 -0
- data/lib/aspera/ascp/management.rb +16 -8
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +12 -19
- data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +38 -33
- data/lib/aspera/cli/command_line.rb +63 -8
- data/lib/aspera/cli/command_registry.rb +9 -1
- data/lib/aspera/cli/command_spec.rb +47 -7
- data/lib/aspera/cli/completion/ascli.bash +19 -5
- data/lib/aspera/cli/completion/ascli.fish +13 -2
- data/lib/aspera/cli/completion/ascli.ps1 +31 -9
- data/lib/aspera/cli/completion/ascli.zsh +22 -6
- data/lib/aspera/cli/completion_actions.rb +303 -69
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +6 -0
- data/lib/aspera/cli/extended_value.rb +66 -31
- data/lib/aspera/cli/formatter.rb +43 -30
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -10
- data/lib/aspera/cli/http.rb +38 -10
- data/lib/aspera/cli/info.rb +12 -7
- data/lib/aspera/cli/mailer.rb +6 -3
- data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
- data/lib/aspera/cli/mcp/secrets.rb +8 -5
- data/lib/aspera/cli/mcp/tool.rb +20 -17
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +77 -18
- data/lib/aspera/cli/options.schema.yaml +641 -80
- data/lib/aspera/cli/parser.rb +71 -35
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +248 -37
- data/lib/aspera/cli/plugins/ats.rb +10 -9
- data/lib/aspera/cli/plugins/base.rb +50 -18
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +210 -104
- data/lib/aspera/cli/plugins/console.rb +6 -3
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +21 -7
- data/lib/aspera/cli/plugins/faspex5.rb +31 -26
- data/lib/aspera/cli/plugins/faspio.rb +7 -5
- data/lib/aspera/cli/plugins/httpgw.rb +7 -4
- data/lib/aspera/cli/plugins/mcp.rb +42 -22
- data/lib/aspera/cli/plugins/node.rb +39 -38
- data/lib/aspera/cli/plugins/oauth.rb +23 -6
- data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
- data/lib/aspera/cli/plugins/preview.rb +72 -45
- data/lib/aspera/cli/plugins/server.rb +66 -14
- data/lib/aspera/cli/plugins/shares.rb +87 -18
- data/lib/aspera/cli/plugins/transferd.rb +228 -0
- data/lib/aspera/cli/poll.rb +236 -0
- data/lib/aspera/cli/preset_actions.rb +144 -30
- data/lib/aspera/cli/preset_manager.rb +113 -35
- data/lib/aspera/cli/prompt.rb +157 -9
- data/lib/aspera/cli/result.rb +12 -8
- data/lib/aspera/cli/runner.rb +89 -48
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +89 -9
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +15 -10
- data/lib/aspera/cli/table_fitter.rb +3 -0
- data/lib/aspera/cli/terminal_formatter.rb +9 -6
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +43 -20
- data/lib/aspera/cli/transfer_progress.rb +14 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +188 -42
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.rb +12 -10
- data/lib/aspera/cli/wizard.rb +42 -39
- data/lib/aspera/command_line_builder.rb +9 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +4 -1
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/environment.rb +23 -12
- data/lib/aspera/exec_spec.rb +3 -0
- data/lib/aspera/faspex_gw.rb +13 -9
- data/lib/aspera/faspex_postproc.rb +15 -12
- data/lib/aspera/formatter_interface.rb +4 -1
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +20 -5
- data/lib/aspera/inflate.rb +43 -0
- data/lib/aspera/json_rpc/client.rb +6 -5
- data/lib/aspera/json_rpc/version.rb +3 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +18 -5
- data/lib/aspera/markdown.rb +5 -2
- data/lib/aspera/mime.rb +3 -0
- data/lib/aspera/nagios.rb +6 -4
- data/lib/aspera/node_emulator.rb +15 -13
- data/lib/aspera/oauth/client/base.rb +124 -73
- data/lib/aspera/oauth/client/boot.rb +7 -4
- data/lib/aspera/oauth/client/factory.rb +41 -18
- data/lib/aspera/oauth/client/generic.rb +5 -1
- data/lib/aspera/oauth/client/json_credentials.rb +7 -3
- data/lib/aspera/oauth/client/jwt.rb +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +94 -0
- data/lib/aspera/oauth/client/url_json.rb +7 -3
- data/lib/aspera/oauth/client/web.rb +40 -13
- data/lib/aspera/oauth/server/pkce.rb +6 -5
- data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +16 -6
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +17 -2
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +3 -0
- data/lib/aspera/rest/aspera_errors.rb +4 -1
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +45 -34
- data/lib/aspera/rest/error_analyzer.rb +5 -2
- data/lib/aspera/rest/list.rb +8 -4
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +8 -5
- data/lib/aspera/rest.rb +9 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +8 -2
- data/lib/aspera/schema/reader.rb +7 -0
- data/lib/aspera/schema/registry.rb +44 -28
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +150 -0
- data/lib/aspera/secret/hider.rb +213 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +112 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +5 -2
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +3 -0
- data/lib/aspera/sync/conf.schema.yaml +9 -4
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +17 -15
- data/lib/aspera/temp_file_manager.rb +8 -4
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +17 -14
- data/lib/aspera/transfer/result.rb +23 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +9 -22
- data/lib/aspera/units/decimal.rb +67 -0
- data/lib/aspera/units/duration.rb +109 -0
- data/lib/aspera/uri_reader.rb +64 -9
- data/lib/aspera/web_auth/index.html.erb +6 -2
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +47 -10
- data/lib/aspera/web_page.rb +7 -2
- data/lib/aspera/web_server_simple.rb +6 -3
- data/lib/aspera/yaml.rb +5 -1
- data/llms-install.md +88 -0
- data/llms.txt +87 -0
- data/skills/ascli/SKILL.md +94 -0
- data.tar.gz.sig +0 -0
- metadata +77 -20
- metadata.gz.sig +0 -0
- data/lib/aspera/duration.rb +0 -42
- data/lib/aspera/keychain/base.rb +0 -30
- data/lib/aspera/keychain/encrypted_hash.rb +0 -192
- data/lib/aspera/keychain/factory.rb +0 -58
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
- data/lib/aspera/keychain/macos_security.rb +0 -277
- data/lib/aspera/keychain/one_password_api.rb +0 -91
- data/lib/aspera/keychain/one_password_base.rb +0 -39
- data/lib/aspera/keychain/one_password_cli.rb +0 -100
- data/lib/aspera/secret_hider.rb +0 -196
- data/lib/aspera/secret_strength.rb +0 -114
data/lib/aspera/dot_container.rb
CHANGED
data/lib/aspera/environment.rb
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'log'
|
|
7
|
+
require_relative 'assert'
|
|
8
|
+
require_relative 'rainbow'
|
|
6
9
|
require 'rbconfig'
|
|
7
10
|
require 'singleton'
|
|
8
11
|
require 'openssl'
|
|
@@ -10,10 +13,9 @@ require 'open3'
|
|
|
10
13
|
require 'English'
|
|
11
14
|
require 'io/console'
|
|
12
15
|
require 'shellwords'
|
|
13
|
-
require 'aspera/rainbow'
|
|
14
16
|
using Rainbow
|
|
15
17
|
|
|
16
|
-
# cspell:words
|
|
18
|
+
# cspell:words bccwin conhost Cmder
|
|
17
19
|
|
|
18
20
|
module Aspera
|
|
19
21
|
# detect OS, architecture, and specific stuff
|
|
@@ -76,9 +78,10 @@ module Aspera
|
|
|
76
78
|
return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
|
|
77
79
|
end
|
|
78
80
|
|
|
79
|
-
#
|
|
81
|
+
# Fresh binding: no local variables, `self` is a plain `Object` (not a sandbox).
|
|
82
|
+
# Constants still resolve through the `Aspera` nesting.
|
|
80
83
|
def empty_binding
|
|
81
|
-
return
|
|
84
|
+
return Object.new.instance_eval { binding }
|
|
82
85
|
end
|
|
83
86
|
|
|
84
87
|
# Secure execution of Ruby code
|
|
@@ -109,8 +112,8 @@ module Aspera
|
|
|
109
112
|
return str if str.match?(%r{\A[A-Za-z0-9_.,:/@+=-]+\z})
|
|
110
113
|
# return str if Shellwords.shellescape(str) == str
|
|
111
114
|
|
|
112
|
-
# Otherwise use single quotes
|
|
113
|
-
"'#{str.gsub("'"
|
|
115
|
+
# Otherwise use single quotes (gsub with block: in a replacement string, \' is the post-match)
|
|
116
|
+
"'#{str.gsub("'") { %q('\'') }}'"
|
|
114
117
|
end
|
|
115
118
|
|
|
116
119
|
# Executes a command without invoking a shell.
|
|
@@ -158,7 +161,7 @@ module Aspera
|
|
|
158
161
|
# https://docs.ruby-lang.org/en/master/Process.html#method-c-spawn
|
|
159
162
|
# https://docs.ruby-lang.org/en/master/Process.html#module-Process-label-Execution+Options
|
|
160
163
|
kwargs[:close_others] = true unless kwargs.key?(:close_others)
|
|
161
|
-
pid = Process.spawn(*build_spawn_argv(cmd, kwargs), **kwargs)
|
|
164
|
+
pid = ::Process.spawn(*build_spawn_argv(cmd, kwargs), **kwargs)
|
|
162
165
|
Log.dump(:pid, pid)
|
|
163
166
|
pid
|
|
164
167
|
when :capture
|
|
@@ -288,7 +291,9 @@ module Aspera
|
|
|
288
291
|
# url_method: :text or :graphical, how URLs, secrets and files are presented to the user
|
|
289
292
|
# browser_command: [Array<String>, nil] command (and arguments) that opens a URL (URL is appended), instead of
|
|
290
293
|
# the system default
|
|
291
|
-
|
|
294
|
+
# web_auth: :port or :paste, how the result of a web login is received: local port, or URL pasted by the user
|
|
295
|
+
# hidden_silent: if true, hidden input shows no mark of its state
|
|
296
|
+
attr_accessor :url_method, :browser_command, :web_auth, :hidden_silent, :file_illegal_characters
|
|
292
297
|
attr_reader :os, :cpu, :default_gui_mode
|
|
293
298
|
|
|
294
299
|
def initialize
|
|
@@ -333,6 +338,8 @@ module Aspera
|
|
|
333
338
|
end
|
|
334
339
|
@url_method = @default_gui_mode
|
|
335
340
|
@browser_command = nil
|
|
341
|
+
@web_auth = :port
|
|
342
|
+
@hidden_silent = false
|
|
336
343
|
@file_illegal_characters = REPLACE_CHARACTER + WINDOWS_FILENAME_INVALID_CHARACTERS
|
|
337
344
|
nil
|
|
338
345
|
end
|
|
@@ -387,7 +394,11 @@ module Aspera
|
|
|
387
394
|
# @return [nil]
|
|
388
395
|
def open_uri_graphical(uri)
|
|
389
396
|
if @browser_command
|
|
390
|
-
|
|
397
|
+
begin
|
|
398
|
+
self.class.secure_execute(*@browser_command, uri.to_s)
|
|
399
|
+
rescue Errno::ENOENT
|
|
400
|
+
raise ParameterError, "Browser command not found (option ui.command): #{@browser_command.first}. Check that it is in PATH, or use its full path."
|
|
401
|
+
end
|
|
391
402
|
else
|
|
392
403
|
case @os
|
|
393
404
|
when Environment::OS_MACOS then self.class.secure_execute('open', uri.to_s)
|
data/lib/aspera/exec_spec.rb
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
3
6
|
module Aspera
|
|
4
7
|
# Holds the executable name, environment variables and command line arguments for a spawned process,
|
|
5
8
|
# and the temporary files it uses (for example a file list), which must exist until it ends.
|
data/lib/aspera/faspex_gw.rb
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'api/faspex'
|
|
7
|
+
require_relative 'log'
|
|
8
|
+
require_relative 'assert'
|
|
9
|
+
require_relative 'hash_ext'
|
|
10
|
+
require_relative 'mime'
|
|
11
|
+
require_relative 'secret/strength'
|
|
12
|
+
require_relative 'servlet_helpers'
|
|
13
|
+
require_relative 'web_server_simple'
|
|
10
14
|
require 'webrick'
|
|
11
15
|
require 'json'
|
|
12
16
|
|
|
@@ -38,8 +42,8 @@ module Aspera
|
|
|
38
42
|
not_allowed = parameters.keys - ALLOWED_PARAMETERS
|
|
39
43
|
Aspera.assert(not_allowed.empty?, type: ParameterError) { "unsupported parameters: #{not_allowed.join(', ')}" }
|
|
40
44
|
Aspera.assert(parameters[:username].nil? == parameters[:password].nil?, type: ParameterError) { 'Parameters username and password must be set together' }
|
|
41
|
-
parameters[:password_min_bits] ||=
|
|
42
|
-
|
|
45
|
+
parameters[:password_min_bits] ||= Secret::Strength::DEFAULT_MIN_BITS
|
|
46
|
+
Secret::Strength.assert_strong(parameters[:password], name: 'password', min_bits: parameters[:password_min_bits]) unless parameters[:password].nil?
|
|
43
47
|
parameters
|
|
44
48
|
end
|
|
45
49
|
|
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'log'
|
|
7
|
+
require_relative 'assert'
|
|
8
|
+
require_relative 'environment'
|
|
9
|
+
require_relative 'hash_ext'
|
|
10
|
+
require_relative 'secret/strength'
|
|
11
|
+
require_relative 'servlet_helpers'
|
|
12
|
+
require_relative 'web_server_simple'
|
|
3
13
|
require 'json'
|
|
4
14
|
require 'timeout'
|
|
5
15
|
require 'English'
|
|
6
16
|
require 'ipaddr'
|
|
7
17
|
require 'openssl'
|
|
8
18
|
require 'webrick'
|
|
9
|
-
require 'aspera/log'
|
|
10
|
-
require 'aspera/assert'
|
|
11
|
-
require 'aspera/environment'
|
|
12
|
-
require 'aspera/hash_ext'
|
|
13
|
-
require 'aspera/secret_strength'
|
|
14
|
-
require 'aspera/servlet_helpers'
|
|
15
|
-
require 'aspera/web_server_simple'
|
|
16
19
|
|
|
17
20
|
module Aspera
|
|
18
21
|
# Start a Faspex-4 style post-processing script using Faspex-5 webhook
|
|
@@ -49,9 +52,9 @@ module Aspera
|
|
|
49
52
|
parameters[:script_folder] ||= '.'
|
|
50
53
|
parameters[:fail_on_error] ||= false
|
|
51
54
|
parameters[:timeout_seconds] ||= 60
|
|
52
|
-
parameters[:secret_token_min_bits] ||=
|
|
55
|
+
parameters[:secret_token_min_bits] ||= Secret::Strength::DEFAULT_MIN_BITS
|
|
53
56
|
Aspera.assert(File.directory?(parameters[:script_folder])) { "script_folder not found: #{parameters[:script_folder]}" }
|
|
54
|
-
|
|
57
|
+
Secret::Strength.assert_strong(parameters[:secret_token], name: 'secret_token', min_bits: parameters[:secret_token_min_bits]) if parameters.key?(:secret_token)
|
|
55
58
|
# allowed_ips: optional IP addresses or networks (CIDR); nil means no restriction
|
|
56
59
|
parameters[:allowed_ips] = Array(parameters[:allowed_ips]).map { |ip| ip_address(ip) } if parameters.key?(:allowed_ips)
|
|
57
60
|
parameters
|
|
@@ -153,7 +156,7 @@ module Aspera
|
|
|
153
156
|
post_proc_pid = Environment.secure_execute(script_path, mode: :background, env: script_environment(webhook_parameters))
|
|
154
157
|
Timeout.timeout(@parameters[:timeout_seconds]) do
|
|
155
158
|
# "wait" for process to avoid zombie
|
|
156
|
-
Process.wait(post_proc_pid)
|
|
159
|
+
::Process.wait(post_proc_pid)
|
|
157
160
|
post_proc_pid = nil
|
|
158
161
|
end
|
|
159
162
|
process_status = $CHILD_STATUS
|
|
@@ -223,8 +226,8 @@ module Aspera
|
|
|
223
226
|
# Kill the script after a timeout or an error
|
|
224
227
|
# @param pid [Integer] Process id of the script
|
|
225
228
|
def kill_script(pid)
|
|
226
|
-
Process.kill('SIGKILL', pid)
|
|
227
|
-
Process.wait(pid)
|
|
229
|
+
::Process.kill('SIGKILL', pid)
|
|
230
|
+
::Process.wait(pid)
|
|
228
231
|
Log.log.error("Killed process: #{pid}")
|
|
229
232
|
rescue SystemCallError
|
|
230
233
|
# already terminated
|
data/lib/aspera/graphql.rb
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'rest/call_error'
|
|
7
|
+
require_relative 'assert'
|
|
4
8
|
|
|
5
9
|
module Aspera
|
|
6
10
|
# Helper to execute GraphQL queries via a Rest::Client object.
|
data/lib/aspera/hash_ext.rb
CHANGED
data/lib/aspera/id_generator.rb
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'environment'
|
|
5
7
|
require 'uri'
|
|
6
8
|
|
|
7
9
|
module Aspera
|
|
@@ -15,12 +17,25 @@ module Aspera
|
|
|
15
17
|
def from_list(*ids)
|
|
16
18
|
safe_char = Environment.instance.safe_filename_character
|
|
17
19
|
# compact: remove nils
|
|
18
|
-
id = ids.flatten.compact.map
|
|
19
|
-
i.is_a?(String) && i.start_with?('http://', 'https://') ? URI.parse(i).host : i.to_s
|
|
20
|
-
end.join(safe_char)
|
|
20
|
+
id = ids.flatten.compact.map { |i| host_or_text(i) }.join(safe_char)
|
|
21
21
|
# keep dot for extension only (nicer)
|
|
22
22
|
return Environment.instance.sanitized_filename(id.gsub('.', safe_char)).downcase
|
|
23
23
|
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
# A valid HTTP(S) URL with a host: its host name.
|
|
28
|
+
# Any other value is kept as text, e.g. an OAuth scope of several URLs separated by spaces, or an invalid URL.
|
|
29
|
+
# @param value [Object] one object ID
|
|
30
|
+
# @return [String] the host name, or the value as text
|
|
31
|
+
def host_or_text(value)
|
|
32
|
+
return value.to_s unless value.is_a?(String)
|
|
33
|
+
# A plain identifier also parses (as URI::Generic, without host): check the class (URI::HTTPS is a URI::HTTP) and the host
|
|
34
|
+
uri = URI.parse(value)
|
|
35
|
+
uri.is_a?(URI::HTTP) && !uri.host.to_s.empty? ? uri.host : value
|
|
36
|
+
rescue URI::InvalidURIError
|
|
37
|
+
value
|
|
38
|
+
end
|
|
24
39
|
end
|
|
25
40
|
end
|
|
26
41
|
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
7
|
+
require 'zlib'
|
|
8
|
+
|
|
9
|
+
module Aspera
|
|
10
|
+
# Decompression of zlib data that comes from outside (e.g. a code pasted by a user, a token given by a client or a server).
|
|
11
|
+
# A small input can inflate to gigabytes (decompression bomb): the size of the result is checked while decompressing, not after.
|
|
12
|
+
module Inflate
|
|
13
|
+
# Size of the slices of compressed data given to the inflater.
|
|
14
|
+
# Deflate expands at most about 1032 times, so one slice gives at most about 4 MiB.
|
|
15
|
+
# The input is sliced because `Zlib::Inflate#inflate` with a block does not yield chunks on JRuby:
|
|
16
|
+
# it yields the whole result once decompressed.
|
|
17
|
+
INPUT_SLICE = 4096
|
|
18
|
+
private_constant :INPUT_SLICE
|
|
19
|
+
class << self
|
|
20
|
+
# @param data [String] Zlib data
|
|
21
|
+
# @param max_size [Integer] Maximum size of the result, in bytes
|
|
22
|
+
# @param what [String] What is decompressed, for error messages, e.g. `Code`
|
|
23
|
+
# @return [String] Inflated data
|
|
24
|
+
# @raise [ParameterError] If the result is larger than `max_size`, or the data is incomplete
|
|
25
|
+
# @raise [Zlib::Error] If the data is not zlib data
|
|
26
|
+
def limited(data, max_size:, what:)
|
|
27
|
+
output = +''
|
|
28
|
+
inflater = Zlib::Inflate.new
|
|
29
|
+
begin
|
|
30
|
+
0.step(data.bytesize - 1, INPUT_SLICE) do |offset|
|
|
31
|
+
output << inflater.inflate(data.byteslice(offset, INPUT_SLICE))
|
|
32
|
+
raise ParameterError, "#{what} is too large when decompressed: more than #{max_size} bytes" if output.bytesize > max_size
|
|
33
|
+
break if inflater.finished?
|
|
34
|
+
end
|
|
35
|
+
raise ParameterError, "#{what} is incomplete" unless inflater.finished?
|
|
36
|
+
ensure
|
|
37
|
+
inflater.close
|
|
38
|
+
end
|
|
39
|
+
output
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
5
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
require_relative '../rest/error_analyzer'
|
|
7
|
+
require_relative '../rest/call_error'
|
|
8
|
+
require_relative '../assert'
|
|
9
|
+
require_relative 'version'
|
|
9
10
|
|
|
10
11
|
# Error in non-2XX response. Error in 2XX response is raised by the client.
|
|
11
12
|
Aspera::Rest::ErrorAnalyzer.instance.add_simple_handler(name: 'JSON RPC', path: %w[error message])
|
data/lib/aspera/line_logger.rb
CHANGED
data/lib/aspera/link_header.rb
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
3
6
|
require 'strscan'
|
|
4
7
|
|
|
5
8
|
module Aspera
|
|
@@ -72,7 +75,7 @@ module Aspera
|
|
|
72
75
|
end
|
|
73
76
|
|
|
74
77
|
# RFC 2616 token: any char except separators
|
|
75
|
-
TOKEN_RE = /[^()
|
|
78
|
+
TOKEN_RE = /[^()<>@,;:"\[\]?={}\s]+/ # RFC 2616 token
|
|
76
79
|
QUOTED_RE = /"((?:[^"\\]|\\.)*)"/ # double-quoted string with backslash escapes
|
|
77
80
|
HREF_RE = /\s*<([^>]*)>\s*;?\s*/ # <URI> possibly followed by ;
|
|
78
81
|
ATTR_RE = /(#{TOKEN_RE})\s*=\s*(#{TOKEN_RE}|#{QUOTED_RE})\s*/ # key=value or key="value"
|
data/lib/aspera/log.rb
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
7
|
+
require_relative 'secret/hider'
|
|
8
|
+
require_relative 'rainbow'
|
|
5
9
|
require 'logger'
|
|
6
10
|
require 'pp'
|
|
7
11
|
require 'json'
|
|
8
12
|
require 'singleton'
|
|
9
13
|
require 'stringio'
|
|
10
|
-
require 'aspera/rainbow'
|
|
11
14
|
using Rainbow
|
|
12
15
|
|
|
13
16
|
# Ignore warnings
|
|
@@ -111,7 +114,7 @@ module Aspera
|
|
|
111
114
|
Aspera.assert(object.nil? || block.nil?, 'Use either object, or block, not both')
|
|
112
115
|
object = yield if block_given?
|
|
113
116
|
# Secrets hidden by key name in the structure, whatever the dump format (the log formatter then hides secrets in text)
|
|
114
|
-
object =
|
|
117
|
+
object = Secret::Hider.instance.hidden_copy(object) unless Secret::Hider.instance.log_secrets
|
|
115
118
|
dump_text =
|
|
116
119
|
case instance.dump_format
|
|
117
120
|
when :json
|
|
@@ -175,6 +178,16 @@ module Aspera
|
|
|
175
178
|
nil
|
|
176
179
|
end
|
|
177
180
|
|
|
181
|
+
# Send logs to the logger of a host application (e.g. `Rails.logger`), until `logger_type` is set.
|
|
182
|
+
# Its level and formatter are not changed: secrets are hidden in dumps (`Log.dump`), not in log lines.
|
|
183
|
+
# @param logger [Logger] Logger of the host application
|
|
184
|
+
# @return [nil]
|
|
185
|
+
def logger=(logger)
|
|
186
|
+
Aspera.assert(%i[add level].all? { |m| logger.respond_to?(m) }) { "not a logger: #{logger.class}" }
|
|
187
|
+
@logger = logger
|
|
188
|
+
nil
|
|
189
|
+
end
|
|
190
|
+
|
|
178
191
|
# Set log level of underlying logger given symbol level
|
|
179
192
|
# @param new_level [Symbol] One of LEVELS
|
|
180
193
|
def level=(new_level)
|
|
@@ -193,7 +206,7 @@ module Aspera
|
|
|
193
206
|
raise Error, 'Formatter must be a String, a Logger::Formatter or a Proc'
|
|
194
207
|
end
|
|
195
208
|
# Update formatter with password hiding
|
|
196
|
-
@logger.formatter =
|
|
209
|
+
@logger.formatter = Secret::Hider.instance.log_formatter(formatter)
|
|
197
210
|
nil
|
|
198
211
|
end
|
|
199
212
|
|
data/lib/aspera/markdown.rb
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
4
7
|
|
|
5
8
|
module Aspera
|
|
6
9
|
# Formatting for Markdown
|
|
@@ -170,8 +173,8 @@ module Aspera
|
|
|
170
173
|
headings = table.shift
|
|
171
174
|
table.unshift(col_widths.map { |col_width| '-' * col_width })
|
|
172
175
|
table.unshift(headings)
|
|
176
|
+
# Separator row with spaces around pipes, as other rows (markdownlint MD060, style `compact`)
|
|
173
177
|
lines = table.map { |line| "| #{line.map { |i| i.to_s.gsub('\\', '\\\\').gsub('|', '\|') }.join(' | ')} |\n" }
|
|
174
|
-
lines[1] = lines[1].tr(' ', '-')
|
|
175
178
|
return lines.join.chomp
|
|
176
179
|
end
|
|
177
180
|
end
|
data/lib/aspera/mime.rb
CHANGED
data/lib/aspera/nagios.rb
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'log'
|
|
7
|
+
require_relative 'assert'
|
|
6
8
|
|
|
7
9
|
module Aspera
|
|
8
10
|
class Nagios
|
|
@@ -46,7 +48,7 @@ module Aspera
|
|
|
46
48
|
# display status for nagios
|
|
47
49
|
puts("#{status} - [#{message}]\n")
|
|
48
50
|
# provide exit code to nagios
|
|
49
|
-
Process.exit(LEVELS.index(data.first['status'].to_sym))
|
|
51
|
+
::Process.exit(LEVELS.index(data.first['status'].to_sym))
|
|
50
52
|
end
|
|
51
53
|
end
|
|
52
54
|
|
data/lib/aspera/node_emulator.rb
CHANGED
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
7
|
+
require_relative 'ascp/installation'
|
|
8
|
+
require_relative 'agent/transferd'
|
|
9
|
+
require_relative 'transfer/spec'
|
|
10
|
+
require_relative 'transfer/error'
|
|
11
|
+
require_relative 'secret/hider'
|
|
12
|
+
require_relative 'string_ext'
|
|
13
|
+
require_relative 'log'
|
|
14
|
+
require_relative 'servlet_helpers'
|
|
13
15
|
require 'webrick'
|
|
14
16
|
require 'etc'
|
|
15
17
|
require 'securerandom'
|
|
@@ -295,7 +297,7 @@ module Aspera
|
|
|
295
297
|
@node_id = SecureRandom.uuid
|
|
296
298
|
@cluster_id = SecureRandom.uuid
|
|
297
299
|
# transferd runs as the user of the emulator
|
|
298
|
-
@transfer_user = Etc.getpwuid(Process.euid)&.name || Etc.getlogin.to_s
|
|
300
|
+
@transfer_user = Etc.getpwuid(::Process.euid)&.name || Etc.getlogin.to_s
|
|
299
301
|
end
|
|
300
302
|
|
|
301
303
|
# Get the node information
|
|
@@ -345,7 +347,7 @@ module Aspera
|
|
|
345
347
|
def start(transfer_spec)
|
|
346
348
|
Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
|
|
347
349
|
# copy without secrets, returned in transfer `start_spec`
|
|
348
|
-
start_spec =
|
|
350
|
+
start_spec = Secret::Hider.instance.hidden_copy(transfer_spec)
|
|
349
351
|
Log.dump(:start_spec, start_spec)
|
|
350
352
|
request = ::Transferd::Api::TransferRequest.new(
|
|
351
353
|
transferType: ::Transferd::Api::TransferType::FILE_REGULAR,
|
|
@@ -434,7 +436,7 @@ module Aspera
|
|
|
434
436
|
|
|
435
437
|
# @return [Float] current time in seconds, for durations
|
|
436
438
|
def monotonic_time
|
|
437
|
-
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
439
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
438
440
|
end
|
|
439
441
|
|
|
440
442
|
# Convert a Node API path to a real path
|