aspera-cli 4.28.0 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +143 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1633 -609
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +20 -20
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +71 -65
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +16 -8
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +12 -19
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -33
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +6 -1
  50. data/lib/aspera/cli/command_spec.rb +34 -7
  51. data/lib/aspera/cli/completion/ascli.bash +19 -5
  52. data/lib/aspera/cli/completion/ascli.fish +13 -2
  53. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  54. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  55. data/lib/aspera/cli/completion_actions.rb +303 -69
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +3 -0
  59. data/lib/aspera/cli/extended_value.rb +61 -27
  60. data/lib/aspera/cli/formatter.rb +29 -22
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -10
  63. data/lib/aspera/cli/http.rb +12 -9
  64. data/lib/aspera/cli/info.rb +12 -7
  65. data/lib/aspera/cli/mailer.rb +6 -3
  66. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  67. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  68. data/lib/aspera/cli/mcp/tool.rb +15 -14
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +65 -12
  73. data/lib/aspera/cli/options.schema.yaml +487 -63
  74. data/lib/aspera/cli/parser.rb +56 -32
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +244 -35
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +17 -12
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +198 -103
  81. data/lib/aspera/cli/plugins/console.rb +6 -3
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +21 -7
  84. data/lib/aspera/cli/plugins/faspex5.rb +28 -25
  85. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  86. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +28 -22
  88. data/lib/aspera/cli/plugins/node.rb +31 -30
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +63 -11
  93. data/lib/aspera/cli/plugins/shares.rb +86 -16
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +144 -30
  96. data/lib/aspera/cli/preset_manager.rb +52 -18
  97. data/lib/aspera/cli/prompt.rb +4 -1
  98. data/lib/aspera/cli/result.rb +12 -8
  99. data/lib/aspera/cli/runner.rb +69 -46
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +79 -9
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +3 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +29 -20
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +188 -42
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.rb +12 -10
  113. data/lib/aspera/cli/wizard.rb +42 -39
  114. data/lib/aspera/command_line_builder.rb +9 -5
  115. data/lib/aspera/command_line_converter.rb +4 -1
  116. data/lib/aspera/coverage.rb +103 -33
  117. data/lib/aspera/data_repository.rb +4 -1
  118. data/lib/aspera/dot_container.rb +4 -1
  119. data/lib/aspera/duration.rb +5 -2
  120. data/lib/aspera/environment.rb +21 -12
  121. data/lib/aspera/exec_spec.rb +3 -0
  122. data/lib/aspera/faspex_gw.rb +13 -9
  123. data/lib/aspera/faspex_postproc.rb +15 -12
  124. data/lib/aspera/formatter_interface.rb +4 -1
  125. data/lib/aspera/graphql.rb +5 -1
  126. data/lib/aspera/hash_ext.rb +3 -0
  127. data/lib/aspera/id_generator.rb +4 -2
  128. data/lib/aspera/json_rpc/client.rb +6 -5
  129. data/lib/aspera/json_rpc/version.rb +3 -0
  130. data/lib/aspera/line_logger.rb +4 -1
  131. data/lib/aspera/link_header.rb +4 -1
  132. data/lib/aspera/log.rb +18 -5
  133. data/lib/aspera/markdown.rb +5 -2
  134. data/lib/aspera/mime.rb +3 -0
  135. data/lib/aspera/nagios.rb +6 -4
  136. data/lib/aspera/node_emulator.rb +15 -13
  137. data/lib/aspera/oauth/client/base.rb +91 -70
  138. data/lib/aspera/oauth/client/boot.rb +7 -4
  139. data/lib/aspera/oauth/client/factory.rb +6 -3
  140. data/lib/aspera/oauth/client/generic.rb +5 -1
  141. data/lib/aspera/oauth/client/json_credentials.rb +4 -1
  142. data/lib/aspera/oauth/client/jwt.rb +6 -3
  143. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  144. data/lib/aspera/oauth/client/url_json.rb +4 -1
  145. data/lib/aspera/oauth/client/web.rb +39 -12
  146. data/lib/aspera/oauth/server/pkce.rb +6 -5
  147. data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
  148. data/lib/aspera/persistency_action_once.rb +5 -3
  149. data/lib/aspera/persistency_folder.rb +6 -3
  150. data/lib/aspera/preview/file_types.rb +6 -3
  151. data/lib/aspera/preview/generator.rb +9 -5
  152. data/lib/aspera/preview/options.schema.yaml +3 -3
  153. data/lib/aspera/preview/terminal.rb +8 -4
  154. data/lib/aspera/preview/utils.rb +6 -4
  155. data/lib/aspera/products/connect.rb +5 -2
  156. data/lib/aspera/products/desktop.rb +4 -1
  157. data/lib/aspera/products/other.rb +4 -2
  158. data/lib/aspera/products/transferd.rb +6 -2
  159. data/lib/aspera/products/transferd_service.rb +610 -0
  160. data/lib/aspera/proxy_auto_config.js +2 -2
  161. data/lib/aspera/proxy_auto_config.rb +5 -2
  162. data/lib/aspera/rainbow.rb +3 -0
  163. data/lib/aspera/rest/aspera_errors.rb +4 -1
  164. data/lib/aspera/rest/call_error.rb +3 -0
  165. data/lib/aspera/rest/client.rb +22 -29
  166. data/lib/aspera/rest/error_analyzer.rb +5 -2
  167. data/lib/aspera/rest/list.rb +8 -4
  168. data/lib/aspera/rest/parameters.rb +3 -0
  169. data/lib/aspera/rest/util.rb +8 -5
  170. data/lib/aspera/rest.rb +9 -6
  171. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  172. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  173. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  174. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  175. data/lib/aspera/schema/async_tables.yaml +11 -10
  176. data/lib/aspera/schema/documentation.rb +8 -2
  177. data/lib/aspera/schema/reader.rb +7 -0
  178. data/lib/aspera/schema/registry.rb +43 -28
  179. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  180. data/lib/aspera/schema/validator.rb +4 -1
  181. data/lib/aspera/sealed_hash.rb +166 -0
  182. data/lib/aspera/secret/hider.rb +201 -0
  183. data/lib/aspera/secret/reference.rb +150 -0
  184. data/lib/aspera/secret/resolver.rb +212 -0
  185. data/lib/aspera/secret/store/base.rb +153 -0
  186. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  187. data/lib/aspera/secret/store/factory.rb +103 -0
  188. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  189. data/lib/aspera/secret/store/keeper.rb +141 -0
  190. data/lib/aspera/secret/store/macos_security.rb +284 -0
  191. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  192. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  193. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  194. data/lib/aspera/secret/store/secret_service.rb +125 -0
  195. data/lib/aspera/secret/strength.rb +119 -0
  196. data/lib/aspera/servlet_helpers.rb +5 -2
  197. data/lib/aspera/ssh.rb +10 -7
  198. data/lib/aspera/ssl.rb +5 -2
  199. data/lib/aspera/string_ext.rb +3 -0
  200. data/lib/aspera/sync/args.schema.yaml +3 -0
  201. data/lib/aspera/sync/conf.schema.yaml +9 -4
  202. data/lib/aspera/sync/database.rb +3 -0
  203. data/lib/aspera/sync/operations.rb +16 -14
  204. data/lib/aspera/temp_file_manager.rb +8 -4
  205. data/lib/aspera/timer_limiter.rb +3 -0
  206. data/lib/aspera/transfer/error.rb +4 -1
  207. data/lib/aspera/transfer/faux_file.rb +4 -1
  208. data/lib/aspera/transfer/parameters.rb +17 -14
  209. data/lib/aspera/transfer/result.rb +3 -0
  210. data/lib/aspera/transfer/resumer.rb +6 -4
  211. data/lib/aspera/transfer/spec.rb +9 -22
  212. data/lib/aspera/transfer/units.rb +75 -0
  213. data/lib/aspera/uri_reader.rb +57 -7
  214. data/lib/aspera/web_auth/index.html.erb +6 -2
  215. data/lib/aspera/web_auth/paste.rb +109 -0
  216. data/lib/aspera/web_auth/receiver.rb +50 -0
  217. data/lib/aspera/web_auth/relay.rb +76 -0
  218. data/lib/aspera/web_auth/reply.html.erb +150 -0
  219. data/lib/aspera/web_auth/server.rb +47 -10
  220. data/lib/aspera/web_page.rb +7 -2
  221. data/lib/aspera/web_server_simple.rb +6 -3
  222. data/lib/aspera/yaml.rb +5 -1
  223. data/llms.txt +81 -0
  224. data/skills/ascli/SKILL.md +88 -0
  225. data.tar.gz.sig +0 -0
  226. metadata +72 -19
  227. metadata.gz.sig +0 -0
  228. data/lib/aspera/keychain/base.rb +0 -30
  229. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  230. data/lib/aspera/keychain/factory.rb +0 -58
  231. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  232. data/lib/aspera/keychain/macos_security.rb +0 -277
  233. data/lib/aspera/keychain/one_password_api.rb +0 -91
  234. data/lib/aspera/keychain/one_password_base.rb +0 -39
  235. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  236. data/lib/aspera/secret_hider.rb +0 -196
  237. data/lib/aspera/secret_strength.rb +0 -114
@@ -1,39 +1,109 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # coverage for tests
4
- if ENV.key?('ENABLE_COVERAGE')
5
- require 'simplecov'
6
- require 'securerandom'
7
- require 'fileutils'
8
- # compute development top folder based on this source location
9
- development_root = File.dirname(File.realpath(__FILE__), 3)
10
- coverage_dir = 'tmp/coverage'
11
- coverage_root = File.join(development_root, coverage_dir)
12
- FileUtils.mkdir_p(coverage_root)
13
- SimpleCov.root(development_root)
14
- SimpleCov.coverage_dir(coverage_dir)
15
- SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
16
- # keep cache data for 1 day (must be longer than time to run the whole test suite)
17
- SimpleCov.merge_timeout(86400)
18
- SimpleCov.command_name(SecureRandom.uuid)
19
- SimpleCov.at_exit do
20
- original_file_descriptor = $stdout
21
- $stdout.reopen(File.open(File.join(coverage_root, 'simplecov.log'), 'a'))
22
- SimpleCov.result.format!
23
- $stdout.reopen(original_file_descriptor)
24
- end
25
- # lines with those words are ignored from coverage
26
- no_cov_functions = %w[error_unreachable_line error_unexpected_value Log.log.trace].freeze
27
- SimpleCov.start do
28
- # assert usually do not trigger
29
- add_filter do |source_file|
30
- source_file.lines.each do |line|
31
- line.skipped! if no_cov_functions.any? { |i| line.src.include?(i) }
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ module Aspera
7
+ # Test coverage with SimpleCov, enabled with env var `ENABLE_COVERAGE`.
8
+ # Each tested process stores only its own result in `parts/<uuid>`: merging all results at each process exit is slow when there are many processes.
9
+ # The report is generated by `report`, which merges all parts (called by the test Rake tasks).
10
+ module Coverage
11
+ # Development top folder, computed from this source location
12
+ ROOT = File.dirname(File.realpath(__FILE__), 3)
13
+ # Report folder, relative to `ROOT`
14
+ DIR = 'tmp/coverage'
15
+ # One sub-folder per process, each with its own `.resultset.json`
16
+ PARTS = File.join(DIR, 'parts')
17
+ # Coverage badge (shields.io endpoint), relative to `ROOT`
18
+ BADGE = 'docs/coverage.json'
19
+ # Badge color: first one whose lower limit is reached
20
+ BADGE_COLORS = {90 => 'brightgreen', 80 => 'green', 70 => 'yellowgreen', 60 => 'yellow', 50 => 'orange'}.freeze
21
+ # Results are kept for 1 day (must be longer than time to run the whole test suite, possibly in several runs)
22
+ MERGE_TIMEOUT = 86_400
23
+ # Lines with those words are ignored from coverage (assertions usually do not trigger)
24
+ NO_COV_FUNCTIONS = %w[error_unreachable_line error_unexpected_value Log.log.trace].freeze
25
+ # No coverage in these files or folders
26
+ NO_COV_PATHS = %w[
27
+ build/
28
+ tmp/
29
+ lib/transferd_pb.rb
30
+ lib/transferd_services_pb.rb
31
+ lib/aspera/secret/store/macos_security.rb
32
+ lib/aspera/assert.rb
33
+ ].freeze
34
+ private_constant :BADGE_COLORS, :MERGE_TIMEOUT, :NO_COV_FUNCTIONS, :NO_COV_PATHS
35
+
36
+ class << self
37
+ # Start coverage of current process.
38
+ # Must be called before any library is loaded.
39
+ def start
40
+ require 'simplecov'
41
+ require 'securerandom'
42
+ name = SecureRandom.uuid
43
+ SimpleCov.root(ROOT)
44
+ SimpleCov.coverage_dir(File.join(PARTS, name))
45
+ SimpleCov.command_name(name)
46
+ SimpleCov.merge_subprocesses(true)
47
+ SimpleCov.merge_timeout(MERGE_TIMEOUT)
48
+ # The trend of a single process is meaningless
49
+ SimpleCov.history_limit(0)
50
+ # Only store this process result (in its own folder): no report
51
+ SimpleCov.at_exit { SimpleCov.result }
52
+ SimpleCov.start { Aspera::Coverage.filters(self) }
53
+ end
54
+
55
+ # Merge the results of all processes and generate the report in `DIR`.
56
+ # Results older than `MERGE_TIMEOUT` are deleted.
57
+ # @return [Float, nil] Percentage of covered lines, or `nil` if there is no result
58
+ def report
59
+ require 'simplecov'
60
+ require 'fileutils'
61
+ require 'etc'
62
+ parts = Dir.glob(File.join(ROOT, PARTS, '*'))
63
+ old, parts = parts.partition { |dir| Time.now - File.mtime(dir) > MERGE_TIMEOUT }
64
+ FileUtils.rm_rf(old)
65
+ results = parts.map { |dir| File.join(dir, '.resultset.json') }.select { |file| File.exist?(file) }
66
+ return if results.empty?
67
+ # Else the merged result would be added to the previous one
68
+ FileUtils.rm_f(File.join(ROOT, DIR, '.resultset.json'))
69
+ SimpleCov.root(ROOT)
70
+ SimpleCov.coverage_dir(DIR)
71
+ percent = nil
72
+ # The merged command name lists all process UUIDs: show their number instead
73
+ SimpleCov.at_exit do
74
+ result = SimpleCov.result
75
+ result.command_name = "#{result.command_names.size} processes"
76
+ result.format!
77
+ percent = result.covered_percent
78
+ end
79
+ SimpleCov.collate(results, processes: Etc.nprocessors) { Aspera::Coverage.filters(self) }
80
+ percent
81
+ end
82
+
83
+ # Write the coverage badge: a shields.io endpoint file, shown in the top `README.md`
84
+ # @param percent [Float] Percentage of covered lines
85
+ def write_badge(percent)
86
+ require 'json'
87
+ color = BADGE_COLORS.find { |limit, _| percent >= limit }&.last || 'red'
88
+ File.write(File.join(ROOT, BADGE), "#{JSON.pretty_generate(schemaVersion: 1, label: 'coverage', message: format('%.1f%%', percent), color: color)}\n")
89
+ end
90
+
91
+ # Configure coverage criteria and filters (in `SimpleCov.start` or `SimpleCov.collate` block)
92
+ # Branch coverage is measured in addition to line coverage (shown in the report, the badge shows lines).
93
+ # @param config [SimpleCov] Receiver of the configuration block
94
+ def filters(config)
95
+ config.enable_coverage(:branch)
96
+ config.skip do |source_file|
97
+ source_file.lines.each do |line|
98
+ line.skipped! if NO_COV_FUNCTIONS.any? { |i| line.src.include?(i) }
99
+ end
100
+ false
101
+ end
102
+ NO_COV_PATHS.each { |path| config.skip(path) }
32
103
  end
33
- false
34
104
  end
35
- # no coverage in these files
36
- add_filter 'lib/aspera/keychain/macos_security.rb'
37
- add_filter 'lib/aspera/assert.rb'
38
105
  end
39
106
  end
107
+
108
+ # Start coverage when this file is loaded, if enabled: must be required before any other library
109
+ Aspera::Coverage.start if ENV.key?('ENABLE_COVERAGE')
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
4
7
  require 'singleton'
5
8
  require 'openssl'
6
9
  require 'zlib'
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
4
7
 
5
8
  module Aspera
6
9
  # Convert dotted-path to/from nested Hash/Array container
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
4
7
  require 'strscan'
5
8
 
6
9
  module Aspera
@@ -29,7 +32,7 @@ module Aspera
29
32
  scanner = StringScanner.new(value)
30
33
  units = []
31
34
  until scanner.eos?
32
- Aspera.assert(scanner.scan(/(\d+)(ms|us|[wdhms])/), type: ParameterError) { "Invalid duration: #{value}" }
35
+ Aspera.assert(scanner.scan(/(\d+)(ms|us|[wdhms])/), type: ParameterError) { "Invalid duration: #{value}" } # cspell:disable-line
33
36
  units.push([scanner[1], scanner[2]])
34
37
  end
35
38
  Aspera.assert(!units.empty?, type: ParameterError) { "Invalid duration: #{value}" }
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore USERPROFILE HOMEDRIVE HOMEPATH LC_CTYPE msys aarch
4
- require 'aspera/log'
5
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'log'
7
+ require_relative 'assert'
8
+ require_relative 'rainbow'
6
9
  require 'rbconfig'
7
10
  require 'singleton'
8
11
  require 'openssl'
@@ -10,10 +13,9 @@ require 'open3'
10
13
  require 'English'
11
14
  require 'io/console'
12
15
  require 'shellwords'
13
- require 'aspera/rainbow'
14
16
  using Rainbow
15
17
 
16
- # cspell:words mswin bccwin conhost mintty Cmder
18
+ # cspell:words bccwin conhost Cmder
17
19
 
18
20
  module Aspera
19
21
  # detect OS, architecture, and specific stuff
@@ -76,9 +78,10 @@ module Aspera
76
78
  return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
77
79
  end
78
80
 
79
- # Empty variable binding for secure eval
81
+ # Fresh binding: no local variables, `self` is a plain `Object` (not a sandbox).
82
+ # Constants still resolve through the `Aspera` nesting.
80
83
  def empty_binding
81
- return Kernel.binding
84
+ return Object.new.instance_eval { binding }
82
85
  end
83
86
 
84
87
  # Secure execution of Ruby code
@@ -109,8 +112,8 @@ module Aspera
109
112
  return str if str.match?(%r{\A[A-Za-z0-9_.,:/@+=-]+\z})
110
113
  # return str if Shellwords.shellescape(str) == str
111
114
 
112
- # Otherwise use single quotes
113
- "'#{str.gsub("'", %q('\'\''))}'"
115
+ # Otherwise use single quotes (gsub with block: in a replacement string, \' is the post-match)
116
+ "'#{str.gsub("'") { %q('\'') }}'"
114
117
  end
115
118
 
116
119
  # Executes a command without invoking a shell.
@@ -158,7 +161,7 @@ module Aspera
158
161
  # https://docs.ruby-lang.org/en/master/Process.html#method-c-spawn
159
162
  # https://docs.ruby-lang.org/en/master/Process.html#module-Process-label-Execution+Options
160
163
  kwargs[:close_others] = true unless kwargs.key?(:close_others)
161
- pid = Process.spawn(*build_spawn_argv(cmd, kwargs), **kwargs)
164
+ pid = ::Process.spawn(*build_spawn_argv(cmd, kwargs), **kwargs)
162
165
  Log.dump(:pid, pid)
163
166
  pid
164
167
  when :capture
@@ -288,7 +291,8 @@ module Aspera
288
291
  # url_method: :text or :graphical, how URLs, secrets and files are presented to the user
289
292
  # browser_command: [Array<String>, nil] command (and arguments) that opens a URL (URL is appended), instead of
290
293
  # the system default
291
- attr_accessor :url_method, :browser_command, :file_illegal_characters
294
+ # web_auth: :port or :paste, how the result of a web login is received: local port, or URL pasted by the user
295
+ attr_accessor :url_method, :browser_command, :web_auth, :file_illegal_characters
292
296
  attr_reader :os, :cpu, :default_gui_mode
293
297
 
294
298
  def initialize
@@ -333,6 +337,7 @@ module Aspera
333
337
  end
334
338
  @url_method = @default_gui_mode
335
339
  @browser_command = nil
340
+ @web_auth = :port
336
341
  @file_illegal_characters = REPLACE_CHARACTER + WINDOWS_FILENAME_INVALID_CHARACTERS
337
342
  nil
338
343
  end
@@ -387,7 +392,11 @@ module Aspera
387
392
  # @return [nil]
388
393
  def open_uri_graphical(uri)
389
394
  if @browser_command
390
- self.class.secure_execute(*@browser_command, uri.to_s)
395
+ begin
396
+ self.class.secure_execute(*@browser_command, uri.to_s)
397
+ rescue Errno::ENOENT
398
+ raise ParameterError, "Browser command not found (option ui.command): #{@browser_command.first}. Check that it is in PATH, or use its full path."
399
+ end
391
400
  else
392
401
  case @os
393
402
  when Environment::OS_MACOS then self.class.secure_execute('open', uri.to_s)
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  # Holds the executable name, environment variables and command line arguments for a spawned process,
5
8
  # and the temporary files it uses (for example a file list), which must exist until it ends.
@@ -1,12 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'aspera/hash_ext'
6
- require 'aspera/mime'
7
- require 'aspera/secret_strength'
8
- require 'aspera/servlet_helpers'
9
- require 'aspera/web_server_simple'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'api/faspex'
7
+ require_relative 'log'
8
+ require_relative 'assert'
9
+ require_relative 'hash_ext'
10
+ require_relative 'mime'
11
+ require_relative 'secret/strength'
12
+ require_relative 'servlet_helpers'
13
+ require_relative 'web_server_simple'
10
14
  require 'webrick'
11
15
  require 'json'
12
16
 
@@ -38,8 +42,8 @@ module Aspera
38
42
  not_allowed = parameters.keys - ALLOWED_PARAMETERS
39
43
  Aspera.assert(not_allowed.empty?, type: ParameterError) { "unsupported parameters: #{not_allowed.join(', ')}" }
40
44
  Aspera.assert(parameters[:username].nil? == parameters[:password].nil?, type: ParameterError) { 'Parameters username and password must be set together' }
41
- parameters[:password_min_bits] ||= SecretStrength::DEFAULT_MIN_BITS
42
- SecretStrength.assert_strong(parameters[:password], name: 'password', min_bits: parameters[:password_min_bits]) unless parameters[:password].nil?
45
+ parameters[:password_min_bits] ||= Secret::Strength::DEFAULT_MIN_BITS
46
+ Secret::Strength.assert_strong(parameters[:password], name: 'password', min_bits: parameters[:password_min_bits]) unless parameters[:password].nil?
43
47
  parameters
44
48
  end
45
49
 
@@ -1,18 +1,21 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'log'
7
+ require_relative 'assert'
8
+ require_relative 'environment'
9
+ require_relative 'hash_ext'
10
+ require_relative 'secret/strength'
11
+ require_relative 'servlet_helpers'
12
+ require_relative 'web_server_simple'
3
13
  require 'json'
4
14
  require 'timeout'
5
15
  require 'English'
6
16
  require 'ipaddr'
7
17
  require 'openssl'
8
18
  require 'webrick'
9
- require 'aspera/log'
10
- require 'aspera/assert'
11
- require 'aspera/environment'
12
- require 'aspera/hash_ext'
13
- require 'aspera/secret_strength'
14
- require 'aspera/servlet_helpers'
15
- require 'aspera/web_server_simple'
16
19
 
17
20
  module Aspera
18
21
  # Start a Faspex-4 style post-processing script using Faspex-5 webhook
@@ -49,9 +52,9 @@ module Aspera
49
52
  parameters[:script_folder] ||= '.'
50
53
  parameters[:fail_on_error] ||= false
51
54
  parameters[:timeout_seconds] ||= 60
52
- parameters[:secret_token_min_bits] ||= SecretStrength::DEFAULT_MIN_BITS
55
+ parameters[:secret_token_min_bits] ||= Secret::Strength::DEFAULT_MIN_BITS
53
56
  Aspera.assert(File.directory?(parameters[:script_folder])) { "script_folder not found: #{parameters[:script_folder]}" }
54
- SecretStrength.assert_strong(parameters[:secret_token], name: 'secret_token', min_bits: parameters[:secret_token_min_bits]) if parameters.key?(:secret_token)
57
+ Secret::Strength.assert_strong(parameters[:secret_token], name: 'secret_token', min_bits: parameters[:secret_token_min_bits]) if parameters.key?(:secret_token)
55
58
  # allowed_ips: optional IP addresses or networks (CIDR); nil means no restriction
56
59
  parameters[:allowed_ips] = Array(parameters[:allowed_ips]).map { |ip| ip_address(ip) } if parameters.key?(:allowed_ips)
57
60
  parameters
@@ -153,7 +156,7 @@ module Aspera
153
156
  post_proc_pid = Environment.secure_execute(script_path, mode: :background, env: script_environment(webhook_parameters))
154
157
  Timeout.timeout(@parameters[:timeout_seconds]) do
155
158
  # "wait" for process to avoid zombie
156
- Process.wait(post_proc_pid)
159
+ ::Process.wait(post_proc_pid)
157
160
  post_proc_pid = nil
158
161
  end
159
162
  process_status = $CHILD_STATUS
@@ -223,8 +226,8 @@ module Aspera
223
226
  # Kill the script after a timeout or an error
224
227
  # @param pid [Integer] Process id of the script
225
228
  def kill_script(pid)
226
- Process.kill('SIGKILL', pid)
227
- Process.wait(pid)
229
+ ::Process.kill('SIGKILL', pid)
230
+ ::Process.wait(pid)
228
231
  Log.log.error("Killed process: #{pid}")
229
232
  rescue SystemCallError
230
233
  # already terminated
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
4
7
 
5
8
  module Aspera
6
9
  # Common interface for documentation formatters
@@ -1,6 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/rest/call_error'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'rest/call_error'
7
+ require_relative 'assert'
4
8
 
5
9
  module Aspera
6
10
  # Helper to execute GraphQL queries via a Rest::Client object.
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  class ::Hash
4
7
  # Recursively merge another hash, with nested hashes merged deeply
5
8
  # @param second [Hash] Hash to merge into self
@@ -1,7 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
4
- require 'aspera/environment'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'environment'
5
7
  require 'uri'
6
8
 
7
9
  module Aspera
@@ -1,11 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore jsonrpc
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
4
5
 
5
- require 'aspera/rest/error_analyzer'
6
- require 'aspera/rest/call_error'
7
- require 'aspera/assert'
8
- require 'aspera/json_rpc/version'
6
+ require_relative '../rest/error_analyzer'
7
+ require_relative '../rest/call_error'
8
+ require_relative '../assert'
9
+ require_relative 'version'
9
10
 
10
11
  # Error in non-2XX response. Error in 2XX response is raised by the client.
11
12
  Aspera::Rest::ErrorAnalyzer.instance.add_simple_handler(name: 'JSON RPC', path: %w[error message])
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  module JsonRpc
5
8
  # Protocol version: value of `jsonrpc` in request and response
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'log'
4
7
 
5
8
  module Aspera
6
9
  # used for logging http
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  require 'strscan'
4
7
 
5
8
  module Aspera
@@ -72,7 +75,7 @@ module Aspera
72
75
  end
73
76
 
74
77
  # RFC 2616 token: any char except separators
75
- TOKEN_RE = /[^()<>@,;:\"\[\]?={}\s]+/ # RFC 2616 token
78
+ TOKEN_RE = /[^()<>@,;:"\[\]?={}\s]+/ # RFC 2616 token
76
79
  QUOTED_RE = /"((?:[^"\\]|\\.)*)"/ # double-quoted string with backslash escapes
77
80
  HREF_RE = /\s*<([^>]*)>\s*;?\s*/ # <URI> possibly followed by ;
78
81
  ATTR_RE = /(#{TOKEN_RE})\s*=\s*(#{TOKEN_RE}|#{QUOTED_RE})\s*/ # key=value or key="value"
data/lib/aspera/log.rb CHANGED
@@ -1,13 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
4
- require 'aspera/secret_hider'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'secret/hider'
8
+ require_relative 'rainbow'
5
9
  require 'logger'
6
10
  require 'pp'
7
11
  require 'json'
8
12
  require 'singleton'
9
13
  require 'stringio'
10
- require 'aspera/rainbow'
11
14
  using Rainbow
12
15
 
13
16
  # Ignore warnings
@@ -111,7 +114,7 @@ module Aspera
111
114
  Aspera.assert(object.nil? || block.nil?, 'Use either object, or block, not both')
112
115
  object = yield if block_given?
113
116
  # Secrets hidden by key name in the structure, whatever the dump format (the log formatter then hides secrets in text)
114
- object = SecretHider.instance.hidden_copy(object) unless SecretHider.instance.log_secrets
117
+ object = Secret::Hider.instance.hidden_copy(object) unless Secret::Hider.instance.log_secrets
115
118
  dump_text =
116
119
  case instance.dump_format
117
120
  when :json
@@ -175,6 +178,16 @@ module Aspera
175
178
  nil
176
179
  end
177
180
 
181
+ # Send logs to the logger of a host application (e.g. `Rails.logger`), until `logger_type` is set.
182
+ # Its level and formatter are not changed: secrets are hidden in dumps (`Log.dump`), not in log lines.
183
+ # @param logger [Logger] Logger of the host application
184
+ # @return [nil]
185
+ def logger=(logger)
186
+ Aspera.assert(%i[add level].all? { |m| logger.respond_to?(m) }) { "not a logger: #{logger.class}" }
187
+ @logger = logger
188
+ nil
189
+ end
190
+
178
191
  # Set log level of underlying logger given symbol level
179
192
  # @param new_level [Symbol] One of LEVELS
180
193
  def level=(new_level)
@@ -193,7 +206,7 @@ module Aspera
193
206
  raise Error, 'Formatter must be a String, a Logger::Formatter or a Proc'
194
207
  end
195
208
  # Update formatter with password hiding
196
- @logger.formatter = SecretHider.instance.log_formatter(formatter)
209
+ @logger.formatter = Secret::Hider.instance.log_formatter(formatter)
197
210
  nil
198
211
  end
199
212
 
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
4
7
 
5
8
  module Aspera
6
9
  # Formatting for Markdown
@@ -170,8 +173,8 @@ module Aspera
170
173
  headings = table.shift
171
174
  table.unshift(col_widths.map { |col_width| '-' * col_width })
172
175
  table.unshift(headings)
176
+ # Separator row with spaces around pipes, as other rows (markdownlint MD060, style `compact`)
173
177
  lines = table.map { |line| "| #{line.map { |i| i.to_s.gsub('\\', '\\\\').gsub('|', '\|') }.join(' | ')} |\n" }
174
- lines[1] = lines[1].tr(' ', '-')
175
178
  return lines.join.chomp
176
179
  end
177
180
  end
data/lib/aspera/mime.rb CHANGED
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  # MIME types used in `Content-Type` and `Accept`
5
8
  module Mime
data/lib/aspera/nagios.rb CHANGED
@@ -1,8 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'date'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'log'
7
+ require_relative 'assert'
6
8
 
7
9
  module Aspera
8
10
  class Nagios
@@ -46,7 +48,7 @@ module Aspera
46
48
  # display status for nagios
47
49
  puts("#{status} - [#{message}]\n")
48
50
  # provide exit code to nagios
49
- Process.exit(LEVELS.index(data.first['status'].to_sym))
51
+ ::Process.exit(LEVELS.index(data.first['status'].to_sym))
50
52
  end
51
53
  end
52
54
 
@@ -1,15 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore precalc noxfer euid gettime
4
- require 'aspera/assert'
5
- require 'aspera/ascp/installation'
6
- require 'aspera/agent/transferd'
7
- require 'aspera/transfer/spec'
8
- require 'aspera/transfer/error'
9
- require 'aspera/secret_hider'
10
- require 'aspera/string_ext'
11
- require 'aspera/log'
12
- require 'aspera/servlet_helpers'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'ascp/installation'
8
+ require_relative 'agent/transferd'
9
+ require_relative 'transfer/spec'
10
+ require_relative 'transfer/error'
11
+ require_relative 'secret/hider'
12
+ require_relative 'string_ext'
13
+ require_relative 'log'
14
+ require_relative 'servlet_helpers'
13
15
  require 'webrick'
14
16
  require 'etc'
15
17
  require 'securerandom'
@@ -295,7 +297,7 @@ module Aspera
295
297
  @node_id = SecureRandom.uuid
296
298
  @cluster_id = SecureRandom.uuid
297
299
  # transferd runs as the user of the emulator
298
- @transfer_user = Etc.getpwuid(Process.euid)&.name || Etc.getlogin.to_s
300
+ @transfer_user = Etc.getpwuid(::Process.euid)&.name || Etc.getlogin.to_s
299
301
  end
300
302
 
301
303
  # Get the node information
@@ -345,7 +347,7 @@ module Aspera
345
347
  def start(transfer_spec)
346
348
  Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
347
349
  # copy without secrets, returned in transfer `start_spec`
348
- start_spec = SecretHider.instance.hidden_copy(transfer_spec)
350
+ start_spec = Secret::Hider.instance.hidden_copy(transfer_spec)
349
351
  Log.dump(:start_spec, start_spec)
350
352
  request = ::Transferd::Api::TransferRequest.new(
351
353
  transferType: ::Transferd::Api::TransferType::FILE_REGULAR,
@@ -434,7 +436,7 @@ module Aspera
434
436
 
435
437
  # @return [Float] current time in seconds, for durations
436
438
  def monotonic_time
437
- Process.clock_gettime(Process::CLOCK_MONOTONIC)
439
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
438
440
  end
439
441
 
440
442
  # Convert a Node API path to a real path