aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -1,9 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
4
7
  require 'singleton'
5
8
  require 'openssl'
6
9
  require 'zlib'
10
+ require 'base64'
7
11
 
8
12
  module Aspera
9
13
  # a simple binary data repository
@@ -14,39 +18,52 @@ module Aspera
14
18
  class DataRepository
15
19
  include Singleton
16
20
 
17
- # in same order as elements in folder
18
- ELEMENTS = %i[dsa rsa uuid aspera.global-cli-client aspera.drive license]
19
- START_INDEX = 1
21
+ # name of data item => name of its file in data folder
22
+ FILES = {
23
+ dsa: '1',
24
+ rsa: '2',
25
+ uuid: '3',
26
+ 'aspera.global-cli-client': '4',
27
+ 'aspera.drive': '5',
28
+ license: '6'
29
+ }.freeze
30
+ # names of data items
31
+ ELEMENTS = FILES.keys.freeze
20
32
  DATA_FOLDER_NAME = 'data'
21
33
 
22
- # decode data as expected as string
23
- # @param name [Symbol] name of the data item
24
- # @return [String] decoded data
34
+ # Get a data item, decoded to its usable form
35
+ # @param name [Symbol] name of the data item, one of ELEMENTS
36
+ # @return [String] PEM private key (dsa, rsa), UUID (uuid), base64url secret (client apps), or text (license)
25
37
  def item(name)
26
- index = ELEMENTS.index(name)
27
- Aspera.assert(index, type: ParameterError) { "Unknown data item #{name} (#{name.class})" }
28
- raw_data = data(START_INDEX + index)
29
- case name
30
- when :dsa, :rsa
31
- # generate PEM from DER
32
- return OpenSSL::PKey.const_get(name.to_s.upcase).new(raw_data).to_pem
33
- when :license
34
- return Zlib::Inflate.inflate(raw_data)
35
- when :uuid
36
- return format('%08x-%04x-%04x-%04x-%04x%08x', *raw_data.unpack('NnnnnN'))
37
- when :'aspera.global-cli-client', :'aspera.drive'
38
- return Base64.urlsafe_encode64(raw_data)
39
- else Aspera.error_unexpected_value(name)
40
- end
38
+ Aspera.assert_values(name, ELEMENTS, type: ParameterError) { 'data item' }
39
+ raw_data = data(name)
40
+ value =
41
+ case name
42
+ when :dsa, :rsa
43
+ # generate PEM from DER
44
+ # Class of key, not `OpenSSL::PKey.read`: on JRuby (jruby-openssl 0.16), it fails on these keys (DER, traditional format)
45
+ {dsa: OpenSSL::PKey::DSA, rsa: OpenSSL::PKey::RSA}[name].new(raw_data).to_pem
46
+ when :license
47
+ Zlib::Inflate.inflate(raw_data)
48
+ when :uuid
49
+ format('%08x-%04x-%04x-%04x-%04x%08x', *raw_data.unpack('NnnnnN'))
50
+ when :'aspera.global-cli-client', :'aspera.drive'
51
+ Base64.urlsafe_encode64(raw_data)
52
+ else Aspera.error_unexpected_value(name)
53
+ end
54
+ # all items are text
55
+ return value.force_encoding(Encoding::UTF_8)
41
56
  end
42
57
 
43
58
  private
44
59
 
45
- private_constant :START_INDEX, :DATA_FOLDER_NAME
60
+ private_constant :FILES, :DATA_FOLDER_NAME
46
61
 
47
62
  # get binary value from data repository
48
- def data(id)
49
- File.read(File.join(__dir__, DATA_FOLDER_NAME, id.to_s), mode: 'rb')
63
+ # @param name [Symbol] name of the data item
64
+ # @return [String] raw binary data
65
+ def data(name)
66
+ File.binread(File.join(__dir__, DATA_FOLDER_NAME, FILES[name]))
50
67
  end
51
68
  end
52
69
  end
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
4
7
 
5
8
  module Aspera
6
9
  # Convert dotted-path to/from nested Hash/Array container
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require 'strscan'
8
+
9
+ module Aspera
10
+ # Duration in the format of Aspera Sync (`async` DURATION)
11
+ module Duration
12
+ # Microseconds per unit
13
+ UNIT_US = {'w' => 604_800_000_000, 'd' => 86_400_000_000, 'h' => 3_600_000_000, 'm' => 60_000_000, 's' => 1_000_000, 'ms' => 1_000, 'us' => 1}.freeze
14
+ private_constant :UNIT_US
15
+
16
+ class << self
17
+ # Convert a duration to an integer in the given time unit
18
+ # @param value [String] `DDd HH:MM:SS.mmm` or `NNw NNd NNh NNm NNs NNms NNus`, leading fields and spaces may be omitted, a plain number is seconds
19
+ # @param unit [String] Time unit of result, e.g. `ms`
20
+ # @return [Integer] Duration in `unit`, rounded down
21
+ def to_unit(value, unit)
22
+ Aspera.assert_values(unit, UNIT_US.keys) { 'duration unit' }
23
+ value = value.to_s.delete(' ')
24
+ if (m = value.match(/\A(?:(\d+)d)?(?:(?:(\d+):)?(\d+):)?(\d+)(?:\.(\d{1,3}))?\z/))
25
+ days, hours, minutes, seconds, millis = m.captures
26
+ hours = (days.to_i * 24) + hours.to_i
27
+ minutes = (hours * 60) + minutes.to_i
28
+ seconds = (minutes * 60) + seconds.to_i
29
+ microseconds = (seconds * UNIT_US['s']) + (millis.to_s.ljust(3, '0').to_i * UNIT_US['ms'])
30
+ else
31
+ # anchored sequential matching (linear time, no backtracking over start positions)
32
+ scanner = StringScanner.new(value)
33
+ units = []
34
+ until scanner.eos?
35
+ Aspera.assert(scanner.scan(/(\d+)(ms|us|[wdhms])/), type: ParameterError) { "Invalid duration: #{value}" } # cspell:disable-line
36
+ units.push([scanner[1], scanner[2]])
37
+ end
38
+ Aspera.assert(!units.empty?, type: ParameterError) { "Invalid duration: #{value}" }
39
+ microseconds = units.sum { |number, number_unit| number.to_i * UNIT_US[number_unit] }
40
+ end
41
+ microseconds / UNIT_US[unit]
42
+ end
43
+ end
44
+ end
45
+ end
@@ -1,18 +1,21 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore USERPROFILE HOMEDRIVE HOMEPATH LC_CTYPE msys aarch
4
- require 'aspera/log'
5
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'log'
7
+ require_relative 'assert'
8
+ require_relative 'rainbow'
6
9
  require 'rbconfig'
7
10
  require 'singleton'
8
11
  require 'openssl'
9
12
  require 'open3'
10
13
  require 'English'
14
+ require 'io/console'
11
15
  require 'shellwords'
12
- require 'aspera/rainbow'
13
16
  using Rainbow
14
17
 
15
- # cspell:words mswin bccwin
18
+ # cspell:words bccwin conhost Cmder
16
19
 
17
20
  module Aspera
18
21
  # detect OS, architecture, and specific stuff
@@ -40,6 +43,19 @@ module Aspera
40
43
 
41
44
  I18N_VARS = %w(LC_ALL LC_CTYPE LANG).freeze
42
45
 
46
+ # Unicode characters used in messages, and their ASCII equivalent when Unicode is not supported (see `Environment.char`)
47
+ # Key: hidden secret, hieroglyph: app switcher menu of web apps, bust: user menu of web apps
48
+ UNICODE_TO_ASCII = {
49
+ '🔑' => '***',
50
+ '✓' => 'Y',
51
+ '✗' => ' ',
52
+ '→' => '->',
53
+ '…' => '...',
54
+ '—' => '-',
55
+ '𓃑' => '(App switcher)',
56
+ '👤' => '(User)'
57
+ }.freeze
58
+
43
59
  # "/" is invalid on both Unix and Windows, other are Windows special characters
44
60
  # See: https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
45
61
  WINDOWS_FILENAME_INVALID_CHARACTERS = '<>:"/\\|?*'
@@ -62,9 +78,10 @@ module Aspera
62
78
  return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
63
79
  end
64
80
 
65
- # Empty variable binding for secure eval
81
+ # Fresh binding: no local variables, `self` is a plain `Object` (not a sandbox).
82
+ # Constants still resolve through the `Aspera` nesting.
66
83
  def empty_binding
67
- return Kernel.binding
84
+ return Object.new.instance_eval { binding }
68
85
  end
69
86
 
70
87
  # Secure execution of Ruby code
@@ -95,8 +112,8 @@ module Aspera
95
112
  return str if str.match?(%r{\A[A-Za-z0-9_.,:/@+=-]+\z})
96
113
  # return str if Shellwords.shellescape(str) == str
97
114
 
98
- # Otherwise use single quotes
99
- "'#{str.gsub("'", %q('\'\''))}'"
115
+ # Otherwise use single quotes (gsub with block: in a replacement string, \' is the post-match)
116
+ "'#{str.gsub("'") { %q('\'') }}'"
100
117
  end
101
118
 
102
119
  # Executes a command without invoking a shell.
@@ -144,7 +161,7 @@ module Aspera
144
161
  # https://docs.ruby-lang.org/en/master/Process.html#method-c-spawn
145
162
  # https://docs.ruby-lang.org/en/master/Process.html#module-Process-label-Execution+Options
146
163
  kwargs[:close_others] = true unless kwargs.key?(:close_others)
147
- pid = Process.spawn(*build_spawn_argv(cmd, kwargs), **kwargs)
164
+ pid = ::Process.spawn(*build_spawn_argv(cmd, kwargs), **kwargs)
148
165
  Log.dump(:pid, pid)
149
166
  pid
150
167
  when :capture
@@ -205,6 +222,15 @@ module Aspera
205
222
  $stdout.tty?
206
223
  end
207
224
 
225
+ # @return [Integer, nil] Number of columns of the terminal, or `nil` if not a terminal or unknown
226
+ def terminal_columns
227
+ return unless terminal?
228
+ columns = $stdout.winsize[1]
229
+ columns.positive? ? columns : nil
230
+ rescue SystemCallError
231
+ nil
232
+ end
233
+
208
234
  # force locale to C so that unicode characters are not used
209
235
  def force_terminal_c
210
236
  I18N_VARS.each { |var| ENV[var] = 'C' }
@@ -213,29 +239,60 @@ module Aspera
213
239
  # Override detection of Unicode support: `true` or `false`, or `nil` for auto-detection
214
240
  attr_writer :unicode
215
241
 
216
- # @return [Boolean] true if we can display Unicode characters
242
+ # @return [Boolean] true if we can display Unicode characters on stdout
243
+ def terminal_supports_unicode?
244
+ return @unicode unless @unicode.nil?
245
+ terminal? && unicode_environment?
246
+ end
247
+
248
+ # @return [Boolean] true if the locale, or the terminal host on Windows, supports Unicode, whether stdout is a terminal or not
217
249
  # Uses Encoding.locale_charmap for OS-independent detection.
218
- # Falls back to locale env vars for systems where charmap is not available.
250
+ # On Windows, also detects terminal hosts that render Unicode whatever the console code page.
251
+ # Falls back to the locale env var with highest precedence, for locales not installed.
219
252
  # https://www.gnu.org/software/libc/manual/html_node/Locale-Categories.html
220
253
  # https://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html
221
- def terminal_supports_unicode?
222
- return @unicode unless @unicode.nil?
223
- return false unless terminal?
224
- locale_charmap_utf8? || I18N_VARS.any? { |var| ENV[var]&.include?('UTF-8') }
254
+ def unicode_environment?
255
+ return true if locale_charmap_utf8?
256
+ return true if instance.os.eql?(OS_WINDOWS) && windows_unicode_terminal?
257
+ locale = ENV.values_at(*I18N_VARS).find { |value| !value.to_s.empty? }
258
+ locale&.match?(/UTF-?8/i) || false
259
+ end
260
+
261
+ # @param char [String] Unicode character, key of UNICODE_TO_ASCII
262
+ # @param unicode [Boolean] `true` if Unicode is supported
263
+ # @return [String] The character, or its ASCII equivalent if Unicode is not supported
264
+ def char(char, unicode: terminal_supports_unicode?)
265
+ Aspera.assert(UNICODE_TO_ASCII.key?(char), "No ASCII equivalent for: #{char}", type: Aspera::InternalError)
266
+ unicode ? char : UNICODE_TO_ASCII[char]
225
267
  end
226
268
 
227
269
  private
228
270
 
271
+ # @return [Boolean] true if the terminal host renders Unicode
272
+ # Ruby writes to the Windows console with WriteConsoleW (UTF-16): the code page does not matter, the font does.
273
+ # Legacy conhost may lack glyphs (e.g. check marks), while these hosts render them.
274
+ def windows_unicode_terminal?
275
+ # Windows Terminal, VS Code, Git Bash, ConEmu or Cmder, MSYS2 or Cygwin or SSH
276
+ ENV.key?('WT_SESSION') ||
277
+ %w[vscode mintty].include?(ENV.fetch('TERM_PROGRAM', nil)) ||
278
+ ENV.fetch('ConEmuANSI', nil).eql?('ON') ||
279
+ ENV.fetch('TERM', '').match?(/xterm|256color/)
280
+ end
281
+
229
282
  # @return [Boolean] true if the locale charmap resolves to UTF-8
230
283
  # Unix: nl_langinfo(CODESET) returns "UTF-8"
231
- # Windows: GetACP returns "CP65001" which Encoding.find resolves as UTF-8
284
+ # Windows: console code page (GetConsoleCP), else ANSI code page (GetACP): "CP65001" resolves as UTF-8
232
285
  def locale_charmap_utf8?
233
286
  Encoding.find(Encoding.locale_charmap) == Encoding::UTF_8
234
287
  rescue ArgumentError
235
288
  false
236
289
  end
237
290
  end
238
- attr_accessor :url_method, :file_illegal_characters
291
+ # url_method: :text or :graphical, how URLs, secrets and files are presented to the user
292
+ # browser_command: [Array<String>, nil] command (and arguments) that opens a URL (URL is appended), instead of
293
+ # the system default
294
+ # web_auth: :port or :paste, how the result of a web login is received: local port, or URL pasted by the user
295
+ attr_accessor :url_method, :browser_command, :web_auth, :file_illegal_characters
239
296
  attr_reader :os, :cpu, :default_gui_mode
240
297
 
241
298
  def initialize
@@ -279,6 +336,8 @@ module Aspera
279
336
  :text
280
337
  end
281
338
  @url_method = @default_gui_mode
339
+ @browser_command = nil
340
+ @web_auth = :port
282
341
  @file_illegal_characters = REPLACE_CHARACTER + WINDOWS_FILENAME_INVALID_CHARACTERS
283
342
  nil
284
343
  end
@@ -327,15 +386,24 @@ module Aspera
327
386
  @default_gui_mode == :graphical
328
387
  end
329
388
 
330
- # Open a URI in the system's default graphical browser (non-blocking)
389
+ # Open a URI in the browser (non-blocking).
390
+ # The browser is, by priority: `browser_command`, the system default.
331
391
  # @param uri [String, URI] the URI to open
332
392
  # @return [nil]
333
393
  def open_uri_graphical(uri)
334
- case @os
335
- when Environment::OS_MACOS then self.class.secure_execute('open', uri.to_s)
336
- when Environment::OS_WINDOWS then self.class.secure_execute('start', 'explorer', %Q{"#{uri}"})
337
- when Environment::OS_LINUX then self.class.secure_execute('xdg-open', uri.to_s)
338
- else Aspera.error_unexpected_value(os) { 'no graphical open method' }
394
+ if @browser_command
395
+ begin
396
+ self.class.secure_execute(*@browser_command, uri.to_s)
397
+ rescue Errno::ENOENT
398
+ raise ParameterError, "Browser command not found (option ui.command): #{@browser_command.first}. Check that it is in PATH, or use its full path."
399
+ end
400
+ else
401
+ case @os
402
+ when Environment::OS_MACOS then self.class.secure_execute('open', uri.to_s)
403
+ when Environment::OS_WINDOWS then self.class.secure_execute('start', 'explorer', %Q{"#{uri}"})
404
+ when Environment::OS_LINUX then self.class.secure_execute('xdg-open', uri.to_s)
405
+ else Aspera.error_unexpected_value(os) { 'no graphical open method' }
406
+ end
339
407
  end
340
408
  nil
341
409
  end
@@ -352,21 +420,26 @@ module Aspera
352
420
  end
353
421
 
354
422
  # Allows a user to open a URL
423
+ # if a browser command is defined (`browser_command`), then it is used, whatever the method
355
424
  # if method is :text, then URL is displayed on terminal
356
425
  # if method is :graphical, then the URL will be opened with the default browser.
357
426
  # this is non blocking
358
427
  def open_uri(the_url)
359
- case @url_method
360
- when :graphical
428
+ if @browser_command
361
429
  open_uri_graphical(the_url)
362
- when :text
363
- case the_url.to_s
364
- when /^http/
365
- puts "USER ACTION: please enter this URL in a browser:\n#{the_url.to_s.red}\n"
366
- else
367
- puts "USER ACTION: open this:\n#{the_url.to_s.red}\n"
430
+ else
431
+ case @url_method
432
+ when :graphical
433
+ open_uri_graphical(the_url)
434
+ when :text
435
+ case the_url.to_s
436
+ when /^http/
437
+ puts "USER ACTION: please enter this URL in a browser:\n#{the_url.to_s.red}\n"
438
+ else
439
+ puts "USER ACTION: open this:\n#{the_url.to_s.red}\n"
440
+ end
441
+ else Aspera.error_unexpected_value(@url_method) { 'URL open method' }
368
442
  end
369
- else Aspera.error_unexpected_value(@url_method) { 'URL open method' }
370
443
  end
371
444
  end
372
445
 
@@ -1,13 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
- # Holds the executable name, environment variables and command line arguments for a spawned process.
5
- ExecSpec = Struct.new(:exec, :env, :args, keyword_init: true) do
6
- def initialize(exec: nil, env: {}, args: [], **) = super
7
+ # Holds the executable name, environment variables and command line arguments for a spawned process,
8
+ # and the temporary files it uses (for example a file list), which must exist until it ends.
9
+ ExecSpec = Struct.new(:exec, :env, :args, :temp_files, keyword_init: true) do
10
+ def initialize(exec: nil, env: {}, args: [], temp_files: [], **) = super
7
11
 
8
- # @return [ExecSpec] deep copy with independent :args and :env
12
+ # @return [ExecSpec] deep copy with independent :args, :env and :temp_files
9
13
  def deep_clone
10
- self.class.new(exec: exec, env: env.dup, args: args.dup)
14
+ self.class.new(exec: exec, env: env.dup, args: args.dup, temp_files: temp_files.dup)
11
15
  end
12
16
  end
13
17
  end
@@ -1,21 +1,71 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'aspera/mime'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'api/faspex'
7
+ require_relative 'log'
8
+ require_relative 'assert'
9
+ require_relative 'hash_ext'
10
+ require_relative 'mime'
11
+ require_relative 'secret/strength'
12
+ require_relative 'servlet_helpers'
13
+ require_relative 'web_server_simple'
6
14
  require 'webrick'
7
15
  require 'json'
8
16
 
9
17
  module Aspera
10
18
  # Simulate the Faspex 4 /send API and creates a package on Aspera on Cloud or Faspex 5
19
+ #
20
+ # Security notes:
21
+ # - Packages are created as the user of the application, to any recipient, and the caller receives an upload transfer spec.
22
+ # - Clients authenticate with HTTP Basic, as on Faspex 4, when `username` and `password` are set:
23
+ # they are required when the server listens on an address that is not loopback (checked by `serve`).
24
+ # - Requests that a web page could send are refused (see ServletHelpers#check_request): the body shall be `application/json`.
11
25
  class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
26
+ include ServletHelpers
27
+
12
28
  AOC_API = 'Aspera::Api::AoC'
13
29
  FX_API = 'Aspera::Api::Faspex'
14
- # @param app_api [Rest::Client] API object
15
- # @param app_context [String] workspace id (aoc only)
16
- def initialize(server, app_api, app_context)
30
+ ALLOWED_PARAMETERS = %i[username password password_min_bits].freeze
31
+ REALM = 'Faspex 4 Gateway'
32
+ private_constant :REALM
33
+
34
+ class << self
35
+ # Check parameters and set defaults
36
+ # Called at startup, so that errors are reported before the first request
37
+ # @param parameters [Hash] Parameters of the servlet
38
+ # @return [Hash] Parameters with defaults
39
+ def normalize_parameters(parameters)
40
+ Aspera.assert_type(parameters, Hash)
41
+ parameters = parameters.symbolize_keys
42
+ not_allowed = parameters.keys - ALLOWED_PARAMETERS
43
+ Aspera.assert(not_allowed.empty?, type: ParameterError) { "unsupported parameters: #{not_allowed.join(', ')}" }
44
+ Aspera.assert(parameters[:username].nil? == parameters[:password].nil?, type: ParameterError) { 'Parameters username and password must be set together' }
45
+ parameters[:password_min_bits] ||= Secret::Strength::DEFAULT_MIN_BITS
46
+ Secret::Strength.assert_strong(parameters[:password], name: 'password', min_bits: parameters[:password_min_bits]) unless parameters[:password].nil?
47
+ parameters
48
+ end
49
+
50
+ # Start the gateway, until signal INT or TERM
51
+ # @param parameters [Hash] Parameters of the server (see `WebServerSimple.url`) and of the servlet (see `normalize_parameters`)
52
+ # @param type [Exception] Exception raised when the server is not protected
53
+ # @yieldreturn [Array] API of the application, and workspace id (AoC only), see `initialize`: called once parameters are checked
54
+ def serve(parameters, type: ParameterError)
55
+ servlet_parameters = normalize_parameters(WebServerSimple.servlet_parameters(parameters))
56
+ # Packages can be sent to anyone by anyone who can reach the server
57
+ WebServerSimple.serve(parameters, self, protected: !servlet_parameters[:username].nil?, what: 'username and password', type: type) { [servlet_parameters, *yield] }
58
+ end
59
+ end
60
+
61
+ # @param server [WEBrick::HTTPServer] Server
62
+ # @param parameters [Hash] Parameters from `normalize_parameters`
63
+ # @param app_api [Rest::Client] API object
64
+ # @param app_context [String] workspace id (aoc only)
65
+ def initialize(server, parameters, app_api, app_context)
17
66
  Aspera.assert_values(app_api.class.name, [AOC_API, FX_API])
18
67
  super(server)
68
+ @expected_auth = "#{parameters[:username]}:#{parameters[:password]}" unless parameters[:username].nil?
19
69
  @app_api = app_api
20
70
  @app_context = app_context
21
71
  end
@@ -66,37 +116,57 @@ module Aspera
66
116
  end
67
117
 
68
118
  def do_POST(request, response)
69
- case request.path
70
- when '/aspera/faspex/send'
119
+ # Refuse requests of web pages
120
+ check_request(request, json: true)
121
+ return json_error(response, 401, 'Invalid or missing credentials', headers: {'WWW-Authenticate' => %Q(Basic realm="#{REALM}")}) unless authorized?(request)
122
+ return json_error(response, 400, 'Unsupported endpoint') unless request.path.eql?('/aspera/faspex/send')
123
+ faspex_pkg_parameters = package_parameters(request.body)
124
+ Log.dump(:faspex_pkg_parameters, faspex_pkg_parameters)
125
+ # compare string, as class is not yet known here
126
+ faspex_package_create_result =
127
+ case @app_api.class.name
128
+ when AOC_API
129
+ faspex4_send_to_aoc(faspex_pkg_parameters)
130
+ when FX_API
131
+ faspex4_send_to_faspex5(faspex_pkg_parameters)
132
+ else Aspera.error_unexpected_value(@app_api.class.name)
133
+ end
134
+ Log.dump(:faspex_package_create_result, faspex_package_create_result)
135
+ json_response(response, 200, faspex_package_create_result)
136
+ rescue WEBrick::HTTPStatus::Error => e
137
+ Log.log.warn { "Rejected request: #{e.message}" }
138
+ json_error(response, e.code, e.message)
139
+ rescue StandardError => e
140
+ # The stack trace is logged only: it is not for the caller
141
+ Log.log.error(e.message)
142
+ Log.log.debug { e.backtrace.join("\n") }
143
+ json_error(response, 500, e.message)
144
+ end
145
+
146
+ private
147
+
148
+ # @param request [WEBrick::HTTPRequest]
149
+ # @return [Boolean] `true` if the request has the expected Basic credentials, or if none are expected
150
+ def authorized?(request)
151
+ @expected_auth.nil? || basic_authorized?(request, @expected_auth)
152
+ end
153
+
154
+ # @param body [String, nil] Body of the request
155
+ # @return [Hash] Parameters of Faspex 4 package creation, with `delivery` and its `recipients`
156
+ # @raise [WEBrick::HTTPStatus::BadRequest] if the body is not a valid package creation request
157
+ def package_parameters(body)
158
+ raise WEBrick::HTTPStatus::BadRequest, 'Empty request' if body.nil?
159
+ parameters =
71
160
  begin
72
- Aspera.assert(!request.body.nil?, 'payload missing')
73
- faspex_pkg_parameters = JSON.parse(request.body)
74
- Log.dump(:faspex_pkg_parameters, faspex_pkg_parameters)
75
- # compare string, as class is not yet known here
76
- faspex_package_create_result =
77
- case @app_api.class.name
78
- when AOC_API
79
- faspex4_send_to_aoc(faspex_pkg_parameters)
80
- when FX_API
81
- faspex4_send_to_faspex5(faspex_pkg_parameters)
82
- else Aspera.error_unexpected_value(@app_api.class.name)
83
- end
84
- Log.dump(:faspex_package_create_result, faspex_package_create_result)
85
- response.status = 200
86
- response.content_type = Mime::JSON
87
- response.body = JSON.generate(faspex_package_create_result)
88
- rescue StandardError => e
89
- response.status = 500
90
- response['Content-Type'] = Mime::JSON
91
- response.body = {error: e.message, stacktrace: e.backtrace}.to_json
92
- Log.log.error(e.message)
93
- Log.log.debug { e.backtrace.join("\n") }
161
+ JSON.parse(body)
162
+ rescue JSON::ParserError
163
+ nil
94
164
  end
95
- else
96
- response.status = 400
97
- response['Content-Type'] = Mime::JSON
98
- response.body = {error: 'Unsupported endpoint'}.to_json
99
- end
165
+ raise WEBrick::HTTPStatus::BadRequest, 'Request body shall be a JSON object' unless parameters.is_a?(Hash)
166
+ raise WEBrick::HTTPStatus::BadRequest, 'Missing object: delivery' unless parameters['delivery'].is_a?(Hash)
167
+ recipients = parameters['delivery']['recipients']
168
+ raise WEBrick::HTTPStatus::BadRequest, 'delivery.recipients shall be a non-empty list of names' unless recipients.is_a?(Array) && !recipients.empty? && recipients.all?(String)
169
+ parameters
100
170
  end
101
171
  end
102
172
  end