aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -0,0 +1,119 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+
9
+ module Aspera
10
+ module Secret
11
+ # Estimate of the strength of a secret (e.g. a token), in bits: log2 of the number of guesses to find it
12
+ #
13
+ # Uses gem `zxcvbn` if installed (dictionaries, l33t, dates...), else a built-in estimate, the lowest of:
14
+ # - alphabet: length x log2(size of character classes used)
15
+ # - Shannon: empirical entropy of characters (catches repeats)
16
+ # - patterns: repeats, sequences and keyboard rows cost 1 bit per character, words and years 2 bits per character
17
+ # Both under-estimate random secrets: a random secret of 32 bytes (e.g. `openssl rand -hex 32`) is far above any usual minimum.
18
+ module Strength
19
+ # Default minimum strength of tokens, in bits: random tokens of 32 bytes are far above
20
+ DEFAULT_MIN_BITS = 80
21
+ # Classes of characters, with their size
22
+ CHARACTER_CLASSES = [[/[a-z]/, 26], [/[A-Z]/, 26], [/\d/, 10], [/[^a-zA-Z\d]/, 33]].freeze
23
+ # Adjacent characters in those rows form a sequence
24
+ KEYBOARD_ROWS = %w[1234567890 qwertyuiop asdfghjkl zxcvbnm azertyuiop qsdfghjklm wxcvbn].freeze
25
+ # Words (3+ letters, optionally capitalized) and years
26
+ LOW_ENTROPY_REGEXES = [/[A-Z]?[a-z]{3,}/, /(?:19|20)\d\d/].freeze
27
+ # Entropy of English text is 1 to 1.5 bit per letter: keep a margin
28
+ LOW_ENTROPY_BITS = 2.0
29
+ # zxcvbn is slow on long secrets: the beginning is enough to assess strength
30
+ ZXCVBN_MAX_LENGTH = 100
31
+ private_constant :CHARACTER_CLASSES, :KEYBOARD_ROWS, :LOW_ENTROPY_REGEXES, :LOW_ENTROPY_BITS, :ZXCVBN_MAX_LENGTH
32
+
33
+ class << self
34
+ # @param secret [String] Secret to assess
35
+ # @return [Float] Estimated strength in bits
36
+ def bits(secret)
37
+ secret = secret.to_s
38
+ return 0.0 if secret.empty?
39
+ zxcvbn_bits(secret) || estimate_bits(secret)
40
+ end
41
+
42
+ # Refuse an empty or weak secret: an empty one would often accept requests without secret
43
+ # @param secret [String] Secret to check
44
+ # @param name [String] Name of the parameter holding the secret, its minimum is in parameter `<name>_min_bits`
45
+ # @param min_bits [Integer] Minimum estimated strength in bits, `0` checks only that the secret is not empty
46
+ # @param type [Exception] Exception to raise
47
+ # @raise [Exception] if the secret is empty or weaker than `min_bits`
48
+ def assert_strong(secret, name:, min_bits: DEFAULT_MIN_BITS, type: ParameterError)
49
+ Aspera.assert(!secret.to_s.empty?, type: type) { "#{name} must not be empty" }
50
+ strength = bits(secret)
51
+ Aspera.assert(strength >= min_bits, type: type) do
52
+ "#{name} is too weak: about #{strength.round} bits, minimum #{min_bits} (#{name}_min_bits). Generate one with: openssl rand -hex 32"
53
+ end
54
+ end
55
+
56
+ # Built-in estimate, without dictionary
57
+ # @param secret [String] Secret to assess
58
+ # @return [Float] Estimated strength in bits
59
+ def estimate_bits(secret)
60
+ return 0.0 if secret.empty?
61
+ bits_per_char = Math.log2(CHARACTER_CLASSES.sum { |regex, size| secret.match?(regex) ? size : 0 })
62
+ [secret.size * bits_per_char, shannon_bits(secret), pattern_bits(secret, bits_per_char)].min
63
+ end
64
+
65
+ private
66
+
67
+ # @return [Float, nil] Strength estimated by gem `zxcvbn`, or `nil` if not installed
68
+ def zxcvbn_bits(secret)
69
+ return unless zxcvbn?
70
+ Math.log2(Zxcvbn.test(secret[0, ZXCVBN_MAX_LENGTH]).guesses)
71
+ rescue StandardError => e
72
+ Log.log.debug { "zxcvbn failed: #{e.message}" }
73
+ nil
74
+ end
75
+
76
+ # @return [Boolean] `true` if gem `zxcvbn` (or `zxcvbn-ruby`) is installed
77
+ def zxcvbn?
78
+ return @zxcvbn unless @zxcvbn.nil?
79
+ @zxcvbn =
80
+ begin
81
+ require 'zxcvbn'
82
+ true
83
+ rescue LoadError
84
+ false
85
+ end
86
+ end
87
+
88
+ # Empirical Shannon entropy of characters of the secret
89
+ def shannon_bits(secret)
90
+ secret.chars.tally.values.sum do |count|
91
+ probability = count.fdiv(secret.size)
92
+ -probability * Math.log2(probability)
93
+ end * secret.size
94
+ end
95
+
96
+ # Sum of bits of each character, depending on patterns
97
+ def pattern_bits(secret, bits_per_char)
98
+ # Bits of characters in words and years
99
+ low = Array.new(secret.size)
100
+ LOW_ENTROPY_REGEXES.each do |regex|
101
+ secret.scan(regex) { low.fill(LOW_ENTROPY_BITS, Regexp.last_match.begin(0)...Regexp.last_match.end(0)) }
102
+ end
103
+ secret.chars.each_with_index.sum do |char, index|
104
+ next 1.0 if index.positive? && sequence?(secret[index - 1], char)
105
+ low[index] || bits_per_char
106
+ end
107
+ end
108
+
109
+ # @return [Boolean] `true` if `char` repeats `previous`, or follows it in alphabet, digits or keyboard row
110
+ def sequence?(previous, char)
111
+ previous = previous.downcase
112
+ char = char.downcase
113
+ return true if (char.ord - previous.ord).abs <= 1
114
+ KEYBOARD_ROWS.any? { |row| (one = row.index(previous)) && (two = row.index(char)) && (one - two).abs.eql?(1) }
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,114 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'mime'
7
+ require_relative 'web_server_simple'
8
+ require 'json'
9
+ require 'openssl'
10
+ require 'uri'
11
+ require 'webrick'
12
+
13
+ module Aspera
14
+ # Mixin for WEBrick servlets of local web servers: JSON responses, and check of requests that a web page could send.
15
+ #
16
+ # A web page open in the browser of the user can send requests to a server that the browser reaches, also on localhost:
17
+ # - Cross-site request forgery: a form, or a `fetch` with a CORS-safelisted `Content-Type` (`text/plain`, form), is sent cross-origin without preflight.
18
+ # - DNS rebinding: a host name of the attacker that resolves to `127.0.0.1` makes the page same-origin with the server, so it also reads responses.
19
+ module ServletHelpers
20
+ class << self
21
+ # @param url [String] URL
22
+ # @return [String] Origin of the URL, as in header `Origin`, e.g. `https://example.com:8443`
23
+ def origin(url)
24
+ uri = URI.parse(url)
25
+ port = ":#{uri.port}" unless uri.port.eql?(uri.default_port)
26
+ "#{uri.scheme}://#{uri.host}#{port}".downcase
27
+ end
28
+
29
+ # @param host [String] Value of header `Host`, e.g. `[::1]:8080`
30
+ # @return [String] Host name, without port and IPv6 brackets
31
+ def host_name(host)
32
+ return host[/\A\[([^\]]*)\]/, 1].to_s if host.start_with?('[')
33
+ host.split(':').first.to_s
34
+ end
35
+
36
+ # Scheme is not compared: header `Host` has none
37
+ # @param origin [String] Value of header `Origin`, e.g. `http://localhost:8080`
38
+ # @param host [String] Value of header `Host`, e.g. `localhost:8080`
39
+ # @return [Boolean] `true` if the origin is the one of the requested server
40
+ def same_origin?(origin, host)
41
+ default_port = origin.start_with?('https://') ? ':443' : ':80'
42
+ origin.sub(%r{\Ahttps?://}, '').delete_suffix(default_port).eql?(host.delete_suffix(default_port))
43
+ end
44
+ end
45
+
46
+ # Set a JSON response
47
+ # @param response [WEBrick::HTTPResponse]
48
+ # @param status [Integer] HTTP status code
49
+ # @param body [Hash, Array] Body of the response
50
+ # @param headers [Hash{String => String}] Other headers of the response, e.g. `Cache-Control`
51
+ # @return [nil]
52
+ def json_response(response, status, body, headers: {})
53
+ response.status = status
54
+ response['Content-Type'] = Mime::JSON
55
+ headers.each { |name, value| response[name] = value }
56
+ response.body = JSON.generate(body)
57
+ nil
58
+ end
59
+
60
+ # Set a JSON error response, in the format of OAuth 2.0 (RFC 6749 §5.2)
61
+ # @param response [WEBrick::HTTPResponse]
62
+ # @param status [Integer] HTTP status code
63
+ # @param error [String] Error code or message
64
+ # @param description [String, nil] Details
65
+ # @param headers [Hash{String => String}] Other headers of the response, e.g. `WWW-Authenticate`
66
+ # @return [nil]
67
+ def json_error(response, status, error, description = nil, headers: {})
68
+ body = {error: error}
69
+ body[:error_description] = description unless description.nil?
70
+ json_response(response, status, body, headers: headers)
71
+ end
72
+
73
+ # @param request [WEBrick::HTTPRequest]
74
+ # @param scheme [String] Authentication scheme, e.g. `Basic` or `Bearer`
75
+ # @return [String, nil] Credentials of header `Authorization` if it has this scheme, else `nil`
76
+ def authorization(request, scheme)
77
+ request_scheme, credentials = request['Authorization'].to_s.split(' ', 2)
78
+ credentials.to_s if request_scheme.to_s.casecmp?(scheme)
79
+ end
80
+
81
+ # Check the credentials of HTTP Basic authentication, in constant time
82
+ # @param request [WEBrick::HTTPRequest]
83
+ # @param expected [String] Expected `username:password`
84
+ # @return [Boolean] `true` if the request has the expected credentials
85
+ def basic_authorized?(request, expected)
86
+ OpenSSL.secure_compare(authorization(request, 'Basic').to_s.unpack1('m').to_s, expected)
87
+ end
88
+
89
+ # Refuse a request that a web page in a browser could send. Clients that are not browsers send no `Origin`, and the `Host` of the URL they call.
90
+ # - `Host`: when the server listens on loopback, a loopback name or address, or one of `allowed_hosts` (DNS rebinding)
91
+ # - `Origin`: when present, the origin of the server, or one of `allowed_origins` (cross-site request)
92
+ # - `Content-Type`: with `json`, a JSON type, which a browser sends cross-origin only after a preflight request `OPTIONS`, not answered (cross-site request)
93
+ # @param request [WEBrick::HTTPRequest]
94
+ # @param json [Boolean] `true` if the body, if any, is parsed as JSON
95
+ # @param allowed_hosts [Array<String>] Other values accepted in `Host`: host name, or `host:port`
96
+ # @param allowed_origins [Array<String>] Other values accepted in `Origin`, e.g. `https://example.com`
97
+ # @raise [WEBrick::HTTPStatus::Forbidden] if `Host` or `Origin` is refused
98
+ # @raise [WEBrick::HTTPStatus::UnsupportedMediaType] if the body is not JSON
99
+ def check_request(request, json: false, allowed_hosts: [], allowed_origins: [])
100
+ host = request['Host']&.downcase
101
+ if host && WebServerSimple.loopback?(@server[:BindAddress].to_s)
102
+ name = ServletHelpers.host_name(host)
103
+ host_allowed = WebServerSimple.loopback?(name) || allowed_hosts.any? { |allowed_host| [name, host].include?(allowed_host.downcase) }
104
+ raise WEBrick::HTTPStatus::Forbidden, "Host is not a loopback address: #{host}" unless host_allowed
105
+ end
106
+ origin = request['Origin']&.downcase
107
+ origin_allowed = origin.nil? || (host && ServletHelpers.same_origin?(origin, host)) || allowed_origins.any? { |allowed_origin| allowed_origin.downcase.eql?(origin) }
108
+ raise WEBrick::HTTPStatus::Forbidden, "Origin is not the one of the server: #{origin}" unless origin_allowed
109
+ return unless json && !request.body.to_s.empty?
110
+ media_type = request['Content-Type'].to_s.split(';').first.to_s.strip.downcase
111
+ raise WEBrick::HTTPStatus::UnsupportedMediaType, "Content-Type shall be #{Mime::JSON}: #{media_type}" unless Mime.json?(media_type)
112
+ end
113
+ end
114
+ end
data/lib/aspera/ssh.rb CHANGED
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'log'
3
8
  require 'net/ssh'
4
- require 'aspera/assert'
5
- require 'aspera/log'
6
9
 
7
10
  module Aspera
8
11
  # A simple wrapper around Net::SSH
@@ -16,8 +19,7 @@ module Aspera
16
19
 
17
20
  class << self
18
21
  # Removes ed25519 keys from the default identity list used by Net::SSH.
19
- # Called when the `ed25519` gem is absent or explicitly disabled via
20
- # `ASCLI_ENABLE_ED25519=false`.
22
+ # Called when explicitly disabled via `ASCLI_ENABLE_ED25519=false`.
21
23
  # @return [void]
22
24
  def disable_ed25519_keys
23
25
  Log.log.debug('Disabling SSH ed25519 user keys')
@@ -99,7 +101,7 @@ module Aspera
99
101
  Log.dump(:data, data, level: :trace1)
100
102
  exit_code = data.read_long
101
103
  end
102
- # send command to SSH channel (execute) cspell: disable-next-line
104
+ # send command to SSH channel (execute)
103
105
  channel.send('cexe'.reverse, cmd) do |_ch, success|
104
106
  raise Error, "could not execute command: #{cmd}" unless success
105
107
  channel.send_data(input) unless input.nil?
@@ -118,6 +120,7 @@ module Aspera
118
120
  end
119
121
  end
120
122
 
121
- # Deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
122
- Aspera::Ssh.disable_ed25519_keys if Gem::Specification.find_all_by_name('ed25519').none? || ENV.fetch('ASCLI_ENABLE_ED25519', 'true').eql?('false')
123
+ # Deactivate ed25519 private keys from SSH identities if requested (ed25519 is natively supported by net-ssh 8 using OpenSSL)
124
+ Aspera::Ssh.disable_ed25519_keys if ENV.fetch('ASCLI_ENABLE_ED25519', 'true').eql?('false')
125
+ # Deactivate ecdsa algorithms with JRuby, as it usually causes problems
123
126
  Aspera::Ssh.disable_ecd_sha2_algorithms if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')
data/lib/aspera/ssl.rb CHANGED
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'log'
3
8
  require 'openssl'
4
- require 'aspera/assert'
5
- require 'aspera/log'
6
9
 
7
10
  module Aspera
8
11
  # Give possibility to globally override SSL options
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  class ::String
4
7
  # Transform capitalized to snake case
5
8
  def capital_to_snake
@@ -10,6 +10,7 @@ type: object
10
10
  properties:
11
11
  alt_logdir:
12
12
  type: string
13
+ x-cli-complete: folder
13
14
  x-cli-option: true
14
15
  watchd:
15
16
  type: string
@@ -37,12 +38,14 @@ properties:
37
38
  x-cli-option: true
38
39
  local_dir:
39
40
  type: string
41
+ x-cli-complete: folder
40
42
  x-cli-option: true
41
43
  remote_dir:
42
44
  type: string
43
45
  x-cli-option: true
44
46
  local_db_dir:
45
47
  type: string
48
+ x-cli-complete: folder
46
49
  x-cli-option: true
47
50
  remote_db_dir:
48
51
  type: string
@@ -77,8 +80,10 @@ properties:
77
80
  type: string
78
81
  x-cli-option: true
79
82
  target_rate:
80
- type: string
83
+ type: integer
81
84
  x-cli-option: true
85
+ x-ts-name: target_rate_kbps
86
+ x-ts-convert: kbps_to_bps
82
87
  cooloff:
83
88
  type: integer
84
89
  x-cli-option: true
@@ -6,6 +6,7 @@ $comment: >-
6
6
  It is modified with special `x-` fields.
7
7
  x-ts-name [Boolean,String] `true` if same name in transfer spec, else real name in transfer spec
8
8
  x-ts-convert [String] Name of methods to convert value from transfer spec to `conf`
9
+ x-unit [String] Time unit of the integer value (e.g. `ms`), the command line option is a DURATION
9
10
  `x-` fields are also documented in `command_line_builder.rb`
10
11
  type: object
11
12
  required:
@@ -16,6 +17,7 @@ properties:
16
17
  ascp_dir:
17
18
  description: Directory containing ascp executable to use.
18
19
  type: string
20
+ x-cli-complete: folder
19
21
  assume_no_mods:
20
22
  description: Assume that the directory structure has not been modified.
21
23
  type: boolean
@@ -163,14 +165,14 @@ properties:
163
165
  x-cli-switch: true
164
166
  ignore_mode:
165
167
  description:
166
- Source files that have had their mode changed after the initial.
168
+ Source files that have had their mode changed after the initial
167
169
  transfer will not update the destination file mode.
168
170
  type: boolean
169
171
  default: false
170
172
  x-cli-option: true
171
173
  x-cli-switch: true
172
174
  ignore_remote_host_sync_name:
173
- description: Do not check that the remote host being used for the current.
175
+ description: Do not check that the remote host being used for the current
174
176
  transfer matches the host used when the local database was created
175
177
  type: boolean
176
178
  default: false
@@ -185,6 +187,7 @@ properties:
185
187
  path:
186
188
  description: The directory to be synchronized on the local host.
187
189
  type: string
190
+ x-cli-complete: folder
188
191
  x-cli-option: --local-dir
189
192
  x-cli-short: -d
190
193
  local_apply_docroot:
@@ -197,6 +200,7 @@ properties:
197
200
  description: Use the specified database directory on the local host.
198
201
  Default is `.private-asp` at the root level of the synchronized directory.
199
202
  type: string
203
+ x-cli-complete: folder
200
204
  x-cli-option: true
201
205
  x-cli-short: -b
202
206
  local_checksum_threads:
@@ -227,6 +231,7 @@ properties:
227
231
  local_keep_dir:
228
232
  description: Move deleted files into the specified directory on the local host.
229
233
  type: string
234
+ x-cli-complete: folder
230
235
  x-cli-option: --keep-dir-local
231
236
  local_mount_signature:
232
237
  description: Verify that the file system is mounted by the existence of this file on the local host.
@@ -260,10 +265,14 @@ properties:
260
265
  local_scan_interval_milliseconds:
261
266
  description:
262
267
  Enable periodic scans on the local host during a continuous sync.
263
- `-1` for disabled
268
+ `-1` for disabled.
269
+ On the command line, the value is a duration, e.g. `1h 30m`, `2s500ms`, `5:30`, `1d 02:00:00.250`.
270
+ A plain number is in seconds.
271
+ Stored in milliseconds.
264
272
  type: integer
265
273
  default: -1
266
274
  x-cli-option: --scan-interval
275
+ x-unit: ms
267
276
  local_scan_threads:
268
277
  description: Number of directory scanning threads on the local host.
269
278
  Value must be between 1 and 99
@@ -291,6 +300,7 @@ properties:
291
300
  local_dir:
292
301
  description: Use the specified logging directory on the local host.
293
302
  type: string
303
+ x-cli-complete: folder
294
304
  x-cli-option: --alt-logdir
295
305
  x-cli-short: -L
296
306
  remote_dir:
@@ -506,8 +516,8 @@ properties:
506
516
  description: Authenticate to the proxy with the specified password.
507
517
  type: string
508
518
  port:
509
- description: Use the specified port, default is 9091 for dnat, 9092.
510
- for dnats
519
+ description: Use the specified port.
520
+ Default is 9091 for dnat, 9092 for dnats.
511
521
  type: integer
512
522
  protocol:
513
523
  description: The protocol to be used.
@@ -602,10 +612,14 @@ properties:
602
612
  remote_scan_interval_milliseconds:
603
613
  description: Enable periodic scans on the remote host.
604
614
  `-1` for disabled.
615
+ On the command line, the value is a duration, e.g. `1h 30m`, `2s500ms`, `5:30`, `1d 02:00:00.250`.
616
+ A plain number is in seconds.
617
+ Stored in milliseconds.
605
618
  type: integer
606
619
  default: -1
607
620
  x-cli-option: --remote-scan-interval
608
621
  x-cli-special: true
622
+ x-unit: ms
609
623
  remote_scan_threads:
610
624
  description: Number of directory scanning threads on the remote host.
611
625
  Value must be between 1 and 99.
@@ -665,7 +679,7 @@ properties:
665
679
  x-cli-option: true
666
680
  x-cli-switch: true
667
681
  scan_dir_rename:
668
- description: Enable the detection of renamed directories and files compared.
682
+ description: Enable the detection of renamed directories and files compared
669
683
  to the previous scan, based on matching inodes
670
684
  type: boolean
671
685
  default: false
@@ -683,11 +697,11 @@ properties:
683
697
  `vhigh` maximizes system activity by continuously scanning files without rest.
684
698
  type: string
685
699
  enum:
686
- - vlow
700
+ - vlow # cspell:disable-line
687
701
  - low
688
702
  - medium
689
703
  - high
690
- - vhigh
704
+ - vhigh # cspell:disable-line
691
705
  default: medium
692
706
  x-cli-option: true
693
707
  x-cli-short: -H
@@ -797,7 +811,7 @@ properties:
797
811
  type: integer
798
812
  x-cli-option: true
799
813
  x-cli-short: -g
800
- rexmsg_size:
814
+ rexmsg_size: # cspell:disable-line
801
815
  description:
802
816
  Use the specified size (in bytes) for a retransmission request.
803
817
  Default is determined by `aspera.conf`.
@@ -806,6 +820,7 @@ properties:
806
820
  x-cli-short: -X
807
821
  target_rate:
808
822
  description: Transfer no faster than the specified rate (in bps).
823
+ A string with suffix `k`, `m` or `g` is converted to bps by `ascli` (e.g. `100m`).
809
824
  type: integer
810
825
  default: 10000000
811
826
  x-ts-name: target_rate_kbps
@@ -836,7 +851,7 @@ properties:
836
851
  enum:
837
852
  - none
838
853
  - redis
839
- - scalekv
854
+ - scalekv # cspell:disable-line
840
855
  default: none
841
856
  domain:
842
857
  description: Specify the domain.
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  if defined?(JRUBY_VERSION)
4
7
  require 'jdbc/sqlite3'
5
8
  Jdbc::SQLite3.load_driver
@@ -1,16 +1,20 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:words logdir bidi watchd cooloff asyncadmin
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
4
5
 
5
- require 'aspera/ascp/installation'
6
- require 'aspera/agent/direct'
7
- require 'aspera/exec_spec'
8
- require 'aspera/command_line_converter'
9
- require 'aspera/command_line_builder'
10
- require 'aspera/log'
11
- require 'aspera/assert'
12
- require 'aspera/environment'
13
- require 'aspera/schema/registry'
6
+ require_relative '../ascp/installation'
7
+ require_relative '../agent/direct'
8
+ require_relative '../transfer/spec'
9
+ require_relative '../transfer/units'
10
+ require_relative '../exec_spec'
11
+ require_relative '../command_line_converter'
12
+ require_relative '../command_line_builder'
13
+ require_relative '../log'
14
+ require_relative '../assert'
15
+ require_relative '../duration'
16
+ require_relative '../environment'
17
+ require_relative '../schema/registry'
14
18
  require 'json'
15
19
  require 'base64'
16
20
  require 'open3'
@@ -98,11 +102,13 @@ module Aspera
98
102
  remote = sync_info['remote']
99
103
  Aspera.assert_type(remote, Hash) { 'remote' }
100
104
  Aspera.assert_type(remote['path'], String) { 'remote path' }
105
+ normalize_target_rate(sync_info['transport'])
101
106
  # get transfer spec if possible, and feed back to new structure
102
107
  if block_given?
103
108
  transfer_spec = yield(direction_sym(sync_info), sync_info['local']['path'], remote['path'])
104
109
  Log.dump(:auth_ts, transfer_spec)
105
110
  transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
111
+ Transfer::Spec.resolve_target_rate(transfer_spec)
106
112
  tspec_to_sync_info(transfer_spec, sync_info, CONF_SCHEMA)
107
113
  update_remote_dir(remote, 'path', transfer_spec)
108
114
  end
@@ -123,6 +129,7 @@ module Aspera
123
129
  sync_info.keys.push('instance').uniq.sort.eql?(CMDLINE_PARAMS_KEYS)
124
130
  Aspera.assert_type(sync_info['sessions'], Array)
125
131
  Aspera.assert_type(sync_info['sessions'].first, Hash)
132
+ sync_info['sessions'].each { |session| normalize_target_rate(session) }
126
133
  if block_given?
127
134
  sync_info['sessions'].each do |session|
128
135
  Aspera.assert_type(session['local_dir'], String) { 'local_dir' }
@@ -130,6 +137,7 @@ module Aspera
130
137
  transfer_spec = yield(direction_sym(session), session['local_dir'], session['remote_dir'])
131
138
  Log.dump(:auth_ts, transfer_spec)
132
139
  transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
140
+ Transfer::Spec.resolve_target_rate(transfer_spec)
133
141
  tspec_to_sync_info(transfer_spec, session, ARGS_SESSION_SCHEMA)
134
142
  session['private_key_paths'] = Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa) if transfer_spec.key?('token')
135
143
  update_remote_dir(session, 'remote_dir', transfer_spec)
@@ -257,6 +265,12 @@ module Aspera
257
265
 
258
266
  # private
259
267
 
268
+ # Convert `target_rate` given as a string with optional suffix (e.g. `100m`) into integer bps
269
+ # @param params [Hash, nil] `transport` (`conf` format) or session (`args` format), modified in place
270
+ def normalize_target_rate(params)
271
+ params['target_rate'] = Transfer::Units.rate_string_to_bps(params['target_rate']) if params.is_a?(Hash) && params['target_rate'].is_a?(String)
272
+ end
273
+
260
274
  # Transfer specification to synchronization information
261
275
  # tag `x-ts-name` in schema is used to map transfer spec parameters to async `sync_info`
262
276
  # @param transfer_spec [Hash] transfer specification
@@ -330,7 +344,15 @@ module Aspera
330
344
  current[key] ||= {}
331
345
  current = current[key]
332
346
  end
333
- current[last_key] = props['x-cli-switch'] ? true : args.shift
347
+ current[last_key] =
348
+ if props['x-cli-switch']
349
+ # a special switch sets the non-default enum value, e.g. `--continuous` sets `mode` to `continuous`
350
+ props.key?('enum') ? (props['enum'] - [props['default']]).first : true
351
+ elsif props.key?('x-unit')
352
+ Duration.to_unit(args.shift, props['x-unit'])
353
+ else
354
+ args.shift
355
+ end
334
356
  end
335
357
  return result
336
358
  end
@@ -343,7 +365,7 @@ module Aspera
343
365
  CONF_SCHEMA = CommandLineBuilder.read_schema(Schema::Registry::SYNC_CONF)
344
366
  CMDLINE_PARAMS_KEYS = %w[instance sessions].freeze
345
367
  ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
346
- PRIVATE_FOLDER = "#{Environment.instance.os.eql?(Environment::OS_WINDOWS) ? '~' : '.'}private-asp"
368
+ PRIVATE_FOLDER = "#{Environment.instance.os.eql?(Environment::OS_WINDOWS) ? '~' : '.'}private-asp".freeze
347
369
  ASYNC_DB = 'snap.db'
348
370
  PARAM_KEYS = %w[local sessions].freeze
349
371