aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -1,26 +1,98 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'log'
7
+ require_relative 'assert'
8
+ require_relative 'environment'
9
+ require_relative 'hash_ext'
10
+ require_relative 'secret/strength'
11
+ require_relative 'servlet_helpers'
12
+ require_relative 'web_server_simple'
3
13
  require 'json'
4
14
  require 'timeout'
5
15
  require 'English'
16
+ require 'ipaddr'
17
+ require 'openssl'
6
18
  require 'webrick'
7
- require 'aspera/log'
8
- require 'aspera/assert'
9
- require 'aspera/mime'
10
19
 
11
20
  module Aspera
12
21
  # Start a Faspex-4 style post-processing script using Faspex-5 webhook
22
+ #
23
+ # Security notes:
24
+ # - By default the listener binds to loopback only (url defaults to http://localhost:...).
25
+ # Use a routable address only when the Faspex containers must reach this host:
26
+ # then `secret_token` or `allowed_ips` is required (checked by `serve`).
27
+ # - Use `secret_token` so the token must be present as a `token` query parameter on every
28
+ # incoming request (configure the webhook URL in Faspex as http://host:port/path?token=VALUE).
29
+ # - Use `allowed_ips` to restrict which source IPs or networks may call the listener.
30
+ # - Requests that a web page could send are refused (see ServletHelpers#check_request): the body shall be `application/json`.
31
+ # - Only webhook values with a key starting with `faspex_` are passed to the script.
32
+ # - With `lambda` in the URL query, a Ruby script (`.rb`) is evaluated in this process:
33
+ # it shall return a lambda, called with the webhook values.
13
34
  class Faspex4PostProcServlet < WEBrick::HTTPServlet::AbstractServlet
14
- ALLOWED_PARAMETERS = %i[root script_folder fail_on_error timeout_seconds].freeze
35
+ include ServletHelpers
36
+
37
+ ALLOWED_PARAMETERS = %i[root script_folder fail_on_error timeout_seconds secret_token secret_token_min_bits allowed_ips].freeze
38
+ # Prefix of keys of webhook values passed as environment variables: Faspex 4 post-processing variables
39
+ ENV_KEY_PREFIX = 'faspex_'
40
+ private_constant :ENV_KEY_PREFIX
41
+
42
+ class << self
43
+ # Check parameters and set defaults
44
+ # Called at startup, so that errors are reported before the first request
45
+ # @param parameters [Hash] Parameters of the servlet
46
+ # @return [Hash] Parameters with defaults, and `allowed_ips` as `IPAddr`
47
+ def normalize_parameters(parameters)
48
+ Aspera.assert_type(parameters, Hash)
49
+ parameters = parameters.symbolize_keys
50
+ not_allowed = parameters.keys - ALLOWED_PARAMETERS
51
+ Aspera.assert(not_allowed.empty?) { "unsupported parameters: #{not_allowed.join(', ')}" }
52
+ parameters[:script_folder] ||= '.'
53
+ parameters[:fail_on_error] ||= false
54
+ parameters[:timeout_seconds] ||= 60
55
+ parameters[:secret_token_min_bits] ||= Secret::Strength::DEFAULT_MIN_BITS
56
+ Aspera.assert(File.directory?(parameters[:script_folder])) { "script_folder not found: #{parameters[:script_folder]}" }
57
+ Secret::Strength.assert_strong(parameters[:secret_token], name: 'secret_token', min_bits: parameters[:secret_token_min_bits]) if parameters.key?(:secret_token)
58
+ # allowed_ips: optional IP addresses or networks (CIDR); nil means no restriction
59
+ parameters[:allowed_ips] = Array(parameters[:allowed_ips]).map { |ip| ip_address(ip) } if parameters.key?(:allowed_ips)
60
+ parameters
61
+ end
62
+
63
+ # Start the server, until signal INT or TERM
64
+ # Parameters are checked before: the servlet is created on each request
65
+ # @param parameters [Hash] Parameters of the server (see `WebServerSimple.url`) and of the servlet (see `normalize_parameters`)
66
+ # @param type [Exception] Exception raised when the server is not protected
67
+ def serve(parameters, type: ParameterError)
68
+ servlet_parameters = normalize_parameters(WebServerSimple.servlet_parameters(parameters).merge(root: WebServerSimple.url(parameters).path))
69
+ # Scripts can be started by anyone who can reach the server
70
+ WebServerSimple.serve(
71
+ parameters,
72
+ self,
73
+ protected: servlet_parameters.key?(:secret_token) || servlet_parameters.key?(:allowed_ips),
74
+ what: 'secret_token or allowed_ips',
75
+ type: type
76
+ ) { [servlet_parameters] }
77
+ end
78
+
79
+ private
80
+
81
+ # @param ip [String, IPAddr] IP address or network (CIDR)
82
+ # @return [IPAddr] Parsed address
83
+ def ip_address(ip)
84
+ return ip if ip.is_a?(IPAddr)
85
+ IPAddr.new(ip)
86
+ rescue IPAddr::Error
87
+ raise ArgumentError, "allowed_ips: invalid IP address or network: #{ip}"
88
+ end
89
+ end
90
+
91
+ # @param server [WEBrick::HTTPServer] Server
92
+ # @param parameters [Hash] Parameters from `normalize_parameters`: the servlet is created on each request
15
93
  def initialize(server, parameters)
16
- Aspera.assert_type(parameters, Hash)
17
- @parameters = parameters.symbolize_keys
94
+ @parameters = parameters
18
95
  Log.dump(:post_proc_parameters, @parameters)
19
- not_allowed = @parameters.keys - ALLOWED_PARAMETERS
20
- Aspera.assert(not_allowed.empty?) { "unsupported parameters: #{not_allowed.join(', ')}" }
21
- @parameters[:script_folder] ||= '.'
22
- @parameters[:fail_on_error] ||= false
23
- @parameters[:timeout_seconds] ||= 60
24
96
  super(server)
25
97
  Log.log.debug { 'Faspex4PostProcServlet initialized' }
26
98
  end
@@ -28,64 +100,138 @@ module Aspera
28
100
  # :reek:UncommunicativeMethodName
29
101
  def do_POST(request, response)
30
102
  Log.log.debug { "request=#{request.path}" }
31
- Log.dump(:query, request.query)
103
+ # `request.query` only parses a form body for POST: parameters in the URL come from the query string
104
+ url_query = WEBrick::HTTPUtils.parse_query(request.query_string)
105
+ Log.dump(:query, url_query)
106
+ post_proc_pid = nil
107
+ script_path = nil
32
108
  begin
33
- # Only accept requests on the root
34
- if !request.path.start_with?(@parameters[:root])
35
- response.status = 400
36
- response['Content-Type'] = Mime::JSON
37
- response.body = {status: 'error', message: 'Request outside domain'}.to_json
38
- return
109
+ # Refuse requests of web pages (also on localhost without protection)
110
+ check_request(request, json: true)
111
+ # Check source IP against allowlist when configured
112
+ if @parameters[:allowed_ips]
113
+ client_ip = request.peeraddr[3]
114
+ unless allowed_ip?(client_ip)
115
+ Log.log.warn { "Rejected request from disallowed IP: #{client_ip}" }
116
+ return error_response(response, 403, 'Forbidden')
117
+ end
39
118
  end
40
- if request.body.nil?
41
- response.status = 400
42
- response['Content-Type'] = Mime::JSON
43
- response.body = {status: 'error', message: 'Empty request'}.to_json
44
- return
119
+ # Verify shared secret when configured (expected as ?token=VALUE in the URL)
120
+ if @parameters[:secret_token] && !OpenSSL.secure_compare(url_query['token'].to_s, @parameters[:secret_token].to_s)
121
+ Log.log.warn { 'Rejected request with invalid or missing token' }
122
+ return error_response(response, 401, 'Unauthorized')
45
123
  end
124
+ # Only accept requests on the root
125
+ return error_response(response, 400, 'Request outside domain') unless request.path.start_with?(@parameters[:root])
126
+ return error_response(response, 400, 'Empty request') if request.body.nil?
127
+ webhook_parameters =
128
+ begin
129
+ JSON.parse(request.body)
130
+ rescue JSON::ParserError
131
+ nil
132
+ end
133
+ return error_response(response, 400, 'Request body shall be a JSON object') unless webhook_parameters.is_a?(Hash)
134
+ Log.dump(:webhook_parameters, webhook_parameters)
46
135
  # build script path by removing domain and adding script folder
47
136
  script_file = request.path[@parameters[:root].size..]
48
137
  Log.log.debug { "script file=#{script_file}" }
49
- script_path = File.join(@parameters[:script_folder], script_file)
50
138
  # Resolve the real path and ensure it stays within script_folder (prevents path traversal)
51
139
  resolved_script_folder = File.realpath(@parameters[:script_folder])
52
- resolved_script_path = File.realpath(script_path)
53
- Aspera.assert(resolved_script_path.start_with?("#{resolved_script_folder}/")) { 'Script path traversal attempt detected' }
54
- script_path = resolved_script_path
140
+ begin
141
+ script_path = File.realpath(File.join(@parameters[:script_folder], script_file))
142
+ rescue Errno::ENOENT, Errno::ENOTDIR
143
+ return error_response(response, 404, 'Script not found')
144
+ end
145
+ unless script_path.start_with?("#{resolved_script_folder}/")
146
+ Log.log.warn { "Rejected script outside script_folder: #{script_path}" }
147
+ return error_response(response, 403, 'Script path traversal attempt detected')
148
+ end
149
+ return error_response(response, 404, 'Script not found') unless File.file?(script_path)
55
150
  Log.log.debug { "script=#{script_path}" }
56
- webhook_parameters = JSON.parse(request.body)
57
- Log.dump(:webhook_parameters, webhook_parameters)
58
151
  process_status = nil
59
- if request.query.key?('lambda')
60
- # Code can throw exception, source code must return a lambda
61
- Environment.secure_eval(File.read(script_path), __FILE__, __LINE__).call(webhook_parameters)
152
+ if url_query.key?('lambda')
153
+ return error_response(response, 400, 'lambda requires a Ruby script (.rb)') unless File.extname(script_path).eql?('.rb')
154
+ run_lambda(script_path, webhook_parameters)
62
155
  else
63
- # env expects only strings
64
- environment = webhook_parameters.each_with_object({}) { |(k, v), h| h[k] = v.to_s }
65
- post_proc_pid = Environment.secure_execute(script_path, mode: :background, env: environment)
156
+ post_proc_pid = Environment.secure_execute(script_path, mode: :background, env: script_environment(webhook_parameters))
66
157
  Timeout.timeout(@parameters[:timeout_seconds]) do
67
158
  # "wait" for process to avoid zombie
68
- Process.wait(post_proc_pid)
159
+ ::Process.wait(post_proc_pid)
69
160
  post_proc_pid = nil
70
161
  end
71
162
  process_status = $CHILD_STATUS
72
163
  Aspera.assert(process_status.success? || !@parameters[:fail_on_error]) { "script #{script_path} failed with code #{process_status.exitstatus}" }
73
164
  end
74
- response.status = 200
75
- response.content_type = Mime::JSON
76
- response.body = JSON.generate({status: 'success', script: script_path, exit_code: process_status&.exitstatus})
165
+ json_response(response, 200, {status: 'success', script: script_path, exit_code: process_status&.exitstatus})
77
166
  Log.log.debug { 'Script executed successfully' }
78
- rescue => e
167
+ rescue WEBrick::HTTPStatus::Error => e
168
+ Log.log.warn { "Rejected request: #{e.message}" }
169
+ error_response(response, e.code, e.message)
170
+ rescue StandardError, ScriptError => e
171
+ # ScriptError: syntax error in lambda script
79
172
  Log.log.error("Script failed: #{e.class}:#{e.message}")
80
- if !post_proc_pid.nil?
81
- Process.kill('SIGKILL', post_proc_pid)
82
- Process.wait(post_proc_pid)
83
- Log.log.error("Killed process: #{post_proc_pid}")
173
+ kill_script(post_proc_pid) unless post_proc_pid.nil?
174
+ error_response(response, 500, e.message, script: script_path)
175
+ end
176
+ end
177
+
178
+ private
179
+
180
+ # Set an error in the response
181
+ # @param response [WEBrick::HTTPResponse] Response to set
182
+ # @param status [Integer] HTTP status code
183
+ # @param message [String] Error message
184
+ # @param info [Hash] Additional fields in body
185
+ # @return nil
186
+ def error_response(response, status, message, **info)
187
+ json_response(response, status, {status: 'error', **info, message: message})
188
+ end
189
+
190
+ # @param client_ip [String] Source IP address of the request
191
+ # @return [Boolean] `true` if the address is in one of `allowed_ips`
192
+ def allowed_ip?(client_ip)
193
+ # IPv4 clients of a dual-stack listener appear as IPv4-mapped IPv6 addresses
194
+ address = IPAddr.new(client_ip).native
195
+ @parameters[:allowed_ips].any? { |allowed| allowed.include?(address) }
196
+ rescue IPAddr::Error
197
+ false
198
+ end
199
+
200
+ # Environment variables of the script: same as Faspex 4 post-processing
201
+ # Other keys are ignored: they could change the behavior of the script (e.g. `PATH`, `LD_PRELOAD`)
202
+ # @param webhook_parameters [Hash] Values received in webhook
203
+ # @return [Hash] Environment variables, values are `String`
204
+ def script_environment(webhook_parameters)
205
+ webhook_parameters.each_with_object({}) do |(key, value), env|
206
+ unless key.start_with?(ENV_KEY_PREFIX)
207
+ Log.log.warn { "Ignoring webhook value with key not starting with #{ENV_KEY_PREFIX}: #{key.inspect}" }
208
+ next
84
209
  end
85
- response.status = 500
86
- response['Content-Type'] = Mime::JSON
87
- response.body = {status: 'error', script: script_path, message: e.message}.to_json
210
+ # Structured values are given as JSON
211
+ env[key] = value.is_a?(Hash) || value.is_a?(Array) ? JSON.generate(value) : value.to_s
212
+ end
213
+ end
214
+
215
+ # Evaluate a Ruby script in this process, and call the lambda it returns
216
+ # @param script_path [String] Path of Ruby script
217
+ # @param webhook_parameters [Hash] Values received in webhook
218
+ def run_lambda(script_path, webhook_parameters)
219
+ Timeout.timeout(@parameters[:timeout_seconds]) do
220
+ code = Environment.secure_eval(File.read(script_path), script_path, 1)
221
+ Aspera.assert(code.respond_to?(:call)) { "script #{script_path} shall return a lambda" }
222
+ code.call(webhook_parameters)
88
223
  end
89
224
  end
225
+
226
+ # Kill the script after a timeout or an error
227
+ # @param pid [Integer] Process id of the script
228
+ def kill_script(pid)
229
+ ::Process.kill('SIGKILL', pid)
230
+ ::Process.wait(pid)
231
+ Log.log.error("Killed process: #{pid}")
232
+ rescue SystemCallError
233
+ # already terminated
234
+ nil
235
+ end
90
236
  end
91
237
  end
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
4
7
 
5
8
  module Aspera
6
9
  # Common interface for documentation formatters
@@ -8,7 +11,6 @@ module Aspera
8
11
  module FormatterInterface
9
12
  Aspera.require_method!(:tick)
10
13
  Aspera.require_method!(:special_format)
11
- Aspera.require_method!(:check_row)
12
14
  Aspera.require_method!(:markdown_text)
13
15
  end
14
16
  end
@@ -1,6 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/rest/call_error'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'rest/call_error'
7
+ require_relative 'assert'
4
8
 
5
9
  module Aspera
6
10
  # Helper to execute GraphQL queries via a Rest::Client object.
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  class ::Hash
4
7
  # Recursively merge another hash, with nested hashes merged deeply
5
8
  # @param second [Hash] Hash to merge into self
@@ -1,7 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
4
- require 'aspera/environment'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'environment'
5
7
  require 'uri'
6
8
 
7
9
  module Aspera
@@ -9,13 +11,14 @@ module Aspera
9
11
  class << self
10
12
  # Generate an ID from a list of object IDs
11
13
  # The generated ID is safe as file name
14
+ # In URLs keep only the host name, we assume only one instance on a given host.
12
15
  # @param ids [Array] the object IDs (can be nested, will be flattened, and nils removed)
13
16
  # @return [String] the generated ID
14
17
  def from_list(*ids)
15
18
  safe_char = Environment.instance.safe_filename_character
16
19
  # compact: remove nils
17
20
  id = ids.flatten.compact.map do |i|
18
- i.is_a?(String) && i.start_with?('https://') ? URI.parse(i).host : i.to_s
21
+ i.is_a?(String) && i.start_with?('http://', 'https://') ? URI.parse(i).host : i.to_s
19
22
  end.join(safe_char)
20
23
  # keep dot for extension only (nicer)
21
24
  return Environment.instance.sanitized_filename(id.gsub('.', safe_char)).downcase
@@ -1,27 +1,55 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore blankslate jsonrpc
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
4
5
 
5
- require 'aspera/rest/error_analyzer'
6
- require 'aspera/assert'
7
- require 'aspera/json_rpc/version'
8
- require 'blankslate'
6
+ require_relative '../rest/error_analyzer'
7
+ require_relative '../rest/call_error'
8
+ require_relative '../assert'
9
+ require_relative 'version'
9
10
 
10
- Aspera::Rest::ErrorAnalyzer.instance.add_simple_handler(name: 'JSON RPC', path: %w[error message], always: true)
11
+ # Error in non-2XX response. Error in 2XX response is raised by the client.
12
+ Aspera::Rest::ErrorAnalyzer.instance.add_simple_handler(name: 'JSON RPC', path: %w[error message])
11
13
 
12
14
  module Aspera
15
+ # JSON-RPC 2.0 over HTTP
13
16
  module JsonRpc
17
+ class << self
18
+ # Send a JSON-RPC request
19
+ # @param api [Rest::Client] Aspera REST object pointing at the JSON-RPC endpoint
20
+ # @param method [String] JSON-RPC method
21
+ # @param args [Array] Arguments of call: a single Hash is sent as named params
22
+ # @param id [Integer] Request id
23
+ # @return [Object] `result` of response
24
+ # @raise [Rest::CallError] on JSON-RPC error, or invalid response
25
+ def request(api, method, args, id)
26
+ args = args.first if args.size == 1 && args.first.is_a?(Hash)
27
+ data = api.create('', {
28
+ jsonrpc: VERSION,
29
+ method: method,
30
+ params: args,
31
+ id: id
32
+ })
33
+ Aspera.assert_type(data, Hash, type: Rest::CallError) { 'JSON-RPC response' }
34
+ Aspera.assert(data['jsonrpc'] == VERSION, type: Rest::CallError) { "JSON-RPC: bad version in response: #{data['jsonrpc']}" }
35
+ Aspera.assert(data.key?('result') ^ data.key?('error'), type: Rest::CallError) { 'JSON-RPC: response must have either result or error' }
36
+ # id is null if server could not read it from request
37
+ Aspera.assert(data['id'] == id || (data.key?('error') && data['id'].nil?), type: Rest::CallError) { "JSON-RPC: bad id in response: #{data['id']}, expected #{id}" }
38
+ return data['result'] if data.key?('result')
39
+ error = data['error']
40
+ Aspera.assert(error.is_a?(Hash) && error['code'].is_a?(Integer) && error['message'].is_a?(String), type: Rest::CallError) { "JSON-RPC: bad error in response: #{error}" }
41
+ raise Rest::CallError, "#{error['message']} (code: #{error['code']})"
42
+ end
43
+ end
44
+
14
45
  # JSON-RPC 2.0 client over an Aspera::Rest::Client HTTP endpoint.
15
46
  # Methods are dispatched dynamically via method_missing.
47
+ # Based on BasicObject, so that methods of Object are sent as JSON-RPC requests.
16
48
  # Example:
17
49
  # client = JsonRpc::Client.new(Rest::Client.new(base_url: 'http://127.0.0.1:33024'))
18
50
  # client.get_info
19
51
  # client.start_transfer(app_id: '...', transfer_spec: {...})
20
- class Client < BlankSlate
21
- reveal :instance_variable_get
22
- reveal :inspect
23
- reveal :to_s
24
-
52
+ class Client < BasicObject
25
53
  # @param api [Rest::Client] Aspera REST object pointing at the JSON-RPC endpoint
26
54
  # @param namespace [String, nil] optional method prefix, e.g. "myns."
27
55
  def initialize(api, namespace = nil)
@@ -31,32 +59,24 @@ module Aspera
31
59
  @request_id = 0
32
60
  end
33
61
 
34
- def respond_to_missing?(_sym, _include_private = false)
35
- true
36
- end
62
+ # @return [String] Class and URL of JSON-RPC endpoint
63
+ def inspect = "#<#{Client.name} #{@api.base_url}>"
64
+
65
+ # @return [String] Same as {#inspect}
66
+ def to_s = inspect
67
+
68
+ # Ruby checks it before implicit conversions (`to_ary`, `to_str`, ...): those are not JSON-RPC methods
69
+ # @param _sym [Symbol] Method name
70
+ # @param _include_private [Boolean] Also private methods
71
+ # @return [false]
72
+ def respond_to_missing?(_sym, _include_private = false) = false
37
73
 
38
74
  # Dispatch any Ruby method call as a JSON-RPC request
39
- def method_missing(method, *args, &block)
40
- args = args.first if args.size == 1 && args.first.is_a?(Hash)
41
- data = @api.create('', {
42
- jsonrpc: VERSION,
43
- method: "#{@namespace}#{method}",
44
- params: args,
45
- id: @request_id += 1
46
- })
47
- Aspera.assert_type(data, Hash) { 'response' }
48
- Aspera.assert(data['jsonrpc'] == VERSION, 'bad version in response')
49
- Aspera.assert(data.key?('id'), 'missing id in response')
50
- Aspera.assert(!(data.key?('error') && data.key?('result')), 'both error and response')
51
- Aspera.assert(
52
- !data.key?('error') ||
53
- data['error'].is_a?(Hash) &&
54
- data['error']['code'].is_a?(Integer) &&
55
- data['error']['message'].is_a?(String),
56
- 'bad error response'
57
- )
58
- return data['result']
59
- end
75
+ # @param method [Symbol] JSON-RPC method, without namespace
76
+ # @param args [Array] Arguments of call: a single Hash is sent as named params, else positional params
77
+ # @return [Object] `result` of response
78
+ # @raise [Rest::CallError] on JSON-RPC error, or invalid response
79
+ def method_missing(method, *args) = JsonRpc.request(@api, "#{@namespace}#{method}", args, @request_id += 1)
60
80
  end
61
81
  end
62
82
  end
@@ -1,7 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  module JsonRpc
8
+ # Protocol version: value of `jsonrpc` in request and response
5
9
  VERSION = '2.0'
6
10
  end
7
11
  end
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'log'
4
7
 
5
8
  module Aspera
6
9
  # used for logging http
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  require 'strscan'
4
7
 
5
8
  module Aspera
@@ -72,7 +75,7 @@ module Aspera
72
75
  end
73
76
 
74
77
  # RFC 2616 token: any char except separators
75
- TOKEN_RE = /[^()<>@,;:\"\[\]?={}\s]+/ # RFC 2616 token
78
+ TOKEN_RE = /[^()<>@,;:"\[\]?={}\s]+/ # RFC 2616 token
76
79
  QUOTED_RE = /"((?:[^"\\]|\\.)*)"/ # double-quoted string with backslash escapes
77
80
  HREF_RE = /\s*<([^>]*)>\s*;?\s*/ # <URI> possibly followed by ;
78
81
  ATTR_RE = /(#{TOKEN_RE})\s*=\s*(#{TOKEN_RE}|#{QUOTED_RE})\s*/ # key=value or key="value"
data/lib/aspera/log.rb CHANGED
@@ -1,13 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
4
- require 'aspera/secret_hider'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'secret/hider'
8
+ require_relative 'rainbow'
5
9
  require 'logger'
6
10
  require 'pp'
7
11
  require 'json'
8
12
  require 'singleton'
9
13
  require 'stringio'
10
- require 'aspera/rainbow'
11
14
  using Rainbow
12
15
 
13
16
  # Ignore warnings
@@ -110,11 +113,8 @@ module Aspera
110
113
  return unless instance.logger.send(:"#{level}?")
111
114
  Aspera.assert(object.nil? || block.nil?, 'Use either object, or block, not both')
112
115
  object = yield if block_given?
113
- instance.logger.send(level, obj_dump(name, object))
114
- end
115
-
116
- # @return [String] Dump of object
117
- def obj_dump(name, object)
116
+ # Secrets hidden by key name in the structure, whatever the dump format (the log formatter then hides secrets in text)
117
+ object = Secret::Hider.instance.hidden_copy(object) unless Secret::Hider.instance.log_secrets
118
118
  dump_text =
119
119
  case instance.dump_format
120
120
  when :json
@@ -123,7 +123,7 @@ module Aspera
123
123
  PP.pp(object, +'')
124
124
  else Aspera.error_unexpected_value(instance.dump_format) { 'dump format' }
125
125
  end
126
- "#{name.to_s.green}(#{instance.dump_format})#{object.class}=\n#{dump_text}"
126
+ instance.logger.send(level, "#{name.to_s.green}(#{instance.dump_format})#{object.class}=\n#{dump_text}")
127
127
  end
128
128
 
129
129
  # Capture the output of $stderr and log it at debug level
@@ -178,6 +178,16 @@ module Aspera
178
178
  nil
179
179
  end
180
180
 
181
+ # Send logs to the logger of a host application (e.g. `Rails.logger`), until `logger_type` is set.
182
+ # Its level and formatter are not changed: secrets are hidden in dumps (`Log.dump`), not in log lines.
183
+ # @param logger [Logger] Logger of the host application
184
+ # @return [nil]
185
+ def logger=(logger)
186
+ Aspera.assert(%i[add level].all? { |m| logger.respond_to?(m) }) { "not a logger: #{logger.class}" }
187
+ @logger = logger
188
+ nil
189
+ end
190
+
181
191
  # Set log level of underlying logger given symbol level
182
192
  # @param new_level [Symbol] One of LEVELS
183
193
  def level=(new_level)
@@ -196,7 +206,7 @@ module Aspera
196
206
  raise Error, 'Formatter must be a String, a Logger::Formatter or a Proc'
197
207
  end
198
208
  # Update formatter with password hiding
199
- @logger.formatter = SecretHider.instance.log_formatter(formatter)
209
+ @logger.formatter = Secret::Hider.instance.log_formatter(formatter)
200
210
  nil
201
211
  end
202
212