aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -1,7 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'log'
3
8
  require 'singleton'
4
9
  require 'fileutils'
10
+ require 'securerandom'
5
11
  require 'etc'
6
12
  require 'tmpdir'
7
13
 
@@ -15,21 +21,34 @@ module Aspera
15
21
  class TempFileManager
16
22
  include Singleton
17
23
 
18
- SEC_IN_DAY = 86_400
19
- # assume no transfer last longer than this
20
- # (garbage collect file list which were not deleted after transfer)
21
- FILE_LIST_AGE_MAX_SEC = SEC_IN_DAY * 5
22
- private_constant :SEC_IN_DAY, :FILE_LIST_AGE_MAX_SEC
24
+ # Assume no transfer lasts longer than this
25
+ # (garbage collect file lists which were not deleted after transfer)
26
+ EXPIRY_DEFAULT_SEC = 5 * 86_400
27
+ private_constant :EXPIRY_DEFAULT_SEC
23
28
 
29
+ # @return [Boolean] `true`: files created by this execution are deleted by `cleanup`
24
30
  attr_accessor :cleanup_on_exit
31
+ # @return [Numeric, nil] Age in seconds of files deleted by `cleanup_expired`, `nil`: never deleted
32
+ attr_reader :expiry_sec
25
33
  attr_reader :global_temp
26
34
 
27
35
  def initialize
28
36
  @created_files = []
37
+ # Number of holders of a created file: not deleted by cleanup while held
38
+ @hold_counts = {}
39
+ # Files are created and released by transfer threads
40
+ @mutex = Mutex.new
29
41
  @cleanup_on_exit = true
42
+ @expiry_sec = EXPIRY_DEFAULT_SEC
30
43
  @global_temp = Etc.systmpdir
31
44
  end
32
45
 
46
+ # @param value [Numeric, nil] Age in seconds, `0`: all files, `nil`: never delete
47
+ def expiry_sec=(value)
48
+ Aspera.assert(value.nil? || (value.is_a?(Numeric) && !value.negative?), type: ParameterError) { "Expiry must be a positive duration: #{value}" }
49
+ @expiry_sec = value
50
+ end
51
+
33
52
  def global_temp=(value)
34
53
  @global_temp = case value
35
54
  when '@env' then Dir.tmpdir
@@ -39,17 +58,45 @@ module Aspera
39
58
  end
40
59
 
41
60
  def delete_file(filepath)
42
- File.delete(filepath) if @cleanup_on_exit
43
- rescue => e
44
- Log.log.warn { "Problem deleting file: #{filepath}: #{e.message}" }
61
+ remove_file(filepath) if @cleanup_on_exit
45
62
  end
46
63
 
47
- # Call this on process exit
64
+ # Call this on process exit, or at the end of a command.
65
+ # Held files are kept: they are deleted when released.
48
66
  def cleanup
49
- @created_files.each do |filepath|
50
- delete_file(filepath) if File.file?(filepath)
67
+ @mutex.synchronize do
68
+ @created_files.each do |filepath|
69
+ delete_file(filepath) if !@hold_counts.key?(filepath) && File.file?(filepath)
70
+ end
71
+ @created_files.select! { |filepath| @hold_counts.key?(filepath) }
72
+ end
73
+ end
74
+
75
+ # Keep created files after `cleanup`, until released, for example a file list used by a transfer
76
+ # that ends after the command (asynchronous transfer).
77
+ # A file can be held several times (multi-session): it is deleted when the last holder releases it.
78
+ # @param paths [Array<String>] Files created by this manager (others are ignored)
79
+ def hold(paths)
80
+ @mutex.synchronize do
81
+ paths.each do |filepath|
82
+ @hold_counts[filepath] = @hold_counts.fetch(filepath, 0) + 1 if @created_files.include?(filepath)
83
+ end
84
+ end
85
+ end
86
+
87
+ # Release files held with `hold`, and delete those that are no longer held.
88
+ # @param paths [Array<String>] Files given to `hold`
89
+ def release(paths)
90
+ @mutex.synchronize do
91
+ paths.each do |filepath|
92
+ next unless @hold_counts.key?(filepath)
93
+ @hold_counts[filepath] -= 1
94
+ next if @hold_counts[filepath].positive?
95
+ @hold_counts.delete(filepath)
96
+ @created_files.delete(filepath)
97
+ delete_file(filepath) if File.file?(filepath)
98
+ end
51
99
  end
52
- @created_files = []
53
100
  end
54
101
 
55
102
  # Ensure that provided folder exists, or create it, atomically create and reserve a unique file.
@@ -63,7 +110,7 @@ module Aspera
63
110
  new_file = Dir::Tmpname.create([basename, suffix ? "-#{suffix}" : ''], temp_folder) do |path|
64
111
  File.open(path, File::WRONLY | File::CREAT | File::EXCL, 0o600) { nil }
65
112
  end
66
- @created_files.push(new_file)
113
+ @mutex.synchronize { @created_files.push(new_file) }
67
114
  new_file
68
115
  end
69
116
 
@@ -71,7 +118,7 @@ module Aspera
71
118
  def new_file_path_global(prefix = nil, suffix: nil)
72
119
  username =
73
120
  begin
74
- Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
121
+ Etc.getlogin || Etc.getpwuid(::Process.uid).name || 'unknown_user'
75
122
  rescue StandardError
76
123
  'unknown_user'
77
124
  end
@@ -85,7 +132,7 @@ module Aspera
85
132
  def new_dir_path_global(prefix = nil)
86
133
  username =
87
134
  begin
88
- Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
135
+ Etc.getlogin || Etc.getpwuid(::Process.uid).name || 'unknown_user'
89
136
  rescue StandardError
90
137
  'unknown_user'
91
138
  end
@@ -93,17 +140,33 @@ module Aspera
93
140
  File.join(@global_temp, dir_name)
94
141
  end
95
142
 
96
- # Garbage collect undeleted files
143
+ # Garbage collect files not deleted by previous executions (e.g. killed), older than `expiry_sec`.
144
+ # Not disabled by `cleanup_on_exit`, which applies to files of this execution.
145
+ # @param temp_folder [String] Folder that contains only temporary files
97
146
  def cleanup_expired(temp_folder)
98
- Dir.entries(temp_folder).each do |name|
147
+ return if @expiry_sec.nil?
148
+ now = Time.now
149
+ Dir.children(temp_folder).each do |name|
99
150
  file_path = File.join(temp_folder, name)
100
- age_sec = (Time.now - File.stat(file_path).mtime).to_i
101
151
  # check age of file, delete too old
102
- if File.file?(file_path) && (age_sec > FILE_LIST_AGE_MAX_SEC)
103
- Log.log.debug { "garbage collecting #{name}" }
104
- delete_file(file_path)
105
- end
152
+ next unless File.file?(file_path) && (now - File.mtime(file_path)) >= @expiry_sec
153
+ Log.log.debug { "garbage collecting #{name}" }
154
+ remove_file(file_path)
155
+ rescue Errno::ENOENT
156
+ # deleted meanwhile by another execution
157
+ nil
106
158
  end
107
159
  end
160
+
161
+ private
162
+
163
+ # Delete a file, a failure is logged
164
+ def remove_file(filepath)
165
+ File.delete(filepath)
166
+ rescue Errno::ENOENT
167
+ nil
168
+ rescue => e
169
+ Log.log.warn { "Problem deleting file: #{filepath}: #{e.message}" }
170
+ end
108
171
  end
109
172
  end
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  # trigger returns true only if the delay has passed since the last trigger
5
8
  class TimerLimiter
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/ascp/management'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../ascp/management'
4
7
 
5
8
  module Aspera
6
9
  module Transfer
@@ -1,12 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  module Transfer
5
8
  # Generates a pseudo file stream.
6
9
  class FauxFile
7
10
  SCHEME = 'faux'
8
11
  # marker for faux file
9
- PREFIX = "#{SCHEME}:///"
12
+ PREFIX = "#{SCHEME}:///".freeze
10
13
  # size units, kilo, mega ...
11
14
  SIZE_UNITS = %w[k m g t p e].freeze
12
15
  private_constant :SCHEME, :PREFIX, :SIZE_UNITS
@@ -1,22 +1,25 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'aspera/exec_spec'
6
- require 'aspera/command_line_builder'
7
- require 'aspera/temp_file_manager'
8
- require 'aspera/transfer/error'
9
- require 'aspera/transfer/spec'
10
- require 'aspera/command_line_converter'
11
- require 'aspera/ascp/installation'
12
- require 'aspera/cli/formatter'
13
- require 'aspera/rest'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative '../exec_spec'
9
+ require_relative '../command_line_builder'
10
+ require_relative '../temp_file_manager'
11
+ require_relative 'error'
12
+ require_relative 'spec'
13
+ require_relative '../command_line_converter'
14
+ require_relative '../ascp/installation'
15
+ require_relative '../cli/formatter'
16
+ require_relative '../rest'
17
+ require_relative '../rainbow'
14
18
  require 'securerandom'
15
19
  require 'base64'
16
20
  require 'json'
17
21
  require 'fileutils'
18
22
  require 'openssl'
19
- require 'aspera/rainbow'
20
23
  using Rainbow
21
24
 
22
25
  module Aspera
@@ -80,6 +83,8 @@ module Aspera
80
83
  Aspera.assert_values(@client_ssh_key, Ascp::Installation::CLIENT_SSH_KEY_OPTIONS)
81
84
  @check_ignore_cb = check_ignore_cb
82
85
  @builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, CommandLineConverter)
86
+ # temporary files used by `ascp`: given to the agent in the ExecSpec
87
+ @temp_files = []
83
88
  end
84
89
 
85
90
  # Place source files on command line or in a temp file list, controlled by @file_list and file_list_folder
@@ -87,7 +92,7 @@ module Aspera
87
92
  # is the file list provided through ascp parameters?
88
93
  ascp_file_list_provided = self.class.ascp_args_file_list?(@ascp_args)
89
94
  # set if paths is mandatory in ts
90
- @builder.required('paths', !@job_spec.key?('keepalive') && !ascp_file_list_provided) # cspell:words keepalive
95
+ @builder.required('paths', !@job_spec.key?('keepalive') && !ascp_file_list_provided)
91
96
  # get paths in transfer spec (after setting if it is mandatory)
92
97
  ts_paths_array = @builder.read_param('paths')
93
98
  file_list_option = nil
@@ -112,6 +117,7 @@ module Aspera
112
117
  lines = ts_paths_array.map { |i| i['source'] }
113
118
  end
114
119
  file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
120
+ @temp_files.push(file_list_file)
115
121
  Log.dump(:file_list, lines)
116
122
  File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
117
123
  Log.log.debug { "#{file_list_option}=\n#{File.read(file_list_file)}".red }
@@ -137,6 +143,7 @@ module Aspera
137
143
  # ignore cert for wss ?
138
144
  if @check_ignore_cb&.call(@job_spec['remote_host'], @job_spec['wss_port'])
139
145
  wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
146
+ @temp_files.push(wss_cert_file)
140
147
  wss_url = "https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}"
141
148
  File.write(wss_cert_file, Rest.remote_certificate_chain(wss_url))
142
149
  # place in front, as more priority
@@ -213,10 +220,11 @@ module Aspera
213
220
  end
214
221
  # disable redis in client, only for ascp, this makes ascp4 fail
215
222
  exec_spec.env['ASPERA_TEST_REDIS_DISABLE'] = 'true' if exec_spec.exec.eql?(:ascp)
223
+ exec_spec.temp_files.concat(@temp_files)
216
224
  Log.log.debug { "ascp args: #{exec_spec}" }
217
225
  return exec_spec
218
226
  end
219
- DELETE_EQUIV = %w[remove_after_transfer remove_empty_directories remove_empty_source_directory]
227
+ DELETE_EQUIV = %w[remove_after_transfer remove_empty_directories remove_empty_source_directory].freeze
220
228
  # `ascp` options to provide a file list
221
229
  FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
222
230
  private_constant :DELETE_EQUIV, :FILE_LIST_OPTIONS
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  module Transfer
5
8
  # Typed result of a transfer operation.
@@ -1,9 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'singleton'
4
- require 'aspera/log'
5
- require 'aspera/assert'
6
- require 'aspera/transfer/error'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative 'error'
7
9
 
8
10
  module Aspera
9
11
  module Transfer
@@ -1,7 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/command_line_builder'
4
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../agent/factory'
7
+ require_relative '../command_line_builder'
8
+ require_relative '../assert'
9
+ require_relative 'units'
5
10
 
6
11
  module Aspera
7
12
  module Transfer
@@ -27,6 +32,8 @@ module Aspera
27
32
  TAG_RESERVED = 'aspera'
28
33
  # Specific fields to not display
29
34
  SPECIFIC = %w{token paths direction source_root destination_root}.freeze
35
+ # Value of agent for documentation with one column per agent
36
+ DOC_OVERVIEW = :overview
30
37
  class << self
31
38
  # wrong def in transferd
32
39
  POLICY_FIX = {
@@ -34,14 +41,8 @@ module Aspera
34
41
  'attrs' => 'attributes',
35
42
  'sparse_csum' => 'sparse_checksum',
36
43
  'full_csum' => 'full_checksum'
37
- }
38
- # Multipliers for target_rate suffix (result is in kbps)
39
- RATE_SUFFIX_KBPS = {
40
- 'k' => 1,
41
- 'm' => 1_000,
42
- 'g' => 1_000_000
43
44
  }.freeze
44
- private_constant :POLICY_FIX, :RATE_SUFFIX_KBPS
45
+ private_constant :POLICY_FIX
45
46
  # translate upload/download to send/receive
46
47
  def transfer_type_to_direction(transfer_type)
47
48
  XFER_TYPE_TO_DIR.fetch(transfer_type)
@@ -52,21 +53,47 @@ module Aspera
52
53
  XFER_DIR_TO_TYPE.fetch(direction)
53
54
  end
54
55
 
55
- # Parse a human-readable rate string (as accepted by ascp -l) into an integer kbps value.
56
- # Accepted: plain integer (kbps), or integer + suffix k/K (kbps), m/M (x1000 kbps), g/G (x1000000 kbps).
57
- # @param value [String] e.g. "100m", "500000", "1g"
58
- # @return [Integer] value in kbps
59
- def rate_string_to_kbps(value)
60
- m = value.to_s.strip.match(/\A(\d+)([kKmMgG])?\z/)
61
- Aspera.assert(m) { "Invalid rate value: #{value.inspect}. Expected integer with optional suffix k/K, m/M or g/G." }
62
- multiplier = m[2] ? RATE_SUFFIX_KBPS.fetch(m[2].downcase) : 1
63
- return m[1].to_i * multiplier
56
+ # Resolve pseudo-parameter `target_rate` (bps) into `target_rate_kbps` (rounded down), overriding it if both are present.
57
+ # @param transfer_spec [Hash] Transfer spec, modified in place
58
+ def resolve_target_rate(transfer_spec)
59
+ return unless transfer_spec.key?('target_rate')
60
+ rate = transfer_spec.delete('target_rate')
61
+ transfer_spec['target_rate_kbps'] = Units.rate_string_to_bps(rate) / 1000 unless rate.nil?
64
62
  end
65
63
 
66
64
  def fix_transferd_resume_policy(transfer_spec)
67
65
  # Fix discrepancy in transfer spec
68
66
  transfer_spec['resume_policy'] = POLICY_FIX[transfer_spec['resume_policy']] if transfer_spec.key?('resume_policy')
69
67
  end
68
+
69
+ # Options of `Schema::Documentation` for the transfer spec: supported agents and `ascp` options
70
+ # @param agent [Symbol, nil] `nil`: all parameters, name of agent: parameters it supports, `DOC_OVERVIEW`: one column per agent
71
+ # @return [Hash] Keyword arguments of `Schema::Documentation.new`
72
+ def doc_options(agent = nil)
73
+ return {annotate: ->(node, formatter) { doc_lines(node, formatter, agent: agent) }} unless agent.eql?(DOC_OVERVIEW)
74
+ {
75
+ annotate: ->(node, formatter) { doc_lines(node, formatter, agents_line: false) },
76
+ extra_columns: Agent::Factory::ALL.sort_by { |_, names| names[:short] }.to_h do |sym, names|
77
+ [names[:short].to_s, ->(node, formatter) { formatter.tick(CommandLineBuilder.supported_by_agent(sym, node)) }]
78
+ end
79
+ }
80
+ end
81
+
82
+ private
83
+
84
+ # Documentation of supported agents and command line option of a parameter: `annotate` of `Schema::Documentation`
85
+ # @param node [Hash] Schema of the parameter
86
+ # @param formatter [FormatterInterface] Formatter of the documentation
87
+ # @param agent [Symbol, nil] Show only parameters supported by this agent, or all if `nil`
88
+ # @param agents_line [Boolean] `true`: add a line with supported agents, if not all
89
+ # @return [Array<String>, nil] Lines added to the description, `nil` to hide the parameter
90
+ def doc_lines(node, formatter, agent: nil, agents_line: true)
91
+ agents = Agent::Factory::ALL.keys.select { |sym| CommandLineBuilder.supported_by_agent(sym, node) }
92
+ return if agents.empty? || (agent && !agents.include?(agent))
93
+ lines = []
94
+ lines.push("(#{agents.map { |i| Agent::Factory::ALL[i][:short].to_s.upcase }.sort.join(', ')})") if agents_line && !agents.length.eql?(Agent::Factory::ALL.length)
95
+ lines.concat(CommandLineBuilder.doc_lines(node, formatter))
96
+ end
70
97
  end
71
98
  SCHEMA = CommandLineBuilder.read_schema(Schema::Registry::TRANSFER_SPEC, ascp: true)
72
99
  # Define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
@@ -757,11 +757,14 @@ properties:
757
757
  x-cli-option: -l
758
758
  target_rate:
759
759
  description: >-
760
- Pseudo-parameter: desired transfer rate as a human-readable string with an optional unit suffix.
761
- Converted to `target_rate_kbps` before the transfer is started, overriding it if both are present.
762
- Accepted formats: plain integer (kbps), or integer followed by `k`/`K` (kbps), `m`/`M` (x1000 kbps), `g`/`G` (x1000000 kbps).
763
- Examples: `100000`, `100000k`, `100m`, `1g`.
764
- type: string
760
+ Pseudo-parameter: desired transfer rate in bps, with an optional unit suffix.
761
+ Converted to `target_rate_kbps` (rounded down) before the transfer is started, overriding it if both are present.
762
+ Also applies to `async` (sync).
763
+ Accepted formats: integer (bps), or integer followed by `k`/`K` (x1000), `m`/`M` (x1000000), `g`/`G` (x1000000000).
764
+ Examples: `100000000`, `100000k`, `100m`, `1g`.
765
+ type:
766
+ - string
767
+ - integer
765
768
  x-cli-special: true
766
769
  title:
767
770
  description: Title of the transfer.
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+
8
+ module Aspera
9
+ module Transfer
10
+ # Rates, sizes and durations of transfers: parsed from user input and formatted for display.
11
+ # Decimal units, like `ascp`. Depends only on `aspera/assert`: usable without the CLI.
12
+ module Units
13
+ # Decimal prefixes: each one is `KILO` times the previous one
14
+ PREFIX = ['', 'K', 'M', 'G', 'T', 'P'].freeze
15
+ KILO = 1000
16
+ # Multiplier of each prefix, e.g. `'M' => 1_000_000`
17
+ MULTIPLIER = PREFIX.each_with_index.to_h { |prefix, power| [prefix, KILO**power] }.freeze
18
+ # Units of sizes: `B`, `KB`, `MB`...
19
+ BYTE_UNITS = PREFIX.map { |prefix| "#{prefix}B" }.freeze
20
+ # Rate typed by a user: integer with an optional prefix, case ignored
21
+ RATE_INPUT = /\A(\d+)([#{PREFIX.join}])?\z/i
22
+ BITS_PER_BYTE = 8
23
+ private_constant :MULTIPLIER, :BYTE_UNITS, :RATE_INPUT, :BITS_PER_BYTE
24
+
25
+ class << self
26
+ # Parse a human-readable rate (as accepted by async -l) into an integer bps value.
27
+ # Accepted: plain integer (bps), or integer + prefix of `PREFIX`, case ignored, e.g. `k` (x1000), `M` (x1000000), `g` (x1000000000).
28
+ # @param value [String, Integer] e.g. "100m", "500000", "1g", 500000
29
+ # @return [Integer] value in bps
30
+ def rate_string_to_bps(value)
31
+ m = RATE_INPUT.match(value.to_s.strip)
32
+ Aspera.assert(m) { "Invalid rate value: #{value.inspect}. Expected integer with optional suffix #{PREFIX.drop(1).join(', ')} (case ignored)." }
33
+ return m[1].to_i * MULTIPLIER.fetch(m[2].to_s.upcase)
34
+ end
35
+
36
+ # Inverse of `rate_string_to_bps`: the shortest exact text, e.g. `100M`, `1G`, `1500K`, `999`
37
+ # @param bps [Integer] Rate in bits per second
38
+ # @return [String] Rate accepted by `rate_string_to_bps`
39
+ def rate_bps_to_string(bps)
40
+ prefix = PREFIX.reverse.find { |candidate| bps.positive? && (bps % MULTIPLIER[candidate]).zero? } || ''
41
+ "#{bps / MULTIPLIER[prefix]}#{prefix}"
42
+ end
43
+
44
+ # @param bytes [Integer] Size in bytes
45
+ # @return [String] Size with decimal unit, e.g. `12.3 MB`
46
+ def format_bytes(bytes)
47
+ value = bytes.to_f
48
+ index = 0
49
+ while value >= KILO && index < BYTE_UNITS.length - 1
50
+ value /= KILO
51
+ index += 1
52
+ end
53
+ return "#{bytes} B" if index.zero?
54
+ format('%.1f %s', value, BYTE_UNITS[index])
55
+ end
56
+
57
+ # @param bytes_per_second [Float, nil] Rate, `nil` if unknown
58
+ # @return [String] Rate in megabits per second (decimal, like `ascp`)
59
+ def format_rate(bytes_per_second)
60
+ return '-- Mbps' if bytes_per_second.nil?
61
+ mbps = bytes_per_second * BITS_PER_BYTE / MULTIPLIER.fetch('M')
62
+ mbps < 10 ? format('%.2f Mbps', mbps) : format('%.0f Mbps', mbps)
63
+ end
64
+
65
+ # @param seconds [Numeric, nil] Duration, `nil` if unknown
66
+ # @return [String] Duration as `hh:mm:ss`
67
+ def format_duration(seconds)
68
+ return '--:--:--' if seconds.nil?
69
+ seconds = seconds.round
70
+ format('%02d:%02d:%02d', seconds / 3600, (seconds / 60) % 60, seconds % 60)
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
@@ -1,10 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'log'
8
+ require_relative 'rest'
9
+ require_relative 'temp_file_manager'
3
10
  require 'uri'
4
11
  require 'base64'
5
- require 'aspera/assert'
6
- require 'aspera/rest'
7
- require 'aspera/temp_file_manager'
8
12
 
9
13
  module Aspera
10
14
  # Read content from a URI; supported schemes: file:, http:, https:, data:
@@ -25,13 +29,15 @@ module Aspera
25
29
  # The canonical form built by {file_url} uses the `file:///` prefix.
26
30
  module UriReader
27
31
  SCHEME_FILE = 'file'
28
- SCHEME_FILE_PFX1 = "#{SCHEME_FILE}:"
32
+ SCHEME_FILE_PFX1 = "#{SCHEME_FILE}:".freeze
29
33
  # Canonical prefix for file: URLs (no host, three slashes).
30
34
  # What follows is the literal path: relative or absolute (starting with a second slash).
31
- SCHEME_FILE_PFX2 = "#{SCHEME_FILE_PFX1}///"
32
- private_constant :SCHEME_FILE, :SCHEME_FILE_PFX1, :SCHEME_FILE_PFX2
35
+ SCHEME_FILE_PFX2 = "#{SCHEME_FILE_PFX1}///".freeze
36
+ # Timeout in seconds when another URL remains to be tried
37
+ FAIL_FAST_TIMEOUT = 10
38
+ private_constant :SCHEME_FILE, :SCHEME_FILE_PFX1, :SCHEME_FILE_PFX2, :FAIL_FAST_TIMEOUT
33
39
  class << self
34
- # @return [Boolean] true if `url` uses the file: scheme recognised by this module
40
+ # @return [Boolean] true if `url` uses the file: scheme recognized by this module
35
41
  def file?(url)
36
42
  url.start_with?(SCHEME_FILE_PFX1)
37
43
  end
@@ -87,6 +93,50 @@ module Aspera
87
93
  end
88
94
  end
89
95
 
96
+ # Read URL with a short timeout and no retry (temporarily changes global HTTP parameters)
97
+ # @param url [String] URL to read
98
+ # @return [String] content
99
+ def read_fail_fast(url)
100
+ parameters = Rest::Parameters.instance
101
+ session_cb = parameters.session_cb
102
+ retry_max = parameters.retry_max
103
+ parameters.retry_max = 0
104
+ parameters.session_cb = lambda do |http_session|
105
+ session_cb&.call(http_session)
106
+ http_session.open_timeout = [http_session.open_timeout, FAIL_FAST_TIMEOUT].compact.min
107
+ http_session.read_timeout = [http_session.read_timeout, FAIL_FAST_TIMEOUT].compact.min
108
+ end
109
+ read(url)
110
+ ensure
111
+ parameters.session_cb = session_cb
112
+ parameters.retry_max = retry_max
113
+ end
114
+
115
+ # Read the first URL that provides valid content.
116
+ # All but the last URL are read with a short timeout and no retry, so that fallback is quick.
117
+ # On failure of a URL (read error, or validation error), a warning is logged and the next URL is tried.
118
+ # The failure of the last URL is raised.
119
+ # @param urls [Array<String>] URLs, in order of preference
120
+ # @param what [String] description of the content, for messages
121
+ # @yield [content] Optional: validates and converts content, shall raise on invalid content
122
+ # @yieldparam content [String] content read
123
+ # @yieldreturn [Object] value returned by this method
124
+ # @return [Object] result of block if given, else content
125
+ def read_first(urls, what: 'content')
126
+ Aspera.assert(!urls.empty?) { "no URL for #{what}" }
127
+ urls.each_with_index do |url, index|
128
+ last = index.eql?(urls.length - 1)
129
+ Log.log.debug { "Retrieving #{what} from #{url}" }
130
+ begin
131
+ content = last ? read(url) : read_fail_fast(url)
132
+ return block_given? ? yield(content) : content
133
+ rescue StandardError => e
134
+ raise if last
135
+ Log.log.warn { "Failed to retrieve #{what} from #{url}: #{e.message}, trying next" }
136
+ end
137
+ end
138
+ end
139
+
90
140
  # Return the local file-system path for the content at `url`, downloading to a temp file if needed.
91
141
  # For file: URLs the path is extracted directly (no download).
92
142
  # For data: and http(s): URLs the content is written to a temporary file and its path is returned.
@@ -0,0 +1,20 @@
1
+ <%# Content of the page displayed after authentication, in layout of Aspera::WebPage %>
2
+ <div class="icon-circle ok">
3
+ <svg xmlns="http://www.w3.org/2000/svg" width="30" height="30" viewBox="0 0 32 32" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round">
4
+ <polyline points="6 16 13 23 26 9"/>
5
+ </svg>
6
+ </div>
7
+
8
+ <h1>Authentication Received</h1>
9
+ <p>Return to your terminal: it completes the login and shows the result. You can then close this window.</p>
10
+
11
+ <% unless additional_info.to_s.empty? %>
12
+ <div class="info-box">
13
+ <%= h additional_info %>
14
+ </div>
15
+ <% end %>
16
+
17
+ <%# The callback URL may contain a code or tokens: remove the query from the address bar and from the entry of the session history %>
18
+ <script>
19
+ if (window.history && history.replaceState) history.replaceState(null, '', location.pathname);
20
+ </script>