aspera-cli 4.28.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +217 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +122 -26
  10. data/SECURITY.md +84 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1895 -632
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +29 -22
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +95 -67
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +370 -0
  41. data/lib/aspera/ascp/log_locations.rb +163 -0
  42. data/lib/aspera/ascp/management.rb +16 -8
  43. data/lib/aspera/ascp/session_errors.rb +217 -0
  44. data/lib/aspera/assert.rb +4 -0
  45. data/lib/aspera/cli/ascp_actions.rb +12 -19
  46. data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
  47. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  48. data/lib/aspera/cli/bootstrapper.rb +38 -33
  49. data/lib/aspera/cli/command_line.rb +63 -8
  50. data/lib/aspera/cli/command_registry.rb +9 -1
  51. data/lib/aspera/cli/command_spec.rb +47 -7
  52. data/lib/aspera/cli/completion/ascli.bash +19 -5
  53. data/lib/aspera/cli/completion/ascli.fish +13 -2
  54. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  55. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  56. data/lib/aspera/cli/completion_actions.rb +303 -69
  57. data/lib/aspera/cli/context.rb +9 -4
  58. data/lib/aspera/cli/deprecation.rb +5 -2
  59. data/lib/aspera/cli/error.rb +6 -0
  60. data/lib/aspera/cli/extended_value.rb +66 -31
  61. data/lib/aspera/cli/formatter.rb +43 -30
  62. data/lib/aspera/cli/gem_checker.rb +9 -6
  63. data/lib/aspera/cli/hints.rb +22 -10
  64. data/lib/aspera/cli/http.rb +38 -10
  65. data/lib/aspera/cli/info.rb +12 -7
  66. data/lib/aspera/cli/mailer.rb +6 -3
  67. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  68. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  69. data/lib/aspera/cli/mcp/tool.rb +20 -17
  70. data/lib/aspera/cli/option_declarator.rb +10 -4
  71. data/lib/aspera/cli/option_registry.rb +6 -3
  72. data/lib/aspera/cli/option_types.rb +17 -2
  73. data/lib/aspera/cli/option_value.rb +77 -18
  74. data/lib/aspera/cli/options.schema.yaml +641 -80
  75. data/lib/aspera/cli/parser.rb +71 -35
  76. data/lib/aspera/cli/plugins/alee.rb +7 -4
  77. data/lib/aspera/cli/plugins/aoc.rb +248 -37
  78. data/lib/aspera/cli/plugins/ats.rb +10 -9
  79. data/lib/aspera/cli/plugins/base.rb +50 -18
  80. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  81. data/lib/aspera/cli/plugins/config.rb +210 -104
  82. data/lib/aspera/cli/plugins/console.rb +6 -3
  83. data/lib/aspera/cli/plugins/cos.rb +8 -4
  84. data/lib/aspera/cli/plugins/factory.rb +21 -7
  85. data/lib/aspera/cli/plugins/faspex5.rb +31 -26
  86. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  87. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  88. data/lib/aspera/cli/plugins/mcp.rb +42 -22
  89. data/lib/aspera/cli/plugins/node.rb +39 -38
  90. data/lib/aspera/cli/plugins/oauth.rb +23 -6
  91. data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
  92. data/lib/aspera/cli/plugins/preview.rb +72 -45
  93. data/lib/aspera/cli/plugins/server.rb +66 -14
  94. data/lib/aspera/cli/plugins/shares.rb +87 -18
  95. data/lib/aspera/cli/plugins/transferd.rb +228 -0
  96. data/lib/aspera/cli/poll.rb +236 -0
  97. data/lib/aspera/cli/preset_actions.rb +144 -30
  98. data/lib/aspera/cli/preset_manager.rb +113 -35
  99. data/lib/aspera/cli/prompt.rb +157 -9
  100. data/lib/aspera/cli/result.rb +12 -8
  101. data/lib/aspera/cli/runner.rb +89 -48
  102. data/lib/aspera/cli/secret_finder.rb +4 -1
  103. data/lib/aspera/cli/security.rb +89 -9
  104. data/lib/aspera/cli/special_values.rb +4 -1
  105. data/lib/aspera/cli/sync_actions.rb +15 -10
  106. data/lib/aspera/cli/table_fitter.rb +3 -0
  107. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  108. data/lib/aspera/cli/transfer_actions.rb +11 -8
  109. data/lib/aspera/cli/transfer_agent.rb +43 -20
  110. data/lib/aspera/cli/transfer_progress.rb +14 -45
  111. data/lib/aspera/cli/vault_actions.rb +45 -0
  112. data/lib/aspera/cli/vault_manager.rb +188 -42
  113. data/lib/aspera/cli/version.rb +4 -1
  114. data/lib/aspera/cli/web_secret.rb +12 -10
  115. data/lib/aspera/cli/wizard.rb +42 -39
  116. data/lib/aspera/command_line_builder.rb +9 -5
  117. data/lib/aspera/command_line_converter.rb +4 -1
  118. data/lib/aspera/coverage.rb +103 -33
  119. data/lib/aspera/data_repository.rb +4 -1
  120. data/lib/aspera/dot_container.rb +4 -1
  121. data/lib/aspera/environment.rb +23 -12
  122. data/lib/aspera/exec_spec.rb +3 -0
  123. data/lib/aspera/faspex_gw.rb +13 -9
  124. data/lib/aspera/faspex_postproc.rb +15 -12
  125. data/lib/aspera/formatter_interface.rb +4 -1
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +20 -5
  129. data/lib/aspera/inflate.rb +43 -0
  130. data/lib/aspera/json_rpc/client.rb +6 -5
  131. data/lib/aspera/json_rpc/version.rb +3 -0
  132. data/lib/aspera/line_logger.rb +4 -1
  133. data/lib/aspera/link_header.rb +4 -1
  134. data/lib/aspera/log.rb +18 -5
  135. data/lib/aspera/markdown.rb +5 -2
  136. data/lib/aspera/mime.rb +3 -0
  137. data/lib/aspera/nagios.rb +6 -4
  138. data/lib/aspera/node_emulator.rb +15 -13
  139. data/lib/aspera/oauth/client/base.rb +124 -73
  140. data/lib/aspera/oauth/client/boot.rb +7 -4
  141. data/lib/aspera/oauth/client/factory.rb +41 -18
  142. data/lib/aspera/oauth/client/generic.rb +5 -1
  143. data/lib/aspera/oauth/client/json_credentials.rb +7 -3
  144. data/lib/aspera/oauth/client/jwt.rb +6 -3
  145. data/lib/aspera/oauth/client/shares_web.rb +94 -0
  146. data/lib/aspera/oauth/client/url_json.rb +7 -3
  147. data/lib/aspera/oauth/client/web.rb +40 -13
  148. data/lib/aspera/oauth/server/pkce.rb +6 -5
  149. data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +16 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +17 -2
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +3 -0
  165. data/lib/aspera/rest/aspera_errors.rb +4 -1
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +45 -34
  168. data/lib/aspera/rest/error_analyzer.rb +5 -2
  169. data/lib/aspera/rest/list.rb +8 -4
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +8 -5
  172. data/lib/aspera/rest.rb +9 -6
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  175. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  176. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  177. data/lib/aspera/schema/async_tables.yaml +11 -10
  178. data/lib/aspera/schema/documentation.rb +8 -2
  179. data/lib/aspera/schema/reader.rb +7 -0
  180. data/lib/aspera/schema/registry.rb +44 -28
  181. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  182. data/lib/aspera/schema/validator.rb +4 -1
  183. data/lib/aspera/sealed_hash.rb +150 -0
  184. data/lib/aspera/secret/hider.rb +213 -0
  185. data/lib/aspera/secret/reference.rb +150 -0
  186. data/lib/aspera/secret/resolver.rb +212 -0
  187. data/lib/aspera/secret/store/base.rb +153 -0
  188. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  189. data/lib/aspera/secret/store/factory.rb +103 -0
  190. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  191. data/lib/aspera/secret/store/keeper.rb +141 -0
  192. data/lib/aspera/secret/store/macos_security.rb +284 -0
  193. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  194. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  195. data/lib/aspera/secret/store/one_password_cli.rb +112 -0
  196. data/lib/aspera/secret/store/secret_service.rb +125 -0
  197. data/lib/aspera/secret/strength.rb +119 -0
  198. data/lib/aspera/servlet_helpers.rb +5 -2
  199. data/lib/aspera/ssh.rb +10 -7
  200. data/lib/aspera/ssl.rb +5 -2
  201. data/lib/aspera/string_ext.rb +3 -0
  202. data/lib/aspera/sync/args.schema.yaml +3 -0
  203. data/lib/aspera/sync/conf.schema.yaml +9 -4
  204. data/lib/aspera/sync/database.rb +3 -0
  205. data/lib/aspera/sync/operations.rb +17 -15
  206. data/lib/aspera/temp_file_manager.rb +8 -4
  207. data/lib/aspera/timer_limiter.rb +3 -0
  208. data/lib/aspera/transfer/error.rb +4 -1
  209. data/lib/aspera/transfer/faux_file.rb +4 -1
  210. data/lib/aspera/transfer/parameters.rb +17 -14
  211. data/lib/aspera/transfer/result.rb +23 -0
  212. data/lib/aspera/transfer/resumer.rb +6 -4
  213. data/lib/aspera/transfer/spec.rb +9 -22
  214. data/lib/aspera/units/decimal.rb +67 -0
  215. data/lib/aspera/units/duration.rb +109 -0
  216. data/lib/aspera/uri_reader.rb +64 -9
  217. data/lib/aspera/web_auth/index.html.erb +6 -2
  218. data/lib/aspera/web_auth/paste.rb +109 -0
  219. data/lib/aspera/web_auth/receiver.rb +50 -0
  220. data/lib/aspera/web_auth/relay.rb +76 -0
  221. data/lib/aspera/web_auth/reply.html.erb +150 -0
  222. data/lib/aspera/web_auth/server.rb +47 -10
  223. data/lib/aspera/web_page.rb +7 -2
  224. data/lib/aspera/web_server_simple.rb +6 -3
  225. data/lib/aspera/yaml.rb +5 -1
  226. data/llms-install.md +88 -0
  227. data/llms.txt +87 -0
  228. data/skills/ascli/SKILL.md +94 -0
  229. data.tar.gz.sig +0 -0
  230. metadata +77 -20
  231. metadata.gz.sig +0 -0
  232. data/lib/aspera/duration.rb +0 -42
  233. data/lib/aspera/keychain/base.rb +0 -30
  234. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  235. data/lib/aspera/keychain/factory.rb +0 -58
  236. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  237. data/lib/aspera/keychain/macos_security.rb +0 -277
  238. data/lib/aspera/keychain/one_password_api.rb +0 -91
  239. data/lib/aspera/keychain/one_password_base.rb +0 -39
  240. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  241. data/lib/aspera/secret_hider.rb +0 -196
  242. data/lib/aspera/secret_strength.rb +0 -114
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+ require_relative 'error'
3
8
  require 'singleton'
4
- require 'aspera/assert'
5
- require 'aspera/cli/error'
6
9
 
7
10
  module Aspera
8
11
  module Cli
@@ -19,10 +22,12 @@ module Aspera
19
22
  # Security levels, from least to most restrictive
20
23
  LEVELS = %i[low medium high].freeze
21
24
  # Extended-value handlers restricted by security level, with the levels that allow them
25
+ # `zlib`: a small value can decompress to gigabytes (decompression bomb), a limit on its size would not limit the number of values
22
26
  EXTENDED_VALUE_LEVELS = {
23
27
  ruby: %i[low],
24
28
  uri: %i[low],
25
29
  env: %i[low],
30
+ zlib: %i[low],
26
31
  file: %i[low medium]
27
32
  }.freeze
28
33
  # Levels that allow processing a secret (e.g. by an extended value): the result could reveal it, as only declared values are hidden
@@ -39,6 +44,7 @@ module Aspera
39
44
  @level = nil
40
45
  @default_level = LEVELS.first
41
46
  @allowed_paths = []
47
+ @writable_paths = []
42
48
  @locked = nil
43
49
  end
44
50
 
@@ -47,7 +53,7 @@ module Aspera
47
53
 
48
54
  # Set by option `security`.
49
55
  # When locked, the value can only be set again to the current one.
50
- # @param value [Hash] `level` and `allowed_paths`, see schema `SecurityOptions`
56
+ # @param value [Hash] `level`, `allowed_paths` and `writable_paths`, see schema `SecurityOptions`
51
57
  # @return [nil]
52
58
  def options=(value)
53
59
  Aspera.assert_type(value, Hash, type: BadArgument) { 'security' }
@@ -56,19 +62,17 @@ module Aspera
56
62
  value = value.transform_keys(&:to_sym)
57
63
  new_level = value[:level]&.to_sym
58
64
  Aspera.assert_values(new_level, LEVELS, type: BadArgument) { 'security.level' } unless new_level.nil?
59
- new_paths = Array(value[:allowed_paths]).map do |path|
60
- path = File.expand_path(path)
61
- Aspera.assert(File.exist?(path), type: BadArgument) { "security.allowed_paths: no such file or folder: #{path}" }
62
- File.realpath(path)
63
- end
65
+ new_paths = existing_real_paths(value[:allowed_paths], 'allowed_paths')
66
+ new_writable_paths = existing_real_paths(value[:writable_paths], 'writable_paths')
64
67
  if @locked
65
- Aspera.assert((new_level || @default_level).eql?(level) && new_paths.eql?(@allowed_paths), type: BadArgument) do
68
+ Aspera.assert((new_level || @default_level).eql?(level) && new_paths.eql?(@allowed_paths) && new_writable_paths.eql?(@writable_paths), type: BadArgument) do
66
69
  "security cannot be changed #{@locked}"
67
70
  end
68
71
  return
69
72
  end
70
73
  @level = new_level
71
74
  @allowed_paths = new_paths
75
+ @writable_paths = new_writable_paths
72
76
  nil
73
77
  end
74
78
 
@@ -83,6 +87,39 @@ module Aspera
83
87
  nil
84
88
  end
85
89
 
90
+ # @param max_level [Symbol, nil] `security:` attribute of an option or command: most restrictive level that allows it, `nil` for any level
91
+ # @return [Boolean] `true` if the feature is available at current level
92
+ def allowed?(max_level)
93
+ max_level.nil? || LEVELS.index(level) <= LEVELS.index(max_level)
94
+ end
95
+
96
+ # Features that execute code (own or downloaded), or load code, declare `security: :low`.
97
+ # @param max_level [Symbol, nil] `security:` attribute of an option or command
98
+ # @param what [String] Feature, e.g. `option notify_template`
99
+ # @raise [BadArgument] If the feature is refused at current level
100
+ def check_level(max_level, what)
101
+ return if allowed?(max_level)
102
+ raise BadArgument, "#{what}: disabled by security level #{level}: use level #{max_level} (#{where_to_set})"
103
+ end
104
+
105
+ # A value stored to be used later (e.g. in a preset) is evaluated when used, maybe at another level (e.g. `low` on the command line).
106
+ # So it must not use an extended value that is refused at current level, e.g. `@val:@ruby:...`.
107
+ # @param value [Object] Value to store (`Hash`, `Array`, `String`)
108
+ # @param what [String] What stores the value, e.g. `preset option ts`
109
+ # @raise [BadArgument] If the value contains an extended value refused at current level
110
+ def check_stored_value(value, what)
111
+ case value
112
+ when Hash then value.each { |k, v| [k, v].each { |item| check_stored_value(item, what) } }
113
+ when Array then value.each { |v| check_stored_value(v, what) }
114
+ when String
115
+ EXTENDED_VALUE_LEVELS.each do |handler, levels|
116
+ next if levels.include?(level) || !value.include?("@#{handler}:")
117
+ raise BadArgument, "#{what}: cannot store @#{handler}: extended value, disabled by security level #{level} (#{where_to_set})"
118
+ end
119
+ end
120
+ nil
121
+ end
122
+
86
123
  # @param handler [Symbol] Extended-value handler
87
124
  # @raise [BadArgument] If the handler is refused at current level, with the level that allows it and where to set it
88
125
  def check_extended_value(handler)
@@ -123,8 +160,51 @@ module Aspera
123
160
  end
124
161
  end
125
162
 
163
+ # Read a local file chosen by the caller, e.g. `@file:`, or a command that reads a file to display something about it.
164
+ # Not for a file that a command sends (e.g. upload, import of a workflow): not restricted, as transfers.
165
+ # @param path [String] Path of a file to read
166
+ # Other arguments are those of `File.read`, e.g. `binmode: true` for bytes
167
+ # @return [String] Content of the file
168
+ # @raise [BadArgument] If reading the file is refused at current level
169
+ def read_file(path, ...) = File.read(readable_path(path), ...)
170
+
171
+ # Writing a file where the client chooses (e.g. downloaded file) could replace a file that is used later (e.g. plugin, shell startup file).
172
+ # At level `medium` and `high`, files can be written only in `writable_paths` (none by default): the real path of the folder is checked,
173
+ # so that neither `..` nor a symbolic link leads outside of them.
174
+ # @param path [String] Local path of a file or folder to write, that may not exist yet
175
+ # (not called for a transfer agent that writes on a remote system, see `Agent::Base#local_destination?`)
176
+ # @return [String] Path to write: unchanged at level `low` (no restriction), else the real path
177
+ # @raise [BadArgument] If writing is refused at current level
178
+ def writable_path(path)
179
+ return path if level.eql?(:low)
180
+ path = File.expand_path(path)
181
+ # The first existing parent gives the real path, the missing part contains no link
182
+ existing = path
183
+ missing = []
184
+ until File.exist?(existing) || File.dirname(existing).eql?(existing)
185
+ missing.unshift(File.basename(existing))
186
+ existing = File.dirname(existing)
187
+ end
188
+ real_path = File.join(File.realpath(existing), *missing)
189
+ Aspera.assert(@writable_paths.any? { |allowed| real_path.eql?(allowed) || real_path.start_with?(File.join(allowed, '')) }, type: BadArgument) do
190
+ "writing is refused by security level #{level}: not in security.writable_paths: #{path} (#{where_to_set})"
191
+ end
192
+ real_path
193
+ end
194
+
126
195
  private
127
196
 
197
+ # @param paths [Array<String>, nil] Paths given to option `security`
198
+ # @param key [String] Name of key, for error messages
199
+ # @return [Array<String>] Real paths
200
+ def existing_real_paths(paths, key)
201
+ Array(paths).map do |path|
202
+ path = File.expand_path(path)
203
+ Aspera.assert(File.exist?(path), type: BadArgument) { "security.#{key}: no such file or folder: #{path}" }
204
+ File.realpath(path)
205
+ end
206
+ end
207
+
128
208
  # @return [String] Where to set the level, for error messages
129
209
  def where_to_set
130
210
  @locked.nil? ? 'option security' : "option security when starting, as it cannot be changed #{@locked}"
@@ -1,11 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  module Cli
5
8
  # base class for plugins modules
6
9
  module SpecialValues
7
10
  # special values
8
- # `INIT` to unitialize with all current IDs
11
+ # `INIT` to initialize with all current IDs
9
12
  INIT = 'INIT'
10
13
  # `ALL` means "all ids"
11
14
  ALL = 'ALL'
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/sync/operations'
4
- require 'aspera/assert'
5
- require 'aspera/environment'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../sync/operations'
7
+ require_relative '../assert'
8
+ require_relative '../environment'
6
9
  require 'pathname'
7
10
 
8
11
  module Aspera
@@ -16,7 +19,7 @@ module Aspera
16
19
  %w[Syncd Error Confl Pconf] +
17
20
  (23..24).map { |i| "P(#{i})" }).freeze
18
21
  # Positional arguments shared by sync transfer commands (push/pull/bidi) and sync admin commands.
19
- PATH_AND_INFO_ARGS = [{name: :path}, {name: :sync_info, type: Hash, mandatory: false, default: {}}].freeze
22
+ PATH_AND_INFO_ARGS = [{name: :path, complete: :folder}, {name: :sync_info, type: Hash, mandatory: false, default: {}}].freeze
20
23
  # When a plugin class includes SyncActions, register the :sql option
21
24
  # in that class's DSL registry so Base#initialize auto-declares it,
22
25
  # and add the class-level DSL methods of ClassMethods.
@@ -34,7 +37,7 @@ module Aspera
34
37
  def sync_admin_commands(parent)
35
38
  commands_under(parent, description: 'Manage sync database (admin operations)') do
36
39
  command(
37
- :find, description: 'Find sync database files', arguments: [{name: :path}],
40
+ :find, description: 'Find sync database files', arguments: [{name: :path, complete: :folder}],
38
41
  action: lambda do |path:, **|
39
42
  dbs = Sync::Operations.list_db_files(path)
40
43
  Result::ObjectList.new(dbs.keys.map { |n| {name: n, path: dbs[n]} })
@@ -47,14 +50,14 @@ module Aspera
47
50
  command(
48
51
  :meta, description: 'Show sync session metadata', arguments: PATH_AND_INFO_ARGS,
49
52
  action: lambda do |path:, sync_info: {}, **|
50
- require 'aspera/sync/database'
53
+ require_relative '../sync/database'
51
54
  Result::SingleObject.new(db_from_args(path: path, sync_info: sync_info).meta(options.get_option(:sql)))
52
55
  end
53
56
  )
54
57
  command(
55
58
  :counters, description: 'Show sync counters', arguments: PATH_AND_INFO_ARGS,
56
59
  action: lambda do |path:, sync_info: {}, **|
57
- require 'aspera/sync/database'
60
+ require_relative '../sync/database'
58
61
  Result::SingleObject.new(db_from_args(path: path, sync_info: sync_info).counters(options.get_option(:sql)))
59
62
  end
60
63
  )
@@ -62,14 +65,14 @@ module Aspera
62
65
  command(
63
66
  :overview, description: 'Show sync database overview', arguments: PATH_AND_INFO_ARGS,
64
67
  action: lambda do |path:, sync_info: {}, **|
65
- require 'aspera/sync/database'
68
+ require_relative '../sync/database'
66
69
  Result::ObjectList.new(db_from_args(path: path, sync_info: sync_info).overview, fields: %w[table name type])
67
70
  end
68
71
  )
69
72
  command(
70
73
  :query, description: 'Execute a raw SQL query', arguments: PATH_AND_INFO_ARGS,
71
74
  action: lambda do |path:, sync_info: {}, **|
72
- require 'aspera/sync/database'
75
+ require_relative '../sync/database'
73
76
  Result.auto(db_from_args(path: path, sync_info: sync_info).execute(options.get_option(:sql, mandatory: true)))
74
77
  end
75
78
  )
@@ -154,7 +157,7 @@ module Aspera
154
157
  end
155
158
 
156
159
  def action_sync_admin_file_info(path:, sync_info: {}, **)
157
- require 'aspera/sync/database'
160
+ require_relative '../sync/database'
158
161
  result = db_from_args(path: path, sync_info: sync_info).file_info(options.get_option(:sql))
159
162
  result.each do |r|
160
163
  r['sstate'] = SyncActions::STATE_STR[r['state']] if r['state']
@@ -168,6 +171,8 @@ module Aspera
168
171
  # @param sync_info [Hash] extra sync configuration (may be empty)
169
172
  # @param block [Proc, nil] block to generate transfer spec; receives (direction, local_dir, remote_dir)
170
173
  def run_sync_transfer(direction, path:, sync_info: {}, &block)
174
+ # Not a transfer of `TransferAgent`: the sync session is started directly
175
+ transfer.assert_no_dry_run
171
176
  Sync::Operations.start(async_info_from_args(path: path, sync_info: sync_info, direction: direction), transfer.user_transfer_spec, &block)
172
177
  Result::Success.new
173
178
  end
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  require 'terminal-table'
4
7
  require 'unicode/display_width'
5
8
 
@@ -1,17 +1,20 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/environment'
4
- require 'aspera/formatter_interface'
5
- require 'aspera/markdown'
6
- require 'aspera/assert'
7
- require 'aspera/rainbow'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../environment'
7
+ require_relative '../formatter_interface'
8
+ require_relative '../markdown'
9
+ require_relative '../assert'
10
+ require_relative '../rainbow'
8
11
  using Rainbow
9
12
 
10
13
  module Aspera
11
14
  module Cli
12
15
  # Terminal formatter with ANSI colors and Unicode support
13
16
  # @see FormatterInterface
14
- # @see MarkdownFormatter (in build/lib/doc_helper.rb)
17
+ # @see MarkdownFormatter (in build/lib/markdown_formatter.rb)
15
18
  module TerminalFormatter
16
19
  include FormatterInterface
17
20
 
@@ -1,15 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/cli/async_transfer_store'
4
- require 'aspera/agent/factory'
5
- require 'aspera/log'
6
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'async_transfer_store'
7
+ require_relative '../agent/factory'
8
+ require_relative '../log'
9
+ require_relative '../assert'
7
10
 
8
11
  module Aspera
9
12
  module Cli
10
13
  # Mixin for Config plugin: async transfer management actions.
11
- # Exposes three sub-commands under `config transfer`:
12
- # status --id=<job_id> Re-query a running/completed transfer
14
+ # Exposes three sub-commands under `config agents jobs` (`config transfer` is a deprecated alias):
15
+ # status <job_id> Re-query a running/completed transfer
13
16
  # list List all persisted async transfer entries
14
17
  # cleanup Remove completed/failed/cancelled entries
15
18
  #
@@ -22,7 +25,7 @@ module Aspera
22
25
  module TransferActions
23
26
  # Re-query the status of a single async transfer job.
24
27
  # @param job_id [String] UUID returned at submission time
25
- def action_transfer_status(job_id:, **)
28
+ def action_agents_jobs_status(job_id:, **)
26
29
  store = async_transfer_store
27
30
  entry = store.read(job_id)
28
31
  Aspera.assert(!entry.nil?, type: Cli::BadArgument) { "Unknown job_id: #{job_id}" }
@@ -43,7 +46,7 @@ module Aspera
43
46
  private_constant :TERMINAL_STATUSES
44
47
 
45
48
  # Delete completed, failed, and cancelled entries from the store.
46
- def action_transfer_cleanup(**)
49
+ def action_agents_jobs_cleanup(**)
47
50
  store = async_transfer_store
48
51
  deleted = store.list
49
52
  .select { |e| TERMINAL_STATUSES.include?(e['status']) }
@@ -1,15 +1,19 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/agent/factory'
4
- require 'aspera/schema/registry'
5
- require 'aspera/cli/option_declarator'
6
- require 'aspera/transfer/result'
7
- require 'aspera/transfer/spec'
8
- require 'aspera/cli/info'
9
- require 'aspera/cli/async_transfer_store'
10
- require 'aspera/log'
11
- require 'aspera/assert'
12
- require 'aspera/secret_hider'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../agent/factory'
7
+ require_relative '../schema/registry'
8
+ require_relative 'option_declarator'
9
+ require_relative '../transfer/result'
10
+ require_relative '../transfer/spec'
11
+ require_relative 'info'
12
+ require_relative 'async_transfer_store'
13
+ require_relative '../log'
14
+ require_relative '../assert'
15
+ require_relative '../secret/hider'
16
+ require_relative 'security'
13
17
  require 'securerandom'
14
18
 
15
19
  module Aspera
@@ -46,7 +50,7 @@ module Aspera
46
50
  extend OptionDeclarator
47
51
 
48
52
  option :ts, description: 'Override transfer spec values', schema: Schema::Registry::TRANSFER_SPEC, on_set: :user_transfer_spec=
49
- option :to_folder, description: 'Destination folder for transferred files'
53
+ option :to_folder, description: 'Destination folder for transferred files', complete: :destination_folder
50
54
  option :sources, description: "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})", default: FILE_LIST_FROM_ARGS
51
55
  option :src_type, description: 'Type of file list', allowed: %i[list pair], default: :list
52
56
  option :transfer, description: 'Transfer agent type, or agent parameters with optional agent key', allowed: [Hash, String], shorthand: 'agent', schema: Schema::Registry::TRANSFER_AGENT_OPTIONS
@@ -224,8 +228,8 @@ module Aspera
224
228
 
225
229
  # Start a transfer and wait for completion, plugins shall use this method
226
230
  # @param transfer_spec [Hash] transfer specification
227
- # @param rest_token [Rest::Client, nil] if oauth token regeneration supported
228
- def start(transfer_spec, rest_token: nil)
231
+ # @param oauth_api [Rest::Client, nil] API whose OAuth token renews the bearer token of the transfer, if rejected
232
+ def start(transfer_spec, oauth_api: nil)
229
233
  # check parameters
230
234
  Aspera.assert_type(transfer_spec, Hash) { 'transfer_spec' }
231
235
  raise "Wrong remote host: #{CP4I_REMOTE_HOST_LB}" if transfer_spec['remote_host'].eql?(CP4I_REMOTE_HOST_LB)
@@ -252,6 +256,9 @@ module Aspera
252
256
  @user_transfer_spec['paths'] = transfer_spec['paths'] || ts_source_paths
253
257
  # updated transfer spec with command line
254
258
  transfer_spec.deep_merge!(@user_transfer_spec)
259
+ # Files are written where the client chooses, see Security: only for an agent that writes on the local system
260
+ receives_locally = transfer_spec['direction'].eql?(Transfer::Spec::DIRECTION_RECEIVE) && agent_instance.local_destination?
261
+ transfer_spec['destination_root'] = Security.instance.writable_path(transfer_spec['destination_root']) if receives_locally && transfer_spec['destination_root']
255
262
  # resolve pseudo-parameter: target_rate -> target_rate_kbps (overrides target_rate_kbps if both are present)
256
263
  Transfer::Spec.resolve_target_rate(transfer_spec)
257
264
  # recursively remove values that are nil (user wants to delete)
@@ -259,9 +266,13 @@ module Aspera
259
266
  # if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
260
267
  transfer_spec['content_protection_password'] = @context.options.prompt_user_input('content protection password', sensitive: true) if transfer_spec['content_protection'].eql?('decrypt') && !transfer_spec.key?('content_protection_password')
261
268
  # Secrets of transfer spec (e.g. `token` from the application) are hidden wherever they appear (logs, results)
262
- SecretHider.instance.add_secrets_in(transfer_spec)
269
+ Secret::Hider.instance.add_secrets_in(transfer_spec)
270
+ # Option `dry_run`: no transfer, the final transfer spec is the result
271
+ return Transfer::Result.dry_run(transfer_spec) if dry_run?
272
+ # Renews the bearer token of the transfer: only possible with OAuth
273
+ token_regenerator = -> { oauth_api.oauth.renewed_authorization } if oauth_api&.auth_params&.fetch(:type, nil).eql?(:oauth2)
263
274
  # create transfer agent
264
- agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
275
+ agent_instance.start_transfer(transfer_spec, token_regenerator: token_regenerator)
265
276
  # --- async mode ---
266
277
  if transfer_options['asynchronous']
267
278
  agent_type = (transfer_options['agent'] || 'direct').to_s
@@ -284,12 +295,12 @@ module Aspera
284
295
  # (e.g. MCP server mode). The ref is stored in AsyncTransferStore's memory map,
285
296
  # never on disk.
286
297
  async_store.register_agent_ref(job_id, agent_instance) if %w[direct httpgw].include?(agent_type)
287
- # transfer_id: in-process agents use job_id; daemon agents expose @transfer_id
298
+ # transfer_id: in-process agents use job_id; transferd has a reader; other daemon agents keep @transfer_id
288
299
  transfer_id =
289
- if %w[direct httpgw].include?(agent_type)
290
- job_id
291
- else
292
- agent_instance.instance_variable_get(:@transfer_id).to_s
300
+ case agent_type
301
+ when 'direct', 'httpgw' then job_id
302
+ when 'transferd' then agent_instance.transfer_id.to_s
303
+ else agent_instance.instance_variable_get(:@transfer_id).to_s
293
304
  end
294
305
  async_store.write(job_id, {
295
306
  'job_id' => job_id,
@@ -311,6 +322,18 @@ module Aspera
311
322
  return result
312
323
  end
313
324
 
325
+ # @return [Boolean] `true` if option `dry_run` is set: transfers are not started (see `start`)
326
+ def dry_run?
327
+ @context.options.get_option(:dry_run)
328
+ end
329
+
330
+ # For a command that changes something before its transfer (e.g. creates a package), or that does not transfer with `start`:
331
+ # fail before any change with option `dry_run`
332
+ # @raise [Cli::BadArgument] if option `dry_run` is set
333
+ def assert_no_dry_run
334
+ Aspera.assert(!dry_run?, type: Cli::BadArgument) { 'Option dry_run is not supported by this command' }
335
+ end
336
+
314
337
  # shut down if agent requires it
315
338
  def shutdown
316
339
  @agent.shutdown if @agent.respond_to?(:shutdown)
@@ -1,9 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative '../units/decimal'
9
+ require_relative '../units/duration'
3
10
  require 'monitor'
4
- require 'aspera/environment'
5
- require 'aspera/log'
6
- require 'aspera/assert'
7
11
  require 'ruby-progressbar'
8
12
 
9
13
  module Aspera
@@ -18,42 +22,7 @@ module Aspera
18
22
  RATE_WINDOW_SEC = 5.0
19
23
  # Rate is not displayed on a shorter period
20
24
  RATE_MIN_SEC = 1.0
21
- # Decimal units, like `ascp`
22
- KILO = 1000
23
- BITS_PER_MEGABIT = KILO * KILO
24
- BYTE_UNITS = %w[B KB MB GB TB PB].freeze
25
- private_constant :RATE_WINDOW_SEC, :RATE_MIN_SEC, :KILO, :BITS_PER_MEGABIT, :BYTE_UNITS
26
-
27
- class << self
28
- # @param bytes [Integer] Size in bytes
29
- # @return [String] Size with decimal unit, e.g. `12.3 MB`
30
- def format_bytes(bytes)
31
- value = bytes.to_f
32
- index = 0
33
- while value >= KILO && index < BYTE_UNITS.length - 1
34
- value /= KILO
35
- index += 1
36
- end
37
- return "#{bytes} B" if index.zero?
38
- format('%.1f %s', value, BYTE_UNITS[index])
39
- end
40
-
41
- # @param bytes_per_second [Float, nil] Rate, `nil` if unknown
42
- # @return [String] Rate in megabits per second (decimal, like `ascp`)
43
- def format_rate(bytes_per_second)
44
- return '-- Mbps' if bytes_per_second.nil?
45
- mbps = bytes_per_second * Environment::BITS_PER_BYTE / BITS_PER_MEGABIT
46
- mbps < 10 ? format('%.2f Mbps', mbps) : format('%.0f Mbps', mbps)
47
- end
48
-
49
- # @param seconds [Numeric, nil] Duration, `nil` if unknown
50
- # @return [String] Duration as `hh:mm:ss`
51
- def format_duration(seconds)
52
- return '--:--:--' if seconds.nil?
53
- seconds = seconds.round
54
- format('%02d:%02d:%02d', seconds / 3600, (seconds / 60) % 60, seconds % 60)
55
- end
56
- end
25
+ private_constant :RATE_WINDOW_SEC, :RATE_MIN_SEC
57
26
 
58
27
  # @param output [IO] Where the progress bar is displayed: not on `$stdout`, which may be redirected for command output
59
28
  def initialize(output: $stderr)
@@ -217,7 +186,7 @@ module Aspera
217
186
 
218
187
  # @return [Float] Monotonic time in seconds
219
188
  def now
220
- Process.clock_gettime(Process::CLOCK_MONOTONIC)
189
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
221
190
  end
222
191
 
223
192
  # Record transferred bytes for rate computation
@@ -251,7 +220,7 @@ module Aspera
251
220
  # Update the progress bar with current state
252
221
  def display
253
222
  if @progress_bar.nil?
254
- @progress_bar = ProgressBar.create(output: @output, format: '%a %B', total: nil, autofinish: false, throttle_rate: 0)
223
+ @progress_bar = ProgressBar.create(output: @output, format: '%a %B', total: nil, autofinish: false, throttle_rate: 0) # cspell:disable-line
255
224
  @format = '%a %B'
256
225
  end
257
226
  current = current_bytes
@@ -261,12 +230,12 @@ module Aspera
261
230
  unless @sessions.empty?
262
231
  bytes_per_second = rate
263
232
  # Fixed width, so that bar does not change size
264
- rate_text = self.class.format_rate(bytes_per_second).rjust(11)
233
+ rate_text = Units::Decimal.format_rate(bytes_per_second).rjust(11)
265
234
  if total.nil?
266
- fields.push(self.class.format_bytes(current).rjust(8), rate_text)
235
+ fields.push(Units::Decimal.format_bytes(current).rjust(8), rate_text)
267
236
  else
268
237
  eta = bytes_per_second&.positive? ? (total - current) / bytes_per_second : nil
269
- fields.push('%j%%', rate_text, "ETA: #{self.class.format_duration(eta)}")
238
+ fields.push('%j%%', rate_text, "ETA: #{eta.nil? ? '--:--:--' : Units::Duration.new(eta).to_hms}")
270
239
  end
271
240
  end
272
241
  update_format(title_text(final: false), fields)
@@ -287,7 +256,7 @@ module Aspera
287
256
  update_bar(total, current)
288
257
  fields.push('%B', '%j%%')
289
258
  end
290
- fields.push(self.class.format_bytes(current), self.class.format_rate(average_rate)) unless @sessions.empty?
259
+ fields.push(Units::Decimal.format_bytes(current), Units::Decimal.format_rate(average_rate)) unless @sessions.empty?
291
260
  # Last status (e.g. `running`) is no more relevant
292
261
  @title = success ? nil : 'failed'
293
262
  # Final line may be shorter (no bar)
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'result'
7
+ require_relative '../assert'
8
+
9
+ module Aspera
10
+ module Cli
11
+ # Mixin for Config plugin: `config vault` actions, on the default store of `context.vault` (`VaultManager`)
12
+ module VaultActions
13
+ # Import secrets from a JSON array; skips entries missing :label
14
+ def action_vault_import(secrets:, **)
15
+ bulk_result(secrets, command: :import, id_result: 'label') do |entry|
16
+ vault_required.set(entry.symbolize_keys)
17
+ {'label' => entry['label'] || entry[:label]}
18
+ end
19
+ end
20
+
21
+ def action_vault_delete(label:, id: nil, **)
22
+ v = vault_required
23
+ kwargs = id && v.method(:delete).parameters.any? { |_t, n| n == :id } ? {id: id} : {}
24
+ v.delete(label: label, **kwargs)
25
+ Result::Status.new("Secret deleted: #{label}")
26
+ end
27
+
28
+ # Checks a reference to a store of option `vault`, without its value
29
+ # @param reference [String] `@vault://<store>/<path>[?<options>][#<field>]`, or without `@vault:`
30
+ def action_vault_check(reference:, **)
31
+ Result::SingleObject.new(context.vault.metadata(reference.delete_prefix('@vault:')))
32
+ end
33
+
34
+ # @return [Secret::Store::Base, nil] Default store of option `vault`, `nil` if not configured
35
+ def vault
36
+ context.vault.store
37
+ end
38
+
39
+ # @return [Secret::Store::Base] Default store of option `vault`, raises if not configured
40
+ def vault_required
41
+ context.vault.store(mandatory: true)
42
+ end
43
+ end
44
+ end
45
+ end