aspera-cli 4.28.0 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +143 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1633 -609
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +20 -20
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +71 -65
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +16 -8
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +12 -19
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -33
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +6 -1
  50. data/lib/aspera/cli/command_spec.rb +34 -7
  51. data/lib/aspera/cli/completion/ascli.bash +19 -5
  52. data/lib/aspera/cli/completion/ascli.fish +13 -2
  53. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  54. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  55. data/lib/aspera/cli/completion_actions.rb +303 -69
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +3 -0
  59. data/lib/aspera/cli/extended_value.rb +61 -27
  60. data/lib/aspera/cli/formatter.rb +29 -22
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -10
  63. data/lib/aspera/cli/http.rb +12 -9
  64. data/lib/aspera/cli/info.rb +12 -7
  65. data/lib/aspera/cli/mailer.rb +6 -3
  66. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  67. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  68. data/lib/aspera/cli/mcp/tool.rb +15 -14
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +65 -12
  73. data/lib/aspera/cli/options.schema.yaml +487 -63
  74. data/lib/aspera/cli/parser.rb +56 -32
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +244 -35
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +17 -12
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +198 -103
  81. data/lib/aspera/cli/plugins/console.rb +6 -3
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +21 -7
  84. data/lib/aspera/cli/plugins/faspex5.rb +28 -25
  85. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  86. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +28 -22
  88. data/lib/aspera/cli/plugins/node.rb +31 -30
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +63 -11
  93. data/lib/aspera/cli/plugins/shares.rb +86 -16
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +144 -30
  96. data/lib/aspera/cli/preset_manager.rb +52 -18
  97. data/lib/aspera/cli/prompt.rb +4 -1
  98. data/lib/aspera/cli/result.rb +12 -8
  99. data/lib/aspera/cli/runner.rb +69 -46
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +79 -9
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +3 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +29 -20
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +188 -42
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.rb +12 -10
  113. data/lib/aspera/cli/wizard.rb +42 -39
  114. data/lib/aspera/command_line_builder.rb +9 -5
  115. data/lib/aspera/command_line_converter.rb +4 -1
  116. data/lib/aspera/coverage.rb +103 -33
  117. data/lib/aspera/data_repository.rb +4 -1
  118. data/lib/aspera/dot_container.rb +4 -1
  119. data/lib/aspera/duration.rb +5 -2
  120. data/lib/aspera/environment.rb +21 -12
  121. data/lib/aspera/exec_spec.rb +3 -0
  122. data/lib/aspera/faspex_gw.rb +13 -9
  123. data/lib/aspera/faspex_postproc.rb +15 -12
  124. data/lib/aspera/formatter_interface.rb +4 -1
  125. data/lib/aspera/graphql.rb +5 -1
  126. data/lib/aspera/hash_ext.rb +3 -0
  127. data/lib/aspera/id_generator.rb +4 -2
  128. data/lib/aspera/json_rpc/client.rb +6 -5
  129. data/lib/aspera/json_rpc/version.rb +3 -0
  130. data/lib/aspera/line_logger.rb +4 -1
  131. data/lib/aspera/link_header.rb +4 -1
  132. data/lib/aspera/log.rb +18 -5
  133. data/lib/aspera/markdown.rb +5 -2
  134. data/lib/aspera/mime.rb +3 -0
  135. data/lib/aspera/nagios.rb +6 -4
  136. data/lib/aspera/node_emulator.rb +15 -13
  137. data/lib/aspera/oauth/client/base.rb +91 -70
  138. data/lib/aspera/oauth/client/boot.rb +7 -4
  139. data/lib/aspera/oauth/client/factory.rb +6 -3
  140. data/lib/aspera/oauth/client/generic.rb +5 -1
  141. data/lib/aspera/oauth/client/json_credentials.rb +4 -1
  142. data/lib/aspera/oauth/client/jwt.rb +6 -3
  143. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  144. data/lib/aspera/oauth/client/url_json.rb +4 -1
  145. data/lib/aspera/oauth/client/web.rb +39 -12
  146. data/lib/aspera/oauth/server/pkce.rb +6 -5
  147. data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
  148. data/lib/aspera/persistency_action_once.rb +5 -3
  149. data/lib/aspera/persistency_folder.rb +6 -3
  150. data/lib/aspera/preview/file_types.rb +6 -3
  151. data/lib/aspera/preview/generator.rb +9 -5
  152. data/lib/aspera/preview/options.schema.yaml +3 -3
  153. data/lib/aspera/preview/terminal.rb +8 -4
  154. data/lib/aspera/preview/utils.rb +6 -4
  155. data/lib/aspera/products/connect.rb +5 -2
  156. data/lib/aspera/products/desktop.rb +4 -1
  157. data/lib/aspera/products/other.rb +4 -2
  158. data/lib/aspera/products/transferd.rb +6 -2
  159. data/lib/aspera/products/transferd_service.rb +610 -0
  160. data/lib/aspera/proxy_auto_config.js +2 -2
  161. data/lib/aspera/proxy_auto_config.rb +5 -2
  162. data/lib/aspera/rainbow.rb +3 -0
  163. data/lib/aspera/rest/aspera_errors.rb +4 -1
  164. data/lib/aspera/rest/call_error.rb +3 -0
  165. data/lib/aspera/rest/client.rb +22 -29
  166. data/lib/aspera/rest/error_analyzer.rb +5 -2
  167. data/lib/aspera/rest/list.rb +8 -4
  168. data/lib/aspera/rest/parameters.rb +3 -0
  169. data/lib/aspera/rest/util.rb +8 -5
  170. data/lib/aspera/rest.rb +9 -6
  171. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  172. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  173. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  174. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  175. data/lib/aspera/schema/async_tables.yaml +11 -10
  176. data/lib/aspera/schema/documentation.rb +8 -2
  177. data/lib/aspera/schema/reader.rb +7 -0
  178. data/lib/aspera/schema/registry.rb +43 -28
  179. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  180. data/lib/aspera/schema/validator.rb +4 -1
  181. data/lib/aspera/sealed_hash.rb +166 -0
  182. data/lib/aspera/secret/hider.rb +201 -0
  183. data/lib/aspera/secret/reference.rb +150 -0
  184. data/lib/aspera/secret/resolver.rb +212 -0
  185. data/lib/aspera/secret/store/base.rb +153 -0
  186. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  187. data/lib/aspera/secret/store/factory.rb +103 -0
  188. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  189. data/lib/aspera/secret/store/keeper.rb +141 -0
  190. data/lib/aspera/secret/store/macos_security.rb +284 -0
  191. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  192. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  193. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  194. data/lib/aspera/secret/store/secret_service.rb +125 -0
  195. data/lib/aspera/secret/strength.rb +119 -0
  196. data/lib/aspera/servlet_helpers.rb +5 -2
  197. data/lib/aspera/ssh.rb +10 -7
  198. data/lib/aspera/ssl.rb +5 -2
  199. data/lib/aspera/string_ext.rb +3 -0
  200. data/lib/aspera/sync/args.schema.yaml +3 -0
  201. data/lib/aspera/sync/conf.schema.yaml +9 -4
  202. data/lib/aspera/sync/database.rb +3 -0
  203. data/lib/aspera/sync/operations.rb +16 -14
  204. data/lib/aspera/temp_file_manager.rb +8 -4
  205. data/lib/aspera/timer_limiter.rb +3 -0
  206. data/lib/aspera/transfer/error.rb +4 -1
  207. data/lib/aspera/transfer/faux_file.rb +4 -1
  208. data/lib/aspera/transfer/parameters.rb +17 -14
  209. data/lib/aspera/transfer/result.rb +3 -0
  210. data/lib/aspera/transfer/resumer.rb +6 -4
  211. data/lib/aspera/transfer/spec.rb +9 -22
  212. data/lib/aspera/transfer/units.rb +75 -0
  213. data/lib/aspera/uri_reader.rb +57 -7
  214. data/lib/aspera/web_auth/index.html.erb +6 -2
  215. data/lib/aspera/web_auth/paste.rb +109 -0
  216. data/lib/aspera/web_auth/receiver.rb +50 -0
  217. data/lib/aspera/web_auth/relay.rb +76 -0
  218. data/lib/aspera/web_auth/reply.html.erb +150 -0
  219. data/lib/aspera/web_auth/server.rb +47 -10
  220. data/lib/aspera/web_page.rb +7 -2
  221. data/lib/aspera/web_server_simple.rb +6 -3
  222. data/lib/aspera/yaml.rb +5 -1
  223. data/llms.txt +81 -0
  224. data/skills/ascli/SKILL.md +88 -0
  225. data.tar.gz.sig +0 -0
  226. metadata +72 -19
  227. metadata.gz.sig +0 -0
  228. data/lib/aspera/keychain/base.rb +0 -30
  229. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  230. data/lib/aspera/keychain/factory.rb +0 -58
  231. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  232. data/lib/aspera/keychain/macos_security.rb +0 -277
  233. data/lib/aspera/keychain/one_password_api.rb +0 -91
  234. data/lib/aspera/keychain/one_password_base.rb +0 -39
  235. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  236. data/lib/aspera/secret_hider.rb +0 -196
  237. data/lib/aspera/secret_strength.rb +0 -114
@@ -1,24 +1,26 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore trevents
4
- require 'aspera/cli/plugins/basic_auth'
5
- require 'aspera/preview/generator'
6
- require 'aspera/preview/utils'
7
- require 'aspera/preview/file_types'
8
- require 'aspera/preview/terminal'
9
- require 'aspera/transfer/spec'
10
- require 'aspera/schema/registry'
11
- require 'aspera/persistency_action_once'
12
- require 'aspera/temp_file_manager'
13
- require 'aspera/api/node'
14
- require 'aspera/hash_ext'
15
- require 'aspera/timer_limiter'
16
- require 'aspera/id_generator'
17
- require 'aspera/uri_reader'
18
- require 'aspera/log'
19
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'basic_auth'
7
+ require_relative '../../preview/generator'
8
+ require_relative '../../preview/utils'
9
+ require_relative '../../preview/file_types'
10
+ require_relative '../../preview/terminal'
11
+ require_relative '../../transfer/spec'
12
+ require_relative '../../schema/registry'
13
+ require_relative '../../persistency_action_once'
14
+ require_relative '../../temp_file_manager'
15
+ require_relative '../../api/node'
16
+ require_relative '../../hash_ext'
17
+ require_relative '../../timer_limiter'
18
+ require_relative '../../id_generator'
19
+ require_relative '../../uri_reader'
20
+ require_relative '../../log'
21
+ require_relative '../../assert'
22
+ require_relative '../../rainbow'
20
23
  require 'securerandom'
21
- require 'aspera/rainbow'
22
24
  using Rainbow
23
25
 
24
26
  module Aspera
@@ -96,7 +98,7 @@ module Aspera
96
98
  Aspera::Preview::Utils.office_tool = @gen_options.office_conversion
97
99
  Api::Node.api_options[:cache] = !@option_folder_reset_cache.eql?(:header)
98
100
  # Start from the full supported format list, then remove any skipped format.
99
- @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
101
+ @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.dup
100
102
  skip = options.get_option(:skip_format)
101
103
  @preview_formats_to_generate.delete(skip) if skip
102
104
  # Created by `tools_cleanup`, only for commands that use it
@@ -165,13 +167,17 @@ module Aspera
165
167
  return if events.empty?
166
168
  events.each do |event|
167
169
  # Ignore non-file events such as folder notifications.
168
- if event.dig('data', 'type').eql?('file')
170
+ if !event.dig('data', 'type').eql?('file')
171
+ nil
172
+ elsif Array(event['types']).include?('file.deleted')
173
+ delete_preview(event['data'])
174
+ else
169
175
  file_entry = @api_node.read("files/#{event['data']['id']}") rescue nil
170
176
  if !file_entry.nil? &&
171
- @option_skip_folders.none? { |d| file_entry['path'].start_with?(d) }
177
+ @option_skip_folders.none? { |d| file_entry['path'].start_with?(d) } &&
178
+ @filter_block.call(file_entry)
172
179
  file_entry['parent_file_id'] = event['data']['parent_file_id']
173
- Log.log.error('TODO'.red) if event['types'].include?('file.deleted')
174
- generate_preview(file_entry) if event['types'].include?('file.deleted')
180
+ generate_preview(file_entry)
175
181
  end
176
182
  end
177
183
  # Periodically log progress and persist the latest processed event.
@@ -186,6 +192,30 @@ module Aspera
186
192
  nil
187
193
  end
188
194
 
195
+ # Delete the preview folder of a deleted file, if it exists.
196
+ # Errors are logged: a failure shall not stop the processing of events.
197
+ # @param entry [Hash] file entry of the event, with at least `id`
198
+ # @return [nil]
199
+ def delete_preview(entry)
200
+ folder_name = entry_preview_folder_name(entry)
201
+ # The identifier comes from the API: it shall not designate another folder
202
+ Aspera.assert(!entry['id'].to_s.empty? && File.basename(folder_name).eql?(folder_name)) { "Invalid file id: #{entry['id']}" }
203
+ if @access_remote
204
+ preview_folder = @api_node.read_folder_content(@previews_folder_entry['id'], {name: folder_name}).find { |item| item['name'].eql?(folder_name) }
205
+ return if preview_folder.nil?
206
+ @api_node.delete("files/#{preview_folder['id']}")
207
+ else
208
+ local_entry_preview_dir = File.join(@local_preview_folder, folder_name)
209
+ return unless File.directory?(local_entry_preview_dir)
210
+ FileUtils.rm_rf(local_entry_preview_dir)
211
+ end
212
+ Log.log.info { "Deleted preview of #{entry['id']}" }
213
+ nil
214
+ rescue StandardError => e
215
+ Log.log.error { "Cannot delete preview of #{entry['id']}: #{e.message}" }
216
+ nil
217
+ end
218
+
189
219
  # Transfer a file to or from the configured Node storage using a tagged transfer spec.
190
220
  #
191
221
  # @param direction [String] transfer direction, typically from [`Transfer::Spec`](lib/aspera/transfer/spec.rb)
@@ -240,16 +270,21 @@ module Aspera
240
270
  # original_mtime=DateTime.parse(entry['modified_time'])
241
271
  # Local directory where previews are generated before being uploaded back.
242
272
  local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
243
- file_info = @api_node.read("files/#{entry['id']}")
244
- # TODO: This does not work with Gen4 because preview folders are hidden by the API.
245
- # this_preview_folder_entries=@api_node.read_folder_content(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
246
- # TODO: Query Gen3 APIs to list preview files and retrieve timestamps.
273
+ file_info = @api_node.read("files/#{entry['id']}", headers: Api::Node.add_cache_control)
274
+ # Check if preview exists either from file_info or by checking preview endpoint
275
+ preview_exist = file_info.key?('preview')
276
+ unless preview_exist
277
+ begin
278
+ @api_node.read("files/#{entry['id']}/preview", headers: {'Accept' => 'image/png'})
279
+ preview_exist = true
280
+ rescue StandardError
281
+ preview_exist = false
282
+ end
283
+ end
247
284
  gen_infos.each do |gen_info|
248
285
  gen_info[:src] = local_original_filepath
249
286
  gen_info[:dst] = File.join(local_entry_preview_dir, gen_info[:base_dest])
250
- # TODO: Reuse `this_preview_folder_entries` once preview folders become visible.
251
- gen_info[:preview_exist] = file_info.key?('preview')
252
- # TODO: Compare source and preview modification times when remote timestamps are available.
287
+ gen_info[:preview_exist] = preview_exist
253
288
  gen_info[:preview_newer_than_original] = gen_info[:preview_exist]
254
289
  end
255
290
  return local_entry_preview_dir
@@ -352,25 +387,15 @@ module Aspera
352
387
 
353
388
  # Scan all files in provided folder entry
354
389
  # @param top_entry [Hash] the top entry to scan
355
- # @param top_path [String, nil] subpath to start folder scan inside
356
- def scan_folder_files(top_entry, top_path = nil)
357
- unless top_path.nil?
358
- # canonical path: start with / and ends with /
359
- top_path = "/#{top_path.split('/').reject(&:empty?).join('/')}/"
360
- end
361
- Log.log.debug { "scan: #{top_entry} : #{top_path}".green }
390
+ def scan_folder_files(top_entry)
391
+ Log.log.debug { "scan: #{top_entry}".green }
362
392
  # don't use recursive call, use list instead
363
393
  entries_to_process = [top_entry]
364
394
  until entries_to_process.empty?
365
395
  entry = entries_to_process.shift
366
- # process this entry only if it is within the top_path
367
396
  entry_path_with_slash = entry['path']
368
397
  Log.log.debug { "processing entry #{entry_path_with_slash}" } if @periodic.trigger?
369
398
  entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
370
- if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
371
- Log.log.debug { "#{entry['path']} folder (skip start)".bg(:red) }
372
- next
373
- end
374
399
  Log.log.debug { "item:#{entry}" }
375
400
  begin
376
401
  case entry['type']
@@ -416,10 +441,10 @@ module Aspera
416
441
  command :trevents, description: 'Process transfer events and generate previews', setup: :setup_node_api, action: ->(**) { run_event_loop(:trevents) }
417
442
  command :check, description: 'Check required tools are installed', action: ->(**) { tools_cleanup { Result::Status.new('Tools validated') } }
418
443
  command :show, description: 'Generate and display preview of a source file',
419
- arguments: [{name: :source_file, description: 'Local file to preview'}]
444
+ arguments: [{name: :source_file, description: 'Local file to preview', complete: :file}]
420
445
  command :test, description: 'Test preview generation for a source file',
421
446
  arguments: [
422
- {name: :source_file, description: 'Local file to preview'},
447
+ {name: :source_file, description: 'Local file to preview', complete: :file},
423
448
  {name: :format, description: 'Preview output format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS, mandatory: false, default: :png}
424
449
  ]
425
450
 
@@ -1,22 +1,28 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore ascmd zmode zuid zgid fasping
4
- require 'aspera/cli/plugins/basic_auth'
5
- require 'aspera/cli/sync_actions'
6
- require 'aspera/transfer/result'
7
- require 'aspera/transfer/spec'
8
- require 'aspera/ascmd'
9
- require 'aspera/ssh'
10
- require 'aspera/nagios'
11
- require 'aspera/log'
12
- require 'aspera/assert'
13
- require 'aspera/environment'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'basic_auth'
7
+ require_relative '../sync_actions'
8
+ require_relative '../ascp_logs_actions'
9
+ require_relative '../../transfer/result'
10
+ require_relative '../../transfer/spec'
11
+ require_relative '../../ascmd'
12
+ require_relative '../../ssh'
13
+ require_relative '../../nagios'
14
+ require_relative '../../log'
15
+ require_relative '../../assert'
16
+ require_relative '../../environment'
17
+ require 'shellwords'
18
+ require 'tmpdir'
14
19
  module Aspera
15
20
  module Cli
16
21
  module Plugins
17
22
  # Operations on HSTS with SSH/FASP (ascmd/ascp)
18
23
  class Server < BasicAuth
19
24
  include SyncActions
25
+ include AscpLogsActions
20
26
 
21
27
  SSH_SCHEME = 'ssh'
22
28
  LOCAL_SCHEME = 'local'
@@ -196,6 +202,9 @@ module Aspera
196
202
  command :transfer, description: 'Check FASP transfer health'
197
203
  end
198
204
 
205
+ # Analysis of ascp logs of the server: same commands as `config ascp logs`, on files read over SSH
206
+ ascp_logs_commands(:logs, arguments: [{name: :paths, description: 'Log files on the server (gzip supported)', multiple: true}])
207
+
199
208
  # AsCmd operations (available only when SSH/local - not WSS)
200
209
  # Arguments follow AsCmd::OPS_ARGS: one path per operation, or source and destination
201
210
  PATHS_ARGS = [{name: :paths, multiple: true}].freeze
@@ -299,6 +308,49 @@ module Aspera
299
308
 
300
309
  private
301
310
 
311
+ # Log files of the server, to analyze with the commands of `logs`.
312
+ # Read over SSH, in a temporary folder removed after the block.
313
+ # @param paths [Array<String>] files on the server
314
+ # @yieldparam files [Array<String>] local copies
315
+ def ascp_logs_files(paths)
316
+ executor = ascmd_executor
317
+ Aspera.assert(executor.is_a?(Ssh), type: Cli::BadArgument) { 'Reading the logs of the server requires an SSH connection (URL scheme ssh)' }
318
+ remaining = ascp_logs_params['remote_max_size']
319
+ Dir.mktmpdir('ascli_logs') do |folder|
320
+ files = paths.each_with_index.map do |remote, index|
321
+ local = File.join(folder, format('%02d_%s', index, File.basename(remote)))
322
+ remaining -= read_remote_log(executor, remote, local, remaining)
323
+ local
324
+ end
325
+ yield(files)
326
+ end
327
+ end
328
+
329
+ # Copy a file of the server over SSH. Shell characters of the path are escaped.
330
+ # The modification date is kept: the year of syslog lines is deduced from it.
331
+ # @param executor [Ssh] SSH connection
332
+ # @param remote [String] file on the server
333
+ # @param local [String] destination
334
+ # @param limit [Integer] maximum size in bytes
335
+ # @return [Integer] size in bytes
336
+ def read_remote_log(executor, remote, local, limit)
337
+ quoted = Shellwords.escape(remote)
338
+ # Force an integer: the value comes from a user option and is interpolated in a shell command
339
+ max_bytes = Integer(limit) + 1
340
+ # First line: modification time (0 if unknown), then the content, read up to one byte over the limit
341
+ output = executor.execute("(stat -c %Y -- #{quoted} 2>/dev/null || echo 0); head -c #{max_bytes} -- #{quoted}").b
342
+ mtime, _, content = output.partition("\n")
343
+ Aspera.assert(content.bytesize <= limit, type: Cli::BadArgument) do
344
+ "Log files from the server exceed the limit of #{ascp_logs_params['remote_max_size']} bytes (option ascp_logs.remote_max_size), at: #{remote}"
345
+ end
346
+ Log.log.debug { "Read #{remote}: #{content.bytesize} bytes" }
347
+ File.binwrite(local, content)
348
+ File.utime(Time.now, Time.at(mtime.to_i), local) if mtime.to_i.positive?
349
+ content.bytesize
350
+ rescue Ssh::Error => e
351
+ raise Cli::BadArgument, "Cannot read #{remote}: #{e.message}"
352
+ end
353
+
302
354
  # AsCmd `cp` and `mv` take source and destination pairs.
303
355
  # With option `to_folder`, paths are only sources, copied or moved into that folder.
304
356
  # @param paths [Array<String>] command line arguments
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/cli/plugins/basic_auth'
4
- require 'aspera/cli/plugins/node'
5
- require 'aspera/assert'
6
- require 'aspera/rest/list'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'basic_auth'
7
+ require_relative 'node'
8
+ require_relative '../../assert'
9
+ require_relative '../../rest/list'
7
10
 
8
11
  module Aspera
9
12
  module Cli
@@ -14,6 +17,13 @@ module Aspera
14
17
  NODE_API_PATH = 'node_api'
15
18
  # path for node admin after base url
16
19
  ADMIN_API_PATH = 'api/v1'
20
+ # path for authentication (SAML login page, OAuth token) after base url
21
+ AUTH_API_PATH = 'auth'
22
+ # default URL that receives the tokens, with auth `web` and `ui.web_auth=port`
23
+ REDIRECT_LOCALHOST = 'http://localhost:12345'
24
+ AUTH_TYPES = %i[basic web].freeze
25
+ option :auth, description: 'Type of authentication: user and password, or Web browser (SAML)', allowed: AUTH_TYPES, default: :basic
26
+ option :redirect_uri, description: "Web authentication with ui.web_auth=port: loopback URL that receives the tokens (default: #{REDIRECT_LOCALHOST})"
17
27
  class << self
18
28
  # Check various endpoints on Shares
19
29
  # @return [Hash] with version, ping, api
@@ -27,12 +37,13 @@ module Aspera
27
37
  .new(base_url: url, redirect_max: 2)
28
38
  .read('', headers: {'Accept'=>'text/html'})
29
39
  Aspera.assert(login_page.include?('aspera-Shares')) { 'not Shares' }
30
- if (m = login_page.match(/\(v([0-9a-f\.]+)\)/))
40
+ if (m = login_page.match(/\(v([0-9a-f.]+)\)/))
31
41
  version = m[1]
32
42
  if (m = login_page.match(/Patch level ([0-9]+)/))
33
43
  version = "#{version} #{m[0]}"
34
44
  end
35
45
  end
46
+ result[:saml] = login_page.include?('/auth/saml')
36
47
  version.nil? ? 'no version' : version
37
48
  rescue => e
38
49
  e
@@ -79,7 +90,8 @@ module Aspera
79
90
  return unless health[:api].is_a?(String)
80
91
  return {
81
92
  version: health[:version].is_a?(String) ? health[:version] : 'unknown',
82
- url: address_or_url
93
+ url: address_or_url,
94
+ saml: health[:saml] || false
83
95
  }
84
96
  end
85
97
  end
@@ -88,16 +100,53 @@ module Aspera
88
100
  # @param app_url [String] Tested URL
89
101
  # @return [Hash] :preset_value, :test_args
90
102
  def wizard(wizard, app_url)
91
- username = options.get_option(:username, mandatory: true)
92
- return {
93
- preset_ids: [username],
94
- preset_value: {
95
- url: app_url,
96
- username: username,
97
- password: options.get_option(:password, mandatory: true)
98
- },
99
- test_args: 'files browse /'
100
- }
103
+ auth_type = if !options.option_def(:auth).source.eql?(:default)
104
+ options.get_option(:auth)
105
+ elsif options.get_option(:password)
106
+ :basic
107
+ else
108
+ saml_detected = begin
109
+ self.class.health_check(app_url)[:saml] || false
110
+ rescue StandardError
111
+ false
112
+ end
113
+ formatter.display_status('SAML authentication is available.') if saml_detected
114
+ if wizard.required
115
+ formatter.display_status('Select authentication method:')
116
+ options.prompt_user_input_in_list('auth', AUTH_TYPES)
117
+ else
118
+ saml_detected ? :web : :basic
119
+ end
120
+ end
121
+
122
+ case auth_type
123
+ when :web
124
+ username = options.get_option(:username)
125
+ preset_ids = username && !username.empty? ? [username] : ['web']
126
+ preset_value = {
127
+ url: app_url,
128
+ auth: :web.to_s
129
+ }
130
+ preset_value[:username] = username if username && !username.empty?
131
+ redirect_uri = options.get_option(:redirect_uri)
132
+ preset_value[:redirect_uri] = redirect_uri if redirect_uri
133
+ {
134
+ preset_ids: preset_ids,
135
+ preset_value: preset_value,
136
+ test_args: 'files browse /'
137
+ }
138
+ when :basic
139
+ username = options.get_option(:username, mandatory: true)
140
+ {
141
+ preset_ids: [username],
142
+ preset_value: {
143
+ url: app_url,
144
+ username: username,
145
+ password: options.get_option(:password, mandatory: true)
146
+ },
147
+ test_args: 'files browse /'
148
+ }
149
+ end
101
150
  end
102
151
 
103
152
  def initialize(**_)
@@ -105,6 +154,27 @@ module Aspera
105
154
  @api_shares_admin = nil
106
155
  end
107
156
 
157
+ # With `auth` `web`, the user authenticates with a browser (SAML), and the REST API uses the OAuth token.
158
+ # Otherwise: user and password.
159
+ # @param subpath [String, nil] Path of API after base URL
160
+ # @return [Hash] Parameters of `Rest::Client`
161
+ def basic_auth_params(subpath = nil)
162
+ return super unless options.get_option(:auth).eql?(:web)
163
+ url = options.get_option(:url, mandatory: true)
164
+ username = options.get_option(:username)
165
+ return {
166
+ base_url: subpath.nil? ? url : "#{url}/#{subpath}",
167
+ auth: {
168
+ type: :oauth2,
169
+ base_url: "#{url}/#{AUTH_API_PATH}",
170
+ grant_method: :shares_web,
171
+ redirect_uri: options.get_option(:redirect_uri) || REDIRECT_LOCALHOST,
172
+ # a token is cached for each user, if known
173
+ cache_ids: [username].compact
174
+ }
175
+ }
176
+ end
177
+
108
178
  SAML_IMPORT_MANDATORY = %w[id name_id].freeze
109
179
  SAML_IMPORT_ALLOWED = (%w[email given_name surname] + SAML_IMPORT_MANDATORY).freeze
110
180
 
@@ -0,0 +1,223 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'base'
7
+ require_relative '../info'
8
+ require_relative '../special_values'
9
+ require_relative '../../ascp/installation'
10
+ require_relative '../../products/transferd_service'
11
+ require_relative '../../schema/documentation'
12
+ require_relative '../../log'
13
+ require_relative '../../assert'
14
+
15
+ module Aspera
16
+ module Cli
17
+ module Plugins
18
+ # Install the Transfer Daemon (`transferd`), run it as a system service, and query a running daemon
19
+ class Transferd < Base
20
+ application_name 'Transfer Daemon'
21
+
22
+ class << self
23
+ # Seam for the client commands: object that gives `info(params)`, `transfer_status(id, params)` and `transfer_cancel(id, params)`,
24
+ # with `params` a Hash with `url`, and optional `api_key`, `ca_file`.
25
+ # `nil` (default): `Agent::Transferd`, loaded on first use, because it requires the optional gem `grpc`.
26
+ # @return [#info, #transfer_status, #transfer_cancel, nil]
27
+ attr_accessor :daemon_client
28
+ end
29
+
30
+ command :install, description: 'Install the transfer daemon',
31
+ arguments: [{name: :version, mandatory: false, default: nil}]
32
+ command :list, description: 'List available SDK locations'
33
+ command :service, description: 'Manage the transfer daemon as a system service'
34
+ commands_under :service do
35
+ service_config_arg = {name: :config, type: Hash, mandatory: false, default: {}, schema: Schema::Registry::TRANSFERD_SERVICE_CONFIG}
36
+ command(
37
+ :spec, description: 'Show the transfer daemon configuration schema',
38
+ action: lambda do |**|
39
+ builder = Schema::Documentation.new(TerminalFormatter, Schema::Registry.instance.reader('transferd')).build
40
+ Result::ObjectList.new(builder.rows, fields: builder.columns)
41
+ end
42
+ )
43
+ command(
44
+ :install, description: 'Install the SDK if needed, then configure and start the service',
45
+ arguments: [service_config_arg],
46
+ action: ->(config:, **) { transferd_service_install(config) }
47
+ )
48
+ command(
49
+ :configure, description: 'Configure the transfer daemon service',
50
+ arguments: [service_config_arg],
51
+ action: lambda do |config:, **|
52
+ transferd_service do |host, sdk_folder, dry_run|
53
+ steps = Products::TransferdService.configure(config, sdk_folder, host: host, dry_run: dry_run)
54
+ Products::TransferdService.check_version(sdk_folder, host: host)
55
+ transferd_service_steps(steps)
56
+ end
57
+ end
58
+ )
59
+ command(
60
+ :status, description: 'Show the state of the service (the API key is not shown)',
61
+ arguments: [service_config_arg],
62
+ action: lambda do |config:, **|
63
+ transferd_service { |host, sdk_folder, _dry_run| Result::SingleObject.new(Products::TransferdService.status(config, sdk_folder, host: host)) }
64
+ end
65
+ )
66
+ command(
67
+ :uninstall, description: 'Stop and remove the service (service.purge: also remove configuration and logs)',
68
+ arguments: [service_config_arg],
69
+ action: lambda do |config:, **|
70
+ transferd_service do |host, _sdk_folder, dry_run|
71
+ purge = Products::TransferdService.service_options(config)['purge'].eql?(true)
72
+ transferd_service_steps(Products::TransferdService.uninstall(config, host: host, dry_run: dry_run, purge: purge))
73
+ end
74
+ end
75
+ )
76
+ command(
77
+ :use, description: 'Use an installed version of the SDK (SDK folder named current), and restart the service',
78
+ arguments: [{name: :version}, service_config_arg],
79
+ action: lambda do |version:, config:, **|
80
+ transferd_service do |host, sdk_folder, dry_run|
81
+ transferd_service_steps(Products::TransferdService.use(config, sdk_folder, version, host: host, dry_run: dry_run))
82
+ end
83
+ end
84
+ )
85
+ command(
86
+ :versions, description: 'List the installed versions of the SDK (SDK folder named current)',
87
+ action: lambda do |**|
88
+ sdk_folder = options.get_option(:sdk_folder, mandatory: true)
89
+ Result::ObjectList.new(Products::TransferdService.versions(sdk_folder), fields: %w[version folder current])
90
+ end
91
+ )
92
+ end
93
+ command :info, description: 'Show information on a running daemon',
94
+ action: ->(**) { Result::SingleObject.new(daemon_client.info(daemon_params)) }
95
+ command :transfers, description: 'Query the transfers of a running daemon'
96
+ commands_under :transfers do
97
+ command(
98
+ :show, description: 'Show the status of a transfer (the daemon keeps it for some time after its end)',
99
+ arguments: [{name: :transfer_id}],
100
+ action: ->(transfer_id:, **) { Result::SingleObject.new(daemon_client.transfer_status(transfer_id, daemon_params)) }
101
+ )
102
+ command(
103
+ :cancel, description: 'Stop a transfer',
104
+ arguments: [{name: :transfer_id}],
105
+ action: lambda do |transfer_id:, **|
106
+ daemon_client.transfer_cancel(transfer_id, daemon_params)
107
+ Result::Status.new("Transfer #{transfer_id} stopped")
108
+ end
109
+ )
110
+ end
111
+
112
+ option :url, description: 'Client commands: endpoint of the running daemon, e.g. 127.0.0.1:55002'
113
+ option :api_key, description: 'Client commands: API key of the daemon, if its authentication is enabled'
114
+ option :ca_file, description: 'Client commands: certificate authority (PEM file) of the daemon, if its TLS is enabled'
115
+
116
+ # Install the transfer SDK (ascp + transferd) from a URL or using the default source.
117
+ # Version defaults to `Info::SDK_VERSION`; pass `LATEST` as argument to install the latest available version.
118
+ # Nothing is done if this version is already installed in the SDK folder (not checked if option `sdk_url` is set).
119
+ # Also called by the deprecated command `config ascp install`.
120
+ # @param version [String, nil] version to install; nil means use the default SDK version
121
+ # @return [Result::Status] installation result message
122
+ def action_install(version: nil, **)
123
+ asked_version = version.nil? ? Info::SDK_VERSION : version
124
+ asked_version = nil if asked_version.eql?(SpecialValues::LATEST)
125
+ installation = Ascp::Installation.instance
126
+ url = sdk_url_option
127
+ if url.nil?
128
+ wanted_version = installation.sdk_location_for_platform(version: asked_version)['version']
129
+ return Result::Status.new("Version #{wanted_version} is already installed in #{Products::Transferd.sdk_directory}") if installation.installed_sdk_version.eql?(wanted_version)
130
+ end
131
+ name, ver, folder = installation.install_sdk(url: url, version: asked_version)
132
+ return Result::Status.new("Installed #{name} version #{ver} in #{folder}")
133
+ end
134
+
135
+ # @return [Result::ObjectList] The SDK locations
136
+ def action_list(**)
137
+ sdk_list = Ascp::Installation.instance.sdk_locations
138
+ Result::ObjectList.new(sdk_list, fields: sdk_list.first.keys - ['url'])
139
+ end
140
+
141
+ private
142
+
143
+ # @return [#info, #transfer_status, #transfer_cancel] The client of the daemon
144
+ # @raise [Cli::Error] if the optional gem `grpc` is missing
145
+ def daemon_client
146
+ injected = self.class.daemon_client
147
+ return injected unless injected.nil?
148
+ begin
149
+ require_relative '../../agent/transferd'
150
+ rescue LoadError => e
151
+ raise Cli::Error, "Commands that query a running daemon require the gem grpc (#{e.message}). Install it with: gem install grpc"
152
+ end
153
+ Agent::Transferd
154
+ end
155
+
156
+ # Parameters of the connection to the daemon: option `url` is mandatory, `api_key` and `ca_file` are optional.
157
+ # Options `url`, `api_key` and `ca_file` have priority, else the same keys of option `transfer` are used, if its agent is `transferd`.
158
+ # @return [Hash]
159
+ def daemon_params
160
+ from_transfer = transfer_agent_params
161
+ params = %w[url api_key ca_file].to_h { |key| [key, options.get_option(key.to_sym) || from_transfer[key]] }.compact
162
+ Aspera.assert(params.key?('url'), type: Cli::BadArgument) do
163
+ "Missing option #{Parser.option_name_to_line(:url)} (or #{Parser.option_name_to_line(:transfer)} with agent transferd and key url)"
164
+ end
165
+ return params
166
+ end
167
+
168
+ # @return [Hash] Keys `url`, `api_key`, `ca_file` of option `transfer`, only if its agent is `transferd`, else empty
169
+ def transfer_agent_params
170
+ transfer_options = transfer.transfer_options
171
+ return {} unless transfer_options['agent'].to_s.eql?('transferd')
172
+ return transfer_options.slice('url', 'api_key', 'ca_file')
173
+ end
174
+
175
+ # @return [String, nil] Value of option `sdk_url`, or nil to select the archive from the SDK locations (`DEF`)
176
+ def sdk_url_option
177
+ sdk_url = options.get_option(:sdk_url, mandatory: true)
178
+ return sdk_url.eql?(SpecialValues::DEF) ? nil : sdk_url
179
+ end
180
+
181
+ # Common processing of the commands of `transferd service` that act on the system.
182
+ # Refuses to run if the system is not supported or the user is not root, unless `--dry-run=yes`.
183
+ # @yieldparam host [Products::TransferdService::Host] Access to the system
184
+ # @yieldparam sdk_folder [String] Option `sdk_folder`, or link `current` of the layout
185
+ # @yieldparam dry_run [Boolean] Option `dry_run`
186
+ def transferd_service
187
+ host = Products::TransferdService::Host.new
188
+ dry_run = options.get_option(:dry_run)
189
+ Products::TransferdService.assert_supported(host, dry_run: dry_run)
190
+ sdk_folder = options.get_option(:sdk_folder, mandatory: true)
191
+ Aspera.assert(!Ascp::Installation.product_selector?(sdk_folder), type: Cli::BadArgument) { 'The service requires a SDK folder, not a product' }
192
+ yield(host, sdk_folder, dry_run)
193
+ end
194
+
195
+ # @param steps [Array<String>] Actions done, or to do with `--dry-run=yes`
196
+ # @return [Result] The list of actions, or a message if there is nothing to do
197
+ def transferd_service_steps(steps)
198
+ steps.empty? ? Result::Status.new('Nothing to do') : Result::ValueList.new(steps, name: 'action')
199
+ end
200
+
201
+ # Implementation of `transferd service install`
202
+ # With the layout (SDK folder named `current`), the SDK is installed in a folder of the version, next to the link.
203
+ # @param config [Hash] `CONFIG`
204
+ def transferd_service_install(config)
205
+ transferd_service do |host, sdk_folder, dry_run|
206
+ version = Products::TransferdService.service_options(config)['version'] || Info::SDK_VERSION
207
+ sdk_url = sdk_url_option
208
+ installation = Ascp::Installation.instance
209
+ install_sdk =
210
+ if Products::TransferdService.layout?(sdk_folder)
211
+ ->(folder) { installation.install_sdk(url: sdk_url, version: version, folder: folder) }
212
+ else
213
+ -> { installation.install_sdk(url: sdk_url, version: version) }
214
+ end
215
+ steps = Products::TransferdService.install(config, sdk_folder, host: host, dry_run: dry_run, install_sdk: install_sdk, version: version)
216
+ Products::TransferdService.check_version(sdk_folder, host: host)
217
+ transferd_service_steps(steps)
218
+ end
219
+ end
220
+ end
221
+ end
222
+ end
223
+ end