aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -1,24 +1,26 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore trevents
4
- require 'aspera/cli/plugins/basic_auth'
5
- require 'aspera/preview/generator'
6
- require 'aspera/preview/utils'
7
- require 'aspera/preview/file_types'
8
- require 'aspera/preview/terminal'
9
- require 'aspera/transfer/spec'
10
- require 'aspera/schema/registry'
11
- require 'aspera/persistency_action_once'
12
- require 'aspera/temp_file_manager'
13
- require 'aspera/api/node'
14
- require 'aspera/hash_ext'
15
- require 'aspera/timer_limiter'
16
- require 'aspera/id_generator'
17
- require 'aspera/uri_reader'
18
- require 'aspera/log'
19
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'basic_auth'
7
+ require_relative '../../preview/generator'
8
+ require_relative '../../preview/utils'
9
+ require_relative '../../preview/file_types'
10
+ require_relative '../../preview/terminal'
11
+ require_relative '../../transfer/spec'
12
+ require_relative '../../schema/registry'
13
+ require_relative '../../persistency_action_once'
14
+ require_relative '../../temp_file_manager'
15
+ require_relative '../../api/node'
16
+ require_relative '../../hash_ext'
17
+ require_relative '../../timer_limiter'
18
+ require_relative '../../id_generator'
19
+ require_relative '../../uri_reader'
20
+ require_relative '../../log'
21
+ require_relative '../../assert'
22
+ require_relative '../../rainbow'
20
23
  require 'securerandom'
21
- require 'aspera/rainbow'
22
24
  using Rainbow
23
25
 
24
26
  module Aspera
@@ -96,7 +98,7 @@ module Aspera
96
98
  Aspera::Preview::Utils.office_tool = @gen_options.office_conversion
97
99
  Api::Node.api_options[:cache] = !@option_folder_reset_cache.eql?(:header)
98
100
  # Start from the full supported format list, then remove any skipped format.
99
- @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
101
+ @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.dup
100
102
  skip = options.get_option(:skip_format)
101
103
  @preview_formats_to_generate.delete(skip) if skip
102
104
  # Created by `tools_cleanup`, only for commands that use it
@@ -165,13 +167,17 @@ module Aspera
165
167
  return if events.empty?
166
168
  events.each do |event|
167
169
  # Ignore non-file events such as folder notifications.
168
- if event.dig('data', 'type').eql?('file')
170
+ if !event.dig('data', 'type').eql?('file')
171
+ nil
172
+ elsif Array(event['types']).include?('file.deleted')
173
+ delete_preview(event['data'])
174
+ else
169
175
  file_entry = @api_node.read("files/#{event['data']['id']}") rescue nil
170
176
  if !file_entry.nil? &&
171
- @option_skip_folders.none? { |d| file_entry['path'].start_with?(d) }
177
+ @option_skip_folders.none? { |d| file_entry['path'].start_with?(d) } &&
178
+ @filter_block.call(file_entry)
172
179
  file_entry['parent_file_id'] = event['data']['parent_file_id']
173
- Log.log.error('TODO'.red) if event['types'].include?('file.deleted')
174
- generate_preview(file_entry) if event['types'].include?('file.deleted')
180
+ generate_preview(file_entry)
175
181
  end
176
182
  end
177
183
  # Periodically log progress and persist the latest processed event.
@@ -186,6 +192,30 @@ module Aspera
186
192
  nil
187
193
  end
188
194
 
195
+ # Delete the preview folder of a deleted file, if it exists.
196
+ # Errors are logged: a failure shall not stop the processing of events.
197
+ # @param entry [Hash] file entry of the event, with at least `id`
198
+ # @return [nil]
199
+ def delete_preview(entry)
200
+ folder_name = entry_preview_folder_name(entry)
201
+ # The identifier comes from the API: it shall not designate another folder
202
+ Aspera.assert(!entry['id'].to_s.empty? && File.basename(folder_name).eql?(folder_name)) { "Invalid file id: #{entry['id']}" }
203
+ if @access_remote
204
+ preview_folder = @api_node.read_folder_content(@previews_folder_entry['id'], {name: folder_name}).find { |item| item['name'].eql?(folder_name) }
205
+ return if preview_folder.nil?
206
+ @api_node.delete("files/#{preview_folder['id']}")
207
+ else
208
+ local_entry_preview_dir = File.join(@local_preview_folder, folder_name)
209
+ return unless File.directory?(local_entry_preview_dir)
210
+ FileUtils.rm_rf(local_entry_preview_dir)
211
+ end
212
+ Log.log.info { "Deleted preview of #{entry['id']}" }
213
+ nil
214
+ rescue StandardError => e
215
+ Log.log.error { "Cannot delete preview of #{entry['id']}: #{e.message}" }
216
+ nil
217
+ end
218
+
189
219
  # Transfer a file to or from the configured Node storage using a tagged transfer spec.
190
220
  #
191
221
  # @param direction [String] transfer direction, typically from [`Transfer::Spec`](lib/aspera/transfer/spec.rb)
@@ -240,16 +270,21 @@ module Aspera
240
270
  # original_mtime=DateTime.parse(entry['modified_time'])
241
271
  # Local directory where previews are generated before being uploaded back.
242
272
  local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
243
- file_info = @api_node.read("files/#{entry['id']}")
244
- # TODO: This does not work with Gen4 because preview folders are hidden by the API.
245
- # this_preview_folder_entries=@api_node.read_folder_content(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
246
- # TODO: Query Gen3 APIs to list preview files and retrieve timestamps.
273
+ file_info = @api_node.read("files/#{entry['id']}", headers: Api::Node.add_cache_control)
274
+ # Check if preview exists either from file_info or by checking preview endpoint
275
+ preview_exist = file_info.key?('preview')
276
+ unless preview_exist
277
+ begin
278
+ @api_node.read("files/#{entry['id']}/preview", headers: {'Accept' => 'image/png'})
279
+ preview_exist = true
280
+ rescue StandardError
281
+ preview_exist = false
282
+ end
283
+ end
247
284
  gen_infos.each do |gen_info|
248
285
  gen_info[:src] = local_original_filepath
249
286
  gen_info[:dst] = File.join(local_entry_preview_dir, gen_info[:base_dest])
250
- # TODO: Reuse `this_preview_folder_entries` once preview folders become visible.
251
- gen_info[:preview_exist] = file_info.key?('preview')
252
- # TODO: Compare source and preview modification times when remote timestamps are available.
287
+ gen_info[:preview_exist] = preview_exist
253
288
  gen_info[:preview_newer_than_original] = gen_info[:preview_exist]
254
289
  end
255
290
  return local_entry_preview_dir
@@ -352,25 +387,15 @@ module Aspera
352
387
 
353
388
  # Scan all files in provided folder entry
354
389
  # @param top_entry [Hash] the top entry to scan
355
- # @param top_path [String, nil] subpath to start folder scan inside
356
- def scan_folder_files(top_entry, top_path = nil)
357
- unless top_path.nil?
358
- # canonical path: start with / and ends with /
359
- top_path = "/#{top_path.split('/').reject(&:empty?).join('/')}/"
360
- end
361
- Log.log.debug { "scan: #{top_entry} : #{top_path}".green }
390
+ def scan_folder_files(top_entry)
391
+ Log.log.debug { "scan: #{top_entry}".green }
362
392
  # don't use recursive call, use list instead
363
393
  entries_to_process = [top_entry]
364
394
  until entries_to_process.empty?
365
395
  entry = entries_to_process.shift
366
- # process this entry only if it is within the top_path
367
396
  entry_path_with_slash = entry['path']
368
397
  Log.log.debug { "processing entry #{entry_path_with_slash}" } if @periodic.trigger?
369
398
  entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
370
- if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
371
- Log.log.debug { "#{entry['path']} folder (skip start)".bg(:red) }
372
- next
373
- end
374
399
  Log.log.debug { "item:#{entry}" }
375
400
  begin
376
401
  case entry['type']
@@ -416,10 +441,10 @@ module Aspera
416
441
  command :trevents, description: 'Process transfer events and generate previews', setup: :setup_node_api, action: ->(**) { run_event_loop(:trevents) }
417
442
  command :check, description: 'Check required tools are installed', action: ->(**) { tools_cleanup { Result::Status.new('Tools validated') } }
418
443
  command :show, description: 'Generate and display preview of a source file',
419
- arguments: [{name: :source_file, description: 'Local file to preview'}]
444
+ arguments: [{name: :source_file, description: 'Local file to preview', complete: :file}]
420
445
  command :test, description: 'Test preview generation for a source file',
421
446
  arguments: [
422
- {name: :source_file, description: 'Local file to preview'},
447
+ {name: :source_file, description: 'Local file to preview', complete: :file},
423
448
  {name: :format, description: 'Preview output format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS, mandatory: false, default: :png}
424
449
  ]
425
450
 
@@ -1,22 +1,28 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore ascmd zmode zuid zgid fasping
4
- require 'aspera/cli/plugins/basic_auth'
5
- require 'aspera/cli/sync_actions'
6
- require 'aspera/transfer/result'
7
- require 'aspera/transfer/spec'
8
- require 'aspera/ascmd'
9
- require 'aspera/ssh'
10
- require 'aspera/nagios'
11
- require 'aspera/log'
12
- require 'aspera/assert'
13
- require 'aspera/environment'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'basic_auth'
7
+ require_relative '../sync_actions'
8
+ require_relative '../ascp_logs_actions'
9
+ require_relative '../../transfer/result'
10
+ require_relative '../../transfer/spec'
11
+ require_relative '../../ascmd'
12
+ require_relative '../../ssh'
13
+ require_relative '../../nagios'
14
+ require_relative '../../log'
15
+ require_relative '../../assert'
16
+ require_relative '../../environment'
17
+ require 'shellwords'
18
+ require 'tmpdir'
14
19
  module Aspera
15
20
  module Cli
16
21
  module Plugins
17
22
  # Operations on HSTS with SSH/FASP (ascmd/ascp)
18
23
  class Server < BasicAuth
19
24
  include SyncActions
25
+ include AscpLogsActions
20
26
 
21
27
  SSH_SCHEME = 'ssh'
22
28
  LOCAL_SCHEME = 'local'
@@ -66,10 +72,12 @@ module Aspera
66
72
  # @param app_url [String] Tested URL
67
73
  # @return [Hash] :preset_value, :test_args
68
74
  def wizard(wizard, app_url)
75
+ username = options.get_option(:username, mandatory: true)
69
76
  return {
77
+ preset_ids: [username],
70
78
  preset_value: {
71
79
  url: app_url,
72
- username: options.get_option(:username, mandatory: true),
80
+ username: username,
73
81
  password: options.get_option(:password, mandatory: true)
74
82
  },
75
83
  test_args: 'browse /'
@@ -194,6 +202,9 @@ module Aspera
194
202
  command :transfer, description: 'Check FASP transfer health'
195
203
  end
196
204
 
205
+ # Analysis of ascp logs of the server: same commands as `config ascp logs`, on files read over SSH
206
+ ascp_logs_commands(:logs, arguments: [{name: :paths, description: 'Log files on the server (gzip supported)', multiple: true}])
207
+
197
208
  # AsCmd operations (available only when SSH/local - not WSS)
198
209
  # Arguments follow AsCmd::OPS_ARGS: one path per operation, or source and destination
199
210
  PATHS_ARGS = [{name: :paths, multiple: true}].freeze
@@ -297,6 +308,49 @@ module Aspera
297
308
 
298
309
  private
299
310
 
311
+ # Log files of the server, to analyze with the commands of `logs`.
312
+ # Read over SSH, in a temporary folder removed after the block.
313
+ # @param paths [Array<String>] files on the server
314
+ # @yieldparam files [Array<String>] local copies
315
+ def ascp_logs_files(paths)
316
+ executor = ascmd_executor
317
+ Aspera.assert(executor.is_a?(Ssh), type: Cli::BadArgument) { 'Reading the logs of the server requires an SSH connection (URL scheme ssh)' }
318
+ remaining = ascp_logs_params['remote_max_size']
319
+ Dir.mktmpdir('ascli_logs') do |folder|
320
+ files = paths.each_with_index.map do |remote, index|
321
+ local = File.join(folder, format('%02d_%s', index, File.basename(remote)))
322
+ remaining -= read_remote_log(executor, remote, local, remaining)
323
+ local
324
+ end
325
+ yield(files)
326
+ end
327
+ end
328
+
329
+ # Copy a file of the server over SSH. Shell characters of the path are escaped.
330
+ # The modification date is kept: the year of syslog lines is deduced from it.
331
+ # @param executor [Ssh] SSH connection
332
+ # @param remote [String] file on the server
333
+ # @param local [String] destination
334
+ # @param limit [Integer] maximum size in bytes
335
+ # @return [Integer] size in bytes
336
+ def read_remote_log(executor, remote, local, limit)
337
+ quoted = Shellwords.escape(remote)
338
+ # Force an integer: the value comes from a user option and is interpolated in a shell command
339
+ max_bytes = Integer(limit) + 1
340
+ # First line: modification time (0 if unknown), then the content, read up to one byte over the limit
341
+ output = executor.execute("(stat -c %Y -- #{quoted} 2>/dev/null || echo 0); head -c #{max_bytes} -- #{quoted}").b
342
+ mtime, _, content = output.partition("\n")
343
+ Aspera.assert(content.bytesize <= limit, type: Cli::BadArgument) do
344
+ "Log files from the server exceed the limit of #{ascp_logs_params['remote_max_size']} bytes (option ascp_logs.remote_max_size), at: #{remote}"
345
+ end
346
+ Log.log.debug { "Read #{remote}: #{content.bytesize} bytes" }
347
+ File.binwrite(local, content)
348
+ File.utime(Time.now, Time.at(mtime.to_i), local) if mtime.to_i.positive?
349
+ content.bytesize
350
+ rescue Ssh::Error => e
351
+ raise Cli::BadArgument, "Cannot read #{remote}: #{e.message}"
352
+ end
353
+
300
354
  # AsCmd `cp` and `mv` take source and destination pairs.
301
355
  # With option `to_folder`, paths are only sources, copied or moved into that folder.
302
356
  # @param paths [Array<String>] command line arguments