aspera-cli 4.28.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +217 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +122 -26
  10. data/SECURITY.md +84 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1895 -632
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +29 -22
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +95 -67
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +370 -0
  41. data/lib/aspera/ascp/log_locations.rb +163 -0
  42. data/lib/aspera/ascp/management.rb +16 -8
  43. data/lib/aspera/ascp/session_errors.rb +217 -0
  44. data/lib/aspera/assert.rb +4 -0
  45. data/lib/aspera/cli/ascp_actions.rb +12 -19
  46. data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
  47. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  48. data/lib/aspera/cli/bootstrapper.rb +38 -33
  49. data/lib/aspera/cli/command_line.rb +63 -8
  50. data/lib/aspera/cli/command_registry.rb +9 -1
  51. data/lib/aspera/cli/command_spec.rb +47 -7
  52. data/lib/aspera/cli/completion/ascli.bash +19 -5
  53. data/lib/aspera/cli/completion/ascli.fish +13 -2
  54. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  55. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  56. data/lib/aspera/cli/completion_actions.rb +303 -69
  57. data/lib/aspera/cli/context.rb +9 -4
  58. data/lib/aspera/cli/deprecation.rb +5 -2
  59. data/lib/aspera/cli/error.rb +6 -0
  60. data/lib/aspera/cli/extended_value.rb +66 -31
  61. data/lib/aspera/cli/formatter.rb +43 -30
  62. data/lib/aspera/cli/gem_checker.rb +9 -6
  63. data/lib/aspera/cli/hints.rb +22 -10
  64. data/lib/aspera/cli/http.rb +38 -10
  65. data/lib/aspera/cli/info.rb +12 -7
  66. data/lib/aspera/cli/mailer.rb +6 -3
  67. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  68. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  69. data/lib/aspera/cli/mcp/tool.rb +20 -17
  70. data/lib/aspera/cli/option_declarator.rb +10 -4
  71. data/lib/aspera/cli/option_registry.rb +6 -3
  72. data/lib/aspera/cli/option_types.rb +17 -2
  73. data/lib/aspera/cli/option_value.rb +77 -18
  74. data/lib/aspera/cli/options.schema.yaml +641 -80
  75. data/lib/aspera/cli/parser.rb +71 -35
  76. data/lib/aspera/cli/plugins/alee.rb +7 -4
  77. data/lib/aspera/cli/plugins/aoc.rb +248 -37
  78. data/lib/aspera/cli/plugins/ats.rb +10 -9
  79. data/lib/aspera/cli/plugins/base.rb +50 -18
  80. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  81. data/lib/aspera/cli/plugins/config.rb +210 -104
  82. data/lib/aspera/cli/plugins/console.rb +6 -3
  83. data/lib/aspera/cli/plugins/cos.rb +8 -4
  84. data/lib/aspera/cli/plugins/factory.rb +21 -7
  85. data/lib/aspera/cli/plugins/faspex5.rb +31 -26
  86. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  87. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  88. data/lib/aspera/cli/plugins/mcp.rb +42 -22
  89. data/lib/aspera/cli/plugins/node.rb +39 -38
  90. data/lib/aspera/cli/plugins/oauth.rb +23 -6
  91. data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
  92. data/lib/aspera/cli/plugins/preview.rb +72 -45
  93. data/lib/aspera/cli/plugins/server.rb +66 -14
  94. data/lib/aspera/cli/plugins/shares.rb +87 -18
  95. data/lib/aspera/cli/plugins/transferd.rb +228 -0
  96. data/lib/aspera/cli/poll.rb +236 -0
  97. data/lib/aspera/cli/preset_actions.rb +144 -30
  98. data/lib/aspera/cli/preset_manager.rb +113 -35
  99. data/lib/aspera/cli/prompt.rb +157 -9
  100. data/lib/aspera/cli/result.rb +12 -8
  101. data/lib/aspera/cli/runner.rb +89 -48
  102. data/lib/aspera/cli/secret_finder.rb +4 -1
  103. data/lib/aspera/cli/security.rb +89 -9
  104. data/lib/aspera/cli/special_values.rb +4 -1
  105. data/lib/aspera/cli/sync_actions.rb +15 -10
  106. data/lib/aspera/cli/table_fitter.rb +3 -0
  107. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  108. data/lib/aspera/cli/transfer_actions.rb +11 -8
  109. data/lib/aspera/cli/transfer_agent.rb +43 -20
  110. data/lib/aspera/cli/transfer_progress.rb +14 -45
  111. data/lib/aspera/cli/vault_actions.rb +45 -0
  112. data/lib/aspera/cli/vault_manager.rb +188 -42
  113. data/lib/aspera/cli/version.rb +4 -1
  114. data/lib/aspera/cli/web_secret.rb +12 -10
  115. data/lib/aspera/cli/wizard.rb +42 -39
  116. data/lib/aspera/command_line_builder.rb +9 -5
  117. data/lib/aspera/command_line_converter.rb +4 -1
  118. data/lib/aspera/coverage.rb +103 -33
  119. data/lib/aspera/data_repository.rb +4 -1
  120. data/lib/aspera/dot_container.rb +4 -1
  121. data/lib/aspera/environment.rb +23 -12
  122. data/lib/aspera/exec_spec.rb +3 -0
  123. data/lib/aspera/faspex_gw.rb +13 -9
  124. data/lib/aspera/faspex_postproc.rb +15 -12
  125. data/lib/aspera/formatter_interface.rb +4 -1
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +20 -5
  129. data/lib/aspera/inflate.rb +43 -0
  130. data/lib/aspera/json_rpc/client.rb +6 -5
  131. data/lib/aspera/json_rpc/version.rb +3 -0
  132. data/lib/aspera/line_logger.rb +4 -1
  133. data/lib/aspera/link_header.rb +4 -1
  134. data/lib/aspera/log.rb +18 -5
  135. data/lib/aspera/markdown.rb +5 -2
  136. data/lib/aspera/mime.rb +3 -0
  137. data/lib/aspera/nagios.rb +6 -4
  138. data/lib/aspera/node_emulator.rb +15 -13
  139. data/lib/aspera/oauth/client/base.rb +124 -73
  140. data/lib/aspera/oauth/client/boot.rb +7 -4
  141. data/lib/aspera/oauth/client/factory.rb +41 -18
  142. data/lib/aspera/oauth/client/generic.rb +5 -1
  143. data/lib/aspera/oauth/client/json_credentials.rb +7 -3
  144. data/lib/aspera/oauth/client/jwt.rb +6 -3
  145. data/lib/aspera/oauth/client/shares_web.rb +94 -0
  146. data/lib/aspera/oauth/client/url_json.rb +7 -3
  147. data/lib/aspera/oauth/client/web.rb +40 -13
  148. data/lib/aspera/oauth/server/pkce.rb +6 -5
  149. data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +16 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +17 -2
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +3 -0
  165. data/lib/aspera/rest/aspera_errors.rb +4 -1
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +45 -34
  168. data/lib/aspera/rest/error_analyzer.rb +5 -2
  169. data/lib/aspera/rest/list.rb +8 -4
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +8 -5
  172. data/lib/aspera/rest.rb +9 -6
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  175. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  176. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  177. data/lib/aspera/schema/async_tables.yaml +11 -10
  178. data/lib/aspera/schema/documentation.rb +8 -2
  179. data/lib/aspera/schema/reader.rb +7 -0
  180. data/lib/aspera/schema/registry.rb +44 -28
  181. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  182. data/lib/aspera/schema/validator.rb +4 -1
  183. data/lib/aspera/sealed_hash.rb +150 -0
  184. data/lib/aspera/secret/hider.rb +213 -0
  185. data/lib/aspera/secret/reference.rb +150 -0
  186. data/lib/aspera/secret/resolver.rb +212 -0
  187. data/lib/aspera/secret/store/base.rb +153 -0
  188. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  189. data/lib/aspera/secret/store/factory.rb +103 -0
  190. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  191. data/lib/aspera/secret/store/keeper.rb +141 -0
  192. data/lib/aspera/secret/store/macos_security.rb +284 -0
  193. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  194. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  195. data/lib/aspera/secret/store/one_password_cli.rb +112 -0
  196. data/lib/aspera/secret/store/secret_service.rb +125 -0
  197. data/lib/aspera/secret/strength.rb +119 -0
  198. data/lib/aspera/servlet_helpers.rb +5 -2
  199. data/lib/aspera/ssh.rb +10 -7
  200. data/lib/aspera/ssl.rb +5 -2
  201. data/lib/aspera/string_ext.rb +3 -0
  202. data/lib/aspera/sync/args.schema.yaml +3 -0
  203. data/lib/aspera/sync/conf.schema.yaml +9 -4
  204. data/lib/aspera/sync/database.rb +3 -0
  205. data/lib/aspera/sync/operations.rb +17 -15
  206. data/lib/aspera/temp_file_manager.rb +8 -4
  207. data/lib/aspera/timer_limiter.rb +3 -0
  208. data/lib/aspera/transfer/error.rb +4 -1
  209. data/lib/aspera/transfer/faux_file.rb +4 -1
  210. data/lib/aspera/transfer/parameters.rb +17 -14
  211. data/lib/aspera/transfer/result.rb +23 -0
  212. data/lib/aspera/transfer/resumer.rb +6 -4
  213. data/lib/aspera/transfer/spec.rb +9 -22
  214. data/lib/aspera/units/decimal.rb +67 -0
  215. data/lib/aspera/units/duration.rb +109 -0
  216. data/lib/aspera/uri_reader.rb +64 -9
  217. data/lib/aspera/web_auth/index.html.erb +6 -2
  218. data/lib/aspera/web_auth/paste.rb +109 -0
  219. data/lib/aspera/web_auth/receiver.rb +50 -0
  220. data/lib/aspera/web_auth/relay.rb +76 -0
  221. data/lib/aspera/web_auth/reply.html.erb +150 -0
  222. data/lib/aspera/web_auth/server.rb +47 -10
  223. data/lib/aspera/web_page.rb +7 -2
  224. data/lib/aspera/web_server_simple.rb +6 -3
  225. data/lib/aspera/yaml.rb +5 -1
  226. data/llms-install.md +88 -0
  227. data/llms.txt +87 -0
  228. data/skills/ascli/SKILL.md +94 -0
  229. data.tar.gz.sig +0 -0
  230. metadata +77 -20
  231. metadata.gz.sig +0 -0
  232. data/lib/aspera/duration.rb +0 -42
  233. data/lib/aspera/keychain/base.rb +0 -30
  234. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  235. data/lib/aspera/keychain/factory.rb +0 -58
  236. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  237. data/lib/aspera/keychain/macos_security.rb +0 -277
  238. data/lib/aspera/keychain/one_password_api.rb +0 -91
  239. data/lib/aspera/keychain/one_password_base.rb +0 -39
  240. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  241. data/lib/aspera/secret_hider.rb +0 -196
  242. data/lib/aspera/secret_strength.rb +0 -114
@@ -1,24 +1,26 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore trevents
4
- require 'aspera/cli/plugins/basic_auth'
5
- require 'aspera/preview/generator'
6
- require 'aspera/preview/utils'
7
- require 'aspera/preview/file_types'
8
- require 'aspera/preview/terminal'
9
- require 'aspera/transfer/spec'
10
- require 'aspera/schema/registry'
11
- require 'aspera/persistency_action_once'
12
- require 'aspera/temp_file_manager'
13
- require 'aspera/api/node'
14
- require 'aspera/hash_ext'
15
- require 'aspera/timer_limiter'
16
- require 'aspera/id_generator'
17
- require 'aspera/uri_reader'
18
- require 'aspera/log'
19
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'basic_auth'
7
+ require_relative '../../preview/generator'
8
+ require_relative '../../preview/utils'
9
+ require_relative '../../preview/file_types'
10
+ require_relative '../../preview/terminal'
11
+ require_relative '../../transfer/spec'
12
+ require_relative '../../schema/registry'
13
+ require_relative '../../persistency_action_once'
14
+ require_relative '../../temp_file_manager'
15
+ require_relative '../../api/node'
16
+ require_relative '../../hash_ext'
17
+ require_relative '../../timer_limiter'
18
+ require_relative '../../id_generator'
19
+ require_relative '../../uri_reader'
20
+ require_relative '../../log'
21
+ require_relative '../../assert'
22
+ require_relative '../../rainbow'
20
23
  require 'securerandom'
21
- require 'aspera/rainbow'
22
24
  using Rainbow
23
25
 
24
26
  module Aspera
@@ -96,7 +98,7 @@ module Aspera
96
98
  Aspera::Preview::Utils.office_tool = @gen_options.office_conversion
97
99
  Api::Node.api_options[:cache] = !@option_folder_reset_cache.eql?(:header)
98
100
  # Start from the full supported format list, then remove any skipped format.
99
- @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
101
+ @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.dup
100
102
  skip = options.get_option(:skip_format)
101
103
  @preview_formats_to_generate.delete(skip) if skip
102
104
  # Created by `tools_cleanup`, only for commands that use it
@@ -165,13 +167,17 @@ module Aspera
165
167
  return if events.empty?
166
168
  events.each do |event|
167
169
  # Ignore non-file events such as folder notifications.
168
- if event.dig('data', 'type').eql?('file')
170
+ if !event.dig('data', 'type').eql?('file')
171
+ nil
172
+ elsif Array(event['types']).include?('file.deleted')
173
+ delete_preview(event['data'])
174
+ else
169
175
  file_entry = @api_node.read("files/#{event['data']['id']}") rescue nil
170
176
  if !file_entry.nil? &&
171
- @option_skip_folders.none? { |d| file_entry['path'].start_with?(d) }
177
+ @option_skip_folders.none? { |d| file_entry['path'].start_with?(d) } &&
178
+ @filter_block.call(file_entry)
172
179
  file_entry['parent_file_id'] = event['data']['parent_file_id']
173
- Log.log.error('TODO'.red) if event['types'].include?('file.deleted')
174
- generate_preview(file_entry) if event['types'].include?('file.deleted')
180
+ generate_preview(file_entry)
175
181
  end
176
182
  end
177
183
  # Periodically log progress and persist the latest processed event.
@@ -186,6 +192,30 @@ module Aspera
186
192
  nil
187
193
  end
188
194
 
195
+ # Delete the preview folder of a deleted file, if it exists.
196
+ # Errors are logged: a failure shall not stop the processing of events.
197
+ # @param entry [Hash] file entry of the event, with at least `id`
198
+ # @return [nil]
199
+ def delete_preview(entry)
200
+ folder_name = entry_preview_folder_name(entry)
201
+ # The identifier comes from the API: it shall not designate another folder
202
+ Aspera.assert(!entry['id'].to_s.empty? && File.basename(folder_name).eql?(folder_name)) { "Invalid file id: #{entry['id']}" }
203
+ if @access_remote
204
+ preview_folder = @api_node.read_folder_content(@previews_folder_entry['id'], {name: folder_name}).find { |item| item['name'].eql?(folder_name) }
205
+ return if preview_folder.nil?
206
+ @api_node.delete("files/#{preview_folder['id']}")
207
+ else
208
+ local_entry_preview_dir = File.join(@local_preview_folder, folder_name)
209
+ return unless File.directory?(local_entry_preview_dir)
210
+ FileUtils.rm_rf(local_entry_preview_dir)
211
+ end
212
+ Log.log.info { "Deleted preview of #{entry['id']}" }
213
+ nil
214
+ rescue StandardError => e
215
+ Log.log.error { "Cannot delete preview of #{entry['id']}: #{e.message}" }
216
+ nil
217
+ end
218
+
189
219
  # Transfer a file to or from the configured Node storage using a tagged transfer spec.
190
220
  #
191
221
  # @param direction [String] transfer direction, typically from [`Transfer::Spec`](lib/aspera/transfer/spec.rb)
@@ -195,6 +225,8 @@ module Aspera
195
225
  # @return [Object] transfer result returned by [`Main.result_transfer`](lib/aspera/cli/main.rb)
196
226
  def do_transfer(direction, folder_id, source_filename, destination = '/')
197
227
  Aspera.assert(!(destination.nil? && direction.eql?(Transfer::Spec::DIRECTION_RECEIVE)), 'destination must be set for receive direction')
228
+ # Previews are generated from downloaded files, and uploaded
229
+ transfer.assert_no_dry_run
198
230
  t_spec = @api_node.transfer_spec_gen4(folder_id, direction, {
199
231
  'paths' => [{'source' => source_filename}],
200
232
  'tags' => {Transfer::Spec::TAG_RESERVED => {PREV_GEN_TAG => true}}
@@ -240,16 +272,21 @@ module Aspera
240
272
  # original_mtime=DateTime.parse(entry['modified_time'])
241
273
  # Local directory where previews are generated before being uploaded back.
242
274
  local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
243
- file_info = @api_node.read("files/#{entry['id']}")
244
- # TODO: This does not work with Gen4 because preview folders are hidden by the API.
245
- # this_preview_folder_entries=@api_node.read_folder_content(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
246
- # TODO: Query Gen3 APIs to list preview files and retrieve timestamps.
275
+ file_info = @api_node.read("files/#{entry['id']}", headers: Api::Node.add_cache_control)
276
+ # Check if preview exists either from file_info or by checking preview endpoint
277
+ preview_exist = file_info.key?('preview')
278
+ unless preview_exist
279
+ begin
280
+ @api_node.read("files/#{entry['id']}/preview", headers: {'Accept' => 'image/png'})
281
+ preview_exist = true
282
+ rescue StandardError
283
+ preview_exist = false
284
+ end
285
+ end
247
286
  gen_infos.each do |gen_info|
248
287
  gen_info[:src] = local_original_filepath
249
288
  gen_info[:dst] = File.join(local_entry_preview_dir, gen_info[:base_dest])
250
- # TODO: Reuse `this_preview_folder_entries` once preview folders become visible.
251
- gen_info[:preview_exist] = file_info.key?('preview')
252
- # TODO: Compare source and preview modification times when remote timestamps are available.
289
+ gen_info[:preview_exist] = preview_exist
253
290
  gen_info[:preview_newer_than_original] = gen_info[:preview_exist]
254
291
  end
255
292
  return local_entry_preview_dir
@@ -352,25 +389,15 @@ module Aspera
352
389
 
353
390
  # Scan all files in provided folder entry
354
391
  # @param top_entry [Hash] the top entry to scan
355
- # @param top_path [String, nil] subpath to start folder scan inside
356
- def scan_folder_files(top_entry, top_path = nil)
357
- unless top_path.nil?
358
- # canonical path: start with / and ends with /
359
- top_path = "/#{top_path.split('/').reject(&:empty?).join('/')}/"
360
- end
361
- Log.log.debug { "scan: #{top_entry} : #{top_path}".green }
392
+ def scan_folder_files(top_entry)
393
+ Log.log.debug { "scan: #{top_entry}".green }
362
394
  # don't use recursive call, use list instead
363
395
  entries_to_process = [top_entry]
364
396
  until entries_to_process.empty?
365
397
  entry = entries_to_process.shift
366
- # process this entry only if it is within the top_path
367
398
  entry_path_with_slash = entry['path']
368
399
  Log.log.debug { "processing entry #{entry_path_with_slash}" } if @periodic.trigger?
369
400
  entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
370
- if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
371
- Log.log.debug { "#{entry['path']} folder (skip start)".bg(:red) }
372
- next
373
- end
374
401
  Log.log.debug { "item:#{entry}" }
375
402
  begin
376
403
  case entry['type']
@@ -415,11 +442,11 @@ module Aspera
415
442
  command :events, description: 'Process file events and generate previews', setup: :setup_node_api, action: ->(**) { run_event_loop(:events) }
416
443
  command :trevents, description: 'Process transfer events and generate previews', setup: :setup_node_api, action: ->(**) { run_event_loop(:trevents) }
417
444
  command :check, description: 'Check required tools are installed', action: ->(**) { tools_cleanup { Result::Status.new('Tools validated') } }
418
- command :show, description: 'Generate and display preview of a source file',
419
- arguments: [{name: :source_file, description: 'Local file to preview'}]
445
+ command :show, read_only: false, description: 'Generate and display preview of a source file',
446
+ arguments: [{name: :source_file, description: 'Local file to preview', complete: :file}]
420
447
  command :test, description: 'Test preview generation for a source file',
421
448
  arguments: [
422
- {name: :source_file, description: 'Local file to preview'},
449
+ {name: :source_file, description: 'Local file to preview', complete: :file},
423
450
  {name: :format, description: 'Preview output format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS, mandatory: false, default: :png}
424
451
  ]
425
452
 
@@ -1,22 +1,28 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore ascmd zmode zuid zgid fasping
4
- require 'aspera/cli/plugins/basic_auth'
5
- require 'aspera/cli/sync_actions'
6
- require 'aspera/transfer/result'
7
- require 'aspera/transfer/spec'
8
- require 'aspera/ascmd'
9
- require 'aspera/ssh'
10
- require 'aspera/nagios'
11
- require 'aspera/log'
12
- require 'aspera/assert'
13
- require 'aspera/environment'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'basic_auth'
7
+ require_relative '../sync_actions'
8
+ require_relative '../ascp_logs_actions'
9
+ require_relative '../../transfer/result'
10
+ require_relative '../../transfer/spec'
11
+ require_relative '../../ascmd'
12
+ require_relative '../../ssh'
13
+ require_relative '../../nagios'
14
+ require_relative '../../log'
15
+ require_relative '../../assert'
16
+ require_relative '../../environment'
17
+ require 'shellwords'
18
+ require 'tmpdir'
14
19
  module Aspera
15
20
  module Cli
16
21
  module Plugins
17
22
  # Operations on HSTS with SSH/FASP (ascmd/ascp)
18
23
  class Server < BasicAuth
19
24
  include SyncActions
25
+ include AscpLogsActions
20
26
 
21
27
  SSH_SCHEME = 'ssh'
22
28
  LOCAL_SCHEME = 'local'
@@ -196,6 +202,9 @@ module Aspera
196
202
  command :transfer, description: 'Check FASP transfer health'
197
203
  end
198
204
 
205
+ # Analysis of ascp logs of the server: same commands as `config ascp logs`, on files read over SSH
206
+ ascp_logs_commands(:logs, arguments: [{name: :paths, description: 'Log files on the server (gzip supported)', multiple: true}])
207
+
199
208
  # AsCmd operations (available only when SSH/local - not WSS)
200
209
  # Arguments follow AsCmd::OPS_ARGS: one path per operation, or source and destination
201
210
  PATHS_ARGS = [{name: :paths, multiple: true}].freeze
@@ -214,15 +223,15 @@ module Aspera
214
223
  command :cp, description: 'Copy files: source destination pairs, or sources with --to-folder', condition: :ascmd_available?, arguments: PATHS_ARGS
215
224
  command :mkdir, description: 'Create directory', condition: :ascmd_available?, arguments: PATHS_ARGS
216
225
  command(
217
- :df, description: 'Show disk usage', condition: :ascmd_available?,
226
+ :df, read_only: true, description: 'Show disk usage', condition: :ascmd_available?,
218
227
  action: lambda do |**|
219
228
  execute_ascmd(:df, []) do |result|
220
229
  Result::ObjectList.new(result.map(&:stringify_keys))
221
230
  end
222
231
  end
223
232
  )
224
- command :du, description: 'Show file sizes', condition: :ascmd_available?, arguments: PATHS_ARGS
225
- command :md5sum, description: 'Compute MD5 checksums', condition: :ascmd_available?, arguments: PATHS_ARGS
233
+ command :du, read_only: true, description: 'Show file sizes', condition: :ascmd_available?, arguments: PATHS_ARGS
234
+ command :md5sum, read_only: true, description: 'Compute MD5 checksums', condition: :ascmd_available?, arguments: PATHS_ARGS
226
235
  command :info, description: 'Show server system information', condition: :ascmd_available?,
227
236
  action: ->(**) { execute_ascmd(:info, []) { |r| Result::SingleObject.new(r.stringify_keys) } }
228
237
 
@@ -299,6 +308,49 @@ module Aspera
299
308
 
300
309
  private
301
310
 
311
+ # Log files of the server, to analyze with the commands of `logs`.
312
+ # Read over SSH, in a temporary folder removed after the block.
313
+ # @param paths [Array<String>] files on the server
314
+ # @yieldparam files [Array<String>] local copies
315
+ def ascp_logs_files(paths)
316
+ executor = ascmd_executor
317
+ Aspera.assert(executor.is_a?(Ssh), type: Cli::BadArgument) { 'Reading the logs of the server requires an SSH connection (URL scheme ssh)' }
318
+ remaining = ascp_logs_params['remote_max_size']
319
+ Dir.mktmpdir('ascli_logs') do |folder|
320
+ files = paths.each_with_index.map do |remote, index|
321
+ local = File.join(folder, format('%02d_%s', index, File.basename(remote)))
322
+ remaining -= read_remote_log(executor, remote, local, remaining)
323
+ local
324
+ end
325
+ yield(files)
326
+ end
327
+ end
328
+
329
+ # Copy a file of the server over SSH. Shell characters of the path are escaped.
330
+ # The modification date is kept: the year of syslog lines is deduced from it.
331
+ # @param executor [Ssh] SSH connection
332
+ # @param remote [String] file on the server
333
+ # @param local [String] destination
334
+ # @param limit [Integer] maximum size in bytes
335
+ # @return [Integer] size in bytes
336
+ def read_remote_log(executor, remote, local, limit)
337
+ quoted = Shellwords.escape(remote)
338
+ # Force an integer: the value comes from a user option and is interpolated in a shell command
339
+ max_bytes = Integer(limit) + 1
340
+ # First line: modification time (0 if unknown), then the content, read up to one byte over the limit
341
+ output = executor.execute("(stat -c %Y -- #{quoted} 2>/dev/null || echo 0); head -c #{max_bytes} -- #{quoted}").b
342
+ mtime, _, content = output.partition("\n")
343
+ Aspera.assert(content.bytesize <= limit, type: Cli::BadArgument) do
344
+ "Log files from the server exceed the limit of #{ascp_logs_params['remote_max_size']} bytes (option ascp_logs.remote_max_size), at: #{remote}"
345
+ end
346
+ Log.log.debug { "Read #{remote}: #{content.bytesize} bytes" }
347
+ File.binwrite(local, content)
348
+ File.utime(Time.now, Time.at(mtime.to_i), local) if mtime.to_i.positive?
349
+ content.bytesize
350
+ rescue Ssh::Error => e
351
+ raise Cli::BadArgument, "Cannot read #{remote}: #{e.message}"
352
+ end
353
+
302
354
  # AsCmd `cp` and `mv` take source and destination pairs.
303
355
  # With option `to_folder`, paths are only sources, copied or moved into that folder.
304
356
  # @param paths [Array<String>] command line arguments
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/cli/plugins/basic_auth'
4
- require 'aspera/cli/plugins/node'
5
- require 'aspera/assert'
6
- require 'aspera/rest/list'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'basic_auth'
7
+ require_relative 'node'
8
+ require_relative '../../assert'
9
+ require_relative '../../rest/list'
7
10
 
8
11
  module Aspera
9
12
  module Cli
@@ -14,6 +17,13 @@ module Aspera
14
17
  NODE_API_PATH = 'node_api'
15
18
  # path for node admin after base url
16
19
  ADMIN_API_PATH = 'api/v1'
20
+ # path for authentication (SAML login page, OAuth token) after base url
21
+ AUTH_API_PATH = 'auth'
22
+ # default URL that receives the tokens, with auth `web` and `ui.web_auth=port`
23
+ REDIRECT_LOCALHOST = 'http://localhost:12345'
24
+ AUTH_TYPES = %i[basic web].freeze
25
+ option :auth, description: 'Type of authentication: user and password, or Web browser (SAML)', allowed: AUTH_TYPES, default: :basic
26
+ option :redirect_uri, description: "Web authentication with ui.web_auth=port: loopback URL that receives the tokens (default: #{REDIRECT_LOCALHOST})"
17
27
  class << self
18
28
  # Check various endpoints on Shares
19
29
  # @return [Hash] with version, ping, api
@@ -27,12 +37,13 @@ module Aspera
27
37
  .new(base_url: url, redirect_max: 2)
28
38
  .read('', headers: {'Accept'=>'text/html'})
29
39
  Aspera.assert(login_page.include?('aspera-Shares')) { 'not Shares' }
30
- if (m = login_page.match(/\(v([0-9a-f\.]+)\)/))
40
+ if (m = login_page.match(/\(v([0-9a-f.]+)\)/))
31
41
  version = m[1]
32
42
  if (m = login_page.match(/Patch level ([0-9]+)/))
33
43
  version = "#{version} #{m[0]}"
34
44
  end
35
45
  end
46
+ result[:saml] = login_page.include?('/auth/saml')
36
47
  version.nil? ? 'no version' : version
37
48
  rescue => e
38
49
  e
@@ -79,7 +90,8 @@ module Aspera
79
90
  return unless health[:api].is_a?(String)
80
91
  return {
81
92
  version: health[:version].is_a?(String) ? health[:version] : 'unknown',
82
- url: address_or_url
93
+ url: address_or_url,
94
+ saml: health[:saml] || false
83
95
  }
84
96
  end
85
97
  end
@@ -88,16 +100,53 @@ module Aspera
88
100
  # @param app_url [String] Tested URL
89
101
  # @return [Hash] :preset_value, :test_args
90
102
  def wizard(wizard, app_url)
91
- username = options.get_option(:username, mandatory: true)
92
- return {
93
- preset_ids: [username],
94
- preset_value: {
95
- url: app_url,
96
- username: username,
97
- password: options.get_option(:password, mandatory: true)
98
- },
99
- test_args: 'files browse /'
100
- }
103
+ auth_type = if !options.option_def(:auth).source.eql?(:default)
104
+ options.get_option(:auth)
105
+ elsif options.get_option(:password)
106
+ :basic
107
+ else
108
+ saml_detected = begin
109
+ self.class.health_check(app_url)[:saml] || false
110
+ rescue StandardError
111
+ false
112
+ end
113
+ formatter.display_status('SAML authentication is available.') if saml_detected
114
+ if wizard.required
115
+ formatter.display_status('Select authentication method:')
116
+ options.prompt_user_input_in_list('auth', AUTH_TYPES)
117
+ else
118
+ saml_detected ? :web : :basic
119
+ end
120
+ end
121
+
122
+ case auth_type
123
+ when :web
124
+ username = options.get_option(:username)
125
+ preset_ids = username && !username.empty? ? [username] : ['web']
126
+ preset_value = {
127
+ url: app_url,
128
+ auth: :web.to_s
129
+ }
130
+ preset_value[:username] = username if username && !username.empty?
131
+ redirect_uri = options.get_option(:redirect_uri)
132
+ preset_value[:redirect_uri] = redirect_uri if redirect_uri
133
+ {
134
+ preset_ids: preset_ids,
135
+ preset_value: preset_value,
136
+ test_args: 'files browse /'
137
+ }
138
+ when :basic
139
+ username = options.get_option(:username, mandatory: true)
140
+ {
141
+ preset_ids: [username],
142
+ preset_value: {
143
+ url: app_url,
144
+ username: username,
145
+ password: options.get_option(:password, mandatory: true)
146
+ },
147
+ test_args: 'files browse /'
148
+ }
149
+ end
101
150
  end
102
151
 
103
152
  def initialize(**_)
@@ -105,6 +154,27 @@ module Aspera
105
154
  @api_shares_admin = nil
106
155
  end
107
156
 
157
+ # With `auth` `web`, the user authenticates with a browser (SAML), and the REST API uses the OAuth token.
158
+ # Otherwise: user and password.
159
+ # @param subpath [String, nil] Path of API after base URL
160
+ # @return [Hash] Parameters of `Rest::Client`
161
+ def basic_auth_params(subpath = nil)
162
+ return super unless options.get_option(:auth).eql?(:web)
163
+ url = options.get_option(:url, mandatory: true)
164
+ username = options.get_option(:username)
165
+ return {
166
+ base_url: subpath.nil? ? url : "#{url}/#{subpath}",
167
+ auth: {
168
+ type: :oauth2,
169
+ base_url: "#{url}/#{AUTH_API_PATH}",
170
+ grant_method: :shares_web,
171
+ redirect_uri: options.get_option(:redirect_uri) || REDIRECT_LOCALHOST,
172
+ # a token is cached for each user, if known
173
+ cache_ids: [username].compact
174
+ }
175
+ }
176
+ end
177
+
108
178
  SAML_IMPORT_MANDATORY = %w[id name_id].freeze
109
179
  SAML_IMPORT_ALLOWED = (%w[email given_name surname] + SAML_IMPORT_MANDATORY).freeze
110
180
 
@@ -287,8 +357,7 @@ module Aspera
287
357
  # entity_plural captured at class-load time in the closure
288
358
  entity_plural_search = entity_type.eql?(:user) ? 'users' : 'groups'
289
359
  commands_under :search do
290
- command :list,
291
- description: "Search #{entity_type}s across all directories (use --query={q: '...'} to filter)",
360
+ command :list, description: "Search #{entity_type}s across all directories (use --query={q: '...'} to filter)",
292
361
  action: ->(**) { entity_list(api: @api_shares_admin, entity: entity_plural_search, items_key: 'members') }
293
362
  command :show,
294
363
  description: "Get directory #{entity_type} details by URN",