aspera-cli 4.26.2 → 4.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +310 -21
  4. data/CONTRIBUTING.md +99 -14
  5. data/TODO.md +50 -0
  6. data/bin/ascli +3 -3
  7. data/docs/README.md +10977 -0
  8. data/docs/test-mcp-with-ai.md +202 -0
  9. data/lib/aspera/agent/base.rb +29 -7
  10. data/lib/aspera/agent/connect.rb +58 -21
  11. data/lib/aspera/agent/desktop.rb +53 -18
  12. data/lib/aspera/agent/direct.rb +71 -40
  13. data/lib/aspera/agent/factory.rb +3 -3
  14. data/lib/aspera/agent/httpgw.rb +55 -14
  15. data/lib/aspera/agent/node.rb +34 -2
  16. data/lib/aspera/agent/transferd.rb +48 -12
  17. data/lib/aspera/api/alee.rb +1 -1
  18. data/lib/aspera/api/aoc.rb +42 -44
  19. data/lib/aspera/api/cos_node.rb +10 -9
  20. data/lib/aspera/api/faspex.rb +16 -15
  21. data/lib/aspera/api/httpgw.rb +23 -23
  22. data/lib/aspera/api/node.rb +51 -70
  23. data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
  24. data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
  25. data/lib/aspera/ascmd.rb +32 -30
  26. data/lib/aspera/ascp/installation.rb +75 -69
  27. data/lib/aspera/ascp/management.rb +7 -8
  28. data/lib/aspera/assert.rb +12 -13
  29. data/lib/aspera/cli/ascp_actions.rb +155 -0
  30. data/lib/aspera/cli/async_transfer_store.rb +125 -0
  31. data/lib/aspera/cli/bootstrapper.rb +197 -0
  32. data/lib/aspera/cli/command_registry.rb +145 -0
  33. data/lib/aspera/cli/command_spec.rb +141 -0
  34. data/lib/aspera/cli/context.rb +19 -6
  35. data/lib/aspera/cli/error.rb +16 -0
  36. data/lib/aspera/cli/extended_value.rb +27 -26
  37. data/lib/aspera/cli/formatter.rb +95 -59
  38. data/lib/aspera/cli/gem_checker.rb +65 -0
  39. data/lib/aspera/cli/hints.rb +3 -3
  40. data/lib/aspera/cli/http.rb +74 -30
  41. data/lib/aspera/cli/info.rb +2 -0
  42. data/lib/aspera/cli/mailer.rb +97 -0
  43. data/lib/aspera/cli/mcp_tool.rb +225 -0
  44. data/lib/aspera/cli/option_declarator.rb +77 -0
  45. data/lib/aspera/cli/options.schema.yaml +634 -7
  46. data/lib/aspera/cli/parser.rb +1217 -0
  47. data/lib/aspera/cli/plugins/alee.rb +20 -22
  48. data/lib/aspera/cli/plugins/aoc.rb +1178 -881
  49. data/lib/aspera/cli/plugins/ats.rb +205 -162
  50. data/lib/aspera/cli/plugins/base.rb +703 -175
  51. data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
  52. data/lib/aspera/cli/plugins/config.rb +434 -757
  53. data/lib/aspera/cli/plugins/console.rb +107 -65
  54. data/lib/aspera/cli/plugins/cos.rb +46 -33
  55. data/lib/aspera/cli/plugins/factory.rb +6 -6
  56. data/lib/aspera/cli/plugins/faspex5.rb +627 -388
  57. data/lib/aspera/cli/plugins/faspio.rb +54 -51
  58. data/lib/aspera/cli/plugins/httpgw.rb +18 -25
  59. data/lib/aspera/cli/plugins/mcp.rb +279 -0
  60. data/lib/aspera/cli/plugins/node.rb +988 -840
  61. data/lib/aspera/cli/plugins/oauth.rb +7 -10
  62. data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
  63. data/lib/aspera/cli/plugins/preview.rb +195 -157
  64. data/lib/aspera/cli/plugins/server.rb +141 -91
  65. data/lib/aspera/cli/plugins/shares.rb +343 -110
  66. data/lib/aspera/cli/preset_actions.rb +159 -0
  67. data/lib/aspera/cli/preset_manager.rb +82 -39
  68. data/lib/aspera/cli/result.rb +78 -26
  69. data/lib/aspera/cli/runner.rb +323 -153
  70. data/lib/aspera/cli/secret_finder.rb +40 -0
  71. data/lib/aspera/cli/special_values.rb +1 -0
  72. data/lib/aspera/cli/sync_actions.rb +89 -64
  73. data/lib/aspera/cli/terminal_formatter.rb +3 -3
  74. data/lib/aspera/cli/transfer_actions.rb +90 -0
  75. data/lib/aspera/cli/transfer_agent.rb +129 -64
  76. data/lib/aspera/cli/transfer_progress.rb +9 -9
  77. data/lib/aspera/cli/vault_manager.rb +79 -0
  78. data/lib/aspera/cli/version.rb +1 -1
  79. data/lib/aspera/cli/wizard.rb +28 -30
  80. data/lib/aspera/colors.rb +3 -3
  81. data/lib/aspera/command_line_builder.rb +34 -34
  82. data/lib/aspera/command_line_converter.rb +1 -1
  83. data/lib/aspera/coverage.rb +1 -2
  84. data/lib/aspera/data_repository.rb +2 -1
  85. data/lib/aspera/dot_container.rb +12 -12
  86. data/lib/aspera/environment.rb +30 -23
  87. data/lib/aspera/exec_spec.rb +13 -0
  88. data/lib/aspera/faspex_gw.rb +5 -5
  89. data/lib/aspera/faspex_postproc.rb +16 -10
  90. data/lib/aspera/graphql.rb +37 -0
  91. data/lib/aspera/hash_ext.rb +8 -2
  92. data/lib/aspera/json_rpc/client.rb +62 -0
  93. data/lib/aspera/json_rpc/version.rb +7 -0
  94. data/lib/aspera/keychain/base.rb +10 -3
  95. data/lib/aspera/keychain/encrypted_hash.rb +92 -17
  96. data/lib/aspera/keychain/factory.rb +20 -8
  97. data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
  98. data/lib/aspera/keychain/macos_security.rb +23 -25
  99. data/lib/aspera/keychain/one_password_api.rb +86 -0
  100. data/lib/aspera/keychain/one_password_base.rb +34 -0
  101. data/lib/aspera/keychain/one_password_cli.rb +98 -0
  102. data/lib/aspera/link_header.rb +82 -0
  103. data/lib/aspera/log.rb +29 -15
  104. data/lib/aspera/markdown.rb +90 -7
  105. data/lib/aspera/nagios.rb +8 -8
  106. data/lib/aspera/node_simulator.rb +35 -27
  107. data/lib/aspera/oauth/base.rb +9 -10
  108. data/lib/aspera/oauth/boot.rb +5 -5
  109. data/lib/aspera/oauth/factory.rb +12 -10
  110. data/lib/aspera/oauth/jwt.rb +9 -11
  111. data/lib/aspera/oauth/web.rb +6 -6
  112. data/lib/aspera/persistency_action_once.rb +12 -12
  113. data/lib/aspera/persistency_folder.rb +18 -11
  114. data/lib/aspera/preview/file_types.rb +5 -5
  115. data/lib/aspera/preview/generator.rb +52 -49
  116. data/lib/aspera/preview/options.rb +3 -2
  117. data/lib/aspera/preview/terminal.rb +5 -5
  118. data/lib/aspera/preview/utils.rb +66 -19
  119. data/lib/aspera/products/connect.rb +2 -2
  120. data/lib/aspera/products/desktop.rb +1 -1
  121. data/lib/aspera/products/other.rb +3 -3
  122. data/lib/aspera/products/transferd.rb +4 -4
  123. data/lib/aspera/proxy_auto_config.rb +15 -14
  124. data/lib/aspera/rest.rb +159 -151
  125. data/lib/aspera/rest_error_analyzer.rb +6 -6
  126. data/lib/aspera/rest_errors_aspera.rb +0 -10
  127. data/lib/aspera/rest_list.rb +11 -10
  128. data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
  129. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
  130. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
  131. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
  132. data/lib/aspera/schema/documentation.rb +65 -28
  133. data/lib/aspera/schema/reader.rb +136 -13
  134. data/lib/aspera/schema/registry.rb +60 -8
  135. data/lib/aspera/secret_hider.rb +12 -7
  136. data/lib/aspera/ssh.rb +64 -31
  137. data/lib/aspera/ssl.rb +5 -5
  138. data/lib/aspera/sync/conf.schema.yaml +2 -2
  139. data/lib/aspera/sync/database.rb +2 -2
  140. data/lib/aspera/sync/operations.rb +22 -25
  141. data/lib/aspera/temp_file_manager.rb +25 -5
  142. data/lib/aspera/timer_limiter.rb +1 -1
  143. data/lib/aspera/transfer/faux_file.rb +24 -11
  144. data/lib/aspera/transfer/parameters.rb +36 -34
  145. data/lib/aspera/transfer/result.rb +74 -0
  146. data/lib/aspera/transfer/resumer.rb +10 -10
  147. data/lib/aspera/transfer/spec.rb +18 -1
  148. data/lib/aspera/transfer/spec.schema.yaml +12 -3
  149. data/lib/aspera/uri_reader.rb +52 -15
  150. data/lib/aspera/web_auth.rb +8 -7
  151. data/lib/aspera/web_server_simple.rb +15 -12
  152. data/lib/aspera/yaml.rb +5 -4
  153. data.tar.gz.sig +0 -0
  154. metadata +34 -6
  155. metadata.gz.sig +0 -0
  156. data/lib/aspera/cli/manager.rb +0 -766
  157. data/lib/aspera/cli/plugins/faspex.rb +0 -539
  158. data/lib/aspera/json_rpc.rb +0 -52
  159. data/lib/aspera/transfer/uri.rb +0 -56
@@ -48,13 +48,30 @@ module Aspera
48
48
 
49
49
  attr_accessor :option_skip_types, :option_previews_folder, :option_folder_reset_cache, :option_skip_folders, :option_overwrite
50
50
 
51
+ # DSL option declarations (Pattern 1 and Pattern 2 - handler: Symbol resolved to self)
52
+ option :skip_format, description: 'Skip this preview format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
53
+ # TODO: use the same option as in `node` plugin
54
+ option :folder_reset_cache, description: 'Force detection of generated preview by refresh cache', allowed: %i[no header read], handler: :option_folder_reset_cache, default: :no
55
+ option :skip_types, description: 'Skip generation for those types of files', allowed: Type::SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES, handler: :option_skip_types
56
+ option :previews_folder, description: 'Preview folder in storage root', handler: :option_previews_folder, default: DEFAULT_PREVIEWS_FOLDER
57
+ option :skip_folders, description: 'List of folder to skip', allowed: Type::STRING_ARRAY, handler: :option_skip_folders
58
+ option :base, description: 'Basename of output for for test'
59
+ option :filter, description: 'File name filter: String (glob), Regexp, or Proc', allowed: [String, Regexp, Proc, NilClass]
60
+ option :mimemagic, description: 'Use Mime type detection of gem mimemagic', allowed: Type::BOOLEAN, default: false
61
+ option :overwrite, description: 'When to overwrite result file', allowed: %i[always never mtime], handler: :option_overwrite, default: :mtime
62
+ option :root_url, description: "How to read and write files on storage (#{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})", default: REMOTE_ACCESS
63
+ # Generator-specific options (Category C - bound to @gen_options via set_handler in initialize)
64
+ Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
65
+ values = if opt.key?(:values)
66
+ opt[:values]
67
+ elsif BoolValue.symbol?(opt[:default])
68
+ BoolValue::TYPES
69
+ end
70
+ option(opt[:name], description: opt[:description].capitalize, allowed: values, default: opt[:default])
71
+ end
72
+
51
73
  def initialize(**_)
52
74
  super
53
- @option_skip_types = []
54
- @option_skip_folders = []
55
- @option_previews_folder = nil
56
- @option_overwrite = nil
57
- @option_folder_reset_cache = nil
58
75
  # Generator configuration populated from CLI options.
59
76
  @gen_options = Aspera::Preview::Options.new
60
77
  # Used to rate-limit periodic progress logging and checkpoint persistence.
@@ -62,40 +79,9 @@ module Aspera
62
79
  # Optional callback used to filter entries before generation.
63
80
  @filter_block = nil
64
81
  @access_remote = true
65
- # Bind CLI options directly to generator option attributes.
66
- options.declare(
67
- :skip_format, 'Skip this preview format',
68
- allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
69
- )
70
- # TODO: use the same option as in `node` plugin
71
- options.declare(
72
- :folder_reset_cache, 'Force detection of generated preview by refresh cache',
73
- allowed: %i[no header read],
74
- handler: {o: self, m: :option_folder_reset_cache},
75
- default: :no
76
- )
77
- options.declare(:skip_types, 'Skip generation for those types of files', handler: {o: self, m: :option_skip_types}, allowed: Allowed::TYPES_SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES)
78
- options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
79
- options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, allowed: Allowed::TYPES_STRING_ARRAY)
80
- options.declare(:base, 'Basename of output for for test')
81
- options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
82
- options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
83
- options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', allowed: Allowed::TYPES_BOOLEAN, default: false)
84
- options.declare(:overwrite, 'When to overwrite result file', handler: {o: self, m: :option_overwrite}, allowed: %i[always never mtime], default: :mtime)
85
- options.declare(
86
- :root_url,
87
- "How to read and write files on storage (<empty>, #{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})",
88
- allowed: Allowed::TYPES_STRING,
89
- default: ''
90
- )
91
- # Declare generator-specific options and apply their default values.
82
+ # Bind generator-specific options to @gen_options (Category C: set_handler after object creation)
92
83
  Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
93
- values = if opt.key?(:values)
94
- opt[:values]
95
- elsif BoolValue.symbol?(opt[:default])
96
- BoolValue::TYPES
97
- end
98
- options.declare(opt[:name], opt[:description].capitalize, allowed: values, handler: {o: @gen_options, m: opt[:name]}, default: opt[:default])
84
+ options.set_handler(opt[:name], object: @gen_options, method: opt[:name])
99
85
  end
100
86
 
101
87
  options.parse_options!
@@ -106,13 +92,13 @@ module Aspera
106
92
  @preview_formats_to_generate.delete(skip) if skip
107
93
  @tmp_folder = File.join(TempFileManager.instance.global_temp, "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
108
94
  FileUtils.mkdir_p(@tmp_folder)
109
- Log.log.debug{"tmpdir: #{@tmp_folder}"}
95
+ Log.log.debug { "tmpdir: #{@tmp_folder}" }
110
96
  end
111
97
 
112
98
  # Process legacy transfer events and trigger preview generation for completed downloads.
113
99
  #
114
100
  # @param iteration_persistency [PersistencyActionOnce, nil] stores the last processed event id
115
- # @return [void]
101
+ # @return [nil]
116
102
  def process_trevents(iteration_persistency)
117
103
  events_filter = {
118
104
  'access_key' => @access_key_self['id'],
@@ -124,7 +110,7 @@ module Aspera
124
110
  events = @api_node.read('events', events_filter)
125
111
  rescue RestCallError => e
126
112
  if e.message.include?('Invalid iteration_token')
127
- Log.log.warn{"Retrying without iteration token: #{e}"}
113
+ Log.log.warn { "Retrying without iteration token: #{e}" }
128
114
  events_filter.delete('iteration_token')
129
115
  retry
130
116
  end
@@ -145,19 +131,20 @@ module Aspera
145
131
  end
146
132
  # Periodically log progress and persist the latest processed event.
147
133
  next unless @periodic.trigger? || event.equal?(events.last)
148
- Log.log.debug{"Processed event #{event['id']}"}
134
+ Log.log.debug { "Processed event #{event['id']}" }
149
135
  # Save a checkpoint to avoid replaying the full batch after a failure.
150
136
  if !iteration_persistency.nil?
151
137
  iteration_persistency.data[0] = event['id'].to_s
152
138
  iteration_persistency.save
153
139
  end
154
140
  end
141
+ nil
155
142
  end
156
143
 
157
144
  # Process recent Node API file events since the last persisted checkpoint.
158
145
  #
159
146
  # @param iteration_persistency [PersistencyActionOnce, nil] stores the last processed event id
160
- # @return [void]
147
+ # @return [nil]
161
148
  def process_events(iteration_persistency)
162
149
  # Restrict the event stream to file-related changes for the current access key.
163
150
  events_filter = {
@@ -173,7 +160,7 @@ module Aspera
173
160
  if event.dig('data', 'type').eql?('file')
174
161
  file_entry = @api_node.read("files/#{event['data']['id']}") rescue nil
175
162
  if !file_entry.nil? &&
176
- @option_skip_folders.none?{ |d| file_entry['path'].start_with?(d)}
163
+ @option_skip_folders.none? { |d| file_entry['path'].start_with?(d) }
177
164
  file_entry['parent_file_id'] = event['data']['parent_file_id']
178
165
  Log.log.error('TODO'.red) if event['types'].include?('file.deleted')
179
166
  generate_preview(file_entry) if event['types'].include?('file.deleted')
@@ -181,13 +168,14 @@ module Aspera
181
168
  end
182
169
  # Periodically log progress and persist the latest processed event.
183
170
  next unless @periodic.trigger? || event.equal?(events.last)
184
- Log.log.debug{"Processing event #{event['id']}"}
171
+ Log.log.debug { "Processing event #{event['id']}" }
185
172
  # Save a checkpoint to avoid replaying the full batch after a failure.
186
173
  if !iteration_persistency.nil?
187
174
  iteration_persistency.data[0] = event['id'].to_s
188
175
  iteration_persistency.save
189
176
  end
190
177
  end
178
+ nil
191
179
  end
192
180
 
193
181
  # Transfer a file to or from the configured Node storage using a tagged transfer spec.
@@ -215,7 +203,11 @@ module Aspera
215
203
  # @param entry [Hash] file entry containing at least the relative path
216
204
  # @return [String] local directory where previews for this entry are stored
217
205
  def get_infos_local(gen_infos, entry)
218
- local_original_filepath = File.join(@local_storage_root, entry['path'])
206
+ # Resolve to a real path and ensure it stays within the storage root (prevents path traversal via API response)
207
+ candidate = File.expand_path(File.join(@local_storage_root, entry['path']))
208
+ resolved_root = File.realpath(@local_storage_root)
209
+ Aspera.assert(candidate.start_with?("#{resolved_root}/")) { 'Preview entry path traversal attempt detected' }
210
+ local_original_filepath = candidate
219
211
  original_mtime = File.mtime(local_original_filepath)
220
212
  # Output directory for previews generated from the local source file.
221
213
  local_entry_preview_dir = File.join(@local_preview_folder, entry_preview_folder_name(entry))
@@ -278,7 +270,7 @@ module Aspera
278
270
  # Remote entries must include `parent_file_id`.
279
271
  #
280
272
  # @param entry [Hash] local or remote file entry to preview
281
- # @return [void]
273
+ # @return [nil]
282
274
  def generate_preview(entry)
283
275
  # prepare generic information
284
276
  gen_infos = @preview_formats_to_generate.map do |preview_format|
@@ -327,14 +319,14 @@ module Aspera
327
319
  # download original file to temp folder
328
320
  do_transfer(Transfer::Spec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
329
321
  end
330
- Log.log.debug{"source: #{entry['id']}: #{entry['path']}"}
322
+ Log.log.debug { "source: #{entry['id']}: #{entry['path']}" }
331
323
  gen_infos.each do |gen_info|
332
324
  gen_info[:generator].generate
333
325
  rescue => e
334
- Log.log.error{"Ignoring: #{e.class} #{e.message}"}
326
+ Log.log.error { "Ignoring: #{e.class} #{e.message}" }
335
327
  Log.log.debug(e.backtrace.join("\n").red)
336
328
  # in case of any error, place a standard error image
337
- FileUtils.cp(gen_info[:generator].error_asset, @destination_file_path)
329
+ FileUtils.cp(gen_info[:generator].error_asset, gen_info[:dst])
338
330
  end
339
331
  if @access_remote
340
332
  # upload
@@ -346,7 +338,7 @@ module Aspera
346
338
  # force read file updated previews
347
339
  @api_node.read("files/#{entry['id']}") if @option_folder_reset_cache.eql?(:read)
348
340
  rescue StandardError => e
349
- Log.log.error{"Ignore: #{e.message}"}
341
+ Log.log.error { "Ignore: #{e.message}" }
350
342
  Log.log.debug(e.backtrace.join("\n").red)
351
343
  end
352
344
 
@@ -358,20 +350,20 @@ module Aspera
358
350
  # canonical path: start with / and ends with /
359
351
  top_path = "/#{top_path.split('/').reject(&:empty?).join('/')}/"
360
352
  end
361
- Log.log.debug{"scan: #{top_entry} : #{top_path}".green}
353
+ Log.log.debug { "scan: #{top_entry} : #{top_path}".green }
362
354
  # don't use recursive call, use list instead
363
355
  entries_to_process = [top_entry]
364
356
  until entries_to_process.empty?
365
357
  entry = entries_to_process.shift
366
358
  # process this entry only if it is within the top_path
367
359
  entry_path_with_slash = entry['path']
368
- Log.log.debug{"processing entry #{entry_path_with_slash}"} if @periodic.trigger?
360
+ Log.log.debug { "processing entry #{entry_path_with_slash}" } if @periodic.trigger?
369
361
  entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
370
362
  if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
371
- Log.log.debug{"#{entry['path']} folder (skip start)".bg_red}
363
+ Log.log.debug { "#{entry['path']} folder (skip start)".bg_red }
372
364
  next
373
365
  end
374
- Log.log.debug{"item:#{entry}"}
366
+ Log.log.debug { "item:#{entry}" }
375
367
  begin
376
368
  case entry['type']
377
369
  when 'file'
@@ -384,9 +376,9 @@ module Aspera
384
376
  Log.log.debug('Ignoring link.')
385
377
  when 'folder'
386
378
  if @option_skip_folders.include?(entry['path'])
387
- Log.log.debug{"#{entry['path']} folder (skip list)".bg_red}
379
+ Log.log.debug { "#{entry['path']} folder (skip list)".bg_red }
388
380
  else
389
- Log.log.debug{"#{entry['path']} folder".green}
381
+ Log.log.debug { "#{entry['path']} folder".green }
390
382
  # get folder content
391
383
  folder_entries = @api_node.read_folder_content(entry['id'])
392
384
  # process all items in current folder
@@ -398,120 +390,166 @@ module Aspera
398
390
  end
399
391
  end
400
392
  else
401
- Log.log.warn{"unknown entry type: #{entry['type']}"}
393
+ Log.log.warn { "unknown entry type: #{entry['type']}" }
402
394
  end
403
395
  rescue StandardError => e
404
- Log.log.warn{"An error occurred: #{e}, ignoring"}
396
+ Log.log.warn { "An error occurred: #{e}, ignoring" }
405
397
  end
406
398
  end
407
399
  end
408
400
 
409
- ACTIONS = %i[scan events trevents check test show].freeze
401
+ # --- DSL ---
402
+
403
+ # scan, events and trevents connect to the Node API (setup: :setup_node_api).
404
+ # `check`, `test` and `show` work without Node API authentication.
405
+ command :scan, description: 'Scan a file or folder and generate previews', setup: :setup_node_api, arguments: [{name: :path, type: String, description: 'File or folder from which to start scanning', mandatory: false, default: '/'}]
406
+ command :events, description: 'Process file events and generate previews', setup: :setup_node_api
407
+ command :trevents, description: 'Process transfer events and generate previews', setup: :setup_node_api
408
+ command :check, description: 'Check required tools are installed'
409
+ command :show, description: 'Generate and display preview of a source file', arguments: [{name: :source_file, type: String, description: 'Local file to preview'}]
410
+ command :test, description: 'Test preview generation for a source file',
411
+ arguments: [
412
+ {name: :source_file, type: String, description: 'Local file to preview'},
413
+ {name: :format, description: 'Preview output format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS, mandatory: false, default: :png}
414
+ ]
415
+
416
+ # --- setup ---
410
417
 
411
- def execute_action
412
- command = options.get_next_command(ACTIONS)
413
- unless %i[check test show].include?(command)
414
- # This will use node api
415
- @api_node = Api::Node.new(**basic_auth_params)
416
- @transfer_server_address = URI.parse(@api_node.base_url).host
417
- # Get current access key information
418
- @access_key_self = @api_node.read('access_keys/self')
419
- # TODO: check events is activated here:
420
- # note that docroot is good to look at as well
421
- node_info = @api_node.read('info')
422
- Log.log.debug{"root: #{node_info['docroot']}"}
423
- # Default storage url to local file if not provided
424
- option_root_url = options.get_option(:root_url, mandatory: true)
425
- option_root_url = UriReader.file_url(@access_key_self['storage']['path']) if option_root_url.empty? && @access_key_self['storage']['type'].eql?('local')
426
- @access_remote = !UriReader.file?(option_root_url)
427
- Log.log.debug{"remote: #{@access_remote}"}
428
- # TODO: can the `previews` folder parameter be read from Node API ?
429
- @option_skip_folders.push("/#{@option_previews_folder}")
430
- if @access_remote
431
- # NOTE: the filter "name", it's why we take the first one
432
- @previews_folder_entry = @api_node.read_folder_content(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
433
- raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
434
- 'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
418
+ # Build the Node API and set up storage references for scan/events/trevents.
419
+ # @return [Hash] empty ctx (state stored in instance variables)
420
+ def setup_node_api(**)
421
+ @api_node = Api::Node.new(**basic_auth_params)
422
+ @transfer_server_address = URI.parse(@api_node.base_url).host
423
+ # Get current access key information
424
+ @access_key_self = @api_node.read('access_keys/self')
425
+ # TODO: check events is activated here:
426
+ # note that docroot is good to look at as well
427
+ node_info = @api_node.read('info')
428
+ Log.log.debug { "root: #{node_info['docroot']}" }
429
+ # Default storage url to local file if not provided
430
+ option_root_url = options.get_option(:root_url, mandatory: true)
431
+ option_root_url = UriReader.file_url(@access_key_self['storage']['path']) if option_root_url.eql?(REMOTE_ACCESS) && @access_key_self['storage']['type'].eql?('local')
432
+ @access_remote = !UriReader.file?(option_root_url)
433
+ Log.log.debug { "remote: #{@access_remote}" }
434
+ # TODO: can the `previews` folder parameter be read from Node API ?
435
+ @option_skip_folders.push("/#{@option_previews_folder}")
436
+ if @access_remote
437
+ # NOTE: the filter "name", it's why we take the first one
438
+ @previews_folder_entry = @api_node.read_folder_content(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
439
+ raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
440
+ 'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
441
+ else
442
+ @local_storage_root = UriReader.file_path(option_root_url)
443
+ # TODO: Windows could have "C:" ?
444
+ Aspera.assert(@local_storage_root.start_with?('/')) { "not local storage: #{@local_storage_root}" }
445
+ Aspera.assert(File.directory?(@local_storage_root), type: Cli::Error) { "Local storage root folder #{@local_storage_root} does not exist." }
446
+ @local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
447
+ Aspera.assert(File.directory?(@local_preview_folder), type: Cli::Error) { "Folder #{@local_preview_folder} does not exist locally. Please create it, or specify an alternate name." }
448
+ # Protection to avoid clash of file id for two different access keys
449
+ marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
450
+ Log.log.debug { "marker file: #{marker_file}" }
451
+ if File.exist?(marker_file)
452
+ ak = File.read(marker_file).chomp
453
+ Aspera.assert(@access_key_self['id'].eql?(ak)) { "mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}" }
435
454
  else
436
- @local_storage_root = UriReader.file_path(option_root_url)
437
- # TODO: Windows could have "C:" ?
438
- Aspera.assert(@local_storage_root.start_with?('/')){"not local storage: #{@local_storage_root}"}
439
- Aspera.assert(File.directory?(@local_storage_root), type: Cli::Error){"Local storage root folder #{@local_storage_root} does not exist."}
440
- @local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
441
- Aspera.assert(File.directory?(@local_preview_folder), type: Cli::Error){"Folder #{@local_preview_folder} does not exist locally. Please create it, or specify an alternate name."}
442
- # Protection to avoid clash of file id for two different access keys
443
- marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
444
- Log.log.debug{"marker file: #{marker_file}"}
445
- if File.exist?(marker_file)
446
- ak = File.read(marker_file).chomp
447
- Aspera.assert(@access_key_self['id'].eql?(ak)){"mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}"}
448
- else
449
- File.write(marker_file, @access_key_self['id'])
450
- end
455
+ File.write(marker_file, @access_key_self['id'])
451
456
  end
452
457
  end
458
+ {}
459
+ end
460
+
461
+ # --- handlers ---
462
+
463
+ def action_scan(path:, **)
464
+ check_tools_and_mimemagic
465
+ apifid =
466
+ if (selector = Parser.percent_selector(path))
467
+ Aspera.assert_values(selector[:field], ['id'], type: BadArgument) { 'file id' }
468
+ Api::NodeFileId.new(@api_node, selector[:value].to_s.empty? ? @access_key_self['root_file_id'] : selector[:value])
469
+ else
470
+ @api_node.resolve_api_fid(@access_key_self['root_file_id'], path)
471
+ end
472
+ @filter_block = Base.file_matcher(options.get_option(:filter))
473
+ scan_folder_files(@api_node.read("files/#{apifid.file_id}"))
474
+ Result::Status.new('scan finished')
475
+ ensure
476
+ cleanup_tmp_folder
477
+ end
478
+
479
+ def action_events(**)
480
+ check_tools_and_mimemagic
481
+ run_event_loop(:events, filter: options.get_option(:filter))
482
+ ensure
483
+ cleanup_tmp_folder
484
+ end
485
+
486
+ def action_trevents(**)
487
+ check_tools_and_mimemagic
488
+ run_event_loop(:trevents, filter: options.get_option(:filter))
489
+ ensure
490
+ cleanup_tmp_folder
491
+ end
492
+
493
+ def action_check
494
+ check_tools_and_mimemagic
495
+ Result::Status.new('Tools validated')
496
+ ensure
497
+ cleanup_tmp_folder
498
+ end
499
+
500
+ def action_test(source_file:, format:, **)
501
+ check_tools_and_mimemagic
502
+ generated_file_path = preview_filename(format, options.get_option(:base))
503
+ Aspera::Preview::Generator.new(source_file, generated_file_path, @gen_options, @tmp_folder).generate
504
+ Result::Status.new("generated: #{generated_file_path}")
505
+ ensure
506
+ cleanup_tmp_folder
507
+ end
508
+
509
+ def action_show(source_file:, **)
510
+ check_tools_and_mimemagic
511
+ generated_file_path = preview_filename(:png, options.get_option(:base))
512
+ Aspera::Preview::Generator.new(source_file, generated_file_path, @gen_options, @tmp_folder).generate
513
+ formatter.display_status("generated: #{generated_file_path}")
514
+ Result::Image.new(UriReader.file_url(generated_file_path))
515
+ ensure
516
+ cleanup_tmp_folder
517
+ end
518
+
519
+ private
520
+
521
+ # Check tools and set mimemagic flag (required for all commands).
522
+ def check_tools_and_mimemagic
453
523
  Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, mandatory: true)
454
- # check tools that are anyway required for all cases
455
524
  Aspera::Preview::Utils.check_tools(@option_skip_types)
456
- case command
457
- when :scan
458
- scan_path = options.get_option(:scan_path)
459
- scan_id = options.get_option(:scan_id)
460
- # by default start at root
461
- folder_info =
462
- if scan_id.nil?
463
- {
464
- 'id' => @access_key_self['root_file_id'],
465
- 'name' => '/',
466
- 'type' => 'folder',
467
- 'path' => '/'
468
- }
469
- else
470
- @api_node.read("files/#{scan_id}")
471
- end
472
- @filter_block = Api::Node.file_matcher_from_argument(options)
473
- scan_folder_files(folder_info, scan_path)
474
- return Result::Status.new('scan finished')
475
- when :events, :trevents
476
- @filter_block = Api::Node.file_matcher_from_argument(options)
477
- iteration_persistency = nil
478
- if options.get_option(:once_only, mandatory: true)
479
- iteration_persistency = PersistencyActionOnce.new(
480
- manager: persistency,
481
- data: [],
482
- id: IdGenerator.from_list(
483
- 'preview_iteration',
484
- command.to_s,
485
- options.get_option(:url, mandatory: true),
486
- options.get_option(:username, mandatory: true)
487
- )
525
+ end
526
+
527
+ # Clean up the temporary folder after each command.
528
+ def cleanup_tmp_folder
529
+ Log.log.debug { "cleaning up temp folder #{@tmp_folder}" }
530
+ FileUtils.rm_rf(@tmp_folder)
531
+ end
532
+
533
+ # Shared event-loop body for :events and :trevents.
534
+ # @param command [:events, :trevents]
535
+ # @param filter [String, Regexp, Proc, nil] optional filter expression
536
+ def run_event_loop(command, filter: nil)
537
+ @filter_block = Base.file_matcher(filter)
538
+ iteration_persistency = nil
539
+ if options.get_option(:once_only, mandatory: true)
540
+ iteration_persistency = PersistencyActionOnce.new(
541
+ manager: persistency,
542
+ data: [],
543
+ id: IdGenerator.from_list(
544
+ 'preview_iteration',
545
+ command.to_s,
546
+ options.get_option(:url, mandatory: true),
547
+ options.get_option(:username, mandatory: true)
488
548
  )
489
- end
490
- # call processing method specified by command line command
491
- send(:"process_#{command}", iteration_persistency)
492
- return Result::Status.new("#{command} finished")
493
- when :check
494
- return Result::Status.new('Tools validated')
495
- when :test
496
- source = options.get_next_argument('source file')
497
- format = options.get_next_argument('format', accept_list: Aspera::Preview::Generator::PREVIEW_FORMATS, default: :png)
498
- generated_file_path = preview_filename(format, options.get_option(:base))
499
- Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder).generate
500
- return Result::Status.new("generated: #{generated_file_path}")
501
- when :show
502
- source = options.get_next_argument('source file')
503
- # terminal_options = options.get_next_argument('options', validation: Hash, default: {}).symbolize_keys
504
- generated_file_path = preview_filename(:png, options.get_option(:base))
505
- Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder).generate
506
- formatter.display_status("generated: #{generated_file_path}")
507
- # formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
508
- # return Result::Status.new("generated: #{generated_file_path}")
509
- return Result::Image.new(UriReader.file_url(generated_file_path))
510
- else Aspera.error_unexpected_value(command)
549
+ )
511
550
  end
512
- ensure
513
- Log.log.debug{"cleaning up temp folder #{@tmp_folder}"}
514
- FileUtils.rm_rf(@tmp_folder)
551
+ send(:"process_#{command}", iteration_persistency)
552
+ Result::Status.new("#{command} finished")
515
553
  end
516
554
  end
517
555
  end