aspera-cli 4.27.0 → 4.27.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +46 -1
  4. data/CONTRIBUTING.md +1 -7
  5. data/TODO.md +50 -0
  6. data/docs/README.md +709 -489
  7. data/lib/aspera/agent/base.rb +10 -2
  8. data/lib/aspera/agent/connect.rb +7 -7
  9. data/lib/aspera/agent/desktop.rb +3 -3
  10. data/lib/aspera/agent/direct.rb +48 -19
  11. data/lib/aspera/agent/factory.rb +3 -3
  12. data/lib/aspera/agent/httpgw.rb +55 -14
  13. data/lib/aspera/agent/node.rb +2 -2
  14. data/lib/aspera/agent/transferd.rb +12 -12
  15. data/lib/aspera/api/aoc.rb +28 -28
  16. data/lib/aspera/api/cos_node.rb +10 -9
  17. data/lib/aspera/api/faspex.rb +2 -2
  18. data/lib/aspera/api/httpgw.rb +20 -20
  19. data/lib/aspera/api/node.rb +31 -53
  20. data/lib/aspera/ascmd.rb +13 -13
  21. data/lib/aspera/ascp/installation.rb +43 -36
  22. data/lib/aspera/ascp/management.rb +6 -6
  23. data/lib/aspera/assert.rb +12 -13
  24. data/lib/aspera/cli/ascp_actions.rb +7 -7
  25. data/lib/aspera/cli/async_transfer_store.rb +49 -5
  26. data/lib/aspera/cli/bootstrapper.rb +9 -9
  27. data/lib/aspera/cli/command_registry.rb +1 -1
  28. data/lib/aspera/cli/command_spec.rb +6 -3
  29. data/lib/aspera/cli/context.rb +3 -3
  30. data/lib/aspera/cli/extended_value.rb +26 -26
  31. data/lib/aspera/cli/formatter.rb +76 -54
  32. data/lib/aspera/cli/gem_checker.rb +1 -1
  33. data/lib/aspera/cli/hints.rb +2 -2
  34. data/lib/aspera/cli/http.rb +22 -29
  35. data/lib/aspera/cli/mailer.rb +7 -7
  36. data/lib/aspera/cli/mcp_tool.rb +31 -4
  37. data/lib/aspera/cli/option_declarator.rb +77 -0
  38. data/lib/aspera/cli/options.schema.yaml +273 -3
  39. data/lib/aspera/cli/parser.rb +539 -280
  40. data/lib/aspera/cli/plugins/alee.rb +1 -1
  41. data/lib/aspera/cli/plugins/aoc.rb +303 -314
  42. data/lib/aspera/cli/plugins/ats.rb +25 -21
  43. data/lib/aspera/cli/plugins/base.rb +357 -186
  44. data/lib/aspera/cli/plugins/basic_auth.rb +0 -12
  45. data/lib/aspera/cli/plugins/config.rb +76 -73
  46. data/lib/aspera/cli/plugins/console.rb +8 -8
  47. data/lib/aspera/cli/plugins/cos.rb +4 -3
  48. data/lib/aspera/cli/plugins/factory.rb +2 -2
  49. data/lib/aspera/cli/plugins/faspex5.rb +259 -248
  50. data/lib/aspera/cli/plugins/faspio.rb +7 -2
  51. data/lib/aspera/cli/plugins/httpgw.rb +1 -1
  52. data/lib/aspera/cli/plugins/mcp.rb +11 -11
  53. data/lib/aspera/cli/plugins/node.rb +227 -283
  54. data/lib/aspera/cli/plugins/orchestrator.rb +23 -23
  55. data/lib/aspera/cli/plugins/preview.rb +61 -64
  56. data/lib/aspera/cli/plugins/server.rb +36 -31
  57. data/lib/aspera/cli/plugins/shares.rb +174 -135
  58. data/lib/aspera/cli/preset_actions.rb +39 -19
  59. data/lib/aspera/cli/preset_manager.rb +74 -32
  60. data/lib/aspera/cli/result.rb +18 -16
  61. data/lib/aspera/cli/runner.rb +75 -39
  62. data/lib/aspera/cli/secret_finder.rb +1 -1
  63. data/lib/aspera/cli/sync_actions.rb +36 -35
  64. data/lib/aspera/cli/terminal_formatter.rb +2 -2
  65. data/lib/aspera/cli/transfer_actions.rb +17 -10
  66. data/lib/aspera/cli/transfer_agent.rb +44 -44
  67. data/lib/aspera/cli/transfer_progress.rb +7 -7
  68. data/lib/aspera/cli/vault_manager.rb +34 -12
  69. data/lib/aspera/cli/version.rb +1 -1
  70. data/lib/aspera/cli/wizard.rb +17 -20
  71. data/lib/aspera/colors.rb +3 -3
  72. data/lib/aspera/command_line_builder.rb +17 -16
  73. data/lib/aspera/command_line_converter.rb +1 -1
  74. data/lib/aspera/coverage.rb +1 -2
  75. data/lib/aspera/data_repository.rb +1 -1
  76. data/lib/aspera/dot_container.rb +8 -8
  77. data/lib/aspera/environment.rb +13 -13
  78. data/lib/aspera/faspex_gw.rb +4 -4
  79. data/lib/aspera/faspex_postproc.rb +14 -9
  80. data/lib/aspera/graphql.rb +4 -2
  81. data/lib/aspera/hash_ext.rb +2 -2
  82. data/lib/aspera/json_rpc/client.rb +1 -1
  83. data/lib/aspera/keychain/base.rb +10 -3
  84. data/lib/aspera/keychain/encrypted_hash.rb +92 -17
  85. data/lib/aspera/keychain/factory.rb +17 -5
  86. data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
  87. data/lib/aspera/keychain/macos_security.rb +23 -25
  88. data/lib/aspera/keychain/one_password_api.rb +86 -0
  89. data/lib/aspera/keychain/one_password_base.rb +34 -0
  90. data/lib/aspera/keychain/one_password_cli.rb +98 -0
  91. data/lib/aspera/link_header.rb +2 -2
  92. data/lib/aspera/log.rb +13 -13
  93. data/lib/aspera/markdown.rb +5 -5
  94. data/lib/aspera/nagios.rb +8 -8
  95. data/lib/aspera/node_simulator.rb +33 -27
  96. data/lib/aspera/oauth/base.rb +4 -5
  97. data/lib/aspera/oauth/boot.rb +3 -3
  98. data/lib/aspera/oauth/factory.rb +5 -5
  99. data/lib/aspera/oauth/jwt.rb +5 -7
  100. data/lib/aspera/oauth/web.rb +4 -4
  101. data/lib/aspera/persistency_action_once.rb +4 -4
  102. data/lib/aspera/persistency_folder.rb +8 -8
  103. data/lib/aspera/preview/file_types.rb +4 -4
  104. data/lib/aspera/preview/generator.rb +49 -46
  105. data/lib/aspera/preview/options.rb +3 -2
  106. data/lib/aspera/preview/terminal.rb +5 -5
  107. data/lib/aspera/preview/utils.rb +65 -18
  108. data/lib/aspera/products/connect.rb +2 -2
  109. data/lib/aspera/products/desktop.rb +1 -1
  110. data/lib/aspera/products/other.rb +2 -2
  111. data/lib/aspera/products/transferd.rb +3 -3
  112. data/lib/aspera/proxy_auto_config.rb +10 -10
  113. data/lib/aspera/rest.rb +149 -115
  114. data/lib/aspera/rest_error_analyzer.rb +3 -3
  115. data/lib/aspera/rest_list.rb +10 -9
  116. data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
  117. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
  118. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +39 -0
  119. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
  120. data/lib/aspera/schema/documentation.rb +17 -11
  121. data/lib/aspera/schema/reader.rb +78 -4
  122. data/lib/aspera/schema/registry.rb +26 -7
  123. data/lib/aspera/secret_hider.rb +5 -5
  124. data/lib/aspera/ssh.rb +64 -31
  125. data/lib/aspera/ssl.rb +3 -3
  126. data/lib/aspera/sync/database.rb +2 -2
  127. data/lib/aspera/sync/operations.rb +13 -13
  128. data/lib/aspera/temp_file_manager.rb +24 -4
  129. data/lib/aspera/transfer/faux_file.rb +2 -2
  130. data/lib/aspera/transfer/parameters.rb +15 -15
  131. data/lib/aspera/transfer/resumer.rb +9 -9
  132. data/lib/aspera/transfer/spec.rb +1 -1
  133. data/lib/aspera/transfer/spec.schema.yaml +1 -0
  134. data/lib/aspera/uri_reader.rb +52 -15
  135. data/lib/aspera/web_auth.rb +6 -5
  136. data/lib/aspera/web_server_simple.rb +9 -9
  137. data/lib/aspera/yaml.rb +4 -4
  138. data.tar.gz.sig +0 -0
  139. metadata +10 -4
  140. metadata.gz.sig +0 -0
  141. data/lib/aspera/cli/plugins/faspex.rb +0 -576
  142. data/lib/aspera/transfer/uri.rb +0 -56
@@ -35,7 +35,7 @@ module Aspera
35
35
  }
36
36
  rescue StandardError => e
37
37
  error = e
38
- Log.log.debug{"detect error: #{e}"}
38
+ Log.log.debug { "detect error: #{e}" }
39
39
  end
40
40
  raise error if error
41
41
  return
@@ -57,7 +57,7 @@ module Aspera
57
57
  end
58
58
 
59
59
  option :result, description: "Specify result value as: 'work_step:parameter'"
60
- option :synchronous, description: 'Wait for completion', allowed: Allowed::TYPES_BOOLEAN, default: false
60
+ option :synchronous, description: 'Wait for completion', allowed: Type::BOOLEAN, default: false
61
61
  option :ret_style, description: 'How return type is requested in api', allowed: %i[header arg ext], default: :arg
62
62
  option :auth_style, description: 'Authentication type', allowed: %i[arg_pass head_basic apikey], default: :head_basic
63
63
 
@@ -104,65 +104,65 @@ module Aspera
104
104
  # --- DSL ---
105
105
 
106
106
  command :health, description: 'Check Orchestrator API health', setup: :setup_api
107
- command :info, description: 'Ping the remote Orchestrator instance', setup: :setup_api, action: ->{Result::SingleObject.new(call_ao('remote_node_ping', format: 'xml', xml_arrays: false))}
108
- command :processes, description: 'Show Orchestrator background process status', setup: :setup_api, action: ->{Result::ObjectList.new(call_ao('processes_status', format: 'xml')['process'])}
109
- command :monitors, description: 'Show Orchestrator monitor snapshot', setup: :setup_api, action: ->{Result::SingleObject.new(call_ao('monitor_snapshot')['monitor'])}
110
- command :plugins, description: 'Show Orchestrator plugin versions', setup: :setup_api, action: ->{Result::ObjectList.new(call_ao('plugin_version')['Plugin'])}
107
+ command :info, description: 'Ping the remote Orchestrator instance', setup: :setup_api, action: -> { Result::SingleObject.new(call_ao('remote_node_ping', format: 'xml', xml_arrays: false)) }
108
+ command :processes, description: 'Show Orchestrator background process status', setup: :setup_api, action: -> { Result::ObjectList.new(call_ao('processes_status', format: 'xml')['process']) }
109
+ command :monitors, description: 'Show Orchestrator monitor snapshot', setup: :setup_api, action: -> { Result::SingleObject.new(call_ao('monitor_snapshot')['monitor']) }
110
+ command :plugins, description: 'Show Orchestrator plugin versions', setup: :setup_api, action: -> { Result::ObjectList.new(call_ao('plugin_version')['Plugin']) }
111
111
  command :workflows, description: 'Manage workflows', setup: :setup_api
112
112
  command :workorders, description: 'Manage work orders', setup: :setup_api
113
113
  command :workstep, description: 'Manage work steps', setup: :setup_api
114
114
 
115
- commands_under(:workflows) do
115
+ commands_under :workflows do
116
116
  command(:list, description: 'List all workflows', action: lambda do
117
117
  Result::ObjectList.new(
118
118
  call_ao('workflows_list')['workflows']['workflow'],
119
119
  fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification]
120
120
  )
121
121
  end)
122
- command :status, description: 'Check running status of workflow(s)',
122
+ command :status, description: 'Check running status of workflows',
123
123
  arguments: [{name: :wf_id, type: :identifier}],
124
- action: ->(wf_id:, **){Result::ObjectList.new(call_ao(wf_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{wf_id}")['workflows']['workflow'])}
124
+ action: ->(wf_id:, **) { Result::ObjectList.new(call_ao(wf_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{wf_id}")['workflows']['workflow']) }
125
125
  command :inputs, description: 'Fetch input specification for a workflow',
126
126
  arguments: [{name: :wf_id, type: :identifier}],
127
- action: ->(wf_id:, **){Result::SingleObject.new(call_ao("workflow_inputs_spec/#{wf_id}")['workflow_inputs_spec'])}
127
+ action: ->(wf_id:, **) { Result::SingleObject.new(call_ao("workflow_inputs_spec/#{wf_id}")['workflow_inputs_spec']) }
128
128
  command :details, description: 'Check detailed running status of a workflow',
129
129
  arguments: [{name: :wf_id, type: :identifier}],
130
- action: ->(wf_id:, **){Result::ObjectList.new(call_ao("workflow_details/#{wf_id}")['workflows']['workflow']['statuses'])}
130
+ action: ->(wf_id:, **) { Result::ObjectList.new(call_ao("workflow_details/#{wf_id}")['workflows']['workflow']['statuses']) }
131
131
  command :start, description: 'Initiate a work order (sync or async)',
132
132
  arguments: [{name: :wf_id, type: :identifier}, {name: :external_parameters, type: Hash, mandatory: false, default: {}}]
133
133
  command :export, description: 'Export a workflow',
134
134
  arguments: [{name: :wf_id, type: :identifier}],
135
- action: ->(wf_id:, **){Result::Text.new(call_ao("export_workflow/#{wf_id}", format: nil, http: true).body)}
135
+ action: ->(wf_id:, **) { Result::Text.new(call_ao("export_workflow/#{wf_id}", format: nil, http: true).body) }
136
136
  command :workorders, description: 'Fetch all work orders from a workflow',
137
137
  arguments: [{name: :wf_id, type: :identifier}],
138
- action: ->(wf_id:, **){Result::ObjectList.new(call_ao("work_orders_list/#{wf_id}")['work_orders'])}
138
+ action: ->(wf_id:, **) { Result::ObjectList.new(call_ao("work_orders_list/#{wf_id}")['work_orders']) }
139
139
  command :outputs, description: 'Fetch output specification for a workflow',
140
140
  arguments: [{name: :wf_id, type: :identifier}],
141
- action: ->(wf_id:, **){Result::ObjectList.new(call_ao("workflow_outputs_spec/#{wf_id}")['workflow_outputs_spec']['output'])}
141
+ action: ->(wf_id:, **) { Result::ObjectList.new(call_ao("workflow_outputs_spec/#{wf_id}")['workflow_outputs_spec']['output']) }
142
142
  end
143
143
 
144
- commands_under(:workorders) do
144
+ commands_under :workorders do
145
145
  command :status, description: 'Check the status of a work order',
146
146
  arguments: [{name: :wo_id, type: :identifier}],
147
- action: ->(wo_id:, **){Result::SingleObject.new(call_ao("work_order_status/#{wo_id}")['work_order'])}
147
+ action: ->(wo_id:, **) { Result::SingleObject.new(call_ao("work_order_status/#{wo_id}")['work_order']) }
148
148
  command :cancel, description: 'Cancel a work order',
149
149
  arguments: [{name: :wo_id, type: :identifier}],
150
- action: ->(wo_id:, **){Result::SingleObject.new(call_ao("work_order_cancel/#{wo_id}")['work_order'])}
150
+ action: ->(wo_id:, **) { Result::SingleObject.new(call_ao("work_order_cancel/#{wo_id}")['work_order']) }
151
151
  command :reset, description: 'Reset a work order',
152
152
  arguments: [{name: :wo_id, type: :identifier}],
153
- action: ->(wo_id:, **){Result::SingleObject.new(call_ao("work_order_reset/#{wo_id}")['work_order'])}
153
+ action: ->(wo_id:, **) { Result::SingleObject.new(call_ao("work_order_reset/#{wo_id}")['work_order']) }
154
154
  command :output, description: 'Fetch output of a work order',
155
155
  arguments: [{name: :wo_id, type: :identifier}],
156
- action: ->(wo_id:, **){Result::ObjectList.new(call_ao("work_order_output/#{wo_id}", format: 'xml')['variable'])}
156
+ action: ->(wo_id:, **) { Result::ObjectList.new(call_ao("work_order_output/#{wo_id}", format: 'xml')['variable']) }
157
157
  end
158
158
 
159
- commands_under(:workstep) do
159
+ commands_under :workstep do
160
160
  command :status, description: 'Check the status of a work step',
161
161
  arguments: [{name: :ws_id, type: :identifier}],
162
- action: ->(ws_id:, **){Result::SingleObject.new(call_ao("work_step_status/#{ws_id}"))}
162
+ action: ->(ws_id:, **) { Result::SingleObject.new(call_ao("work_step_status/#{ws_id}")) }
163
163
  command :cancel, description: 'Cancel a work step',
164
164
  arguments: [{name: :ws_id, type: :identifier}],
165
- action: ->(ws_id:, **){Result::SingleObject.new(call_ao("work_step_cancel/#{ws_id}"))}
165
+ action: ->(ws_id:, **) { Result::SingleObject.new(call_ao("work_step_cancel/#{ws_id}")) }
166
166
  end
167
167
 
168
168
  # --- setup ---
@@ -221,7 +221,7 @@ module Aspera
221
221
  result_location = options.get_option(:result)
222
222
  unless result_location.nil?
223
223
  fields = result_location.split(':')
224
- Aspera.assert(fields.length == 2, type: Cli::BadArgument){"Expects: work_step:result_name : #{result_location}"}
224
+ Aspera.assert(fields.length == 2, type: Cli::BadArgument) { "Expects: work_step:result_name : #{result_location}" }
225
225
  call_params['explicit_output_step'] = fields[0]
226
226
  call_params['explicit_output_variable'] = fields[1]
227
227
  # implicitly, call is synchronous
@@ -52,15 +52,14 @@ module Aspera
52
52
  option :skip_format, description: 'Skip this preview format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
53
53
  # TODO: use the same option as in `node` plugin
54
54
  option :folder_reset_cache, description: 'Force detection of generated preview by refresh cache', allowed: %i[no header read], handler: :option_folder_reset_cache, default: :no
55
- option :skip_types, description: 'Skip generation for those types of files', allowed: Allowed::TYPES_SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES, handler: :option_skip_types
55
+ option :skip_types, description: 'Skip generation for those types of files', allowed: Type::SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES, handler: :option_skip_types
56
56
  option :previews_folder, description: 'Preview folder in storage root', handler: :option_previews_folder, default: DEFAULT_PREVIEWS_FOLDER
57
- option :skip_folders, description: 'List of folder to skip', allowed: Allowed::TYPES_STRING_ARRAY, handler: :option_skip_folders
57
+ option :skip_folders, description: 'List of folder to skip', allowed: Type::STRING_ARRAY, handler: :option_skip_folders
58
58
  option :base, description: 'Basename of output for for test'
59
- option :scan_path, description: 'Subpath in folder id to start scan in (default=/)'
60
- option :scan_id, description: 'Folder id in storage to start scan in, default is access key main folder id'
61
- option :mimemagic, description: 'Use Mime type detection of gem mimemagic', allowed: Allowed::TYPES_BOOLEAN, default: false
59
+ option :filter, description: 'File name filter: String (glob), Regexp, or Proc', allowed: [String, Regexp, Proc, NilClass]
60
+ option :mimemagic, description: 'Use Mime type detection of gem mimemagic', allowed: Type::BOOLEAN, default: false
62
61
  option :overwrite, description: 'When to overwrite result file', allowed: %i[always never mtime], handler: :option_overwrite, default: :mtime
63
- option :root_url, description: "How to read and write files on storage (<empty>, #{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})", default: ''
62
+ option :root_url, description: "How to read and write files on storage (#{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})", default: REMOTE_ACCESS
64
63
  # Generator-specific options (Category C - bound to @gen_options via set_handler in initialize)
65
64
  Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
66
65
  values = if opt.key?(:values)
@@ -93,7 +92,7 @@ module Aspera
93
92
  @preview_formats_to_generate.delete(skip) if skip
94
93
  @tmp_folder = File.join(TempFileManager.instance.global_temp, "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
95
94
  FileUtils.mkdir_p(@tmp_folder)
96
- Log.log.debug{"tmpdir: #{@tmp_folder}"}
95
+ Log.log.debug { "tmpdir: #{@tmp_folder}" }
97
96
  end
98
97
 
99
98
  # Process legacy transfer events and trigger preview generation for completed downloads.
@@ -111,7 +110,7 @@ module Aspera
111
110
  events = @api_node.read('events', events_filter)
112
111
  rescue RestCallError => e
113
112
  if e.message.include?('Invalid iteration_token')
114
- Log.log.warn{"Retrying without iteration token: #{e}"}
113
+ Log.log.warn { "Retrying without iteration token: #{e}" }
115
114
  events_filter.delete('iteration_token')
116
115
  retry
117
116
  end
@@ -132,7 +131,7 @@ module Aspera
132
131
  end
133
132
  # Periodically log progress and persist the latest processed event.
134
133
  next unless @periodic.trigger? || event.equal?(events.last)
135
- Log.log.debug{"Processed event #{event['id']}"}
134
+ Log.log.debug { "Processed event #{event['id']}" }
136
135
  # Save a checkpoint to avoid replaying the full batch after a failure.
137
136
  if !iteration_persistency.nil?
138
137
  iteration_persistency.data[0] = event['id'].to_s
@@ -161,7 +160,7 @@ module Aspera
161
160
  if event.dig('data', 'type').eql?('file')
162
161
  file_entry = @api_node.read("files/#{event['data']['id']}") rescue nil
163
162
  if !file_entry.nil? &&
164
- @option_skip_folders.none?{ |d| file_entry['path'].start_with?(d)}
163
+ @option_skip_folders.none? { |d| file_entry['path'].start_with?(d) }
165
164
  file_entry['parent_file_id'] = event['data']['parent_file_id']
166
165
  Log.log.error('TODO'.red) if event['types'].include?('file.deleted')
167
166
  generate_preview(file_entry) if event['types'].include?('file.deleted')
@@ -169,7 +168,7 @@ module Aspera
169
168
  end
170
169
  # Periodically log progress and persist the latest processed event.
171
170
  next unless @periodic.trigger? || event.equal?(events.last)
172
- Log.log.debug{"Processing event #{event['id']}"}
171
+ Log.log.debug { "Processing event #{event['id']}" }
173
172
  # Save a checkpoint to avoid replaying the full batch after a failure.
174
173
  if !iteration_persistency.nil?
175
174
  iteration_persistency.data[0] = event['id'].to_s
@@ -204,7 +203,11 @@ module Aspera
204
203
  # @param entry [Hash] file entry containing at least the relative path
205
204
  # @return [String] local directory where previews for this entry are stored
206
205
  def get_infos_local(gen_infos, entry)
207
- local_original_filepath = File.join(@local_storage_root, entry['path'])
206
+ # Resolve to a real path and ensure it stays within the storage root (prevents path traversal via API response)
207
+ candidate = File.expand_path(File.join(@local_storage_root, entry['path']))
208
+ resolved_root = File.realpath(@local_storage_root)
209
+ Aspera.assert(candidate.start_with?("#{resolved_root}/")) { 'Preview entry path traversal attempt detected' }
210
+ local_original_filepath = candidate
208
211
  original_mtime = File.mtime(local_original_filepath)
209
212
  # Output directory for previews generated from the local source file.
210
213
  local_entry_preview_dir = File.join(@local_preview_folder, entry_preview_folder_name(entry))
@@ -316,14 +319,14 @@ module Aspera
316
319
  # download original file to temp folder
317
320
  do_transfer(Transfer::Spec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
318
321
  end
319
- Log.log.debug{"source: #{entry['id']}: #{entry['path']}"}
322
+ Log.log.debug { "source: #{entry['id']}: #{entry['path']}" }
320
323
  gen_infos.each do |gen_info|
321
324
  gen_info[:generator].generate
322
325
  rescue => e
323
- Log.log.error{"Ignoring: #{e.class} #{e.message}"}
326
+ Log.log.error { "Ignoring: #{e.class} #{e.message}" }
324
327
  Log.log.debug(e.backtrace.join("\n").red)
325
328
  # in case of any error, place a standard error image
326
- FileUtils.cp(gen_info[:generator].error_asset, @destination_file_path)
329
+ FileUtils.cp(gen_info[:generator].error_asset, gen_info[:dst])
327
330
  end
328
331
  if @access_remote
329
332
  # upload
@@ -335,7 +338,7 @@ module Aspera
335
338
  # force read file updated previews
336
339
  @api_node.read("files/#{entry['id']}") if @option_folder_reset_cache.eql?(:read)
337
340
  rescue StandardError => e
338
- Log.log.error{"Ignore: #{e.message}"}
341
+ Log.log.error { "Ignore: #{e.message}" }
339
342
  Log.log.debug(e.backtrace.join("\n").red)
340
343
  end
341
344
 
@@ -347,20 +350,20 @@ module Aspera
347
350
  # canonical path: start with / and ends with /
348
351
  top_path = "/#{top_path.split('/').reject(&:empty?).join('/')}/"
349
352
  end
350
- Log.log.debug{"scan: #{top_entry} : #{top_path}".green}
353
+ Log.log.debug { "scan: #{top_entry} : #{top_path}".green }
351
354
  # don't use recursive call, use list instead
352
355
  entries_to_process = [top_entry]
353
356
  until entries_to_process.empty?
354
357
  entry = entries_to_process.shift
355
358
  # process this entry only if it is within the top_path
356
359
  entry_path_with_slash = entry['path']
357
- Log.log.debug{"processing entry #{entry_path_with_slash}"} if @periodic.trigger?
360
+ Log.log.debug { "processing entry #{entry_path_with_slash}" } if @periodic.trigger?
358
361
  entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
359
362
  if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
360
- Log.log.debug{"#{entry['path']} folder (skip start)".bg_red}
363
+ Log.log.debug { "#{entry['path']} folder (skip start)".bg_red }
361
364
  next
362
365
  end
363
- Log.log.debug{"item:#{entry}"}
366
+ Log.log.debug { "item:#{entry}" }
364
367
  begin
365
368
  case entry['type']
366
369
  when 'file'
@@ -373,9 +376,9 @@ module Aspera
373
376
  Log.log.debug('Ignoring link.')
374
377
  when 'folder'
375
378
  if @option_skip_folders.include?(entry['path'])
376
- Log.log.debug{"#{entry['path']} folder (skip list)".bg_red}
379
+ Log.log.debug { "#{entry['path']} folder (skip list)".bg_red }
377
380
  else
378
- Log.log.debug{"#{entry['path']} folder".green}
381
+ Log.log.debug { "#{entry['path']} folder".green }
379
382
  # get folder content
380
383
  folder_entries = @api_node.read_folder_content(entry['id'])
381
384
  # process all items in current folder
@@ -387,26 +390,28 @@ module Aspera
387
390
  end
388
391
  end
389
392
  else
390
- Log.log.warn{"unknown entry type: #{entry['type']}"}
393
+ Log.log.warn { "unknown entry type: #{entry['type']}" }
391
394
  end
392
395
  rescue StandardError => e
393
- Log.log.warn{"An error occurred: #{e}, ignoring"}
396
+ Log.log.warn { "An error occurred: #{e}, ignoring" }
394
397
  end
395
398
  end
396
399
  end
397
400
 
398
401
  # --- DSL ---
399
402
 
400
- # Commands that connect to the Node API share a common setup.
401
- # :check, :test and :show work without Node API authentication.
402
- command :scan, description: 'Scan all files and generate previews', setup: :setup_node_api
403
- command :events, description: 'Process file events and generate previews', setup: :setup_node_api
404
- command :trevents, description: 'Process transfer events and generate previews', setup: :setup_node_api
403
+ # scan, events and trevents connect to the Node API (setup: :setup_node_api).
404
+ # `check`, `test` and `show` work without Node API authentication.
405
+ command :scan, description: 'Scan a file or folder and generate previews', setup: :setup_node_api, arguments: [{name: :path, type: String, description: 'File or folder from which to start scanning', mandatory: false, default: '/'}]
406
+ command :events, description: 'Process file events and generate previews', setup: :setup_node_api
407
+ command :trevents, description: 'Process transfer events and generate previews', setup: :setup_node_api
405
408
  command :check, description: 'Check required tools are installed'
409
+ command :show, description: 'Generate and display preview of a source file', arguments: [{name: :source_file, type: String, description: 'Local file to preview'}]
406
410
  command :test, description: 'Test preview generation for a source file',
407
- arguments: [{name: :source_file, type: String}, {name: :format, allowed: Aspera::Preview::Generator::PREVIEW_FORMATS, mandatory: false, default: :png}]
408
- command :show, description: 'Generate and display preview of a source file',
409
- arguments: [{name: :source_file, type: String}]
411
+ arguments: [
412
+ {name: :source_file, type: String, description: 'Local file to preview'},
413
+ {name: :format, description: 'Preview output format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS, mandatory: false, default: :png}
414
+ ]
410
415
 
411
416
  # --- setup ---
412
417
 
@@ -420,12 +425,12 @@ module Aspera
420
425
  # TODO: check events is activated here:
421
426
  # note that docroot is good to look at as well
422
427
  node_info = @api_node.read('info')
423
- Log.log.debug{"root: #{node_info['docroot']}"}
428
+ Log.log.debug { "root: #{node_info['docroot']}" }
424
429
  # Default storage url to local file if not provided
425
430
  option_root_url = options.get_option(:root_url, mandatory: true)
426
- option_root_url = UriReader.file_url(@access_key_self['storage']['path']) if option_root_url.empty? && @access_key_self['storage']['type'].eql?('local')
431
+ option_root_url = UriReader.file_url(@access_key_self['storage']['path']) if option_root_url.eql?(REMOTE_ACCESS) && @access_key_self['storage']['type'].eql?('local')
427
432
  @access_remote = !UriReader.file?(option_root_url)
428
- Log.log.debug{"remote: #{@access_remote}"}
433
+ Log.log.debug { "remote: #{@access_remote}" }
429
434
  # TODO: can the `previews` folder parameter be read from Node API ?
430
435
  @option_skip_folders.push("/#{@option_previews_folder}")
431
436
  if @access_remote
@@ -436,16 +441,16 @@ module Aspera
436
441
  else
437
442
  @local_storage_root = UriReader.file_path(option_root_url)
438
443
  # TODO: Windows could have "C:" ?
439
- Aspera.assert(@local_storage_root.start_with?('/')){"not local storage: #{@local_storage_root}"}
440
- Aspera.assert(File.directory?(@local_storage_root), type: Cli::Error){"Local storage root folder #{@local_storage_root} does not exist."}
444
+ Aspera.assert(@local_storage_root.start_with?('/')) { "not local storage: #{@local_storage_root}" }
445
+ Aspera.assert(File.directory?(@local_storage_root), type: Cli::Error) { "Local storage root folder #{@local_storage_root} does not exist." }
441
446
  @local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
442
- Aspera.assert(File.directory?(@local_preview_folder), type: Cli::Error){"Folder #{@local_preview_folder} does not exist locally. Please create it, or specify an alternate name."}
447
+ Aspera.assert(File.directory?(@local_preview_folder), type: Cli::Error) { "Folder #{@local_preview_folder} does not exist locally. Please create it, or specify an alternate name." }
443
448
  # Protection to avoid clash of file id for two different access keys
444
449
  marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
445
- Log.log.debug{"marker file: #{marker_file}"}
450
+ Log.log.debug { "marker file: #{marker_file}" }
446
451
  if File.exist?(marker_file)
447
452
  ak = File.read(marker_file).chomp
448
- Aspera.assert(@access_key_self['id'].eql?(ak)){"mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}"}
453
+ Aspera.assert(@access_key_self['id'].eql?(ak)) { "mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}" }
449
454
  else
450
455
  File.write(marker_file, @access_key_self['id'])
451
456
  end
@@ -455,38 +460,32 @@ module Aspera
455
460
 
456
461
  # --- handlers ---
457
462
 
458
- def action_scan
463
+ def action_scan(path:, **)
459
464
  check_tools_and_mimemagic
460
- scan_path = options.get_option(:scan_path)
461
- scan_id = options.get_option(:scan_id)
462
- folder_info =
463
- if scan_id.nil?
464
- {
465
- 'id' => @access_key_self['root_file_id'],
466
- 'name' => '/',
467
- 'type' => 'folder',
468
- 'path' => '/'
469
- }
465
+ apifid =
466
+ if (selector = Parser.percent_selector(path))
467
+ Aspera.assert_values(selector[:field], ['id'], type: BadArgument) { 'file id' }
468
+ Api::NodeFileId.new(@api_node, selector[:value].to_s.empty? ? @access_key_self['root_file_id'] : selector[:value])
470
469
  else
471
- @api_node.read("files/#{scan_id}")
470
+ @api_node.resolve_api_fid(@access_key_self['root_file_id'], path)
472
471
  end
473
- @filter_block = Api::Node.file_matcher_from_argument(options)
474
- scan_folder_files(folder_info, scan_path)
472
+ @filter_block = Base.file_matcher(options.get_option(:filter))
473
+ scan_folder_files(@api_node.read("files/#{apifid.file_id}"))
475
474
  Result::Status.new('scan finished')
476
475
  ensure
477
476
  cleanup_tmp_folder
478
477
  end
479
478
 
480
- def action_events
479
+ def action_events(**)
481
480
  check_tools_and_mimemagic
482
- run_event_loop(:events)
481
+ run_event_loop(:events, filter: options.get_option(:filter))
483
482
  ensure
484
483
  cleanup_tmp_folder
485
484
  end
486
485
 
487
- def action_trevents
486
+ def action_trevents(**)
488
487
  check_tools_and_mimemagic
489
- run_event_loop(:trevents)
488
+ run_event_loop(:trevents, filter: options.get_option(:filter))
490
489
  ensure
491
490
  cleanup_tmp_folder
492
491
  end
@@ -509,12 +508,9 @@ module Aspera
509
508
 
510
509
  def action_show(source_file:, **)
511
510
  check_tools_and_mimemagic
512
- # terminal_options = options.get_next_argument('options', validation: Hash, default: {}).symbolize_keys
513
511
  generated_file_path = preview_filename(:png, options.get_option(:base))
514
512
  Aspera::Preview::Generator.new(source_file, generated_file_path, @gen_options, @tmp_folder).generate
515
513
  formatter.display_status("generated: #{generated_file_path}")
516
- # formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
517
- # Result::Status.new("generated: #{generated_file_path}")
518
514
  Result::Image.new(UriReader.file_url(generated_file_path))
519
515
  ensure
520
516
  cleanup_tmp_folder
@@ -530,14 +526,15 @@ module Aspera
530
526
 
531
527
  # Clean up the temporary folder after each command.
532
528
  def cleanup_tmp_folder
533
- Log.log.debug{"cleaning up temp folder #{@tmp_folder}"}
529
+ Log.log.debug { "cleaning up temp folder #{@tmp_folder}" }
534
530
  FileUtils.rm_rf(@tmp_folder)
535
531
  end
536
532
 
537
533
  # Shared event-loop body for :events and :trevents.
538
534
  # @param command [:events, :trevents]
539
- def run_event_loop(command)
540
- @filter_block = Api::Node.file_matcher_from_argument(options)
535
+ # @param filter [String, Regexp, Proc, nil] optional filter expression
536
+ def run_event_loop(command, filter: nil)
537
+ @filter_block = Base.file_matcher(filter)
541
538
  iteration_persistency = nil
542
539
  if options.get_option(:once_only, mandatory: true)
543
540
  iteration_persistency = PersistencyActionOnce.new(
@@ -47,7 +47,7 @@ module Aspera
47
47
  error = nil
48
48
  urls.each do |base_url|
49
49
  server_uri = URI.parse(base_url)
50
- Log.log.debug{"URI=#{server_uri}, host=#{server_uri.hostname}, port=#{server_uri.port}, scheme=#{server_uri.scheme}"}
50
+ Log.log.debug { "URI=#{server_uri}, host=#{server_uri.hostname}, port=#{server_uri.port}, scheme=#{server_uri.scheme}" }
51
51
  next unless server_uri.scheme.eql?(SSH_SCHEME)
52
52
  socket = TCPSocket.new(server_uri.hostname, server_uri.port)
53
53
  socket.puts('SSH-2.0-Ascli_0.0')
@@ -55,7 +55,7 @@ module Aspera
55
55
  return {version: version.gsub(/^SSH-2.0-/, ''), url: base_url} if version.match?(/^SSH-2.0-/)
56
56
  rescue StandardError => e
57
57
  error = e
58
- Log.log.debug{"detect error: #{e}"}
58
+ Log.log.debug { "detect error: #{e}" }
59
59
  end
60
60
  raise error if error
61
61
  return
@@ -91,10 +91,10 @@ module Aspera
91
91
  url = options.get_option(:url, mandatory: true)
92
92
  server_transfer_spec = {}
93
93
  server_uri = URI.parse(url)
94
- Log.log.debug{"URI=#{server_uri}, host=#{server_uri.hostname}, port=#{server_uri.port}, scheme=#{server_uri.scheme}"}
94
+ Log.log.debug { "URI=#{server_uri}, host=#{server_uri.hostname}, port=#{server_uri.port}, scheme=#{server_uri.scheme}" }
95
95
  server_transfer_spec['remote_host'] = server_uri.hostname
96
96
  unless URI_SCHEMES.include?(server_uri.scheme)
97
- Log.log.warn{"Scheme [#{server_uri.scheme}] not supported in #{url}, use one of: #{URI_SCHEMES.join(', ')}. Defaulting to #{SSH_SCHEME}."}
97
+ Log.log.warn { "Scheme [#{server_uri.scheme}] not supported in #{url}, use one of: #{URI_SCHEMES.join(', ')}. Defaulting to #{SSH_SCHEME}." }
98
98
  server_uri.scheme = SSH_SCHEME
99
99
  end
100
100
  if server_uri.scheme.eql?(LOCAL_SCHEME)
@@ -115,13 +115,13 @@ module Aspera
115
115
  Log.log.warn('URL scheme is https but no token was provided in transfer spec.')
116
116
  Log.log.warn("If you want to access the server, not using WSS for session, then use a URL with scheme \"#{SSH_SCHEME}\" and proper SSH port")
117
117
  assumed_url = "#{SSH_SCHEME}://#{server_transfer_spec['remote_host']}:#{Transfer::Spec::SSH_PORT}"
118
- Log.log.warn{"Assuming proper URL is: #{assumed_url}"}
118
+ Log.log.warn { "Assuming proper URL is: #{assumed_url}" }
119
119
  server_uri = URI.parse(assumed_url)
120
120
  end
121
121
  # Scheme is SSH
122
122
  if options.get_option(:username).nil?
123
123
  options.set_option(:username, Transfer::Spec::ACCESS_KEY_TRANSFER_USER)
124
- Log.log.info{"No username provided: Assuming default transfer user: #{Transfer::Spec::ACCESS_KEY_TRANSFER_USER}"}
124
+ Log.log.info { "No username provided: Assuming default transfer user: #{Transfer::Spec::ACCESS_KEY_TRANSFER_USER}" }
125
125
  end
126
126
  server_transfer_spec['remote_user'] = options.get_option(:username, mandatory: true)
127
127
  if !server_uri.port.nil?
@@ -137,14 +137,14 @@ module Aspera
137
137
  end
138
138
  ssh_key_list = options.get_option(:ssh_keys)
139
139
  if !ssh_key_list.nil?
140
- Aspera.assert_array_all(ssh_key_list, String){'ssh_keys'}
141
- ssh_key_list.map!{ |p| File.expand_path(p)}
142
- Log.log.debug{"SSH keys=#{ssh_key_list}"}
140
+ Aspera.assert_array_all(ssh_key_list, String) { 'ssh_keys' }
141
+ ssh_key_list.map! { |p| File.expand_path(p) }
142
+ Log.dump(:ssh_keys, ssh_key_list)
143
143
  if !ssh_key_list.empty?
144
144
  @ssh_opts[:keys] = ssh_key_list
145
145
  # PEM as per RFC 7468
146
146
  server_transfer_spec['ssh_private_key'] = File.read(ssh_key_list.first).strip
147
- Log.log.warn{'Using only first SSH key for transfers'} unless ssh_key_list.length.eql?(1)
147
+ Log.log.warn { 'Using only first SSH key for transfers' } unless ssh_key_list.length.eql?(1)
148
148
  cred_set = true
149
149
  end
150
150
  end
@@ -160,12 +160,12 @@ module Aspera
160
160
  end
161
161
 
162
162
  Sync::Operations::DIRECTIONS.each do |dir|
163
- define_method(:"action_sync_#{dir}"){ |**| run_sync_transfer(dir){@server_transfer_spec}}
163
+ define_method(:"action_sync_#{dir}") { |path:, sync_info: {}, **| run_sync_transfer(dir, path: path, sync_info: sync_info) { @server_transfer_spec } }
164
164
  end
165
165
 
166
166
  # --- DSL ---
167
167
 
168
- option :ssh_keys, description: 'SSH key path list', allowed: Allowed::TYPES_STRING_ARRAY
168
+ option :ssh_keys, description: 'SSH key path list', allowed: Type::STRING_ARRAY
169
169
  option :passphrase, description: 'SSH private key passphrase'
170
170
  option :ssh_options, description: 'SSH options', allowed: Hash
171
171
 
@@ -178,20 +178,25 @@ module Aspera
178
178
  @server_transfer_spec['direction'] = Transfer::Spec.transfer_type_to_direction(:upload)
179
179
  Runner.result_transfer(transfer.start(@server_transfer_spec))
180
180
  end)
181
- command(:download, description: 'Download files from server', transfer_paths: :receive, action: lambda do
182
- @server_transfer_spec['direction'] = Transfer::Spec.transfer_type_to_direction(:download)
183
- Runner.result_transfer(transfer.start(@server_transfer_spec))
184
- end)
181
+ command(
182
+ :download,
183
+ description: 'Download files from server',
184
+ transfer_paths: :receive,
185
+ action: lambda do
186
+ @server_transfer_spec['direction'] = Transfer::Spec.transfer_type_to_direction(:download)
187
+ Runner.result_transfer(transfer.start(@server_transfer_spec))
188
+ end
189
+ )
185
190
  command :sync, description: 'Synchronize files with server'
186
- commands_under(:sync) do
191
+ commands_under :sync do
187
192
  Sync::Operations::DIRECTIONS.each do |dir|
188
- command(dir, description: "#{dir.capitalize}-sync with server", transfer_paths: :send)
193
+ command dir, description: "#{dir.capitalize}-sync with server", transfer_paths: :send, arguments: SyncActions::PATH_AND_INFO_ARGS
189
194
  end
190
195
  command :admin, description: 'Manage sync database (admin operations)'
191
- SyncActions.register_sync_admin_commands(self, %i[sync admin])
196
+ SyncActions.register_sync_admin_commands(self, :admin)
192
197
  end
193
198
 
194
- commands_under(:health) do
199
+ commands_under :health do
195
200
  command :transfer, description: 'Check FASP transfer health'
196
201
  end
197
202
 
@@ -199,26 +204,26 @@ module Aspera
199
204
  ASCMD_ARGS = [{name: :command_arguments, multiple: true, mandatory: false, default: nil}].freeze
200
205
  private_constant :ASCMD_ARGS
201
206
 
202
- command :ls, description: 'List files on server', condition: :ascmd_available?, aliases: [:browse], arguments: ASCMD_ARGS
203
- command :rm, description: 'Delete file(s) on server', condition: :ascmd_available?, aliases: [:delete], arguments: ASCMD_ARGS
204
- command :mv, description: 'Rename/move file(s) on server', condition: :ascmd_available?, aliases: [:rename], arguments: ASCMD_ARGS
205
- command :cp, description: 'Copy file(s) on server', condition: :ascmd_available?, arguments: ASCMD_ARGS
206
- command :mkdir, description: 'Create directory on server', condition: :ascmd_available?, arguments: ASCMD_ARGS
207
- command :df, description: 'Show disk usage on server', condition: :ascmd_available?, arguments: ASCMD_ARGS
208
- command :du, description: 'Show file sizes on server', condition: :ascmd_available?, arguments: ASCMD_ARGS
209
- command :md5sum, description: 'Compute MD5 checksums on server', condition: :ascmd_available?, arguments: ASCMD_ARGS
207
+ command :ls, description: 'List files', condition: :ascmd_available?, aliases: [:browse], arguments: ASCMD_ARGS
208
+ command :rm, description: 'Delete files', condition: :ascmd_available?, aliases: [:delete], arguments: ASCMD_ARGS
209
+ command :mv, description: 'Rename/move files', condition: :ascmd_available?, aliases: [:rename], arguments: ASCMD_ARGS
210
+ command :cp, description: 'Copy files', condition: :ascmd_available?, arguments: ASCMD_ARGS
211
+ command :mkdir, description: 'Create directory', condition: :ascmd_available?, arguments: ASCMD_ARGS
212
+ command :df, description: 'Show disk usage', condition: :ascmd_available?, arguments: ASCMD_ARGS
213
+ command :du, description: 'Show file sizes', condition: :ascmd_available?, arguments: ASCMD_ARGS
214
+ command :md5sum, description: 'Compute MD5 checksums', condition: :ascmd_available?, arguments: ASCMD_ARGS
210
215
  command :info, description: 'Show server system information', condition: :ascmd_available?, arguments: ASCMD_ARGS
211
216
 
212
217
  # Generate ascmd handlers - convention: action_<op>
213
218
  %i[rm mv cp mkdir].each do |op|
214
219
  define_action_method([op]) do |command_arguments:, **|
215
- execute_ascmd(op, command_arguments){Result::Success.new}
220
+ execute_ascmd(op, command_arguments) { Result::Success.new }
216
221
  end
217
222
  end
218
223
 
219
224
  %i[du md5sum info].each do |op|
220
225
  define_action_method([op]) do |command_arguments:, **|
221
- execute_ascmd(op, command_arguments){ |r| Result::SingleObject.new(r.stringify_keys)}
226
+ execute_ascmd(op, command_arguments) { |r| Result::SingleObject.new(r.stringify_keys) }
222
227
  end
223
228
  end
224
229
 
@@ -238,7 +243,7 @@ module Aspera
238
243
  when :local then LocalExecutor.new
239
244
  when :wss then nil
240
245
  when :ssh then Ssh.new(@server_transfer_spec['remote_host'], @server_transfer_spec['remote_user'], @ssh_opts)
241
- else Aspera.error_unexpected_value(@connection_type){'connection type'}
246
+ else Aspera.error_unexpected_value(@connection_type) { 'connection type' }
242
247
  end
243
248
  {}
244
249
  end