aspera-cli 4.9.0 → 4.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +20 -0
  4. data/CHANGELOG.md +509 -0
  5. data/CONTRIBUTING.md +118 -0
  6. data/README.md +1241 -916
  7. data/bin/ascli +4 -4
  8. data/bin/asession +11 -11
  9. data/docs/test_env.conf +32 -21
  10. data/examples/aoc.rb +4 -4
  11. data/examples/dascli +16 -9
  12. data/examples/faspex4.rb +8 -8
  13. data/examples/node.rb +12 -12
  14. data/examples/server.rb +10 -10
  15. data/lib/aspera/aoc.rb +273 -266
  16. data/lib/aspera/ascmd.rb +56 -54
  17. data/lib/aspera/ats_api.rb +4 -4
  18. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  19. data/lib/aspera/cli/extended_value.rb +5 -5
  20. data/lib/aspera/cli/formater.rb +64 -64
  21. data/lib/aspera/cli/info.rb +2 -2
  22. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  23. data/lib/aspera/cli/listener/logger.rb +1 -1
  24. data/lib/aspera/cli/listener/progress.rb +5 -6
  25. data/lib/aspera/cli/listener/progress_multi.rb +14 -19
  26. data/lib/aspera/cli/main.rb +66 -67
  27. data/lib/aspera/cli/manager.rb +112 -110
  28. data/lib/aspera/cli/plugin.rb +57 -36
  29. data/lib/aspera/cli/plugins/alee.rb +4 -4
  30. data/lib/aspera/cli/plugins/aoc.rb +309 -670
  31. data/lib/aspera/cli/plugins/ats.rb +44 -46
  32. data/lib/aspera/cli/plugins/bss.rb +10 -10
  33. data/lib/aspera/cli/plugins/config.rb +497 -378
  34. data/lib/aspera/cli/plugins/console.rb +12 -12
  35. data/lib/aspera/cli/plugins/cos.rb +18 -20
  36. data/lib/aspera/cli/plugins/faspex.rb +112 -114
  37. data/lib/aspera/cli/plugins/faspex5.rb +71 -46
  38. data/lib/aspera/cli/plugins/node.rb +379 -283
  39. data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
  40. data/lib/aspera/cli/plugins/preview.rb +122 -114
  41. data/lib/aspera/cli/plugins/server.rb +137 -83
  42. data/lib/aspera/cli/plugins/shares.rb +30 -29
  43. data/lib/aspera/cli/plugins/sync.rb +13 -33
  44. data/lib/aspera/cli/transfer_agent.rb +60 -59
  45. data/lib/aspera/cli/version.rb +1 -1
  46. data/lib/aspera/colors.rb +3 -3
  47. data/lib/aspera/command_line_builder.rb +27 -27
  48. data/lib/aspera/cos_node.rb +22 -20
  49. data/lib/aspera/data_repository.rb +1 -1
  50. data/lib/aspera/environment.rb +35 -15
  51. data/lib/aspera/fasp/agent_base.rb +15 -15
  52. data/lib/aspera/fasp/agent_connect.rb +23 -21
  53. data/lib/aspera/fasp/agent_direct.rb +66 -64
  54. data/lib/aspera/fasp/agent_httpgw.rb +141 -78
  55. data/lib/aspera/fasp/agent_node.rb +23 -21
  56. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  57. data/lib/aspera/fasp/error.rb +3 -2
  58. data/lib/aspera/fasp/error_info.rb +11 -8
  59. data/lib/aspera/fasp/installation.rb +79 -79
  60. data/lib/aspera/fasp/listener.rb +1 -1
  61. data/lib/aspera/fasp/parameters.rb +86 -71
  62. data/lib/aspera/fasp/parameters.yaml +7 -4
  63. data/lib/aspera/fasp/resume_policy.rb +8 -8
  64. data/lib/aspera/fasp/transfer_spec.rb +35 -2
  65. data/lib/aspera/fasp/uri.rb +7 -7
  66. data/lib/aspera/faspex_gw.rb +7 -5
  67. data/lib/aspera/hash_ext.rb +3 -3
  68. data/lib/aspera/id_generator.rb +5 -5
  69. data/lib/aspera/keychain/encrypted_hash.rb +38 -105
  70. data/lib/aspera/keychain/macos_security.rb +128 -57
  71. data/lib/aspera/log.rb +7 -7
  72. data/lib/aspera/nagios.rb +19 -18
  73. data/lib/aspera/node.rb +209 -35
  74. data/lib/aspera/oauth.rb +37 -36
  75. data/lib/aspera/open_application.rb +19 -11
  76. data/lib/aspera/persistency_action_once.rb +4 -4
  77. data/lib/aspera/persistency_folder.rb +16 -15
  78. data/lib/aspera/preview/file_types.rb +8 -8
  79. data/lib/aspera/preview/generator.rb +67 -67
  80. data/lib/aspera/preview/utils.rb +27 -27
  81. data/lib/aspera/proxy_auto_config.js +41 -41
  82. data/lib/aspera/proxy_auto_config.rb +21 -14
  83. data/lib/aspera/rest.rb +72 -67
  84. data/lib/aspera/rest_call_error.rb +2 -1
  85. data/lib/aspera/rest_error_analyzer.rb +18 -17
  86. data/lib/aspera/rest_errors_aspera.rb +16 -16
  87. data/lib/aspera/secret_hider.rb +15 -13
  88. data/lib/aspera/ssh.rb +11 -10
  89. data/lib/aspera/sync.rb +158 -44
  90. data/lib/aspera/temp_file_manager.rb +2 -2
  91. data/lib/aspera/uri_reader.rb +4 -4
  92. data/lib/aspera/web_auth.rb +14 -13
  93. data.tar.gz.sig +0 -0
  94. metadata +11 -36
  95. metadata.gz.sig +0 -0
@@ -17,7 +17,7 @@ require 'securerandom'
17
17
  module Aspera
18
18
  module Cli
19
19
  module Plugins
20
- class Preview < BasicAuthPlugin
20
+ class Preview < Aspera::Cli::BasicAuthPlugin
21
21
  # special tag to identify transfers related to generator
22
22
  PREV_GEN_TAG = 'preview_generator'
23
23
  # defined by node API: suffix for folder containing previews
@@ -30,12 +30,19 @@ module Aspera
30
30
  AK_MARKER_FILE = '.aspera_access_key'
31
31
  LOCAL_STORAGE_PCVL = 'file:///'
32
32
  LOG_LIMITER_SEC = 30.0
33
- private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER,
34
- :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE, :LOG_LIMITER_SEC
33
+ private_constant :PREV_GEN_TAG,
34
+ :PREVIEW_FOLDER_SUFFIX,
35
+ :PREVIEW_BASENAME,
36
+ :TMP_DIR_PREFIX,
37
+ :DEFAULT_PREVIEWS_FOLDER,
38
+ :LOCAL_STORAGE_PCVL,
39
+ :AK_MARKER_FILE,
40
+ :LOG_LIMITER_SEC
35
41
 
36
42
  # option_skip_format has special accessors
37
43
  attr_accessor :option_previews_folder
38
44
  attr_accessor :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
45
+
39
46
  def initialize(env)
40
47
  super(env)
41
48
  @skip_types = []
@@ -47,45 +54,45 @@ module Aspera
47
54
  # used to trigger periodic processing
48
55
  @periodic = TimerLimiter.new(LOG_LIMITER_SEC)
49
56
  # link CLI options to gen_info attributes
50
- options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
51
- options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
52
- options.set_obj_attr(:skip_types,self,:option_skip_types)
53
- options.set_obj_attr(:previews_folder,self,:option_previews_folder,DEFAULT_PREVIEWS_FOLDER)
54
- options.set_obj_attr(:skip_folders,self,:option_skip_folders,[]) # no skip
55
- options.set_obj_attr(:overwrite,self,:option_overwrite,:mtime)
56
- options.set_obj_attr(:file_access,self,:option_file_access,:local)
57
- options.add_opt_list(:skip_format,Aspera::Preview::Generator::PREVIEW_FORMATS,'skip this preview format (multiple possible)')
58
- options.add_opt_list(:folder_reset_cache,%i[no header read],'force detection of generated preview by refresh cache')
59
- options.add_opt_simple(:skip_types,'skip types in comma separated list')
60
- options.add_opt_simple(:previews_folder,'preview folder in storage root')
61
- options.add_opt_simple(:temp_folder,'path to temp folder')
62
- options.add_opt_simple(:skip_folders,'list of folder to skip')
63
- options.add_opt_simple(:case,'basename of output for for test')
64
- options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
65
- options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
66
- options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
67
- options.add_opt_list(:overwrite,%i[always never mtime],'when to overwrite result file')
68
- options.add_opt_list(:file_access,%i[local remote],'how to read and write files in repository')
69
- options.set_option(:temp_folder,Dir.tmpdir)
70
- options.set_option(:mimemagic,false)
57
+ options.set_obj_attr(:skip_format, self, :option_skip_format, []) # no skip
58
+ options.set_obj_attr(:folder_reset_cache, self, :option_folder_reset_cache, :no)
59
+ options.set_obj_attr(:skip_types, self, :option_skip_types)
60
+ options.set_obj_attr(:previews_folder, self, :option_previews_folder, DEFAULT_PREVIEWS_FOLDER)
61
+ options.set_obj_attr(:skip_folders, self, :option_skip_folders, []) # no skip
62
+ options.set_obj_attr(:overwrite, self, :option_overwrite, :mtime)
63
+ options.set_obj_attr(:file_access, self, :option_file_access, :local)
64
+ options.add_opt_list(:skip_format, Aspera::Preview::Generator::PREVIEW_FORMATS, 'skip this preview format (multiple possible)')
65
+ options.add_opt_list(:folder_reset_cache, %i[no header read], 'force detection of generated preview by refresh cache')
66
+ options.add_opt_simple(:skip_types, 'skip types in comma separated list')
67
+ options.add_opt_simple(:previews_folder, 'preview folder in storage root')
68
+ options.add_opt_simple(:temp_folder, 'path to temp folder')
69
+ options.add_opt_simple(:skip_folders, 'list of folder to skip')
70
+ options.add_opt_simple(:case, 'basename of output for for test')
71
+ options.add_opt_simple(:scan_path, 'subpath in folder id to start scan in (default=/)')
72
+ options.add_opt_simple(:scan_id, 'forder id in storage to start scan in, default is access key main folder id')
73
+ options.add_opt_boolean(:mimemagic, 'use Mime type detection of gem mimemagic')
74
+ options.add_opt_list(:overwrite, %i[always never mtime], 'when to overwrite result file')
75
+ options.add_opt_list(:file_access, %i[local remote], 'how to read and write files in repository')
76
+ options.set_option(:temp_folder, Dir.tmpdir)
77
+ options.set_option(:mimemagic, false)
71
78
 
72
79
  # add other options for generator (and set default values)
73
80
  Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
74
- options.set_obj_attr(opt[:name],@gen_options,opt[:name],opt[:default])
75
- if opt.has_key?(:values)
76
- options.add_opt_list(opt[:name],opt[:values],opt[:description])
81
+ options.set_obj_attr(opt[:name], @gen_options, opt[:name], opt[:default])
82
+ if opt.key?(:values)
83
+ options.add_opt_list(opt[:name], opt[:values], opt[:description])
77
84
  elsif Cli::Manager::BOOLEAN_SIMPLE.include?(opt[:default])
78
- options.add_opt_boolean(opt[:name],opt[:description])
85
+ options.add_opt_boolean(opt[:name], opt[:description])
79
86
  else
80
- options.add_opt_simple(opt[:name],opt[:description])
87
+ options.add_opt_simple(opt[:name], opt[:description])
81
88
  end
82
89
  end
83
90
 
84
91
  options.parse_options!
85
92
  raise 'skip_folder shall be an Array, use @json:[...]' unless @option_skip_folders.is_a?(Array)
86
- @tmp_folder = File.join(options.get_option(:temp_folder,is_type: :mandatory),"#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
93
+ @tmp_folder = File.join(options.get_option(:temp_folder, is_type: :mandatory), "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
87
94
  FileUtils.mkdir_p(@tmp_folder)
88
- Log.log.debug("tmpdir: #{@tmp_folder}")
95
+ Log.log.debug{"tmpdir: #{@tmp_folder}"}
89
96
  end
90
97
 
91
98
  def option_skip_types=(value)
@@ -111,11 +118,11 @@ module Aspera
111
118
 
112
119
  # /files/id/files is normally cached in redis, but we can discard the cache
113
120
  # but /files/id is not cached
114
- def get_folder_entries(file_id,request_args=nil)
121
+ def get_folder_entries(file_id, request_args=nil)
115
122
  headers = {'Accept' => 'application/json'}
116
123
  headers['X-Aspera-Cache-Control'] = 'no-cache' if @option_folder_reset_cache.eql?(:header)
117
- return @api_node.call({operation: 'GET',subpath: "files/#{file_id}/files",headers: headers,url_params: request_args})[:data]
118
- #return @api_node.read("files/#{file_id}/files",request_args)[:data]
124
+ return @api_node.call({operation: 'GET', subpath: "files/#{file_id}/files", headers: headers, url_params: request_args})[:data]
125
+ # return @api_node.read("files/#{file_id}/files",request_args)[:data]
119
126
  end
120
127
 
121
128
  # old version based on folders
@@ -128,10 +135,10 @@ module Aspera
128
135
  # optionally add iteration token from persistency
129
136
  events_filter['iteration_token'] = iteration_persistency.data.first unless iteration_persistency.nil?
130
137
  begin
131
- events = @api_node.read('events',events_filter)[:data]
138
+ events = @api_node.read('events', events_filter)[:data]
132
139
  rescue RestCallError => e
133
140
  if e.message.include?('Invalid iteration_token')
134
- Log.log.warn("Retrying without iteration token: #{e}")
141
+ Log.log.warn{"Retrying without iteration token: #{e}"}
135
142
  events_filter.delete('iteration_token')
136
143
  retry
137
144
  end
@@ -140,11 +147,11 @@ module Aspera
140
147
  return if events.empty?
141
148
  events.each do |event|
142
149
  if event['data']['direction'].eql?(Fasp::TransferSpec::DIRECTION_RECEIVE) &&
143
- event['data']['status'].eql?('completed') &&
144
- event['data']['error_code'].eql?(0) &&
145
- event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
146
- folder_id = event.dig('data','tags','aspera','node','file_id')
147
- folder_id ||= event.dig('data','file_id')
150
+ event['data']['status'].eql?('completed') &&
151
+ event['data']['error_code'].eql?(0) &&
152
+ event['data'].dig('tags', 'aspera', PREV_GEN_TAG).nil?
153
+ folder_id = event.dig('data', 'tags', 'aspera', 'node', 'file_id')
154
+ folder_id ||= event.dig('data', 'file_id')
148
155
  if !folder_id.nil?
149
156
  folder_entry = @api_node.read("files/#{folder_id}")[:data] rescue nil
150
157
  scan_folder_files(folder_entry) unless folder_entry.nil?
@@ -152,7 +159,7 @@ module Aspera
152
159
  end
153
160
  # log/persist periodically or last one
154
161
  next unless @periodic.trigger? || event.equal?(events.last)
155
- Log.log.info("Processed event #{event['id']}")
162
+ Log.log.info{"Processed event #{event['id']}"}
156
163
  # save checkpoint to avoid losing processing in case of error
157
164
  if !iteration_persistency.nil?
158
165
  iteration_persistency.data[0] = event['id'].to_s
@@ -170,14 +177,14 @@ module Aspera
170
177
  }
171
178
  # optionally add iteration token from persistency
172
179
  events_filter['iteration_token'] = iteration_persistency.data.first unless iteration_persistency.nil?
173
- events = @api_node.read('events',events_filter)[:data]
180
+ events = @api_node.read('events', events_filter)[:data]
174
181
  return if events.empty?
175
182
  events.each do |event|
176
183
  # process only files
177
- if event.dig('data','type').eql?('file')
184
+ if event.dig('data', 'type').eql?('file')
178
185
  file_entry = @api_node.read("files/#{event['data']['id']}")[:data] rescue nil
179
186
  if !file_entry.nil? &&
180
- @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
187
+ @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
181
188
  file_entry['parent_file_id'] = event['data']['parent_file_id']
182
189
  if event['types'].include?('file.deleted')
183
190
  Log.log.error('TODO'.red)
@@ -189,7 +196,7 @@ module Aspera
189
196
  end
190
197
  # log/persist periodically or last one
191
198
  next unless @periodic.trigger? || event.equal?(events.last)
192
- Log.log.info("Processing event #{event['id']}")
199
+ Log.log.info{"Processing event #{event['id']}"}
193
200
  # save checkpoint to avoid losing processing in case of error
194
201
  if !iteration_persistency.nil?
195
202
  iteration_persistency.data[0] = event['id'].to_s
@@ -198,40 +205,41 @@ module Aspera
198
205
  end
199
206
  end
200
207
 
201
- def do_transfer(direction,folder_id,source_filename,destination='/')
208
+ def do_transfer(direction, folder_id, source_filename, destination='/')
202
209
  raise 'error' if destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)
203
210
  if @default_transfer_spec.nil?
204
211
  # make a dummy call to get some default transfer parameters
205
- res = @api_node.create('files/upload_setup',{'transfer_requests' => [{'transfer_request' => {'paths' => [{}],'destination_root' => '/'}}]})
212
+ res = @api_node.create('files/upload_setup', {'transfer_requests' => [{'transfer_request' => {'paths' => [{}], 'destination_root' => '/'}}]})
206
213
  template_ts = res[:data]['transfer_specs'].first['transfer_spec']
207
214
  # get ports, anyway that should be 33001 for both. add remote_user ?
208
- @default_transfer_spec = %w[ssh_port fasp_port].each_with_object({}){|e,h|h[e] = template_ts[e];}
215
+ @default_transfer_spec = %w[ssh_port fasp_port].each_with_object({}){|e, h|h[e] = template_ts[e]; }
209
216
  if !@default_transfer_spec['remote_user'].eql?(Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
210
217
  Log.log.warn('remote_user shall be xfer')
211
218
  @default_transfer_spec['remote_user'] = Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER
212
219
  end
213
- Aspera::Node.set_ak_basic_token(@default_transfer_spec,@access_key_self['id'],options.get_option(:password,is_type: :mandatory))
214
- # note: we use the same address for ascp than for node api instead of the one from upload_setup
220
+ @api_node.ts_basic_token(@default_transfer_spec)
221
+ # NOTE: we use the same address for ascp than for node api instead of the one from upload_setup
215
222
  # TODO: configurable ? useful ?
216
223
  @default_transfer_spec['remote_host'] = @transfer_server_address
217
224
  end
218
225
  tspec = @default_transfer_spec.merge({
219
226
  'direction' => direction,
220
227
  'paths' => [{'source' => source_filename}],
221
- 'tags' => { 'aspera' => {
222
- PREV_GEN_TAG => true,
223
- 'node' => {
224
- 'access_key' => @access_key_self['id'],
225
- 'file_id' => folder_id }}}
228
+ 'tags' => {
229
+ 'aspera' => {
230
+ PREV_GEN_TAG => true,
231
+ 'node' => {
232
+ 'access_key' => @access_key_self['id'],
233
+ 'file_id' => folder_id }}}
226
234
  })
227
235
  # force destination
228
236
  # tspec['destination_root']=destination
229
237
  transfer.option_transfer_spec_deep_merge({'destination_root' => destination})
230
- Main.result_transfer(transfer.start(tspec,{src: :node_gen4}))
238
+ Main.result_transfer(transfer.start(tspec))
231
239
  end
232
240
 
233
- def get_infos_local(gen_infos,entry)
234
- local_original_filepath = File.join(@local_storage_root,entry['path'])
241
+ def get_infos_local(gen_infos, entry)
242
+ local_original_filepath = File.join(@local_storage_root, entry['path'])
235
243
  original_mtime = File.mtime(local_original_filepath)
236
244
  # out
237
245
  local_entry_preview_dir = File.join(@local_preview_folder, entry_preview_folder_name(entry))
@@ -244,22 +252,21 @@ module Aspera
244
252
  return local_entry_preview_dir
245
253
  end
246
254
 
247
- def get_infos_remote(gen_infos,entry)
248
- #Log.log.debug(">>>> get_infos_remote #{entry}".red)
255
+ def get_infos_remote(gen_infos, entry)
249
256
  # store source directly here
250
- local_original_filepath = File.join(@tmp_folder,entry['name'])
251
- #original_mtime=DateTime.parse(entry['modified_time'])
257
+ local_original_filepath = File.join(@tmp_folder, entry['name'])
258
+ # original_mtime=DateTime.parse(entry['modified_time'])
252
259
  # out: where previews are generated
253
- local_entry_preview_dir = File.join(@tmp_folder,entry_preview_folder_name(entry))
260
+ local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
254
261
  file_info = @api_node.read("files/#{entry['id']}")[:data]
255
- #TODO: this does not work because previews is hidden in api (gen4)
256
- #this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
262
+ # TODO: this does not work because previews is hidden in api (gen4)
263
+ # this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
257
264
  # TODO: use gen3 api to list files and get date
258
265
  gen_infos.each do |gen_info|
259
266
  gen_info[:src] = local_original_filepath
260
267
  gen_info[:dst] = File.join(local_entry_preview_dir, gen_info[:base_dest])
261
268
  # TODO: use this_preview_folder_entries (but it's hidden)
262
- gen_info[:preview_exist] = file_info.has_key?('preview')
269
+ gen_info[:preview_exist] = file_info.key?('preview')
263
270
  # TODO: get change time and compare, useful ?
264
271
  gen_info[:preview_newer_than_original] = gen_info[:preview_exist]
265
272
  end
@@ -271,7 +278,7 @@ module Aspera
271
278
  "#{entry['id']}#{PREVIEW_FOLDER_SUFFIX}"
272
279
  end
273
280
 
274
- def preview_filename(preview_format,filename=nil)
281
+ def preview_filename(preview_format, filename=nil)
275
282
  filename ||= PREVIEW_BASENAME
276
283
  return "#{filename}.#{preview_format}"
277
284
  end
@@ -289,7 +296,7 @@ module Aspera
289
296
  # lets gather some infos on possibly existing previews
290
297
  # it depends if files access locally or remotely
291
298
  # folder where previews will be generated for this particular entry
292
- local_entry_preview_dir = @access_remote ? get_infos_remote(gen_infos,entry) : get_infos_local(gen_infos,entry)
299
+ local_entry_preview_dir = @access_remote ? get_infos_remote(gen_infos, entry) : get_infos_local(gen_infos, entry)
293
300
  # here we have the status on preview files
294
301
  # let's find if they need generation
295
302
  gen_infos.select! do |gen_info|
@@ -307,7 +314,7 @@ module Aspera
307
314
  end
308
315
  end
309
316
  # need generator for further checks
310
- gen_info[:generator] = Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'])
317
+ gen_info[:generator] = Aspera::Preview::Generator.new(@gen_options, gen_info[:src], gen_info[:dst], @tmp_folder, entry['content_type'])
311
318
  # get conversion_type (if known) and check if supported
312
319
  next false unless gen_info[:generator].supported?
313
320
  # shall we skip it ?
@@ -321,51 +328,51 @@ module Aspera
321
328
  if @access_remote
322
329
  raise 'missing parent_file_id in entry' if entry['parent_file_id'].nil?
323
330
  # download original file to temp folder
324
- do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE,entry['parent_file_id'],entry['name'],@tmp_folder)
331
+ do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
325
332
  end
326
- Log.log.info("source: #{entry['id']}: #{entry['path']})")
333
+ Log.log.info{"source: #{entry['id']}: #{entry['path']})"}
327
334
  gen_infos.each do |gen_info|
328
335
  gen_info[:generator].generate rescue nil
329
336
  end
330
337
  if @access_remote
331
338
  # upload
332
- do_transfer(Fasp::TransferSpec::DIRECTION_SEND,@previews_folder_entry['id'],local_entry_preview_dir)
339
+ do_transfer(Fasp::TransferSpec::DIRECTION_SEND, @previews_folder_entry['id'], local_entry_preview_dir)
333
340
  # cleanup after upload
334
341
  FileUtils.rm_rf(local_entry_preview_dir)
335
- File.delete(File.join(@tmp_folder,entry['name']))
342
+ File.delete(File.join(@tmp_folder, entry['name']))
336
343
  end
337
344
  # force read file updated previews
338
345
  if @option_folder_reset_cache.eql?(:read)
339
346
  @api_node.read("files/#{entry['id']}")
340
347
  end
341
348
  rescue StandardError => e
342
- Log.log.error("Ignore: #{e.message}")
349
+ Log.log.error{"Ignore: #{e.message}"}
343
350
  Log.log.debug(e.backtrace.join("\n").red)
344
351
  end # generate_preview
345
352
 
346
353
  # scan all files in provided folder entry
347
354
  # @param scan_start subpath to start folder scan inside
348
- def scan_folder_files(top_entry,scan_start=nil)
355
+ def scan_folder_files(top_entry, scan_start=nil)
349
356
  if !scan_start.nil?
350
357
  # canonical path: start with / and ends with /
351
358
  scan_start = '/' + scan_start.split('/').reject(&:empty?).join('/')
352
- scan_start = "#{scan_start}/" #unless scan_start.end_with?('/')
359
+ scan_start = "#{scan_start}/" # unless scan_start.end_with?('/')
353
360
  end
354
361
  filter_block = Aspera::Node.file_matcher(options.get_option(:value))
355
- Log.log.debug("scan: #{top_entry} : #{scan_start}".green)
362
+ Log.log.debug{"scan: #{top_entry} : #{scan_start}".green}
356
363
  # don't use recursive call, use list instead
357
364
  entries_to_process = [top_entry]
358
- while !entries_to_process.empty?
365
+ until entries_to_process.empty?
359
366
  entry = entries_to_process.shift
360
367
  # process this entry only if it is within the scan_start
361
368
  entry_path_with_slash = entry['path']
362
- Log.log.info("processing entry #{entry_path_with_slash}") if @periodic.trigger?
369
+ Log.log.info{"processing entry #{entry_path_with_slash}"} if @periodic.trigger?
363
370
  entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
364
371
  if !scan_start.nil? && !scan_start.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(scan_start)
365
- Log.log.debug("#{entry['path']} folder (skip start)".bg_red)
372
+ Log.log.debug{"#{entry['path']} folder (skip start)".bg_red}
366
373
  next
367
374
  end
368
- Log.log.debug("item:#{entry}")
375
+ Log.log.debug{"item:#{entry}"}
369
376
  begin
370
377
  case entry['type']
371
378
  when 'file'
@@ -378,15 +385,15 @@ module Aspera
378
385
  Log.log.debug('Ignoring link.')
379
386
  when 'folder'
380
387
  if @option_skip_folders.include?(entry['path'])
381
- Log.log.debug("#{entry['path']} folder (skip list)".bg_red)
388
+ Log.log.debug{"#{entry['path']} folder (skip list)".bg_red}
382
389
  else
383
- Log.log.debug("#{entry['path']} folder".green)
390
+ Log.log.debug{"#{entry['path']} folder".green}
384
391
  # get folder content
385
392
  folder_entries = get_folder_entries(entry['id'])
386
393
  # process all items in current folder
387
394
  folder_entries.each do |folder_entry|
388
395
  # add path for older versions of ES
389
- if !folder_entry.has_key?('path')
396
+ if !folder_entry.key?('path')
390
397
  folder_entry['path'] = entry_path_with_slash + folder_entry['name']
391
398
  end
392
399
  folder_entry['parent_file_id'] = entry['id']
@@ -394,10 +401,10 @@ module Aspera
394
401
  end
395
402
  end
396
403
  else
397
- Log.log.warn("unknown entry type: #{entry['type']}")
404
+ Log.log.warn{"unknown entry type: #{entry['type']}"}
398
405
  end
399
406
  rescue StandardError => e
400
- Log.log.warn("An error occured: #{e}, ignoring")
407
+ Log.log.warn{"An error occurred: #{e}, ignoring"}
401
408
  end
402
409
  end
403
410
  end
@@ -408,47 +415,47 @@ module Aspera
408
415
  command = options.get_next_command(ACTIONS)
409
416
  unless %i[check test].include?(command)
410
417
  # this will use node api
411
- @api_node = basic_auth_api
418
+ @api_node = Aspera::Node.new(params: basic_auth_params)
412
419
  @transfer_server_address = URI.parse(@api_node.params[:base_url]).host
413
420
  # get current access key
414
421
  @access_key_self = @api_node.read('access_keys/self')[:data]
415
422
  # TODO: check events is activated here:
416
423
  # note that docroot is good to look at as well
417
424
  node_info = @api_node.read('info')[:data]
418
- Log.log.debug("root: #{node_info['docroot']}")
425
+ Log.log.debug{"root: #{node_info['docroot']}"}
419
426
  @access_remote = @option_file_access.eql?(:remote)
420
- Log.log.debug("remote: #{@access_remote}")
421
- Log.log.debug("access key info: #{@access_key_self}")
422
- #TODO: can the previews folder parameter be read from node api ?
427
+ Log.log.debug{"remote: #{@access_remote}"}
428
+ Log.log.debug{"access key info: #{@access_key_self}"}
429
+ # TODO: can the previews folder parameter be read from node api ?
423
430
  @option_skip_folders.push('/' + @option_previews_folder)
424
431
  if @access_remote
425
- # note the filter "name", it's why we take the first one
426
- @previews_folder_entry = get_folder_entries(@access_key_self['root_file_id'],{name: @option_previews_folder}).first
427
- raise CliError,"Folder #{@option_previews_folder} does not exist on node. "\
432
+ # NOTE: the filter "name", it's why we take the first one
433
+ @previews_folder_entry = get_folder_entries(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
434
+ raise CliError, "Folder #{@option_previews_folder} does not exist on node. "\
428
435
  'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
429
436
  else
430
437
  raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
431
438
  @local_storage_root = @access_key_self['storage']['path']
432
- #TODO: option to override @local_storage_root='xxx'
439
+ # TODO: option to override @local_storage_root='xxx'
433
440
  @local_storage_root = @local_storage_root[LOCAL_STORAGE_PCVL.length..-1] if @local_storage_root.start_with?(LOCAL_STORAGE_PCVL)
434
- #TODO: windows could have "C:" ?
441
+ # TODO: windows could have "C:" ?
435
442
  raise "not local storage: #{@local_storage_root}" unless @local_storage_root.start_with?('/')
436
- raise CliError,"Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
437
- @local_preview_folder = File.join(@local_storage_root,@option_previews_folder)
438
- raise CliError,"Folder #{@local_preview_folder} does not exist locally. "\
443
+ raise CliError, "Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
444
+ @local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
445
+ raise CliError, "Folder #{@local_preview_folder} does not exist locally. "\
439
446
  'Please create it, or specify an alternate name.' unless File.directory?(@local_preview_folder)
440
447
  # protection to avoid clash of file id for two different access keys
441
- marker_file = File.join(@local_preview_folder,AK_MARKER_FILE)
442
- Log.log.debug("marker file: #{marker_file}")
448
+ marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
449
+ Log.log.debug{"marker file: #{marker_file}"}
443
450
  if File.exist?(marker_file)
444
451
  ak = File.read(marker_file).chomp
445
452
  raise "mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}" unless @access_key_self['id'].eql?(ak)
446
453
  else
447
- File.write(marker_file,@access_key_self['id'])
454
+ File.write(marker_file, @access_key_self['id'])
448
455
  end
449
456
  end
450
457
  end
451
- Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic,is_type: :mandatory)
458
+ Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, is_type: :mandatory)
452
459
  case command
453
460
  when :scan
454
461
  scan_path = options.get_option(:scan_path)
@@ -456,39 +463,40 @@ module Aspera
456
463
  # by default start at root
457
464
  folder_info =
458
465
  if scan_id.nil?
459
- { 'id' => @access_key_self['root_file_id'],
466
+ {
467
+ 'id' => @access_key_self['root_file_id'],
460
468
  'name' => '/',
461
469
  'type' => 'folder',
462
470
  'path' => '/' }
463
471
  else
464
472
  @api_node.read("files/#{scan_id}")[:data]
465
473
  end
466
- scan_folder_files(folder_info,scan_path)
474
+ scan_folder_files(folder_info, scan_path)
467
475
  return Main.result_status('scan finished')
468
- when :events,:trevents
476
+ when :events, :trevents
469
477
  iteration_persistency = nil
470
- if options.get_option(:once_only,is_type: :mandatory)
478
+ if options.get_option(:once_only, is_type: :mandatory)
471
479
  iteration_persistency = PersistencyActionOnce.new(
472
480
  manager: @agents[:persistency],
473
481
  data: [],
474
482
  id: IdGenerator.from_list([
475
483
  'preview_iteration',
476
484
  command.to_s,
477
- options.get_option(:url,is_type: :mandatory),
478
- options.get_option(:username,is_type: :mandatory)
485
+ options.get_option(:url, is_type: :mandatory),
486
+ options.get_option(:username, is_type: :mandatory)
479
487
  ]))
480
488
  end
481
489
  # call processing method specified by command line command
482
- send("process_#{command}",iteration_persistency)
490
+ send("process_#{command}", iteration_persistency)
483
491
  return Main.result_status("#{command} finished")
484
492
  when :check
485
493
  Aspera::Preview::Utils.check_tools(@skip_types)
486
494
  return Main.result_status('tools validated')
487
495
  when :test
488
- format = options.get_next_argument('format',expected: Aspera::Preview::Generator::PREVIEW_FORMATS)
496
+ format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS)
489
497
  source = options.get_next_argument('source file')
490
- dest = preview_filename(format,options.get_option(:case))
491
- g = Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
498
+ dest = preview_filename(format, options.get_option(:case))
499
+ g = Aspera::Preview::Generator.new(@gen_options, source, dest, @tmp_folder, nil)
492
500
  raise "cannot find file type for #{source}" if g.conversion_type.nil?
493
501
  raise "out format #{format} not supported" unless g.supported?
494
502
  g.generate