aspera-cli 4.10.0 → 4.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +19 -0
- data/CHANGELOG.md +528 -0
- data/CONTRIBUTING.md +143 -0
- data/README.md +977 -589
- data/bin/ascli +4 -4
- data/bin/asession +12 -12
- data/docs/test_env.conf +29 -19
- data/examples/aoc.rb +6 -6
- data/examples/dascli +18 -16
- data/examples/faspex4.rb +15 -15
- data/examples/node.rb +12 -12
- data/examples/proxy.pac +2 -2
- data/examples/server.rb +12 -12
- data/lib/aspera/aoc.rb +344 -272
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +9 -9
- data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +16 -21
- data/lib/aspera/cli/main.rb +72 -73
- data/lib/aspera/cli/manager.rb +112 -112
- data/lib/aspera/cli/plugin.rb +68 -48
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +322 -720
- data/lib/aspera/cli/plugins/ats.rb +50 -52
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +514 -410
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +134 -136
- data/lib/aspera/cli/plugins/faspex5.rb +235 -70
- data/lib/aspera/cli/plugins/node.rb +378 -309
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
- data/lib/aspera/cli/plugins/preview.rb +129 -120
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +77 -52
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +61 -61
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +78 -74
- data/lib/aspera/cos_node.rb +31 -29
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +17 -15
- data/lib/aspera/fasp/agent_connect.rb +34 -32
- data/lib/aspera/fasp/agent_direct.rb +70 -73
- data/lib/aspera/fasp/agent_httpgw.rb +79 -74
- data/lib/aspera/fasp/agent_node.rb +26 -26
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +80 -80
- data/lib/aspera/fasp/listener.rb +2 -2
- data/lib/aspera/fasp/parameters.rb +103 -92
- data/lib/aspera/fasp/parameters.yaml +313 -214
- data/lib/aspera/fasp/resume_policy.rb +10 -10
- data/lib/aspera/fasp/transfer_spec.rb +22 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +80 -159
- data/lib/aspera/faspex_postproc.rb +77 -0
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/nagios.rb +24 -23
- data/lib/aspera/node.rb +217 -38
- data/lib/aspera/oauth.rb +78 -74
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +63 -63
- data/lib/aspera/proxy_auto_config.rb +19 -19
- data/lib/aspera/rest.rb +65 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +22 -21
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +17 -14
- data/lib/aspera/ssh.rb +15 -14
- data/lib/aspera/sync.rb +177 -62
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +13 -64
- data/lib/aspera/web_server_simple.rb +76 -0
- data.tar.gz.sig +0 -0
- metadata +11 -6
- metadata.gz.sig +0 -0
@@ -17,7 +17,7 @@ require 'securerandom'
|
|
17
17
|
module Aspera
|
18
18
|
module Cli
|
19
19
|
module Plugins
|
20
|
-
class Preview < BasicAuthPlugin
|
20
|
+
class Preview < Aspera::Cli::BasicAuthPlugin
|
21
21
|
# special tag to identify transfers related to generator
|
22
22
|
PREV_GEN_TAG = 'preview_generator'
|
23
23
|
# defined by node API: suffix for folder containing previews
|
@@ -28,14 +28,21 @@ module Aspera
|
|
28
28
|
TMP_DIR_PREFIX = 'prev_tmp'
|
29
29
|
DEFAULT_PREVIEWS_FOLDER = 'previews'
|
30
30
|
AK_MARKER_FILE = '.aspera_access_key'
|
31
|
-
|
31
|
+
PVCL_LOCAL_STORAGE = 'file:///'
|
32
32
|
LOG_LIMITER_SEC = 30.0
|
33
|
-
private_constant :PREV_GEN_TAG,
|
34
|
-
:
|
33
|
+
private_constant :PREV_GEN_TAG,
|
34
|
+
:PREVIEW_FOLDER_SUFFIX,
|
35
|
+
:PREVIEW_BASENAME,
|
36
|
+
:TMP_DIR_PREFIX,
|
37
|
+
:DEFAULT_PREVIEWS_FOLDER,
|
38
|
+
:PVCL_LOCAL_STORAGE,
|
39
|
+
:AK_MARKER_FILE,
|
40
|
+
:LOG_LIMITER_SEC
|
35
41
|
|
36
42
|
# option_skip_format has special accessors
|
37
43
|
attr_accessor :option_previews_folder
|
38
44
|
attr_accessor :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
|
45
|
+
|
39
46
|
def initialize(env)
|
40
47
|
super(env)
|
41
48
|
@skip_types = []
|
@@ -47,45 +54,45 @@ module Aspera
|
|
47
54
|
# used to trigger periodic processing
|
48
55
|
@periodic = TimerLimiter.new(LOG_LIMITER_SEC)
|
49
56
|
# link CLI options to gen_info attributes
|
50
|
-
options.set_obj_attr(:skip_format,self
|
51
|
-
options.set_obj_attr(:folder_reset_cache,self
|
52
|
-
options.set_obj_attr(:skip_types,self
|
53
|
-
options.set_obj_attr(:previews_folder,self
|
54
|
-
options.set_obj_attr(:skip_folders,self
|
55
|
-
options.set_obj_attr(:overwrite,self
|
56
|
-
options.set_obj_attr(:file_access,self
|
57
|
-
options.add_opt_list(:skip_format,Aspera::Preview::Generator::PREVIEW_FORMATS,'skip this preview format (multiple possible)')
|
58
|
-
options.add_opt_list(:folder_reset_cache
|
59
|
-
options.add_opt_simple(:skip_types,'skip types in comma separated list')
|
60
|
-
options.add_opt_simple(:previews_folder,'preview folder in storage root')
|
61
|
-
options.add_opt_simple(:temp_folder,'path to temp folder')
|
62
|
-
options.add_opt_simple(:skip_folders,'list of folder to skip')
|
63
|
-
options.add_opt_simple(:case,'basename of output for for test')
|
64
|
-
options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
|
65
|
-
options.add_opt_simple(:scan_id,'
|
66
|
-
options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
|
67
|
-
options.add_opt_list(:overwrite
|
68
|
-
options.add_opt_list(:file_access
|
69
|
-
options.set_option(:temp_folder,Dir.tmpdir)
|
70
|
-
options.set_option(:mimemagic,false)
|
57
|
+
options.set_obj_attr(:skip_format, self, :option_skip_format, []) # no skip
|
58
|
+
options.set_obj_attr(:folder_reset_cache, self, :option_folder_reset_cache, :no)
|
59
|
+
options.set_obj_attr(:skip_types, self, :option_skip_types)
|
60
|
+
options.set_obj_attr(:previews_folder, self, :option_previews_folder, DEFAULT_PREVIEWS_FOLDER)
|
61
|
+
options.set_obj_attr(:skip_folders, self, :option_skip_folders, []) # no skip
|
62
|
+
options.set_obj_attr(:overwrite, self, :option_overwrite, :mtime)
|
63
|
+
options.set_obj_attr(:file_access, self, :option_file_access, :local)
|
64
|
+
options.add_opt_list(:skip_format, Aspera::Preview::Generator::PREVIEW_FORMATS, 'skip this preview format (multiple possible)')
|
65
|
+
options.add_opt_list(:folder_reset_cache, %i[no header read], 'force detection of generated preview by refresh cache')
|
66
|
+
options.add_opt_simple(:skip_types, 'skip types in comma separated list')
|
67
|
+
options.add_opt_simple(:previews_folder, 'preview folder in storage root')
|
68
|
+
options.add_opt_simple(:temp_folder, 'path to temp folder')
|
69
|
+
options.add_opt_simple(:skip_folders, 'list of folder to skip')
|
70
|
+
options.add_opt_simple(:case, 'basename of output for for test')
|
71
|
+
options.add_opt_simple(:scan_path, 'subpath in folder id to start scan in (default=/)')
|
72
|
+
options.add_opt_simple(:scan_id, 'folder id in storage to start scan in, default is access key main folder id')
|
73
|
+
options.add_opt_boolean(:mimemagic, 'use Mime type detection of gem mimemagic')
|
74
|
+
options.add_opt_list(:overwrite, %i[always never mtime], 'when to overwrite result file')
|
75
|
+
options.add_opt_list(:file_access, %i[local remote], 'how to read and write files in repository')
|
76
|
+
options.set_option(:temp_folder, Dir.tmpdir)
|
77
|
+
options.set_option(:mimemagic, false)
|
71
78
|
|
72
79
|
# add other options for generator (and set default values)
|
73
80
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
74
|
-
options.set_obj_attr(opt[:name]
|
75
|
-
if opt.
|
76
|
-
options.add_opt_list(opt[:name],opt[:values],opt[:description])
|
81
|
+
options.set_obj_attr(opt[:name], @gen_options, opt[:name], opt[:default])
|
82
|
+
if opt.key?(:values)
|
83
|
+
options.add_opt_list(opt[:name], opt[:values], opt[:description])
|
77
84
|
elsif Cli::Manager::BOOLEAN_SIMPLE.include?(opt[:default])
|
78
|
-
options.add_opt_boolean(opt[:name],opt[:description])
|
85
|
+
options.add_opt_boolean(opt[:name], opt[:description])
|
79
86
|
else
|
80
|
-
options.add_opt_simple(opt[:name],opt[:description])
|
87
|
+
options.add_opt_simple(opt[:name], opt[:description])
|
81
88
|
end
|
82
89
|
end
|
83
90
|
|
84
91
|
options.parse_options!
|
85
92
|
raise 'skip_folder shall be an Array, use @json:[...]' unless @option_skip_folders.is_a?(Array)
|
86
|
-
@tmp_folder = File.join(options.get_option(:temp_folder,is_type: :mandatory),"#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
|
93
|
+
@tmp_folder = File.join(options.get_option(:temp_folder, is_type: :mandatory), "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
|
87
94
|
FileUtils.mkdir_p(@tmp_folder)
|
88
|
-
Log.log.debug
|
95
|
+
Log.log.debug{"tmpdir: #{@tmp_folder}"}
|
89
96
|
end
|
90
97
|
|
91
98
|
def option_skip_types=(value)
|
@@ -111,11 +118,11 @@ module Aspera
|
|
111
118
|
|
112
119
|
# /files/id/files is normally cached in redis, but we can discard the cache
|
113
120
|
# but /files/id is not cached
|
114
|
-
def get_folder_entries(file_id,request_args=nil)
|
121
|
+
def get_folder_entries(file_id, request_args=nil)
|
115
122
|
headers = {'Accept' => 'application/json'}
|
116
123
|
headers['X-Aspera-Cache-Control'] = 'no-cache' if @option_folder_reset_cache.eql?(:header)
|
117
|
-
return @api_node.call({operation: 'GET',subpath: "files/#{file_id}/files",headers: headers,url_params: request_args})[:data]
|
118
|
-
#return @api_node.read("files/#{file_id}/files",request_args)[:data]
|
124
|
+
return @api_node.call({operation: 'GET', subpath: "files/#{file_id}/files", headers: headers, url_params: request_args})[:data]
|
125
|
+
# return @api_node.read("files/#{file_id}/files",request_args)[:data]
|
119
126
|
end
|
120
127
|
|
121
128
|
# old version based on folders
|
@@ -128,10 +135,10 @@ module Aspera
|
|
128
135
|
# optionally add iteration token from persistency
|
129
136
|
events_filter['iteration_token'] = iteration_persistency.data.first unless iteration_persistency.nil?
|
130
137
|
begin
|
131
|
-
events = @api_node.read('events',events_filter)[:data]
|
138
|
+
events = @api_node.read('events', events_filter)[:data]
|
132
139
|
rescue RestCallError => e
|
133
140
|
if e.message.include?('Invalid iteration_token')
|
134
|
-
Log.log.warn
|
141
|
+
Log.log.warn{"Retrying without iteration token: #{e}"}
|
135
142
|
events_filter.delete('iteration_token')
|
136
143
|
retry
|
137
144
|
end
|
@@ -140,11 +147,11 @@ module Aspera
|
|
140
147
|
return if events.empty?
|
141
148
|
events.each do |event|
|
142
149
|
if event['data']['direction'].eql?(Fasp::TransferSpec::DIRECTION_RECEIVE) &&
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
folder_id = event.dig('data','tags','aspera','node','file_id')
|
147
|
-
folder_id ||= event.dig('data','file_id')
|
150
|
+
event['data']['status'].eql?('completed') &&
|
151
|
+
event['data']['error_code'].eql?(0) &&
|
152
|
+
event['data'].dig('tags', 'aspera', PREV_GEN_TAG).nil?
|
153
|
+
folder_id = event.dig('data', 'tags', 'aspera', 'node', 'file_id')
|
154
|
+
folder_id ||= event.dig('data', 'file_id')
|
148
155
|
if !folder_id.nil?
|
149
156
|
folder_entry = @api_node.read("files/#{folder_id}")[:data] rescue nil
|
150
157
|
scan_folder_files(folder_entry) unless folder_entry.nil?
|
@@ -152,7 +159,7 @@ module Aspera
|
|
152
159
|
end
|
153
160
|
# log/persist periodically or last one
|
154
161
|
next unless @periodic.trigger? || event.equal?(events.last)
|
155
|
-
Log.log.info
|
162
|
+
Log.log.info{"Processed event #{event['id']}"}
|
156
163
|
# save checkpoint to avoid losing processing in case of error
|
157
164
|
if !iteration_persistency.nil?
|
158
165
|
iteration_persistency.data[0] = event['id'].to_s
|
@@ -170,14 +177,14 @@ module Aspera
|
|
170
177
|
}
|
171
178
|
# optionally add iteration token from persistency
|
172
179
|
events_filter['iteration_token'] = iteration_persistency.data.first unless iteration_persistency.nil?
|
173
|
-
events = @api_node.read('events',events_filter)[:data]
|
180
|
+
events = @api_node.read('events', events_filter)[:data]
|
174
181
|
return if events.empty?
|
175
182
|
events.each do |event|
|
176
183
|
# process only files
|
177
|
-
if event.dig('data','type').eql?('file')
|
184
|
+
if event.dig('data', 'type').eql?('file')
|
178
185
|
file_entry = @api_node.read("files/#{event['data']['id']}")[:data] rescue nil
|
179
186
|
if !file_entry.nil? &&
|
180
|
-
|
187
|
+
@option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
|
181
188
|
file_entry['parent_file_id'] = event['data']['parent_file_id']
|
182
189
|
if event['types'].include?('file.deleted')
|
183
190
|
Log.log.error('TODO'.red)
|
@@ -189,7 +196,7 @@ module Aspera
|
|
189
196
|
end
|
190
197
|
# log/persist periodically or last one
|
191
198
|
next unless @periodic.trigger? || event.equal?(events.last)
|
192
|
-
Log.log.info
|
199
|
+
Log.log.info{"Processing event #{event['id']}"}
|
193
200
|
# save checkpoint to avoid losing processing in case of error
|
194
201
|
if !iteration_persistency.nil?
|
195
202
|
iteration_persistency.data[0] = event['id'].to_s
|
@@ -198,40 +205,41 @@ module Aspera
|
|
198
205
|
end
|
199
206
|
end
|
200
207
|
|
201
|
-
def do_transfer(direction,folder_id,source_filename,destination='/')
|
208
|
+
def do_transfer(direction, folder_id, source_filename, destination='/')
|
202
209
|
raise 'error' if destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)
|
203
210
|
if @default_transfer_spec.nil?
|
204
211
|
# make a dummy call to get some default transfer parameters
|
205
|
-
res = @api_node.create('files/upload_setup',{'transfer_requests' => [{'transfer_request' => {'paths' => [{}],'destination_root' => '/'}}]})
|
212
|
+
res = @api_node.create('files/upload_setup', {'transfer_requests' => [{'transfer_request' => {'paths' => [{}], 'destination_root' => '/'}}]})
|
206
213
|
template_ts = res[:data]['transfer_specs'].first['transfer_spec']
|
207
214
|
# get ports, anyway that should be 33001 for both. add remote_user ?
|
208
|
-
@default_transfer_spec = %w[ssh_port fasp_port].each_with_object({}){|e,h|h[e] = template_ts[e];}
|
215
|
+
@default_transfer_spec = %w[ssh_port fasp_port].each_with_object({}){|e, h|h[e] = template_ts[e]; }
|
209
216
|
if !@default_transfer_spec['remote_user'].eql?(Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
210
217
|
Log.log.warn('remote_user shall be xfer')
|
211
218
|
@default_transfer_spec['remote_user'] = Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER
|
212
219
|
end
|
213
|
-
|
214
|
-
#
|
220
|
+
@api_node.ts_basic_token(@default_transfer_spec)
|
221
|
+
# NOTE: we use the same address for ascp than for node api instead of the one from upload_setup
|
215
222
|
# TODO: configurable ? useful ?
|
216
223
|
@default_transfer_spec['remote_host'] = @transfer_server_address
|
217
224
|
end
|
218
|
-
|
225
|
+
t_spec = @default_transfer_spec.merge({
|
219
226
|
'direction' => direction,
|
220
227
|
'paths' => [{'source' => source_filename}],
|
221
|
-
'tags' => {
|
222
|
-
|
223
|
-
|
224
|
-
'
|
225
|
-
|
228
|
+
'tags' => {
|
229
|
+
'aspera' => {
|
230
|
+
PREV_GEN_TAG => true,
|
231
|
+
'node' => {
|
232
|
+
'access_key' => @access_key_self['id'],
|
233
|
+
'file_id' => folder_id }}}
|
226
234
|
})
|
227
235
|
# force destination
|
228
|
-
#
|
236
|
+
# t_spec['destination_root']=destination
|
229
237
|
transfer.option_transfer_spec_deep_merge({'destination_root' => destination})
|
230
|
-
Main.result_transfer(transfer.start(
|
238
|
+
Main.result_transfer(transfer.start(t_spec))
|
231
239
|
end
|
232
240
|
|
233
|
-
def get_infos_local(gen_infos,entry)
|
234
|
-
local_original_filepath = File.join(@local_storage_root,entry['path'])
|
241
|
+
def get_infos_local(gen_infos, entry)
|
242
|
+
local_original_filepath = File.join(@local_storage_root, entry['path'])
|
235
243
|
original_mtime = File.mtime(local_original_filepath)
|
236
244
|
# out
|
237
245
|
local_entry_preview_dir = File.join(@local_preview_folder, entry_preview_folder_name(entry))
|
@@ -244,22 +252,21 @@ module Aspera
|
|
244
252
|
return local_entry_preview_dir
|
245
253
|
end
|
246
254
|
|
247
|
-
def get_infos_remote(gen_infos,entry)
|
248
|
-
#Log.log.debug(">>>> get_infos_remote #{entry}".red)
|
255
|
+
def get_infos_remote(gen_infos, entry)
|
249
256
|
# store source directly here
|
250
|
-
local_original_filepath = File.join(@tmp_folder,entry['name'])
|
251
|
-
#original_mtime=DateTime.parse(entry['modified_time'])
|
257
|
+
local_original_filepath = File.join(@tmp_folder, entry['name'])
|
258
|
+
# original_mtime=DateTime.parse(entry['modified_time'])
|
252
259
|
# out: where previews are generated
|
253
|
-
local_entry_preview_dir = File.join(@tmp_folder,entry_preview_folder_name(entry))
|
260
|
+
local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
|
254
261
|
file_info = @api_node.read("files/#{entry['id']}")[:data]
|
255
|
-
#TODO: this does not work because previews is hidden in api (gen4)
|
256
|
-
#this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
|
262
|
+
# TODO: this does not work because previews is hidden in api (gen4)
|
263
|
+
# this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
|
257
264
|
# TODO: use gen3 api to list files and get date
|
258
265
|
gen_infos.each do |gen_info|
|
259
266
|
gen_info[:src] = local_original_filepath
|
260
267
|
gen_info[:dst] = File.join(local_entry_preview_dir, gen_info[:base_dest])
|
261
268
|
# TODO: use this_preview_folder_entries (but it's hidden)
|
262
|
-
gen_info[:preview_exist] = file_info.
|
269
|
+
gen_info[:preview_exist] = file_info.key?('preview')
|
263
270
|
# TODO: get change time and compare, useful ?
|
264
271
|
gen_info[:preview_newer_than_original] = gen_info[:preview_exist]
|
265
272
|
end
|
@@ -271,7 +278,7 @@ module Aspera
|
|
271
278
|
"#{entry['id']}#{PREVIEW_FOLDER_SUFFIX}"
|
272
279
|
end
|
273
280
|
|
274
|
-
def preview_filename(preview_format,filename=nil)
|
281
|
+
def preview_filename(preview_format, filename=nil)
|
275
282
|
filename ||= PREVIEW_BASENAME
|
276
283
|
return "#{filename}.#{preview_format}"
|
277
284
|
end
|
@@ -289,7 +296,7 @@ module Aspera
|
|
289
296
|
# lets gather some infos on possibly existing previews
|
290
297
|
# it depends if files access locally or remotely
|
291
298
|
# folder where previews will be generated for this particular entry
|
292
|
-
local_entry_preview_dir = @access_remote ? get_infos_remote(gen_infos,entry) : get_infos_local(gen_infos,entry)
|
299
|
+
local_entry_preview_dir = @access_remote ? get_infos_remote(gen_infos, entry) : get_infos_local(gen_infos, entry)
|
293
300
|
# here we have the status on preview files
|
294
301
|
# let's find if they need generation
|
295
302
|
gen_infos.select! do |gen_info|
|
@@ -307,7 +314,7 @@ module Aspera
|
|
307
314
|
end
|
308
315
|
end
|
309
316
|
# need generator for further checks
|
310
|
-
gen_info[:generator] = Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst]
|
317
|
+
gen_info[:generator] = Aspera::Preview::Generator.new(@gen_options, gen_info[:src], gen_info[:dst], @tmp_folder, entry['content_type'])
|
311
318
|
# get conversion_type (if known) and check if supported
|
312
319
|
next false unless gen_info[:generator].supported?
|
313
320
|
# shall we skip it ?
|
@@ -321,51 +328,51 @@ module Aspera
|
|
321
328
|
if @access_remote
|
322
329
|
raise 'missing parent_file_id in entry' if entry['parent_file_id'].nil?
|
323
330
|
# download original file to temp folder
|
324
|
-
do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE,entry['parent_file_id'],entry['name']
|
331
|
+
do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
|
325
332
|
end
|
326
|
-
Log.log.info
|
333
|
+
Log.log.info{"source: #{entry['id']}: #{entry['path']})"}
|
327
334
|
gen_infos.each do |gen_info|
|
328
335
|
gen_info[:generator].generate rescue nil
|
329
336
|
end
|
330
337
|
if @access_remote
|
331
338
|
# upload
|
332
|
-
do_transfer(Fasp::TransferSpec::DIRECTION_SEND
|
339
|
+
do_transfer(Fasp::TransferSpec::DIRECTION_SEND, @previews_folder_entry['id'], local_entry_preview_dir)
|
333
340
|
# cleanup after upload
|
334
341
|
FileUtils.rm_rf(local_entry_preview_dir)
|
335
|
-
File.delete(File.join(@tmp_folder,entry['name']))
|
342
|
+
File.delete(File.join(@tmp_folder, entry['name']))
|
336
343
|
end
|
337
344
|
# force read file updated previews
|
338
345
|
if @option_folder_reset_cache.eql?(:read)
|
339
346
|
@api_node.read("files/#{entry['id']}")
|
340
347
|
end
|
341
348
|
rescue StandardError => e
|
342
|
-
Log.log.error
|
349
|
+
Log.log.error{"Ignore: #{e.message}"}
|
343
350
|
Log.log.debug(e.backtrace.join("\n").red)
|
344
351
|
end # generate_preview
|
345
352
|
|
346
353
|
# scan all files in provided folder entry
|
347
354
|
# @param scan_start subpath to start folder scan inside
|
348
|
-
def scan_folder_files(top_entry,scan_start=nil)
|
355
|
+
def scan_folder_files(top_entry, scan_start=nil)
|
349
356
|
if !scan_start.nil?
|
350
357
|
# canonical path: start with / and ends with /
|
351
358
|
scan_start = '/' + scan_start.split('/').reject(&:empty?).join('/')
|
352
|
-
scan_start = "#{scan_start}/" #unless scan_start.end_with?('/')
|
359
|
+
scan_start = "#{scan_start}/" # unless scan_start.end_with?('/')
|
353
360
|
end
|
354
361
|
filter_block = Aspera::Node.file_matcher(options.get_option(:value))
|
355
|
-
Log.log.debug
|
362
|
+
Log.log.debug{"scan: #{top_entry} : #{scan_start}".green}
|
356
363
|
# don't use recursive call, use list instead
|
357
364
|
entries_to_process = [top_entry]
|
358
|
-
|
365
|
+
until entries_to_process.empty?
|
359
366
|
entry = entries_to_process.shift
|
360
367
|
# process this entry only if it is within the scan_start
|
361
368
|
entry_path_with_slash = entry['path']
|
362
|
-
Log.log.info
|
369
|
+
Log.log.info{"processing entry #{entry_path_with_slash}"} if @periodic.trigger?
|
363
370
|
entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
|
364
371
|
if !scan_start.nil? && !scan_start.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(scan_start)
|
365
|
-
Log.log.debug
|
372
|
+
Log.log.debug{"#{entry['path']} folder (skip start)".bg_red}
|
366
373
|
next
|
367
374
|
end
|
368
|
-
Log.log.debug
|
375
|
+
Log.log.debug{"item:#{entry}"}
|
369
376
|
begin
|
370
377
|
case entry['type']
|
371
378
|
when 'file'
|
@@ -378,15 +385,15 @@ module Aspera
|
|
378
385
|
Log.log.debug('Ignoring link.')
|
379
386
|
when 'folder'
|
380
387
|
if @option_skip_folders.include?(entry['path'])
|
381
|
-
Log.log.debug
|
388
|
+
Log.log.debug{"#{entry['path']} folder (skip list)".bg_red}
|
382
389
|
else
|
383
|
-
Log.log.debug
|
390
|
+
Log.log.debug{"#{entry['path']} folder".green}
|
384
391
|
# get folder content
|
385
392
|
folder_entries = get_folder_entries(entry['id'])
|
386
393
|
# process all items in current folder
|
387
394
|
folder_entries.each do |folder_entry|
|
388
395
|
# add path for older versions of ES
|
389
|
-
if !folder_entry.
|
396
|
+
if !folder_entry.key?('path')
|
390
397
|
folder_entry['path'] = entry_path_with_slash + folder_entry['name']
|
391
398
|
end
|
392
399
|
folder_entry['parent_file_id'] = entry['id']
|
@@ -394,10 +401,10 @@ module Aspera
|
|
394
401
|
end
|
395
402
|
end
|
396
403
|
else
|
397
|
-
Log.log.warn
|
404
|
+
Log.log.warn{"unknown entry type: #{entry['type']}"}
|
398
405
|
end
|
399
406
|
rescue StandardError => e
|
400
|
-
Log.log.warn
|
407
|
+
Log.log.warn{"An error occurred: #{e}, ignoring"}
|
401
408
|
end
|
402
409
|
end
|
403
410
|
end
|
@@ -408,47 +415,49 @@ module Aspera
|
|
408
415
|
command = options.get_next_command(ACTIONS)
|
409
416
|
unless %i[check test].include?(command)
|
410
417
|
# this will use node api
|
411
|
-
@api_node =
|
418
|
+
@api_node = Aspera::Node.new(params: basic_auth_params)
|
412
419
|
@transfer_server_address = URI.parse(@api_node.params[:base_url]).host
|
413
420
|
# get current access key
|
414
421
|
@access_key_self = @api_node.read('access_keys/self')[:data]
|
415
422
|
# TODO: check events is activated here:
|
416
423
|
# note that docroot is good to look at as well
|
417
424
|
node_info = @api_node.read('info')[:data]
|
418
|
-
Log.log.debug
|
425
|
+
Log.log.debug{"root: #{node_info['docroot']}"}
|
419
426
|
@access_remote = @option_file_access.eql?(:remote)
|
420
|
-
Log.log.debug
|
421
|
-
Log.log.debug
|
422
|
-
#TODO: can the previews folder parameter be read from node api ?
|
427
|
+
Log.log.debug{"remote: #{@access_remote}"}
|
428
|
+
Log.log.debug{"access key info: #{@access_key_self}"}
|
429
|
+
# TODO: can the previews folder parameter be read from node api ?
|
423
430
|
@option_skip_folders.push('/' + @option_previews_folder)
|
424
431
|
if @access_remote
|
425
|
-
#
|
426
|
-
@previews_folder_entry = get_folder_entries(@access_key_self['root_file_id'],{name: @option_previews_folder}).first
|
427
|
-
raise CliError,"Folder #{@option_previews_folder} does not exist on node. "\
|
432
|
+
# NOTE: the filter "name", it's why we take the first one
|
433
|
+
@previews_folder_entry = get_folder_entries(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
|
434
|
+
raise CliError, "Folder #{@option_previews_folder} does not exist on node. "\
|
428
435
|
'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
|
429
436
|
else
|
430
437
|
raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
|
431
438
|
@local_storage_root = @access_key_self['storage']['path']
|
432
|
-
#TODO: option to override @local_storage_root='xxx'
|
433
|
-
@local_storage_root = @local_storage_root[
|
434
|
-
#TODO: windows could have "C:" ?
|
439
|
+
# TODO: option to override @local_storage_root='xxx'
|
440
|
+
@local_storage_root = @local_storage_root[PVCL_LOCAL_STORAGE.length..-1] if @local_storage_root.start_with?(PVCL_LOCAL_STORAGE)
|
441
|
+
# TODO: windows could have "C:" ?
|
435
442
|
raise "not local storage: #{@local_storage_root}" unless @local_storage_root.start_with?('/')
|
436
|
-
raise CliError,"Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
|
437
|
-
@local_preview_folder = File.join(@local_storage_root
|
438
|
-
raise CliError,"Folder #{@local_preview_folder} does not exist locally. "\
|
443
|
+
raise CliError, "Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
|
444
|
+
@local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
|
445
|
+
raise CliError, "Folder #{@local_preview_folder} does not exist locally. "\
|
439
446
|
'Please create it, or specify an alternate name.' unless File.directory?(@local_preview_folder)
|
440
447
|
# protection to avoid clash of file id for two different access keys
|
441
|
-
marker_file = File.join(@local_preview_folder,AK_MARKER_FILE)
|
442
|
-
Log.log.debug
|
448
|
+
marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
|
449
|
+
Log.log.debug{"marker file: #{marker_file}"}
|
443
450
|
if File.exist?(marker_file)
|
444
451
|
ak = File.read(marker_file).chomp
|
445
452
|
raise "mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}" unless @access_key_self['id'].eql?(ak)
|
446
453
|
else
|
447
|
-
File.write(marker_file
|
454
|
+
File.write(marker_file, @access_key_self['id'])
|
448
455
|
end
|
449
456
|
end
|
450
457
|
end
|
451
|
-
Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic,is_type: :mandatory)
|
458
|
+
Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, is_type: :mandatory)
|
459
|
+
# check tools that are anyway required for all cases
|
460
|
+
Aspera::Preview::Utils.check_tools(@skip_types)
|
452
461
|
case command
|
453
462
|
when :scan
|
454
463
|
scan_path = options.get_option(:scan_path)
|
@@ -456,39 +465,39 @@ module Aspera
|
|
456
465
|
# by default start at root
|
457
466
|
folder_info =
|
458
467
|
if scan_id.nil?
|
459
|
-
{
|
468
|
+
{
|
469
|
+
'id' => @access_key_self['root_file_id'],
|
460
470
|
'name' => '/',
|
461
471
|
'type' => 'folder',
|
462
472
|
'path' => '/' }
|
463
473
|
else
|
464
474
|
@api_node.read("files/#{scan_id}")[:data]
|
465
475
|
end
|
466
|
-
scan_folder_files(folder_info,scan_path)
|
476
|
+
scan_folder_files(folder_info, scan_path)
|
467
477
|
return Main.result_status('scan finished')
|
468
|
-
when :events
|
478
|
+
when :events, :trevents
|
469
479
|
iteration_persistency = nil
|
470
|
-
if options.get_option(:once_only,is_type: :mandatory)
|
480
|
+
if options.get_option(:once_only, is_type: :mandatory)
|
471
481
|
iteration_persistency = PersistencyActionOnce.new(
|
472
482
|
manager: @agents[:persistency],
|
473
483
|
data: [],
|
474
484
|
id: IdGenerator.from_list([
|
475
485
|
'preview_iteration',
|
476
486
|
command.to_s,
|
477
|
-
options.get_option(:url,is_type: :mandatory),
|
478
|
-
options.get_option(:username,is_type: :mandatory)
|
487
|
+
options.get_option(:url, is_type: :mandatory),
|
488
|
+
options.get_option(:username, is_type: :mandatory)
|
479
489
|
]))
|
480
490
|
end
|
481
491
|
# call processing method specified by command line command
|
482
|
-
send("process_#{command}",iteration_persistency)
|
492
|
+
send("process_#{command}", iteration_persistency)
|
483
493
|
return Main.result_status("#{command} finished")
|
484
494
|
when :check
|
485
|
-
|
486
|
-
return Main.result_status('tools validated')
|
495
|
+
return Main.result_status('Tools validated')
|
487
496
|
when :test
|
488
|
-
format = options.get_next_argument('format',expected: Aspera::Preview::Generator::PREVIEW_FORMATS)
|
497
|
+
format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS)
|
489
498
|
source = options.get_next_argument('source file')
|
490
|
-
dest = preview_filename(format,options.get_option(:case))
|
491
|
-
g = Aspera::Preview::Generator.new(@gen_options,source,dest
|
499
|
+
dest = preview_filename(format, options.get_option(:case))
|
500
|
+
g = Aspera::Preview::Generator.new(@gen_options, source, dest, @tmp_folder, nil)
|
492
501
|
raise "cannot find file type for #{source}" if g.conversion_type.nil?
|
493
502
|
raise "out format #{format} not supported" unless g.supported?
|
494
503
|
g.generate
|