aspera-cli 4.8.0 → 4.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/README.md +445 -160
- data/docs/test_env.conf +1 -5
- data/examples/dascli +1 -4
- data/examples/{transfer.rb → node.rb} +17 -46
- data/examples/server.rb +93 -0
- data/lib/aspera/aoc.rb +4 -2
- data/lib/aspera/ats_api.rb +3 -1
- data/lib/aspera/cli/extended_value.rb +1 -0
- data/lib/aspera/cli/formater.rb +3 -1
- data/lib/aspera/cli/info.rb +1 -1
- data/lib/aspera/cli/main.rb +14 -11
- data/lib/aspera/cli/manager.rb +4 -4
- data/lib/aspera/cli/plugin.rb +50 -9
- data/lib/aspera/cli/plugins/aoc.rb +88 -52
- data/lib/aspera/cli/plugins/config.rb +5 -0
- data/lib/aspera/cli/plugins/faspex.rb +5 -4
- data/lib/aspera/cli/plugins/node.rb +3 -2
- data/lib/aspera/cli/plugins/server.rb +7 -108
- data/lib/aspera/cli/plugins/shares.rb +21 -1
- data/lib/aspera/cli/transfer_agent.rb +21 -14
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/environment.rb +15 -2
- data/lib/aspera/fasp/agent_base.rb +9 -7
- data/lib/aspera/fasp/installation.rb +15 -19
- data/lib/aspera/fasp/parameters.rb +38 -30
- data/lib/aspera/fasp/parameters.yaml +69 -17
- data/lib/aspera/hash_ext.rb +14 -2
- data/lib/aspera/id_generator.rb +12 -10
- data/lib/aspera/keychain/encrypted_hash.rb +3 -3
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/nagios.rb +26 -19
- data/lib/aspera/oauth.rb +4 -4
- data/lib/aspera/open_application.rb +21 -19
- data/lib/aspera/persistency_folder.rb +3 -0
- data/lib/aspera/preview/image_error.png +0 -0
- data/lib/aspera/preview/video_error.png +0 -0
- data/lib/aspera/proxy_auto_config.rb +6 -4
- data/lib/aspera/rest_error_analyzer.rb +15 -13
- data/lib/aspera/rest_errors_aspera.rb +42 -40
- data/lib/aspera/secret_hider.rb +11 -5
- data/lib/aspera/ssh.rb +1 -0
- data/lib/aspera/uri_reader.rb +15 -13
- data.tar.gz.sig +0 -0
- metadata +4 -3
- metadata.gz.sig +0 -0
@@ -55,13 +55,11 @@ module Aspera
|
|
55
55
|
options.add_opt_simple(:name,'resource name')
|
56
56
|
options.add_opt_simple(:path,'file or folder path')
|
57
57
|
options.add_opt_simple(:link,'public link to shared resource')
|
58
|
-
options.add_opt_simple(:new_user_option,'new user creation option')
|
58
|
+
options.add_opt_simple(:new_user_option,'new user creation option for unknown package recipients')
|
59
59
|
options.add_opt_simple(:from_folder,'share to share source folder')
|
60
60
|
options.add_opt_simple(:scope,'OAuth scope for AoC API calls')
|
61
|
-
options.add_opt_boolean(:bulk,'bulk operation')
|
62
61
|
options.add_opt_boolean(:default_ports,'use standard FASP ports or get from node api')
|
63
62
|
options.add_opt_boolean(:validate_metadata,'validate shared inbox metadata')
|
64
|
-
options.set_option(:bulk,:no)
|
65
63
|
options.set_option(:default_ports,:yes)
|
66
64
|
options.set_option(:validate_metadata,:yes)
|
67
65
|
options.set_option(:new_user_option,{'package_contact' => true})
|
@@ -91,7 +89,8 @@ module Aspera
|
|
91
89
|
return transfer.start(*aoc_api.tr_spec(app,direction,node_file,ts_add))
|
92
90
|
end
|
93
91
|
|
94
|
-
|
92
|
+
NODE4_CMD_PATH = %i[bearer_token_node node_info browse find]
|
93
|
+
NODE4_COMMANDS = [NODE4_CMD_PATH,%i[mkdir rename delete upload download transfer http_node_download v3 file]].flatten.freeze
|
95
94
|
|
96
95
|
def execute_node_gen4_command(command_repo,top_node_file)
|
97
96
|
case command_repo
|
@@ -145,7 +144,7 @@ module Aspera
|
|
145
144
|
return Main.result_status("renamed #{thepath} to #{newname}")
|
146
145
|
when :delete
|
147
146
|
thepath = options.get_next_argument('path')
|
148
|
-
return do_bulk_operation(thepath,'deleted','path') do |l_path|
|
147
|
+
return do_bulk_operation(thepath,'deleted',id_result: 'path') do |l_path|
|
149
148
|
raise "expecting String (path), got #{l_path.class.name} (#{l_path})" unless l_path.is_a?(String)
|
150
149
|
node_file = aoc_api.resolve_node_file(top_node_file,l_path)
|
151
150
|
node_api = aoc_api.get_node_api(node_file[:node_info])
|
@@ -232,8 +231,10 @@ module Aspera
|
|
232
231
|
file_path = options.get_option(:path)
|
233
232
|
node_file =
|
234
233
|
if !file_path.nil?
|
234
|
+
# directly returns node_file info
|
235
235
|
aoc_api.resolve_node_file(top_node_file,file_path) # TODO: allow follow link ?
|
236
236
|
else
|
237
|
+
# build node_file info from next argument on command line
|
237
238
|
{node_info: top_node_file[:node_info],file_id: instance_identifier}
|
238
239
|
end
|
239
240
|
node_api = aoc_api.get_node_api(node_file[:node_info])
|
@@ -246,7 +247,7 @@ module Aspera
|
|
246
247
|
res = node_api.update("files/#{node_file[:file_id]}",update_param)[:data]
|
247
248
|
return {type: :single_object,data: res}
|
248
249
|
when :permission
|
249
|
-
command_perm = options.get_next_command(%i[list create])
|
250
|
+
command_perm = options.get_next_command(%i[list create delete])
|
250
251
|
case command_perm
|
251
252
|
when :list
|
252
253
|
# generic options : TODO: as arg ? option_url_query
|
@@ -259,15 +260,20 @@ module Aspera
|
|
259
260
|
end
|
260
261
|
items = node_api.read('permissions',list_options)[:data]
|
261
262
|
return {type: :object_list,data: items}
|
263
|
+
when :delete
|
264
|
+
perm_id=instance_identifier
|
265
|
+
return do_bulk_operation(perm_id,'deleted') do |one_id|
|
266
|
+
node_api.delete("permissions/#{perm_id}")
|
267
|
+
{'id' => one_id}
|
268
|
+
end
|
269
|
+
#node_api.delete("permissions/#{perm_id}")
|
262
270
|
when :create
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
node_file[:node_info]
|
267
|
-
params = {
|
271
|
+
create_param=options.get_next_argument('creation data (Hash)')
|
272
|
+
#access_id = "#{ID_AK_ADMIN}_WS_#{@workspace_info['id']}"
|
273
|
+
default_params = {
|
268
274
|
'file_id' => node_file[:file_id], # mandatory
|
269
|
-
'access_type' => 'user', # mandatory: user or group
|
270
|
-
'access_id' => access_id, # id of user or group
|
275
|
+
#'access_type' => 'user', # mandatory: user or group
|
276
|
+
#'access_id' => access_id, # id of user or group
|
271
277
|
'access_levels' => Aspera::Node::ACCESS_LEVELS,
|
272
278
|
'tags' => {'aspera' => {'files' => {'workspace' => {
|
273
279
|
'id' => @workspace_info['id'],
|
@@ -276,12 +282,39 @@ module Aspera
|
|
276
282
|
'shared_by_user_id' => aoc_api.user_info['id'],
|
277
283
|
'shared_by_name' => aoc_api.user_info['name'],
|
278
284
|
'shared_by_email' => aoc_api.user_info['email'],
|
279
|
-
'shared_with_name' => access_id,
|
285
|
+
#'shared_with_name' => access_id,
|
280
286
|
'access_key' => node_file[:node_info]['access_key'],
|
281
287
|
'node' => node_file[:node_info]['name']}}}}}
|
288
|
+
create_param = default_params.deep_merge(create_param)
|
289
|
+
if create_param.has_key?('with')
|
290
|
+
contact_info = aoc_api.lookup_entity_by_name(
|
291
|
+
'contacts',
|
292
|
+
create_param['with'],
|
293
|
+
{'current_workspace_id' => @workspace_info['id'],'context'=> 'share_folder'})
|
294
|
+
create_param.delete('with')
|
295
|
+
create_param['access_type']=contact_info['source_type']
|
296
|
+
create_param['access_id']=contact_info['source_id']
|
297
|
+
create_param['tags']['aspera']['files']['workspace']['shared_with_name']=contact_info['email']
|
298
|
+
end
|
299
|
+
opt_link_name=nil
|
300
|
+
if create_param.has_key?('link_name')
|
301
|
+
opt_link_name=create_param['link_name']
|
302
|
+
create_param.delete('link_name')
|
303
|
+
end
|
304
|
+
# for admin type:
|
305
|
+
#node_file[:node_info]
|
282
306
|
#node_api = aoc_api.get_node_api(node_file[:node_info])
|
283
|
-
|
284
|
-
|
307
|
+
created_data = node_api.create('permissions',create_param)[:data]
|
308
|
+
event_creation={
|
309
|
+
'types' => ['permission.created'],
|
310
|
+
'node_id' => node_file[:node_info]['id'],
|
311
|
+
'workspace_id' => @workspace_info['id'],
|
312
|
+
'data' => created_data # Response from previous step
|
313
|
+
}
|
314
|
+
#(optional). The name of the folder to be displayed to the destination user. Use it if its value is different from the "share_as" field.
|
315
|
+
event_creation['link_name']=opt_link_name unless opt_link_name.nil?
|
316
|
+
aoc_api.create('events',event_creation)
|
317
|
+
return { type: :single_object, data: created_data}
|
285
318
|
else raise "internal error:shall not reach here (#{command_perm})"
|
286
319
|
end
|
287
320
|
else raise "internal error:shall not reach here (#{command_node_file})"
|
@@ -346,7 +379,7 @@ module Aspera
|
|
346
379
|
end
|
347
380
|
home_node_id ||= @workspace_info['home_node_id'] || @workspace_info['node_id']
|
348
381
|
home_file_id ||= @workspace_info['home_file_id']
|
349
|
-
raise
|
382
|
+
raise "Cannot get user's home node id, check your default workspace or specify one" if home_node_id.to_s.empty?
|
350
383
|
@home_node_file = {
|
351
384
|
node_info: aoc_api.read("nodes/#{home_node_id}")[:data],
|
352
385
|
file_id: home_file_id
|
@@ -356,25 +389,6 @@ module Aspera
|
|
356
389
|
return nil
|
357
390
|
end
|
358
391
|
|
359
|
-
def do_bulk_operation(ids_or_one,success_msg,id_result='id')
|
360
|
-
raise 'missing block' unless block_given?
|
361
|
-
ids_or_one = [ids_or_one] unless options.get_option(:bulk)
|
362
|
-
raise 'expecting Array' unless ids_or_one.is_a?(Array)
|
363
|
-
result_list = []
|
364
|
-
ids_or_one.each do |id|
|
365
|
-
one = {id_result => id}
|
366
|
-
begin
|
367
|
-
res = yield(id)
|
368
|
-
one = res if id.is_a?(Hash) # if block returns a has, let's use this
|
369
|
-
one['status'] = success_msg
|
370
|
-
rescue StandardError => e
|
371
|
-
one['status'] = e.to_s
|
372
|
-
end
|
373
|
-
result_list.push(one)
|
374
|
-
end
|
375
|
-
return {type: :object_list,data: result_list,fields: [id_result,'status']}
|
376
|
-
end
|
377
|
-
|
378
392
|
# get identifier or name from command line
|
379
393
|
# @return identifier
|
380
394
|
def get_resource_id_from_args(resource_class_path)
|
@@ -395,7 +409,12 @@ module Aspera
|
|
395
409
|
return "#{resource_class_path}/#{get_resource_id_from_args(resource_class_path)}"
|
396
410
|
end
|
397
411
|
|
398
|
-
# package creation
|
412
|
+
# Normalize package creation recipient lists as expected by AoC API
|
413
|
+
# AoC expects {type: , id: }, but ascli allows providing either the native values or just a name
|
414
|
+
# in that case, the name is resolved and replaced with {type: , id: }
|
415
|
+
# @param package_data The whole package creation payload
|
416
|
+
# @param recipient_list_field The field in structure, i.e. recipients or bcc_recipients
|
417
|
+
# @return nil package_data is modified
|
399
418
|
def resolve_package_recipients(package_data,recipient_list_field)
|
400
419
|
return unless package_data.has_key?(recipient_list_field)
|
401
420
|
raise CliBadArgument,"#{recipient_list_field} must be an Array" unless package_data[recipient_list_field].is_a?(Array)
|
@@ -404,17 +423,21 @@ module Aspera
|
|
404
423
|
resolved_list = []
|
405
424
|
package_data[recipient_list_field].each do |short_recipient_info|
|
406
425
|
case short_recipient_info
|
407
|
-
when Hash # native
|
426
|
+
when Hash # native API information, check keys
|
408
427
|
raise "#{recipient_list_field} element shall have fields: id and type" unless short_recipient_info.keys.sort.eql?(%w[id type])
|
409
|
-
when String # need to resolve name to type/id
|
428
|
+
when String # CLI helper: need to resolve provided name to type/id
|
410
429
|
# email: user, else dropbox
|
411
430
|
entity_type = short_recipient_info.include?('@') ? 'contacts' : 'dropboxes'
|
412
431
|
begin
|
413
432
|
full_recipient_info = aoc_api.lookup_entity_by_name(entity_type,short_recipient_info,{'current_workspace_id' => @workspace_info['id']})
|
414
433
|
rescue RuntimeError => e
|
415
434
|
raise e unless e.message.start_with?(Aspera::AoC::ENTITY_NOT_FOUND)
|
416
|
-
|
417
|
-
|
435
|
+
# dropboxes cannot be created on the fly
|
436
|
+
raise "no such shared inbox in workspace #{@workspace_info['name']}" if entity_type.eql?('dropboxes')
|
437
|
+
# unknown user: create it as external user
|
438
|
+
full_recipient_info = aoc_api.create('contacts',{
|
439
|
+
'current_workspace_id' => @workspace_info['id'],
|
440
|
+
'email' => short_recipient_info}.merge(new_user_option))[:data]
|
418
441
|
end
|
419
442
|
short_recipient_info = if entity_type.eql?('dropboxes')
|
420
443
|
{'id' => full_recipient_info['id'],'type' => 'dropbox'}
|
@@ -429,6 +452,7 @@ module Aspera
|
|
429
452
|
end
|
430
453
|
# replace with resolved elements
|
431
454
|
package_data[recipient_list_field] = resolved_list
|
455
|
+
return nil
|
432
456
|
end
|
433
457
|
|
434
458
|
def normalize_metadata(pkg_data)
|
@@ -503,6 +527,7 @@ module Aspera
|
|
503
527
|
|
504
528
|
# Call aoc_api.read with same parameters.
|
505
529
|
# Use paging if necessary to get all results
|
530
|
+
# @return {list: , total: }
|
506
531
|
def read_with_paging(resource_class_path,base_query)
|
507
532
|
raise 'Query must be Hash' unless base_query.is_a?(Hash)
|
508
533
|
# set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
|
@@ -530,7 +555,7 @@ module Aspera
|
|
530
555
|
break if !max_pages.nil? && page_count > max_pages
|
531
556
|
break if !max_items.nil? && item_list.count > max_items
|
532
557
|
end
|
533
|
-
return item_list,total_count
|
558
|
+
return {list: item_list,total: total_count}
|
534
559
|
end
|
535
560
|
|
536
561
|
def execute_admin_action
|
@@ -688,7 +713,7 @@ module Aspera
|
|
688
713
|
# TODO: report inconsistency: creation url is !=, and does not return id.
|
689
714
|
resource_class_path = 'admin/client_registration/token' if resource_class_path.eql?('admin/client_registration_tokens')
|
690
715
|
list_or_one = options.get_next_argument('creation data (Hash)')
|
691
|
-
return do_bulk_operation(list_or_one,'created',id_result) do |params|
|
716
|
+
return do_bulk_operation(list_or_one,'created',id_result: id_result) do |params|
|
692
717
|
raise 'expecting Hash' unless params.is_a?(Hash)
|
693
718
|
aoc_api.create(resource_class_path,params)[:data]
|
694
719
|
end
|
@@ -708,11 +733,11 @@ module Aspera
|
|
708
733
|
when :group_membership then default_fields.push(*%w[group_id member_type member_id])
|
709
734
|
when :workspace_membership then default_fields.push(*%w[workspace_id member_type member_id])
|
710
735
|
end
|
711
|
-
|
712
|
-
count_msg = "Items: #{
|
713
|
-
count_msg = count_msg.bg_red unless
|
736
|
+
items = read_with_paging(resource_class_path,option_url_query(default_query))
|
737
|
+
count_msg = "Items: #{items[:list].length}/#{items[:total]}"
|
738
|
+
count_msg = count_msg.bg_red unless items[:list].length.eql?(items[:total].to_i)
|
714
739
|
self.format.display_status(count_msg)
|
715
|
-
return {type: :object_list,data:
|
740
|
+
return {type: :object_list,data: items[:list],fields: default_fields}
|
716
741
|
when :show
|
717
742
|
object = aoc_api.read(resource_instance_path)[:data]
|
718
743
|
fields = object.keys.reject{|k|k.eql?('certificate')}
|
@@ -740,9 +765,9 @@ module Aspera
|
|
740
765
|
command_repo = options.get_next_command(NODE4_COMMANDS)
|
741
766
|
return execute_node_gen4_command(command_repo,{node_info: res_data, file_id: ak_root_file_id})
|
742
767
|
when :shared_folder
|
743
|
-
Log.log.warn('ATTENTION: under development')
|
768
|
+
Log.log.warn('ATTENTION: alpha, under development, do not use')
|
744
769
|
# inside a workspace
|
745
|
-
command_shared = options.get_next_command(%i[list create
|
770
|
+
command_shared = options.get_next_command(%i[list create delete]) # member
|
746
771
|
core_api=Rest.new(aoc_api.params.merge(base_url: aoc_api.params[:base_url].gsub('/api.','/sedemo.')))
|
747
772
|
# generic permission created for each shared folder
|
748
773
|
access_id = "#{ID_AK_ADMIN}_WS_#{res_id}"
|
@@ -787,7 +812,7 @@ module Aspera
|
|
787
812
|
# remove from creation data if present, as it is not a standard argument
|
788
813
|
shared_create_data.delete('node_id')
|
789
814
|
# get optional link_name
|
790
|
-
opt_link_name=shared_create_data['link_name']
|
815
|
+
#opt_link_name=shared_create_data['link_name']
|
791
816
|
shared_create_data.delete('link_name')
|
792
817
|
raise 'missing node information: path' unless shared_create_data.has_key?('path')
|
793
818
|
folder_path=shared_create_data['path']
|
@@ -815,6 +840,7 @@ module Aspera
|
|
815
840
|
}}}}
|
816
841
|
shared_create_data = default_create_data.deep_merge(default_create_data) # ?aspera-node-basic=#{node_id}&aspera-node-prefer-basic=#{node_id}
|
817
842
|
created_data = node_api.create('permissions',shared_create_data)[:data]
|
843
|
+
# new API:
|
818
844
|
#created_data=aoc_api.create("node/#{node_id}/permissions",shared_create_data)[:data]
|
819
845
|
# TODO: send event
|
820
846
|
event_creation={
|
@@ -824,7 +850,7 @@ module Aspera
|
|
824
850
|
'data' => created_data # Response from previous step
|
825
851
|
}
|
826
852
|
#(optional). The name of the folder to be displayed to the destination user. Use it if its value is different from the "share_as" field.
|
827
|
-
event_creation['link_name']=opt_link_name unless opt_link_name.nil?
|
853
|
+
#event_creation['link_name']=opt_link_name unless opt_link_name.nil?
|
828
854
|
aoc_api.create('events',event_creation)
|
829
855
|
return { type: :single_object, data: created_data}
|
830
856
|
end
|
@@ -877,7 +903,8 @@ module Aspera
|
|
877
903
|
end
|
878
904
|
when :packages
|
879
905
|
set_workspace_info if @url_token_data.nil?
|
880
|
-
|
906
|
+
package_command = options.get_next_command([%i[shared_inboxes send recv list show delete],NODE4_CMD_PATH].flatten)
|
907
|
+
case package_command
|
881
908
|
when :shared_inboxes
|
882
909
|
case options.get_next_command(%i[list show])
|
883
910
|
when :list
|
@@ -1004,6 +1031,15 @@ module Aspera
|
|
1004
1031
|
raise 'expecting String identifier' unless id.is_a?(String) || id.is_a?(Integer)
|
1005
1032
|
aoc_api.delete("packages/#{id}")[:data]
|
1006
1033
|
end
|
1034
|
+
when *NODE4_CMD_PATH
|
1035
|
+
package_id = options.get_next_argument('package ID')
|
1036
|
+
#path = options.get_next_argument('path', mandatory: false) || '/'
|
1037
|
+
package_info = aoc_api.read("packages/#{package_id}")[:data]
|
1038
|
+
package_node_file = {
|
1039
|
+
node_info: aoc_api.read("nodes/#{package_info['node_id']}")[:data],
|
1040
|
+
file_id: package_info['file_id']
|
1041
|
+
}
|
1042
|
+
return execute_node_gen4_command(package_command,package_node_file)
|
1007
1043
|
end
|
1008
1044
|
when :files
|
1009
1045
|
# get workspace related information
|
@@ -251,7 +251,9 @@ module Aspera
|
|
251
251
|
require 'openssl'
|
252
252
|
priv_key = OpenSSL::PKey::RSA.new(length)
|
253
253
|
File.write(private_key_path,priv_key.to_s)
|
254
|
+
File.chmod(0400,private_key_path)
|
254
255
|
File.write(private_key_path + '.pub',priv_key.public_key.to_s)
|
256
|
+
File.chmod(0400,private_key_path + '.pub')
|
255
257
|
nil
|
256
258
|
end
|
257
259
|
|
@@ -773,6 +775,7 @@ module Aspera
|
|
773
775
|
result = {type: :other_struct, data: options.get_next_argument('value')}
|
774
776
|
# special for csv
|
775
777
|
result[:type] = :object_list if result[:data].is_a?(Array) && result[:data].first.is_a?(Hash)
|
778
|
+
result[:type] = :single_object if result[:data].is_a?(Hash)
|
776
779
|
return result
|
777
780
|
when :flush_tokens
|
778
781
|
deleted_files = Oauth.flush_tokens
|
@@ -1020,8 +1023,10 @@ module Aspera
|
|
1020
1023
|
def save_presets_to_config_file
|
1021
1024
|
raise 'no configuration loaded' if @config_presets.nil?
|
1022
1025
|
FileUtils.mkdir_p(@main_folder) unless Dir.exist?(@main_folder)
|
1026
|
+
File.chmod(0700,@main_folder)
|
1023
1027
|
Log.log.debug("Writing #{@option_config_file}")
|
1024
1028
|
File.write(@option_config_file,@config_presets.to_yaml)
|
1029
|
+
File.chmod(0600,@option_config_file)
|
1025
1030
|
end
|
1026
1031
|
|
1027
1032
|
# returns [String] name if config_presets has default
|
@@ -326,9 +326,6 @@ module Aspera
|
|
326
326
|
recipient = options.get_option(:recipient)
|
327
327
|
if delivid.eql?(VAL_ALL)
|
328
328
|
pkg_id_uri = mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD],uri: self.class.get_fasp_uri_from_entry(i, raise_no_link: false)}}
|
329
|
-
# TODO : remove ids from skip not present in inbox to avoid growing too big
|
330
|
-
# skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
|
331
|
-
pkg_id_uri.reject!{|i|skip_ids_data.include?(i[:id])}
|
332
329
|
elsif !recipient.nil? && recipient.start_with?('*')
|
333
330
|
found_package_link = mailbox_filtered_entries(stop_at_id: delivid).find{|p|p[PACKAGE_MATCH_FIELD].eql?(delivid)}['link'].first['href']
|
334
331
|
raise 'Not Found. Dropbox and Workgroup packages can use the link option with faspe:' if found_package_link.nil?
|
@@ -367,8 +364,12 @@ module Aspera
|
|
367
364
|
transfer_uri = self.class.get_fasp_uri_from_entry(package_entry)
|
368
365
|
pkg_id_uri = [{id: package_entry['id'],uri: transfer_uri}]
|
369
366
|
end # public link
|
367
|
+
# prune packages already downloaded
|
368
|
+
# TODO : remove ids from skip not present in inbox to avoid growing too big
|
369
|
+
# skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
|
370
|
+
pkg_id_uri.reject!{|i|skip_ids_data.include?(i[:id])}
|
370
371
|
Log.dump(:pkg_id_uri,pkg_id_uri)
|
371
|
-
return Main.result_status('no package') if pkg_id_uri.empty?
|
372
|
+
return Main.result_status('no new package') if pkg_id_uri.empty?
|
372
373
|
result_transfer = []
|
373
374
|
pkg_id_uri.each do |id_uri|
|
374
375
|
if id_uri[:uri].nil?
|
@@ -274,6 +274,7 @@ module Aspera
|
|
274
274
|
# @return {.api,.file_id}
|
275
275
|
def resolve_api_fid(id, path)
|
276
276
|
# TODO: implement
|
277
|
+
Log.log.debug("TODO #{path}")
|
277
278
|
return {api: @api_node, file_id: id}
|
278
279
|
end
|
279
280
|
|
@@ -569,8 +570,8 @@ module Aspera
|
|
569
570
|
# change API if needed
|
570
571
|
if !access_key.eql?('self')
|
571
572
|
secret=config.vault.get(username: access_key)[:secret] #, url: @api_node.params[:base_url] : TODO: better handle vault
|
572
|
-
@api_node.params[:username]=access_key
|
573
|
-
@api_node.params[:password]=secret
|
573
|
+
@api_node.params[:auth][:username]=access_key
|
574
|
+
@api_node.params[:auth][:password]=secret
|
574
575
|
end
|
575
576
|
command_repo = options.get_next_command(NODE4_COMMANDS)
|
576
577
|
return execute_node_gen4_command(command_repo,ak_info['root_file_id'])
|
@@ -21,83 +21,18 @@ module Aspera
|
|
21
21
|
|
22
22
|
def initialize(env)
|
23
23
|
super(env)
|
24
|
-
options.add_opt_simple(:ssh_keys,'
|
25
|
-
options.add_opt_simple(:ssh_options,'
|
26
|
-
options.add_opt_simple(:cmd_prefix,'prefix to add for as cmd execution, e.g. sudo or /opt/aspera/bin ')
|
24
|
+
options.add_opt_simple(:ssh_keys,'SSH key path list (Array or single)')
|
25
|
+
options.add_opt_simple(:ssh_options,'SSH options (Hash)')
|
27
26
|
options.set_option(:ssh_keys,[])
|
28
27
|
options.set_option(:ssh_options,{})
|
29
28
|
options.parse_options!
|
30
29
|
end
|
31
30
|
|
32
|
-
def key_symb_to_str_single(source)
|
33
|
-
return source.each_with_object({}){|(k,v),memo| memo[k.to_s] = v; }
|
34
|
-
end
|
35
|
-
|
36
31
|
def key_symb_to_str_list(source)
|
37
|
-
return source.map
|
32
|
+
return source.map(&:stringify_keys)
|
38
33
|
end
|
39
34
|
|
40
|
-
|
41
|
-
# normal separator
|
42
|
-
r = /:\s*/
|
43
|
-
result = []
|
44
|
-
text.split("\n").each do |line|
|
45
|
-
# console: missing space
|
46
|
-
line.gsub!(/(SessionDataCollector)/,'\1 ')
|
47
|
-
# orchestrator
|
48
|
-
line.gsub!(/ with pid:.*/,'')
|
49
|
-
line.gsub!(/ is /,': ')
|
50
|
-
items = line.split(r)
|
51
|
-
next unless items.length.eql?(2)
|
52
|
-
state = {'process' => items.first,'state' => items.last}
|
53
|
-
# console
|
54
|
-
state['state'].gsub!(/\.+$/,'')
|
55
|
-
# console
|
56
|
-
state['process'].gsub!(/^.+::/,'')
|
57
|
-
# faspex
|
58
|
-
state['process'].gsub!(/^Faspex /,'')
|
59
|
-
# faspex
|
60
|
-
state['process'].gsub!(/ Background/,'')
|
61
|
-
state['process'].gsub!(/serving orchestrator on port /,'')
|
62
|
-
# console
|
63
|
-
r = /\s+/ if state['process'].eql?('Console')
|
64
|
-
# orchestrator
|
65
|
-
state['process'].gsub!(/^ -> /,'')
|
66
|
-
state['process'].gsub!(/ Process/,'')
|
67
|
-
result.push(state)
|
68
|
-
end
|
69
|
-
return result
|
70
|
-
end
|
71
|
-
|
72
|
-
def asconfigurator_parse(result)
|
73
|
-
lines = result.split("\n")
|
74
|
-
# not windows
|
75
|
-
Log.log.debug(%x(type asconfigurator))
|
76
|
-
status=lines.shift
|
77
|
-
case status
|
78
|
-
when 'success'
|
79
|
-
result = []
|
80
|
-
lines.each do |line|
|
81
|
-
Log.log.debug(line.to_s)
|
82
|
-
# normalize values
|
83
|
-
data = line.
|
84
|
-
gsub(/^"/,'').
|
85
|
-
gsub(/,""$/,',"AS_EMPTY"').
|
86
|
-
gsub(/"$/,'').
|
87
|
-
split('","').
|
88
|
-
map{|i|case i;when 'AS_NULL' then nil;when 'AS_EMPTY' then '';when 'true' then true;when 'false' then false;else i;end}
|
89
|
-
data.insert(1,'') if data.length.eql?(4)
|
90
|
-
titles=%w[level section parameter value default]
|
91
|
-
result.push(titles.each_with_object({}){|t,o|o[t]=data.shift})
|
92
|
-
end
|
93
|
-
return result
|
94
|
-
when 'failure'
|
95
|
-
raise lines.join("\n")
|
96
|
-
else raise "Unexpected: #{status}"
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
|
-
ACTIONS = %i[health nodeadmin userdata configurator ctl download upload browse delete rename].concat(Aspera::AsCmd::OPERATIONS).freeze
|
35
|
+
ACTIONS = %i[health download upload browse delete rename].concat(Aspera::AsCmd::OPERATIONS).freeze
|
101
36
|
|
102
37
|
def execute_action
|
103
38
|
server_uri = URI.parse(options.get_option(:url,is_type: :mandatory))
|
@@ -120,6 +55,7 @@ module Aspera
|
|
120
55
|
server_transfer_spec['remote_user'] = options.get_option(:username,is_type: :mandatory)
|
121
56
|
ssh_options = options.get_option(:ssh_options)
|
122
57
|
raise 'expecting a Hash for ssh_options' unless ssh_options.is_a?(Hash)
|
58
|
+
ssh_options = ssh_options.symbolize_keys
|
123
59
|
if !server_uri.port.nil?
|
124
60
|
ssh_options[:port] = server_uri.port
|
125
61
|
server_transfer_spec['ssh_port'] = server_uri.port
|
@@ -160,17 +96,8 @@ module Aspera
|
|
160
96
|
case command
|
161
97
|
when :health
|
162
98
|
nagios = Nagios.new
|
163
|
-
command_nagios = options.get_next_command(%i[
|
99
|
+
command_nagios = options.get_next_command(%i[transfer])
|
164
100
|
case command_nagios
|
165
|
-
when :app_services
|
166
|
-
# will not work with aspshell, requires Linux/bash
|
167
|
-
procs = shell_executor.execute('ps -A -o comm').split("\n")
|
168
|
-
Log.log.debug("found: #{procs}")
|
169
|
-
%w[asperanoded asperaredisd].each do |name|
|
170
|
-
nagios.add_critical('general',"missing process #{name}") unless procs.include?(name)
|
171
|
-
end
|
172
|
-
nagios.add_ok('daemons','ok') if nagios.data.empty?
|
173
|
-
return nagios.result
|
174
101
|
when :transfer
|
175
102
|
file = Tempfile.new('transfer_test')
|
176
103
|
filepath = file.path
|
@@ -189,37 +116,9 @@ module Aspera
|
|
189
116
|
else
|
190
117
|
nagios.add_critical('transfer',statuses.reject{|i|i.eql?(:success)}.first.to_s)
|
191
118
|
end
|
192
|
-
when :asctlstatus
|
193
|
-
realcmd = 'asctl'
|
194
|
-
prefix = options.get_option(:cmd_prefix)
|
195
|
-
realcmd = "#{prefix}#{realcmd} all:status" unless prefix.nil?
|
196
|
-
result = shell_executor.execute(realcmd.split)
|
197
|
-
data = asctl_parse(result)
|
198
|
-
data.each do |i|
|
199
|
-
if i['state'].eql?('running')
|
200
|
-
nagios.add_ok(i['process'],i['state'])
|
201
|
-
else
|
202
|
-
nagios.add_critical(i['process'],i['state'])
|
203
|
-
end
|
204
|
-
end
|
205
119
|
else raise 'ERROR'
|
206
120
|
end
|
207
121
|
return nagios.result
|
208
|
-
when :nodeadmin,:userdata,:configurator,:ctl
|
209
|
-
realcmd = "as#{command}"
|
210
|
-
prefix = options.get_option(:cmd_prefix)
|
211
|
-
realcmd = "#{prefix}#{realcmd}" unless prefix.nil?
|
212
|
-
args = options.get_next_argument("#{realcmd} arguments",expected: :multiple)
|
213
|
-
args.unshift('-x') if command.eql?(:configurator)
|
214
|
-
result = shell_executor.execute(args.unshift(realcmd))
|
215
|
-
case command
|
216
|
-
when :ctl
|
217
|
-
return {type: :object_list,data: asctl_parse(result)}
|
218
|
-
when :configurator
|
219
|
-
result=asconfigurator_parse(result)
|
220
|
-
return {type: :object_list,data: result} # ,option_expand_last: true
|
221
|
-
end
|
222
|
-
return Main.result_status(result)
|
223
122
|
when :upload
|
224
123
|
return Main.result_transfer(transfer.start(server_transfer_spec.merge('direction' => Fasp::TransferSpec::DIRECTION_SEND),{src: :direct}))
|
225
124
|
when :download
|
@@ -233,7 +132,7 @@ module Aspera
|
|
233
132
|
when :mkdir,:mv,:cp,:rm then return Main.result_success
|
234
133
|
when :ls then return {type: :object_list,data: key_symb_to_str_list(result),fields: %w[zmode zuid zgid size mtime name]}
|
235
134
|
when :df then return {type: :object_list,data: key_symb_to_str_list(result)}
|
236
|
-
when :du,:md5sum,:info then return {type: :single_object,data:
|
135
|
+
when :du,:md5sum,:info then return {type: :single_object,data: result.stringify_keys}
|
237
136
|
end
|
238
137
|
rescue Aspera::AsCmd::Error => e
|
239
138
|
raise CliBadArgument,e.extended_message
|
@@ -26,6 +26,9 @@ module Aspera
|
|
26
26
|
# super(env)
|
27
27
|
# end
|
28
28
|
|
29
|
+
SAML_IMPORT_MANDATORY=%w[id name_id].freeze
|
30
|
+
SAML_IMPORT_ALLOWED=[SAML_IMPORT_MANDATORY,%w[email given_name surname]].flatten.freeze
|
31
|
+
|
29
32
|
ACTIONS = %i[health repository admin].freeze
|
30
33
|
|
31
34
|
def execute_action
|
@@ -58,7 +61,7 @@ module Aspera
|
|
58
61
|
command = options.get_next_command(%i[user share])
|
59
62
|
case command
|
60
63
|
when :user
|
61
|
-
command = options.get_next_command(%i[list app_authorizations share_permissions])
|
64
|
+
command = options.get_next_command(%i[list app_authorizations share_permissions saml_import ldap_import])
|
62
65
|
user_id = instance_identifier if %i[app_authorizations share_permissions].include?(command)
|
63
66
|
case command
|
64
67
|
when :list
|
@@ -70,6 +73,23 @@ module Aspera
|
|
70
73
|
#all_shares = api_shares_admin.read('data/shares')[:data]
|
71
74
|
#share_id = all_shares.find{|s| s['name'].eql?(share_name)}['id']
|
72
75
|
return {type: :object_list,data: api_shares_admin.read("data/users/#{user_id}/share_permissions")[:data]}
|
76
|
+
when :saml_import
|
77
|
+
parameters = options.get_option(:value)
|
78
|
+
return do_bulk_operation(parameters,'created') do |user_params|
|
79
|
+
user_params=user_params.transform_keys{|k|k.gsub(/\s+/,'_').downcase}
|
80
|
+
raise 'expecting Hash' unless user_params.is_a?(Hash)
|
81
|
+
SAML_IMPORT_MANDATORY.each{|p|raise "missing mandatory field: #{p}" if user_params[p].nil?}
|
82
|
+
user_params.keys.each do |p|
|
83
|
+
raise "unsupported field: #{p}, use: #{SAML_IMPORT_ALLOWED.join(',')}" unless SAML_IMPORT_ALLOWED.include?(p)
|
84
|
+
end
|
85
|
+
api_shares_admin.create('data/saml_users/import',user_params)[:data]
|
86
|
+
end
|
87
|
+
when :ldap_import
|
88
|
+
parameters = options.get_option(:value)
|
89
|
+
return do_bulk_operation(parameters,'created') do |user_name|
|
90
|
+
raise 'expecting string (user name), have #{user_params.class}' unless user_params.is_a?(String)
|
91
|
+
api_shares_admin.create('data/ldap_users',{'user'=>user_name})[:data]
|
92
|
+
end
|
73
93
|
end
|
74
94
|
when :share
|
75
95
|
command = options.get_next_command(%i[list user_permissions])
|