aspera-cli 4.8.0 → 4.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/README.md +445 -160
- data/docs/test_env.conf +1 -5
- data/examples/dascli +1 -4
- data/examples/{transfer.rb → node.rb} +17 -46
- data/examples/server.rb +93 -0
- data/lib/aspera/aoc.rb +4 -2
- data/lib/aspera/ats_api.rb +3 -1
- data/lib/aspera/cli/extended_value.rb +1 -0
- data/lib/aspera/cli/formater.rb +3 -1
- data/lib/aspera/cli/info.rb +1 -1
- data/lib/aspera/cli/main.rb +14 -11
- data/lib/aspera/cli/manager.rb +4 -4
- data/lib/aspera/cli/plugin.rb +50 -9
- data/lib/aspera/cli/plugins/aoc.rb +88 -52
- data/lib/aspera/cli/plugins/config.rb +5 -0
- data/lib/aspera/cli/plugins/faspex.rb +5 -4
- data/lib/aspera/cli/plugins/node.rb +3 -2
- data/lib/aspera/cli/plugins/server.rb +7 -108
- data/lib/aspera/cli/plugins/shares.rb +21 -1
- data/lib/aspera/cli/transfer_agent.rb +21 -14
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/environment.rb +15 -2
- data/lib/aspera/fasp/agent_base.rb +9 -7
- data/lib/aspera/fasp/installation.rb +15 -19
- data/lib/aspera/fasp/parameters.rb +38 -30
- data/lib/aspera/fasp/parameters.yaml +69 -17
- data/lib/aspera/hash_ext.rb +14 -2
- data/lib/aspera/id_generator.rb +12 -10
- data/lib/aspera/keychain/encrypted_hash.rb +3 -3
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/nagios.rb +26 -19
- data/lib/aspera/oauth.rb +4 -4
- data/lib/aspera/open_application.rb +21 -19
- data/lib/aspera/persistency_folder.rb +3 -0
- data/lib/aspera/preview/image_error.png +0 -0
- data/lib/aspera/preview/video_error.png +0 -0
- data/lib/aspera/proxy_auto_config.rb +6 -4
- data/lib/aspera/rest_error_analyzer.rb +15 -13
- data/lib/aspera/rest_errors_aspera.rb +42 -40
- data/lib/aspera/secret_hider.rb +11 -5
- data/lib/aspera/ssh.rb +1 -0
- data/lib/aspera/uri_reader.rb +15 -13
- data.tar.gz.sig +0 -0
- metadata +4 -3
- metadata.gz.sig +0 -0
@@ -55,13 +55,11 @@ module Aspera
|
|
55
55
|
options.add_opt_simple(:name,'resource name')
|
56
56
|
options.add_opt_simple(:path,'file or folder path')
|
57
57
|
options.add_opt_simple(:link,'public link to shared resource')
|
58
|
-
options.add_opt_simple(:new_user_option,'new user creation option')
|
58
|
+
options.add_opt_simple(:new_user_option,'new user creation option for unknown package recipients')
|
59
59
|
options.add_opt_simple(:from_folder,'share to share source folder')
|
60
60
|
options.add_opt_simple(:scope,'OAuth scope for AoC API calls')
|
61
|
-
options.add_opt_boolean(:bulk,'bulk operation')
|
62
61
|
options.add_opt_boolean(:default_ports,'use standard FASP ports or get from node api')
|
63
62
|
options.add_opt_boolean(:validate_metadata,'validate shared inbox metadata')
|
64
|
-
options.set_option(:bulk,:no)
|
65
63
|
options.set_option(:default_ports,:yes)
|
66
64
|
options.set_option(:validate_metadata,:yes)
|
67
65
|
options.set_option(:new_user_option,{'package_contact' => true})
|
@@ -91,7 +89,8 @@ module Aspera
|
|
91
89
|
return transfer.start(*aoc_api.tr_spec(app,direction,node_file,ts_add))
|
92
90
|
end
|
93
91
|
|
94
|
-
|
92
|
+
NODE4_CMD_PATH = %i[bearer_token_node node_info browse find]
|
93
|
+
NODE4_COMMANDS = [NODE4_CMD_PATH,%i[mkdir rename delete upload download transfer http_node_download v3 file]].flatten.freeze
|
95
94
|
|
96
95
|
def execute_node_gen4_command(command_repo,top_node_file)
|
97
96
|
case command_repo
|
@@ -145,7 +144,7 @@ module Aspera
|
|
145
144
|
return Main.result_status("renamed #{thepath} to #{newname}")
|
146
145
|
when :delete
|
147
146
|
thepath = options.get_next_argument('path')
|
148
|
-
return do_bulk_operation(thepath,'deleted','path') do |l_path|
|
147
|
+
return do_bulk_operation(thepath,'deleted',id_result: 'path') do |l_path|
|
149
148
|
raise "expecting String (path), got #{l_path.class.name} (#{l_path})" unless l_path.is_a?(String)
|
150
149
|
node_file = aoc_api.resolve_node_file(top_node_file,l_path)
|
151
150
|
node_api = aoc_api.get_node_api(node_file[:node_info])
|
@@ -232,8 +231,10 @@ module Aspera
|
|
232
231
|
file_path = options.get_option(:path)
|
233
232
|
node_file =
|
234
233
|
if !file_path.nil?
|
234
|
+
# directly returns node_file info
|
235
235
|
aoc_api.resolve_node_file(top_node_file,file_path) # TODO: allow follow link ?
|
236
236
|
else
|
237
|
+
# build node_file info from next argument on command line
|
237
238
|
{node_info: top_node_file[:node_info],file_id: instance_identifier}
|
238
239
|
end
|
239
240
|
node_api = aoc_api.get_node_api(node_file[:node_info])
|
@@ -246,7 +247,7 @@ module Aspera
|
|
246
247
|
res = node_api.update("files/#{node_file[:file_id]}",update_param)[:data]
|
247
248
|
return {type: :single_object,data: res}
|
248
249
|
when :permission
|
249
|
-
command_perm = options.get_next_command(%i[list create])
|
250
|
+
command_perm = options.get_next_command(%i[list create delete])
|
250
251
|
case command_perm
|
251
252
|
when :list
|
252
253
|
# generic options : TODO: as arg ? option_url_query
|
@@ -259,15 +260,20 @@ module Aspera
|
|
259
260
|
end
|
260
261
|
items = node_api.read('permissions',list_options)[:data]
|
261
262
|
return {type: :object_list,data: items}
|
263
|
+
when :delete
|
264
|
+
perm_id=instance_identifier
|
265
|
+
return do_bulk_operation(perm_id,'deleted') do |one_id|
|
266
|
+
node_api.delete("permissions/#{perm_id}")
|
267
|
+
{'id' => one_id}
|
268
|
+
end
|
269
|
+
#node_api.delete("permissions/#{perm_id}")
|
262
270
|
when :create
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
node_file[:node_info]
|
267
|
-
params = {
|
271
|
+
create_param=options.get_next_argument('creation data (Hash)')
|
272
|
+
#access_id = "#{ID_AK_ADMIN}_WS_#{@workspace_info['id']}"
|
273
|
+
default_params = {
|
268
274
|
'file_id' => node_file[:file_id], # mandatory
|
269
|
-
'access_type' => 'user', # mandatory: user or group
|
270
|
-
'access_id' => access_id, # id of user or group
|
275
|
+
#'access_type' => 'user', # mandatory: user or group
|
276
|
+
#'access_id' => access_id, # id of user or group
|
271
277
|
'access_levels' => Aspera::Node::ACCESS_LEVELS,
|
272
278
|
'tags' => {'aspera' => {'files' => {'workspace' => {
|
273
279
|
'id' => @workspace_info['id'],
|
@@ -276,12 +282,39 @@ module Aspera
|
|
276
282
|
'shared_by_user_id' => aoc_api.user_info['id'],
|
277
283
|
'shared_by_name' => aoc_api.user_info['name'],
|
278
284
|
'shared_by_email' => aoc_api.user_info['email'],
|
279
|
-
'shared_with_name' => access_id,
|
285
|
+
#'shared_with_name' => access_id,
|
280
286
|
'access_key' => node_file[:node_info]['access_key'],
|
281
287
|
'node' => node_file[:node_info]['name']}}}}}
|
288
|
+
create_param = default_params.deep_merge(create_param)
|
289
|
+
if create_param.has_key?('with')
|
290
|
+
contact_info = aoc_api.lookup_entity_by_name(
|
291
|
+
'contacts',
|
292
|
+
create_param['with'],
|
293
|
+
{'current_workspace_id' => @workspace_info['id'],'context'=> 'share_folder'})
|
294
|
+
create_param.delete('with')
|
295
|
+
create_param['access_type']=contact_info['source_type']
|
296
|
+
create_param['access_id']=contact_info['source_id']
|
297
|
+
create_param['tags']['aspera']['files']['workspace']['shared_with_name']=contact_info['email']
|
298
|
+
end
|
299
|
+
opt_link_name=nil
|
300
|
+
if create_param.has_key?('link_name')
|
301
|
+
opt_link_name=create_param['link_name']
|
302
|
+
create_param.delete('link_name')
|
303
|
+
end
|
304
|
+
# for admin type:
|
305
|
+
#node_file[:node_info]
|
282
306
|
#node_api = aoc_api.get_node_api(node_file[:node_info])
|
283
|
-
|
284
|
-
|
307
|
+
created_data = node_api.create('permissions',create_param)[:data]
|
308
|
+
event_creation={
|
309
|
+
'types' => ['permission.created'],
|
310
|
+
'node_id' => node_file[:node_info]['id'],
|
311
|
+
'workspace_id' => @workspace_info['id'],
|
312
|
+
'data' => created_data # Response from previous step
|
313
|
+
}
|
314
|
+
#(optional). The name of the folder to be displayed to the destination user. Use it if its value is different from the "share_as" field.
|
315
|
+
event_creation['link_name']=opt_link_name unless opt_link_name.nil?
|
316
|
+
aoc_api.create('events',event_creation)
|
317
|
+
return { type: :single_object, data: created_data}
|
285
318
|
else raise "internal error:shall not reach here (#{command_perm})"
|
286
319
|
end
|
287
320
|
else raise "internal error:shall not reach here (#{command_node_file})"
|
@@ -346,7 +379,7 @@ module Aspera
|
|
346
379
|
end
|
347
380
|
home_node_id ||= @workspace_info['home_node_id'] || @workspace_info['node_id']
|
348
381
|
home_file_id ||= @workspace_info['home_file_id']
|
349
|
-
raise
|
382
|
+
raise "Cannot get user's home node id, check your default workspace or specify one" if home_node_id.to_s.empty?
|
350
383
|
@home_node_file = {
|
351
384
|
node_info: aoc_api.read("nodes/#{home_node_id}")[:data],
|
352
385
|
file_id: home_file_id
|
@@ -356,25 +389,6 @@ module Aspera
|
|
356
389
|
return nil
|
357
390
|
end
|
358
391
|
|
359
|
-
def do_bulk_operation(ids_or_one,success_msg,id_result='id')
|
360
|
-
raise 'missing block' unless block_given?
|
361
|
-
ids_or_one = [ids_or_one] unless options.get_option(:bulk)
|
362
|
-
raise 'expecting Array' unless ids_or_one.is_a?(Array)
|
363
|
-
result_list = []
|
364
|
-
ids_or_one.each do |id|
|
365
|
-
one = {id_result => id}
|
366
|
-
begin
|
367
|
-
res = yield(id)
|
368
|
-
one = res if id.is_a?(Hash) # if block returns a has, let's use this
|
369
|
-
one['status'] = success_msg
|
370
|
-
rescue StandardError => e
|
371
|
-
one['status'] = e.to_s
|
372
|
-
end
|
373
|
-
result_list.push(one)
|
374
|
-
end
|
375
|
-
return {type: :object_list,data: result_list,fields: [id_result,'status']}
|
376
|
-
end
|
377
|
-
|
378
392
|
# get identifier or name from command line
|
379
393
|
# @return identifier
|
380
394
|
def get_resource_id_from_args(resource_class_path)
|
@@ -395,7 +409,12 @@ module Aspera
|
|
395
409
|
return "#{resource_class_path}/#{get_resource_id_from_args(resource_class_path)}"
|
396
410
|
end
|
397
411
|
|
398
|
-
# package creation
|
412
|
+
# Normalize package creation recipient lists as expected by AoC API
|
413
|
+
# AoC expects {type: , id: }, but ascli allows providing either the native values or just a name
|
414
|
+
# in that case, the name is resolved and replaced with {type: , id: }
|
415
|
+
# @param package_data The whole package creation payload
|
416
|
+
# @param recipient_list_field The field in structure, i.e. recipients or bcc_recipients
|
417
|
+
# @return nil package_data is modified
|
399
418
|
def resolve_package_recipients(package_data,recipient_list_field)
|
400
419
|
return unless package_data.has_key?(recipient_list_field)
|
401
420
|
raise CliBadArgument,"#{recipient_list_field} must be an Array" unless package_data[recipient_list_field].is_a?(Array)
|
@@ -404,17 +423,21 @@ module Aspera
|
|
404
423
|
resolved_list = []
|
405
424
|
package_data[recipient_list_field].each do |short_recipient_info|
|
406
425
|
case short_recipient_info
|
407
|
-
when Hash # native
|
426
|
+
when Hash # native API information, check keys
|
408
427
|
raise "#{recipient_list_field} element shall have fields: id and type" unless short_recipient_info.keys.sort.eql?(%w[id type])
|
409
|
-
when String # need to resolve name to type/id
|
428
|
+
when String # CLI helper: need to resolve provided name to type/id
|
410
429
|
# email: user, else dropbox
|
411
430
|
entity_type = short_recipient_info.include?('@') ? 'contacts' : 'dropboxes'
|
412
431
|
begin
|
413
432
|
full_recipient_info = aoc_api.lookup_entity_by_name(entity_type,short_recipient_info,{'current_workspace_id' => @workspace_info['id']})
|
414
433
|
rescue RuntimeError => e
|
415
434
|
raise e unless e.message.start_with?(Aspera::AoC::ENTITY_NOT_FOUND)
|
416
|
-
|
417
|
-
|
435
|
+
# dropboxes cannot be created on the fly
|
436
|
+
raise "no such shared inbox in workspace #{@workspace_info['name']}" if entity_type.eql?('dropboxes')
|
437
|
+
# unknown user: create it as external user
|
438
|
+
full_recipient_info = aoc_api.create('contacts',{
|
439
|
+
'current_workspace_id' => @workspace_info['id'],
|
440
|
+
'email' => short_recipient_info}.merge(new_user_option))[:data]
|
418
441
|
end
|
419
442
|
short_recipient_info = if entity_type.eql?('dropboxes')
|
420
443
|
{'id' => full_recipient_info['id'],'type' => 'dropbox'}
|
@@ -429,6 +452,7 @@ module Aspera
|
|
429
452
|
end
|
430
453
|
# replace with resolved elements
|
431
454
|
package_data[recipient_list_field] = resolved_list
|
455
|
+
return nil
|
432
456
|
end
|
433
457
|
|
434
458
|
def normalize_metadata(pkg_data)
|
@@ -503,6 +527,7 @@ module Aspera
|
|
503
527
|
|
504
528
|
# Call aoc_api.read with same parameters.
|
505
529
|
# Use paging if necessary to get all results
|
530
|
+
# @return {list: , total: }
|
506
531
|
def read_with_paging(resource_class_path,base_query)
|
507
532
|
raise 'Query must be Hash' unless base_query.is_a?(Hash)
|
508
533
|
# set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
|
@@ -530,7 +555,7 @@ module Aspera
|
|
530
555
|
break if !max_pages.nil? && page_count > max_pages
|
531
556
|
break if !max_items.nil? && item_list.count > max_items
|
532
557
|
end
|
533
|
-
return item_list,total_count
|
558
|
+
return {list: item_list,total: total_count}
|
534
559
|
end
|
535
560
|
|
536
561
|
def execute_admin_action
|
@@ -688,7 +713,7 @@ module Aspera
|
|
688
713
|
# TODO: report inconsistency: creation url is !=, and does not return id.
|
689
714
|
resource_class_path = 'admin/client_registration/token' if resource_class_path.eql?('admin/client_registration_tokens')
|
690
715
|
list_or_one = options.get_next_argument('creation data (Hash)')
|
691
|
-
return do_bulk_operation(list_or_one,'created',id_result) do |params|
|
716
|
+
return do_bulk_operation(list_or_one,'created',id_result: id_result) do |params|
|
692
717
|
raise 'expecting Hash' unless params.is_a?(Hash)
|
693
718
|
aoc_api.create(resource_class_path,params)[:data]
|
694
719
|
end
|
@@ -708,11 +733,11 @@ module Aspera
|
|
708
733
|
when :group_membership then default_fields.push(*%w[group_id member_type member_id])
|
709
734
|
when :workspace_membership then default_fields.push(*%w[workspace_id member_type member_id])
|
710
735
|
end
|
711
|
-
|
712
|
-
count_msg = "Items: #{
|
713
|
-
count_msg = count_msg.bg_red unless
|
736
|
+
items = read_with_paging(resource_class_path,option_url_query(default_query))
|
737
|
+
count_msg = "Items: #{items[:list].length}/#{items[:total]}"
|
738
|
+
count_msg = count_msg.bg_red unless items[:list].length.eql?(items[:total].to_i)
|
714
739
|
self.format.display_status(count_msg)
|
715
|
-
return {type: :object_list,data:
|
740
|
+
return {type: :object_list,data: items[:list],fields: default_fields}
|
716
741
|
when :show
|
717
742
|
object = aoc_api.read(resource_instance_path)[:data]
|
718
743
|
fields = object.keys.reject{|k|k.eql?('certificate')}
|
@@ -740,9 +765,9 @@ module Aspera
|
|
740
765
|
command_repo = options.get_next_command(NODE4_COMMANDS)
|
741
766
|
return execute_node_gen4_command(command_repo,{node_info: res_data, file_id: ak_root_file_id})
|
742
767
|
when :shared_folder
|
743
|
-
Log.log.warn('ATTENTION: under development')
|
768
|
+
Log.log.warn('ATTENTION: alpha, under development, do not use')
|
744
769
|
# inside a workspace
|
745
|
-
command_shared = options.get_next_command(%i[list create
|
770
|
+
command_shared = options.get_next_command(%i[list create delete]) # member
|
746
771
|
core_api=Rest.new(aoc_api.params.merge(base_url: aoc_api.params[:base_url].gsub('/api.','/sedemo.')))
|
747
772
|
# generic permission created for each shared folder
|
748
773
|
access_id = "#{ID_AK_ADMIN}_WS_#{res_id}"
|
@@ -787,7 +812,7 @@ module Aspera
|
|
787
812
|
# remove from creation data if present, as it is not a standard argument
|
788
813
|
shared_create_data.delete('node_id')
|
789
814
|
# get optional link_name
|
790
|
-
opt_link_name=shared_create_data['link_name']
|
815
|
+
#opt_link_name=shared_create_data['link_name']
|
791
816
|
shared_create_data.delete('link_name')
|
792
817
|
raise 'missing node information: path' unless shared_create_data.has_key?('path')
|
793
818
|
folder_path=shared_create_data['path']
|
@@ -815,6 +840,7 @@ module Aspera
|
|
815
840
|
}}}}
|
816
841
|
shared_create_data = default_create_data.deep_merge(default_create_data) # ?aspera-node-basic=#{node_id}&aspera-node-prefer-basic=#{node_id}
|
817
842
|
created_data = node_api.create('permissions',shared_create_data)[:data]
|
843
|
+
# new API:
|
818
844
|
#created_data=aoc_api.create("node/#{node_id}/permissions",shared_create_data)[:data]
|
819
845
|
# TODO: send event
|
820
846
|
event_creation={
|
@@ -824,7 +850,7 @@ module Aspera
|
|
824
850
|
'data' => created_data # Response from previous step
|
825
851
|
}
|
826
852
|
#(optional). The name of the folder to be displayed to the destination user. Use it if its value is different from the "share_as" field.
|
827
|
-
event_creation['link_name']=opt_link_name unless opt_link_name.nil?
|
853
|
+
#event_creation['link_name']=opt_link_name unless opt_link_name.nil?
|
828
854
|
aoc_api.create('events',event_creation)
|
829
855
|
return { type: :single_object, data: created_data}
|
830
856
|
end
|
@@ -877,7 +903,8 @@ module Aspera
|
|
877
903
|
end
|
878
904
|
when :packages
|
879
905
|
set_workspace_info if @url_token_data.nil?
|
880
|
-
|
906
|
+
package_command = options.get_next_command([%i[shared_inboxes send recv list show delete],NODE4_CMD_PATH].flatten)
|
907
|
+
case package_command
|
881
908
|
when :shared_inboxes
|
882
909
|
case options.get_next_command(%i[list show])
|
883
910
|
when :list
|
@@ -1004,6 +1031,15 @@ module Aspera
|
|
1004
1031
|
raise 'expecting String identifier' unless id.is_a?(String) || id.is_a?(Integer)
|
1005
1032
|
aoc_api.delete("packages/#{id}")[:data]
|
1006
1033
|
end
|
1034
|
+
when *NODE4_CMD_PATH
|
1035
|
+
package_id = options.get_next_argument('package ID')
|
1036
|
+
#path = options.get_next_argument('path', mandatory: false) || '/'
|
1037
|
+
package_info = aoc_api.read("packages/#{package_id}")[:data]
|
1038
|
+
package_node_file = {
|
1039
|
+
node_info: aoc_api.read("nodes/#{package_info['node_id']}")[:data],
|
1040
|
+
file_id: package_info['file_id']
|
1041
|
+
}
|
1042
|
+
return execute_node_gen4_command(package_command,package_node_file)
|
1007
1043
|
end
|
1008
1044
|
when :files
|
1009
1045
|
# get workspace related information
|
@@ -251,7 +251,9 @@ module Aspera
|
|
251
251
|
require 'openssl'
|
252
252
|
priv_key = OpenSSL::PKey::RSA.new(length)
|
253
253
|
File.write(private_key_path,priv_key.to_s)
|
254
|
+
File.chmod(0400,private_key_path)
|
254
255
|
File.write(private_key_path + '.pub',priv_key.public_key.to_s)
|
256
|
+
File.chmod(0400,private_key_path + '.pub')
|
255
257
|
nil
|
256
258
|
end
|
257
259
|
|
@@ -773,6 +775,7 @@ module Aspera
|
|
773
775
|
result = {type: :other_struct, data: options.get_next_argument('value')}
|
774
776
|
# special for csv
|
775
777
|
result[:type] = :object_list if result[:data].is_a?(Array) && result[:data].first.is_a?(Hash)
|
778
|
+
result[:type] = :single_object if result[:data].is_a?(Hash)
|
776
779
|
return result
|
777
780
|
when :flush_tokens
|
778
781
|
deleted_files = Oauth.flush_tokens
|
@@ -1020,8 +1023,10 @@ module Aspera
|
|
1020
1023
|
def save_presets_to_config_file
|
1021
1024
|
raise 'no configuration loaded' if @config_presets.nil?
|
1022
1025
|
FileUtils.mkdir_p(@main_folder) unless Dir.exist?(@main_folder)
|
1026
|
+
File.chmod(0700,@main_folder)
|
1023
1027
|
Log.log.debug("Writing #{@option_config_file}")
|
1024
1028
|
File.write(@option_config_file,@config_presets.to_yaml)
|
1029
|
+
File.chmod(0600,@option_config_file)
|
1025
1030
|
end
|
1026
1031
|
|
1027
1032
|
# returns [String] name if config_presets has default
|
@@ -326,9 +326,6 @@ module Aspera
|
|
326
326
|
recipient = options.get_option(:recipient)
|
327
327
|
if delivid.eql?(VAL_ALL)
|
328
328
|
pkg_id_uri = mailbox_filtered_entries.map{|i|{id: i[PACKAGE_MATCH_FIELD],uri: self.class.get_fasp_uri_from_entry(i, raise_no_link: false)}}
|
329
|
-
# TODO : remove ids from skip not present in inbox to avoid growing too big
|
330
|
-
# skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
|
331
|
-
pkg_id_uri.reject!{|i|skip_ids_data.include?(i[:id])}
|
332
329
|
elsif !recipient.nil? && recipient.start_with?('*')
|
333
330
|
found_package_link = mailbox_filtered_entries(stop_at_id: delivid).find{|p|p[PACKAGE_MATCH_FIELD].eql?(delivid)}['link'].first['href']
|
334
331
|
raise 'Not Found. Dropbox and Workgroup packages can use the link option with faspe:' if found_package_link.nil?
|
@@ -367,8 +364,12 @@ module Aspera
|
|
367
364
|
transfer_uri = self.class.get_fasp_uri_from_entry(package_entry)
|
368
365
|
pkg_id_uri = [{id: package_entry['id'],uri: transfer_uri}]
|
369
366
|
end # public link
|
367
|
+
# prune packages already downloaded
|
368
|
+
# TODO : remove ids from skip not present in inbox to avoid growing too big
|
369
|
+
# skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
|
370
|
+
pkg_id_uri.reject!{|i|skip_ids_data.include?(i[:id])}
|
370
371
|
Log.dump(:pkg_id_uri,pkg_id_uri)
|
371
|
-
return Main.result_status('no package') if pkg_id_uri.empty?
|
372
|
+
return Main.result_status('no new package') if pkg_id_uri.empty?
|
372
373
|
result_transfer = []
|
373
374
|
pkg_id_uri.each do |id_uri|
|
374
375
|
if id_uri[:uri].nil?
|
@@ -274,6 +274,7 @@ module Aspera
|
|
274
274
|
# @return {.api,.file_id}
|
275
275
|
def resolve_api_fid(id, path)
|
276
276
|
# TODO: implement
|
277
|
+
Log.log.debug("TODO #{path}")
|
277
278
|
return {api: @api_node, file_id: id}
|
278
279
|
end
|
279
280
|
|
@@ -569,8 +570,8 @@ module Aspera
|
|
569
570
|
# change API if needed
|
570
571
|
if !access_key.eql?('self')
|
571
572
|
secret=config.vault.get(username: access_key)[:secret] #, url: @api_node.params[:base_url] : TODO: better handle vault
|
572
|
-
@api_node.params[:username]=access_key
|
573
|
-
@api_node.params[:password]=secret
|
573
|
+
@api_node.params[:auth][:username]=access_key
|
574
|
+
@api_node.params[:auth][:password]=secret
|
574
575
|
end
|
575
576
|
command_repo = options.get_next_command(NODE4_COMMANDS)
|
576
577
|
return execute_node_gen4_command(command_repo,ak_info['root_file_id'])
|
@@ -21,83 +21,18 @@ module Aspera
|
|
21
21
|
|
22
22
|
def initialize(env)
|
23
23
|
super(env)
|
24
|
-
options.add_opt_simple(:ssh_keys,'
|
25
|
-
options.add_opt_simple(:ssh_options,'
|
26
|
-
options.add_opt_simple(:cmd_prefix,'prefix to add for as cmd execution, e.g. sudo or /opt/aspera/bin ')
|
24
|
+
options.add_opt_simple(:ssh_keys,'SSH key path list (Array or single)')
|
25
|
+
options.add_opt_simple(:ssh_options,'SSH options (Hash)')
|
27
26
|
options.set_option(:ssh_keys,[])
|
28
27
|
options.set_option(:ssh_options,{})
|
29
28
|
options.parse_options!
|
30
29
|
end
|
31
30
|
|
32
|
-
def key_symb_to_str_single(source)
|
33
|
-
return source.each_with_object({}){|(k,v),memo| memo[k.to_s] = v; }
|
34
|
-
end
|
35
|
-
|
36
31
|
def key_symb_to_str_list(source)
|
37
|
-
return source.map
|
32
|
+
return source.map(&:stringify_keys)
|
38
33
|
end
|
39
34
|
|
40
|
-
|
41
|
-
# normal separator
|
42
|
-
r = /:\s*/
|
43
|
-
result = []
|
44
|
-
text.split("\n").each do |line|
|
45
|
-
# console: missing space
|
46
|
-
line.gsub!(/(SessionDataCollector)/,'\1 ')
|
47
|
-
# orchestrator
|
48
|
-
line.gsub!(/ with pid:.*/,'')
|
49
|
-
line.gsub!(/ is /,': ')
|
50
|
-
items = line.split(r)
|
51
|
-
next unless items.length.eql?(2)
|
52
|
-
state = {'process' => items.first,'state' => items.last}
|
53
|
-
# console
|
54
|
-
state['state'].gsub!(/\.+$/,'')
|
55
|
-
# console
|
56
|
-
state['process'].gsub!(/^.+::/,'')
|
57
|
-
# faspex
|
58
|
-
state['process'].gsub!(/^Faspex /,'')
|
59
|
-
# faspex
|
60
|
-
state['process'].gsub!(/ Background/,'')
|
61
|
-
state['process'].gsub!(/serving orchestrator on port /,'')
|
62
|
-
# console
|
63
|
-
r = /\s+/ if state['process'].eql?('Console')
|
64
|
-
# orchestrator
|
65
|
-
state['process'].gsub!(/^ -> /,'')
|
66
|
-
state['process'].gsub!(/ Process/,'')
|
67
|
-
result.push(state)
|
68
|
-
end
|
69
|
-
return result
|
70
|
-
end
|
71
|
-
|
72
|
-
def asconfigurator_parse(result)
|
73
|
-
lines = result.split("\n")
|
74
|
-
# not windows
|
75
|
-
Log.log.debug(%x(type asconfigurator))
|
76
|
-
status=lines.shift
|
77
|
-
case status
|
78
|
-
when 'success'
|
79
|
-
result = []
|
80
|
-
lines.each do |line|
|
81
|
-
Log.log.debug(line.to_s)
|
82
|
-
# normalize values
|
83
|
-
data = line.
|
84
|
-
gsub(/^"/,'').
|
85
|
-
gsub(/,""$/,',"AS_EMPTY"').
|
86
|
-
gsub(/"$/,'').
|
87
|
-
split('","').
|
88
|
-
map{|i|case i;when 'AS_NULL' then nil;when 'AS_EMPTY' then '';when 'true' then true;when 'false' then false;else i;end}
|
89
|
-
data.insert(1,'') if data.length.eql?(4)
|
90
|
-
titles=%w[level section parameter value default]
|
91
|
-
result.push(titles.each_with_object({}){|t,o|o[t]=data.shift})
|
92
|
-
end
|
93
|
-
return result
|
94
|
-
when 'failure'
|
95
|
-
raise lines.join("\n")
|
96
|
-
else raise "Unexpected: #{status}"
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
|
-
ACTIONS = %i[health nodeadmin userdata configurator ctl download upload browse delete rename].concat(Aspera::AsCmd::OPERATIONS).freeze
|
35
|
+
ACTIONS = %i[health download upload browse delete rename].concat(Aspera::AsCmd::OPERATIONS).freeze
|
101
36
|
|
102
37
|
def execute_action
|
103
38
|
server_uri = URI.parse(options.get_option(:url,is_type: :mandatory))
|
@@ -120,6 +55,7 @@ module Aspera
|
|
120
55
|
server_transfer_spec['remote_user'] = options.get_option(:username,is_type: :mandatory)
|
121
56
|
ssh_options = options.get_option(:ssh_options)
|
122
57
|
raise 'expecting a Hash for ssh_options' unless ssh_options.is_a?(Hash)
|
58
|
+
ssh_options = ssh_options.symbolize_keys
|
123
59
|
if !server_uri.port.nil?
|
124
60
|
ssh_options[:port] = server_uri.port
|
125
61
|
server_transfer_spec['ssh_port'] = server_uri.port
|
@@ -160,17 +96,8 @@ module Aspera
|
|
160
96
|
case command
|
161
97
|
when :health
|
162
98
|
nagios = Nagios.new
|
163
|
-
command_nagios = options.get_next_command(%i[
|
99
|
+
command_nagios = options.get_next_command(%i[transfer])
|
164
100
|
case command_nagios
|
165
|
-
when :app_services
|
166
|
-
# will not work with aspshell, requires Linux/bash
|
167
|
-
procs = shell_executor.execute('ps -A -o comm').split("\n")
|
168
|
-
Log.log.debug("found: #{procs}")
|
169
|
-
%w[asperanoded asperaredisd].each do |name|
|
170
|
-
nagios.add_critical('general',"missing process #{name}") unless procs.include?(name)
|
171
|
-
end
|
172
|
-
nagios.add_ok('daemons','ok') if nagios.data.empty?
|
173
|
-
return nagios.result
|
174
101
|
when :transfer
|
175
102
|
file = Tempfile.new('transfer_test')
|
176
103
|
filepath = file.path
|
@@ -189,37 +116,9 @@ module Aspera
|
|
189
116
|
else
|
190
117
|
nagios.add_critical('transfer',statuses.reject{|i|i.eql?(:success)}.first.to_s)
|
191
118
|
end
|
192
|
-
when :asctlstatus
|
193
|
-
realcmd = 'asctl'
|
194
|
-
prefix = options.get_option(:cmd_prefix)
|
195
|
-
realcmd = "#{prefix}#{realcmd} all:status" unless prefix.nil?
|
196
|
-
result = shell_executor.execute(realcmd.split)
|
197
|
-
data = asctl_parse(result)
|
198
|
-
data.each do |i|
|
199
|
-
if i['state'].eql?('running')
|
200
|
-
nagios.add_ok(i['process'],i['state'])
|
201
|
-
else
|
202
|
-
nagios.add_critical(i['process'],i['state'])
|
203
|
-
end
|
204
|
-
end
|
205
119
|
else raise 'ERROR'
|
206
120
|
end
|
207
121
|
return nagios.result
|
208
|
-
when :nodeadmin,:userdata,:configurator,:ctl
|
209
|
-
realcmd = "as#{command}"
|
210
|
-
prefix = options.get_option(:cmd_prefix)
|
211
|
-
realcmd = "#{prefix}#{realcmd}" unless prefix.nil?
|
212
|
-
args = options.get_next_argument("#{realcmd} arguments",expected: :multiple)
|
213
|
-
args.unshift('-x') if command.eql?(:configurator)
|
214
|
-
result = shell_executor.execute(args.unshift(realcmd))
|
215
|
-
case command
|
216
|
-
when :ctl
|
217
|
-
return {type: :object_list,data: asctl_parse(result)}
|
218
|
-
when :configurator
|
219
|
-
result=asconfigurator_parse(result)
|
220
|
-
return {type: :object_list,data: result} # ,option_expand_last: true
|
221
|
-
end
|
222
|
-
return Main.result_status(result)
|
223
122
|
when :upload
|
224
123
|
return Main.result_transfer(transfer.start(server_transfer_spec.merge('direction' => Fasp::TransferSpec::DIRECTION_SEND),{src: :direct}))
|
225
124
|
when :download
|
@@ -233,7 +132,7 @@ module Aspera
|
|
233
132
|
when :mkdir,:mv,:cp,:rm then return Main.result_success
|
234
133
|
when :ls then return {type: :object_list,data: key_symb_to_str_list(result),fields: %w[zmode zuid zgid size mtime name]}
|
235
134
|
when :df then return {type: :object_list,data: key_symb_to_str_list(result)}
|
236
|
-
when :du,:md5sum,:info then return {type: :single_object,data:
|
135
|
+
when :du,:md5sum,:info then return {type: :single_object,data: result.stringify_keys}
|
237
136
|
end
|
238
137
|
rescue Aspera::AsCmd::Error => e
|
239
138
|
raise CliBadArgument,e.extended_message
|
@@ -26,6 +26,9 @@ module Aspera
|
|
26
26
|
# super(env)
|
27
27
|
# end
|
28
28
|
|
29
|
+
SAML_IMPORT_MANDATORY=%w[id name_id].freeze
|
30
|
+
SAML_IMPORT_ALLOWED=[SAML_IMPORT_MANDATORY,%w[email given_name surname]].flatten.freeze
|
31
|
+
|
29
32
|
ACTIONS = %i[health repository admin].freeze
|
30
33
|
|
31
34
|
def execute_action
|
@@ -58,7 +61,7 @@ module Aspera
|
|
58
61
|
command = options.get_next_command(%i[user share])
|
59
62
|
case command
|
60
63
|
when :user
|
61
|
-
command = options.get_next_command(%i[list app_authorizations share_permissions])
|
64
|
+
command = options.get_next_command(%i[list app_authorizations share_permissions saml_import ldap_import])
|
62
65
|
user_id = instance_identifier if %i[app_authorizations share_permissions].include?(command)
|
63
66
|
case command
|
64
67
|
when :list
|
@@ -70,6 +73,23 @@ module Aspera
|
|
70
73
|
#all_shares = api_shares_admin.read('data/shares')[:data]
|
71
74
|
#share_id = all_shares.find{|s| s['name'].eql?(share_name)}['id']
|
72
75
|
return {type: :object_list,data: api_shares_admin.read("data/users/#{user_id}/share_permissions")[:data]}
|
76
|
+
when :saml_import
|
77
|
+
parameters = options.get_option(:value)
|
78
|
+
return do_bulk_operation(parameters,'created') do |user_params|
|
79
|
+
user_params=user_params.transform_keys{|k|k.gsub(/\s+/,'_').downcase}
|
80
|
+
raise 'expecting Hash' unless user_params.is_a?(Hash)
|
81
|
+
SAML_IMPORT_MANDATORY.each{|p|raise "missing mandatory field: #{p}" if user_params[p].nil?}
|
82
|
+
user_params.keys.each do |p|
|
83
|
+
raise "unsupported field: #{p}, use: #{SAML_IMPORT_ALLOWED.join(',')}" unless SAML_IMPORT_ALLOWED.include?(p)
|
84
|
+
end
|
85
|
+
api_shares_admin.create('data/saml_users/import',user_params)[:data]
|
86
|
+
end
|
87
|
+
when :ldap_import
|
88
|
+
parameters = options.get_option(:value)
|
89
|
+
return do_bulk_operation(parameters,'created') do |user_name|
|
90
|
+
raise 'expecting string (user name), have #{user_params.class}' unless user_params.is_a?(String)
|
91
|
+
api_shares_admin.create('data/ldap_users',{'user'=>user_name})[:data]
|
92
|
+
end
|
73
93
|
end
|
74
94
|
when :share
|
75
95
|
command = options.get_next_command(%i[list user_permissions])
|