aspera-cli 4.10.0 → 4.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +20 -0
- data/CHANGELOG.md +509 -0
- data/CONTRIBUTING.md +118 -0
- data/README.md +621 -378
- data/bin/ascli +4 -4
- data/bin/asession +11 -11
- data/docs/test_env.conf +28 -19
- data/examples/aoc.rb +4 -4
- data/examples/dascli +11 -9
- data/examples/faspex4.rb +8 -8
- data/examples/node.rb +11 -11
- data/examples/server.rb +9 -9
- data/lib/aspera/aoc.rb +273 -266
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formater.rb +64 -64
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +14 -19
- data/lib/aspera/cli/main.rb +66 -67
- data/lib/aspera/cli/manager.rb +110 -110
- data/lib/aspera/cli/plugin.rb +54 -37
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +308 -669
- data/lib/aspera/cli/plugins/ats.rb +44 -46
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +447 -344
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +110 -112
- data/lib/aspera/cli/plugins/faspex5.rb +67 -46
- data/lib/aspera/cli/plugins/node.rb +364 -288
- data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
- data/lib/aspera/cli/plugins/preview.rb +122 -114
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +30 -29
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +57 -57
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +27 -27
- data/lib/aspera/cos_node.rb +22 -20
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +15 -15
- data/lib/aspera/fasp/agent_connect.rb +23 -21
- data/lib/aspera/fasp/agent_direct.rb +65 -67
- data/lib/aspera/fasp/agent_httpgw.rb +72 -68
- data/lib/aspera/fasp/agent_node.rb +23 -21
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +78 -78
- data/lib/aspera/fasp/listener.rb +1 -1
- data/lib/aspera/fasp/parameters.rb +75 -72
- data/lib/aspera/fasp/parameters.yaml +2 -2
- data/lib/aspera/fasp/resume_policy.rb +8 -8
- data/lib/aspera/fasp/transfer_spec.rb +35 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +7 -7
- data/lib/aspera/nagios.rb +19 -18
- data/lib/aspera/node.rb +209 -35
- data/lib/aspera/oauth.rb +37 -36
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +41 -41
- data/lib/aspera/proxy_auto_config.rb +16 -16
- data/lib/aspera/rest.rb +56 -60
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +18 -17
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +15 -13
- data/lib/aspera/ssh.rb +11 -10
- data/lib/aspera/sync.rb +158 -44
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +14 -13
- data.tar.gz.sig +0 -0
- metadata +8 -5
- metadata.gz.sig +0 -0
@@ -6,26 +6,26 @@ require 'aspera/ats_api'
|
|
6
6
|
module Aspera
|
7
7
|
module Cli
|
8
8
|
module Plugins
|
9
|
-
#
|
9
|
+
# Access Aspera Transfer Service
|
10
10
|
# https://52.44.83.163/docs/
|
11
11
|
# https://developer.ibm.com/aspera/docs/ats-api-reference/creating-ats-api-keys/
|
12
|
-
class Ats < Plugin
|
12
|
+
class Ats < Aspera::Cli::Plugin
|
13
13
|
def initialize(env)
|
14
14
|
super(env)
|
15
|
-
options.add_opt_simple(:ibm_api_key,'IBM API key, see https://cloud.ibm.com/iam/apikeys')
|
16
|
-
options.add_opt_simple(:instance,'ATS instance in ibm cloud')
|
17
|
-
options.add_opt_simple(:ats_key,'ATS key identifier (ats_xxx)')
|
18
|
-
options.add_opt_simple(:ats_secret,'ATS key secret')
|
19
|
-
options.add_opt_simple(:params,'Parameters access key creation (@json:)')
|
20
|
-
options.add_opt_simple(:cloud,'Cloud provider')
|
21
|
-
options.add_opt_simple(:region,'Cloud region')
|
15
|
+
options.add_opt_simple(:ibm_api_key, 'IBM API key, see https://cloud.ibm.com/iam/apikeys')
|
16
|
+
options.add_opt_simple(:instance, 'ATS instance in ibm cloud')
|
17
|
+
options.add_opt_simple(:ats_key, 'ATS key identifier (ats_xxx)')
|
18
|
+
options.add_opt_simple(:ats_secret, 'ATS key secret')
|
19
|
+
options.add_opt_simple(:params, 'Parameters access key creation (@json:)')
|
20
|
+
options.add_opt_simple(:cloud, 'Cloud provider')
|
21
|
+
options.add_opt_simple(:region, 'Cloud region')
|
22
22
|
options.parse_options!
|
23
23
|
end
|
24
24
|
|
25
25
|
def server_by_cloud_region
|
26
|
-
#
|
27
|
-
cloud = options.get_option(:cloud,is_type: :mandatory).upcase
|
28
|
-
region = options.get_option(:region,is_type: :mandatory)
|
26
|
+
# TODO: provide list ?
|
27
|
+
cloud = options.get_option(:cloud, is_type: :mandatory).upcase
|
28
|
+
region = options.get_option(:region, is_type: :mandatory)
|
29
29
|
return @ats_api_pub.read("servers/#{cloud}/#{region}")[:data]
|
30
30
|
end
|
31
31
|
|
@@ -36,8 +36,8 @@ module Aspera
|
|
36
36
|
base_url: AtsApi.base_url + '/pub/v1',
|
37
37
|
auth: {
|
38
38
|
type: :basic,
|
39
|
-
username: options.get_option(:ats_key,is_type: :mandatory),
|
40
|
-
password: options.get_option(:ats_secret,is_type: :mandatory)}
|
39
|
+
username: options.get_option(:ats_key, is_type: :mandatory),
|
40
|
+
password: options.get_option(:ats_secret, is_type: :mandatory)}
|
41
41
|
})
|
42
42
|
end
|
43
43
|
|
@@ -51,12 +51,12 @@ module Aspera
|
|
51
51
|
params = options.get_option(:params) || {}
|
52
52
|
server_data = nil
|
53
53
|
# if transfer_server_id not provided, get it from command line options
|
54
|
-
if !params.
|
54
|
+
if !params.key?('transfer_server_id')
|
55
55
|
server_data = server_by_cloud_region
|
56
56
|
params['transfer_server_id'] = server_data['id']
|
57
57
|
end
|
58
|
-
Log.log.debug
|
59
|
-
if params.
|
58
|
+
Log.log.debug{"using params: #{params}".bg_red.gray}
|
59
|
+
if params.key?('storage')
|
60
60
|
case params['storage']['type']
|
61
61
|
# here we need somehow to map storage type to field to get for auth end point
|
62
62
|
when 'ibm-s3'
|
@@ -67,36 +67,36 @@ module Aspera
|
|
67
67
|
else
|
68
68
|
server_data2 = @ats_api_pub.all_servers.find do |s|
|
69
69
|
s['cloud'].eql?(server_data['cloud']) &&
|
70
|
-
|
71
|
-
|
70
|
+
s['region'].eql?(server_data['region']) &&
|
71
|
+
s.key?('s3_authentication_endpoint')
|
72
72
|
end
|
73
73
|
raise "no such transfer server id: #{params['transfer_server_id']}" if server_data2.nil?
|
74
74
|
# specific one do not have s3 end point in id
|
75
75
|
params['transfer_server_id'] = server_data2['id']
|
76
76
|
end
|
77
|
-
if !params['storage'].
|
77
|
+
if !params['storage'].key?('authentication_endpoint')
|
78
78
|
params['storage']['endpoint'] = server_data2['s3_authentication_endpoint']
|
79
79
|
end
|
80
80
|
end
|
81
81
|
end
|
82
|
-
res = ats_api_pub_v1.create('access_keys',params)
|
82
|
+
res = ats_api_pub_v1.create('access_keys', params)
|
83
83
|
return {type: :single_object, data: res[:data]}
|
84
84
|
# TODO : action : modify, with "PUT"
|
85
85
|
when :list
|
86
|
-
params = options.get_option(:params) || {'offset' => 0,'max_results' => 1000}
|
87
|
-
res = ats_api_pub_v1.read('access_keys',params)
|
88
|
-
return {type: :object_list, data: res[:data]['data'], fields: ['name','id','created.at','modified.at']}
|
86
|
+
params = options.get_option(:params) || {'offset' => 0, 'max_results' => 1000}
|
87
|
+
res = ats_api_pub_v1.read('access_keys', params)
|
88
|
+
return {type: :object_list, data: res[:data]['data'], fields: ['name', 'id', 'created.at', 'modified.at']}
|
89
89
|
when :show
|
90
90
|
res = ats_api_pub_v1.read("access_keys/#{access_key_id}")
|
91
91
|
return {type: :single_object, data: res[:data]}
|
92
92
|
when :modify
|
93
|
-
params = options.get_option(:value,is_type: :mandatory)
|
93
|
+
params = options.get_option(:value, is_type: :mandatory)
|
94
94
|
params['id'] = access_key_id
|
95
|
-
ats_api_pub_v1.update("access_keys/#{access_key_id}",params)
|
95
|
+
ats_api_pub_v1.update("access_keys/#{access_key_id}", params)
|
96
96
|
return Main.result_status('modified')
|
97
97
|
when :entitlement
|
98
98
|
ak = ats_api_pub_v1.read("access_keys/#{access_key_id}")[:data]
|
99
|
-
api_bss = AoC.metering_api(ak['license']['entitlement_id'],ak['license']['customer_id'])
|
99
|
+
api_bss = AoC.metering_api(ak['license']['entitlement_id'], ak['license']['customer_id'])
|
100
100
|
return {type: :single_object, data: api_bss.read('entitlement')[:data]}
|
101
101
|
when :delete
|
102
102
|
ats_api_pub_v1.delete("access_keys/#{access_key_id}")
|
@@ -104,25 +104,25 @@ module Aspera
|
|
104
104
|
when :node
|
105
105
|
ak_data = ats_api_pub_v1.read("access_keys/#{access_key_id}")[:data]
|
106
106
|
server_data = @ats_api_pub.all_servers.find {|i| i['id'].start_with?(ak_data['transfer_server_id'])}
|
107
|
-
raise CliError,'no such server found' if server_data.nil?
|
108
|
-
|
109
|
-
api_node =
|
110
|
-
base_url:
|
107
|
+
raise CliError, 'no such server found' if server_data.nil?
|
108
|
+
node_url = server_data['transfer_setup_url']
|
109
|
+
api_node = Aspera::Node.new(params: {
|
110
|
+
base_url: node_url,
|
111
111
|
auth: {
|
112
112
|
type: :basic,
|
113
113
|
username: access_key_id,
|
114
|
-
password: @agents[:config].
|
114
|
+
password: @agents[:config].lookup_secret(url: node_url, username: access_key_id)
|
115
115
|
}})
|
116
|
-
command = options.get_next_command(Node::
|
117
|
-
return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).
|
116
|
+
command = options.get_next_command(Node::COMMANDS_GEN4)
|
117
|
+
return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).execute_command_gen4(command, ak_data['root_file_id'])
|
118
118
|
when :cluster
|
119
|
-
|
119
|
+
ats_url = ats_api_pub_v1.params[:base_url]
|
120
120
|
rest_params = {
|
121
|
-
base_url:
|
121
|
+
base_url: ats_url,
|
122
122
|
auth: {
|
123
123
|
type: :basic,
|
124
124
|
username: access_key_id,
|
125
|
-
password: @agents[:config].
|
125
|
+
password: @agents[:config].lookup_secret(url: ats_url, username: access_key_id)
|
126
126
|
}}
|
127
127
|
api_ak_auth = Rest.new(rest_params)
|
128
128
|
return {type: :single_object, data: api_ak_auth.read('servers')[:data]}
|
@@ -138,7 +138,7 @@ module Aspera
|
|
138
138
|
when :list
|
139
139
|
return {type: :object_list, data: @ats_api_pub.all_servers, fields: %w[id cloud region]}
|
140
140
|
when :show
|
141
|
-
if options.get_option(:cloud) || options.get_option(:region,is_type: :optional)
|
141
|
+
if options.get_option(:cloud) || options.get_option(:region, is_type: :optional)
|
142
142
|
server_data = server_by_cloud_region
|
143
143
|
else
|
144
144
|
server_id = instance_identifier
|
@@ -156,12 +156,12 @@ module Aspera
|
|
156
156
|
auth: {
|
157
157
|
type: :oauth2,
|
158
158
|
base_url: 'https://iam.bluemix.net/identity',
|
159
|
-
#does not work: base_url: 'https://iam.cloud.ibm.com/identity',
|
159
|
+
# does not work: base_url: 'https://iam.cloud.ibm.com/identity',
|
160
160
|
crtype: :generic,
|
161
161
|
generic: {
|
162
162
|
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
163
163
|
response_type: 'cloud_iam',
|
164
|
-
apikey: options.get_option(:ibm_api_key,is_type: :mandatory)
|
164
|
+
apikey: options.get_option(:ibm_api_key, is_type: :mandatory)
|
165
165
|
}}})
|
166
166
|
end
|
167
167
|
|
@@ -173,27 +173,25 @@ module Aspera
|
|
173
173
|
rest_add_header = {}
|
174
174
|
if !command.eql?(:instances)
|
175
175
|
instance = options.get_option(:instance)
|
176
|
-
#Log.log.error("1>>#{instance}".red)
|
177
176
|
if instance.nil?
|
178
177
|
# Take the first Aspera on Cloud transfer service instance ID if not provided by user
|
179
178
|
instance = ats_api_v2_auth_ibm.read('instances')[:data]['data'].first
|
180
179
|
self.format.display_status("using first instance: #{instance}")
|
181
180
|
end
|
182
|
-
#Log.log.error("2>>#{instance}".red)
|
183
181
|
rest_add_header = {'X-ATS-Service-Instance-Id' => instance}
|
184
182
|
end
|
185
183
|
ats_ibm_api = ats_api_v2_auth_ibm(rest_add_header)
|
186
184
|
case command
|
187
185
|
when :instances
|
188
186
|
instances = ats_ibm_api.read('instances')[:data]
|
189
|
-
Log.log.warn
|
187
|
+
Log.log.warn{"more instances remaining: #{instances['remaining']}"} unless instances['remaining'].to_i.eql?(0)
|
190
188
|
return {type: :value_list, data: instances['data'], name: 'instance'}
|
191
189
|
when :create
|
192
190
|
create_value = options.get_option(:value) || {}
|
193
|
-
created_key = ats_ibm_api.create('api_keys',create_value)[:data]
|
191
|
+
created_key = ats_ibm_api.create('api_keys', create_value)[:data]
|
194
192
|
return {type: :single_object, data: created_key}
|
195
193
|
when :list # list known api keys in ATS (this require an api_key ...)
|
196
|
-
res = ats_ibm_api.read('api_keys',{'offset' => 0,'max_results' => 1000})
|
194
|
+
res = ats_ibm_api.read('api_keys', {'offset' => 0, 'max_results' => 1000})
|
197
195
|
return {type: :value_list, data: res[:data]['data'], name: 'ats_id'}
|
198
196
|
when :show # show one of api_key in ATS
|
199
197
|
res = ats_ibm_api.read("api_keys/#{concerned_id}")
|
@@ -223,7 +221,7 @@ module Aspera
|
|
223
221
|
when :api_key # manage credential to access ATS API
|
224
222
|
return execute_action_api_key
|
225
223
|
when :aws_trust_policy
|
226
|
-
res = ats_api_pub_v1.read('aws/trustpolicy',{region: options.get_option(:region,is_type: :mandatory)})[:data]
|
224
|
+
res = ats_api_pub_v1.read('aws/trustpolicy', {region: options.get_option(:region, is_type: :mandatory)})[:data]
|
227
225
|
return {type: :single_object, data: res}
|
228
226
|
else raise 'ERROR'
|
229
227
|
end
|
@@ -5,7 +5,7 @@ require 'aspera/rest'
|
|
5
5
|
module Aspera
|
6
6
|
module Cli
|
7
7
|
module Plugins
|
8
|
-
class Bss < BasicAuthPlugin
|
8
|
+
class Bss < Aspera::Cli::BasicAuthPlugin
|
9
9
|
ACTIONS = %i[subscription].freeze
|
10
10
|
|
11
11
|
FIELDS = {
|
@@ -14,7 +14,7 @@ module Aspera
|
|
14
14
|
|
15
15
|
def initialize(env)
|
16
16
|
super(env)
|
17
|
-
@api_bss = env[:bss_api] if env.
|
17
|
+
@api_bss = env[:bss_api] if env.key?(:bss_api)
|
18
18
|
end
|
19
19
|
|
20
20
|
def all_fields(name)
|
@@ -23,7 +23,7 @@ module Aspera
|
|
23
23
|
|
24
24
|
def execute_action
|
25
25
|
if @api_bss.nil?
|
26
|
-
key = options.get_option(:password,is_type: :mandatory)
|
26
|
+
key = options.get_option(:password, is_type: :mandatory)
|
27
27
|
@api_bss = Rest.new(
|
28
28
|
base_url: 'https://dashboard.bss.asperasoft.com/platform',
|
29
29
|
headers: {cookie: "_dashboard_key=#{key}"})
|
@@ -35,15 +35,15 @@ module Aspera
|
|
35
35
|
object = 'bssSubscriptions'
|
36
36
|
case command
|
37
37
|
when :find
|
38
|
-
query = options.get_option(:query,is_type: :mandatory) # AOC_ORGANIZATION_QUERY AOC_USER_EMAIL
|
39
|
-
value = options.get_option(:value,is_type: :mandatory)
|
38
|
+
query = options.get_option(:query, is_type: :mandatory) # AOC_ORGANIZATION_QUERY AOC_USER_EMAIL
|
39
|
+
value = options.get_option(:value, is_type: :mandatory)
|
40
40
|
request = {
|
41
|
-
'variables' => {'filter' => {'key' => query,'value' => value}},
|
41
|
+
'variables' => {'filter' => {'key' => query, 'value' => value}},
|
42
42
|
'query' => "query($filter: BssSubscriptionFilter!) {#{object}(filter: $filter) { #{all_fields('bssSubscriptions')} } }"
|
43
43
|
}
|
44
|
-
result = @api_bss.create('graphql',request)[:data]
|
44
|
+
result = @api_bss.create('graphql', request)[:data]
|
45
45
|
# give fields to keep order
|
46
|
-
return {type: :object_list, data: result['data'][object],fields: FIELDS['bssSubscriptions']}
|
46
|
+
return {type: :object_list, data: result['data'][object], fields: FIELDS['bssSubscriptions']}
|
47
47
|
when :show
|
48
48
|
id = instance_identifier
|
49
49
|
request = {
|
@@ -52,7 +52,7 @@ module Aspera
|
|
52
52
|
'instances { id state planId serviceId ssmSubscriptionId entitlement { id } aocOrganization { id subdomainName name status tier urlId trialExpiresAt '\
|
53
53
|
'users(organizationAdmin: true) { id name email atsAdmin subscriptionAdmin } } } } }'
|
54
54
|
}
|
55
|
-
result = @api_bss.create('graphql',request)[:data]['data'][object].first
|
55
|
+
result = @api_bss.create('graphql', request)[:data]['data'][object].first
|
56
56
|
result.delete('instances')
|
57
57
|
return {type: :single_object, data: result}
|
58
58
|
when :instances
|
@@ -61,7 +61,7 @@ module Aspera
|
|
61
61
|
'variables' => {'id' => id},
|
62
62
|
'query' => "query($id: ID!) {#{object}(id: $id) { aocOrganization { id subdomainName name status tier urlId trialExpiresAt } } } }"
|
63
63
|
}
|
64
|
-
result = @api_bss.create('graphql',request)[:data]['data'][object].first
|
64
|
+
result = @api_bss.create('graphql', request)[:data]['data'][object].first
|
65
65
|
return {type: :object_list, data: result['instances']}
|
66
66
|
end
|
67
67
|
end
|