aspera-cli 4.4.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/cli/plugins/node'
|
2
3
|
require 'aspera/ats_api'
|
3
4
|
|
@@ -10,21 +11,20 @@ module Aspera
|
|
10
11
|
class Ats < Plugin
|
11
12
|
def initialize(env)
|
12
13
|
super(env)
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
14
|
+
options.add_opt_simple(:ibm_api_key,'IBM API key, see https://cloud.ibm.com/iam/apikeys')
|
15
|
+
options.add_opt_simple(:instance,'ATS instance in ibm cloud')
|
16
|
+
options.add_opt_simple(:ats_key,'ATS key identifier (ats_xxx)')
|
17
|
+
options.add_opt_simple(:ats_secret,'ATS key secret')
|
18
|
+
options.add_opt_simple(:params,'Parameters access key creation (@json:)')
|
19
|
+
options.add_opt_simple(:cloud,'Cloud provider')
|
20
|
+
options.add_opt_simple(:region,'Cloud region')
|
21
|
+
options.parse_options!
|
21
22
|
end
|
22
23
|
|
23
|
-
#
|
24
24
|
def server_by_cloud_region
|
25
25
|
# todo: provide list ?
|
26
|
-
cloud=
|
27
|
-
region=
|
26
|
+
cloud=options.get_option(:cloud,:mandatory).upcase
|
27
|
+
region=options.get_option(:region,:mandatory)
|
28
28
|
return @ats_api_pub.read("servers/#{cloud}/#{region}")[:data]
|
29
29
|
end
|
30
30
|
|
@@ -32,24 +32,22 @@ module Aspera
|
|
32
32
|
def ats_api_pub_v1
|
33
33
|
return @ats_api_pub_v1_cache unless @ats_api_pub_v1_cache.nil?
|
34
34
|
@ats_api_pub_v1_cache=Rest.new({
|
35
|
-
:
|
36
|
-
:
|
37
|
-
|
38
|
-
:
|
39
|
-
:
|
35
|
+
base_url: AtsApi.base_url+'/pub/v1',
|
36
|
+
auth: {
|
37
|
+
type: :basic,
|
38
|
+
username: options.get_option(:ats_key,:mandatory),
|
39
|
+
password: options.get_option(:ats_secret,:mandatory)}
|
40
40
|
})
|
41
41
|
end
|
42
42
|
|
43
43
|
def execute_action_access_key
|
44
44
|
commands=[:create,:list,:show,:modify,:delete,:node,:cluster,:entitlement]
|
45
|
-
command=
|
45
|
+
command=options.get_next_command(commands)
|
46
46
|
# those dont require access key id
|
47
|
-
unless [:create,:list].include?(command)
|
48
|
-
access_key_id=self.options.get_option(:id,:mandatory)
|
49
|
-
end
|
47
|
+
access_key_id=instance_identifier() unless [:create,:list].include?(command)
|
50
48
|
case command
|
51
49
|
when :create
|
52
|
-
params=
|
50
|
+
params=options.get_option(:params,:optional) || {}
|
53
51
|
server_data=nil
|
54
52
|
# if transfer_server_id not provided, get it from command line options
|
55
53
|
if !params.has_key?('transfer_server_id')
|
@@ -66,7 +64,11 @@ module Aspera
|
|
66
64
|
server_data2=@ats_api_pub.all_servers.select{|s|s['id'].eql?(params['transfer_server_id'])}.first
|
67
65
|
raise "no such transfer server id: #{params['transfer_server_id']}" if server_data2.nil?
|
68
66
|
else
|
69
|
-
server_data2=@ats_api_pub.all_servers.select
|
67
|
+
server_data2=@ats_api_pub.all_servers.select do |s|
|
68
|
+
s['cloud'].eql?(server_data['cloud']) &&
|
69
|
+
s['region'].eql?(server_data['region']) &&
|
70
|
+
s.has_key?('s3_authentication_endpoint')
|
71
|
+
end.first
|
70
72
|
raise "no such transfer server id: #{params['transfer_server_id']}" if server_data2.nil?
|
71
73
|
# specific one do not have s3 end point in id
|
72
74
|
params['transfer_server_id']=server_data2['id']
|
@@ -77,94 +79,99 @@ module Aspera
|
|
77
79
|
end
|
78
80
|
end
|
79
81
|
res=ats_api_pub_v1.create('access_keys',params)
|
80
|
-
return {:
|
82
|
+
return {type: :single_object, data: res[:data]}
|
81
83
|
# TODO : action : modify, with "PUT"
|
82
84
|
when :list
|
83
|
-
params=
|
84
|
-
res=ats_api_pub_v1.read(
|
85
|
-
return {:
|
85
|
+
params=options.get_option(:params,:optional) || {'offset'=>0,'max_results'=>1000}
|
86
|
+
res=ats_api_pub_v1.read('access_keys',params)
|
87
|
+
return {type: :object_list, data: res[:data]['data'], fields: ['name','id','created.at','modified.at']}
|
86
88
|
when :show
|
87
89
|
res=ats_api_pub_v1.read("access_keys/#{access_key_id}")
|
88
|
-
return {:
|
90
|
+
return {type: :single_object, data: res[:data]}
|
89
91
|
when :modify
|
90
|
-
params=
|
91
|
-
params[
|
92
|
-
|
92
|
+
params=options.get_option(:value,:mandatory)
|
93
|
+
params['id']=access_key_id
|
94
|
+
ats_api_pub_v1.update("access_keys/#{access_key_id}",params)
|
93
95
|
return Main.result_status('modified')
|
94
96
|
when :entitlement
|
95
97
|
ak=ats_api_pub_v1.read("access_keys/#{access_key_id}")[:data]
|
96
98
|
api_bss=AoC.metering_api(ak['license']['entitlement_id'],ak['license']['customer_id'])
|
97
|
-
return {:
|
99
|
+
return {type: :single_object, data: api_bss.read('entitlement')[:data]}
|
98
100
|
when :delete
|
99
|
-
|
101
|
+
ats_api_pub_v1.delete("access_keys/#{access_key_id}")
|
100
102
|
return Main.result_status("deleted #{access_key_id}")
|
101
103
|
when :node
|
102
104
|
ak_data=ats_api_pub_v1.read("access_keys/#{access_key_id}")[:data]
|
103
105
|
server_data=@ats_api_pub.all_servers.select {|i| i['id'].start_with?(ak_data['transfer_server_id'])}.first
|
104
|
-
raise CliError,
|
106
|
+
raise CliError,'no such server found' if server_data.nil?
|
107
|
+
base_url=server_data['transfer_setup_url']
|
105
108
|
api_node=Rest.new({
|
106
|
-
:base_url
|
107
|
-
:
|
108
|
-
|
109
|
-
:
|
110
|
-
:
|
111
|
-
|
109
|
+
base_url: base_url,
|
110
|
+
auth: {
|
111
|
+
type: :basic,
|
112
|
+
username: access_key_id,
|
113
|
+
password: @agents[:config].get_secret(url: base_url,username: access_key_id)
|
114
|
+
}})
|
115
|
+
command=options.get_next_command(Node::COMMON_ACTIONS)
|
112
116
|
return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).execute_action(command)
|
113
117
|
when :cluster
|
118
|
+
base_url=ats_api_pub_v1.params[:base_url]
|
114
119
|
rest_params={
|
115
|
-
|
116
|
-
:
|
117
|
-
|
118
|
-
:
|
119
|
-
:
|
120
|
+
base_url: base_url,
|
121
|
+
auth: {
|
122
|
+
type: :basic,
|
123
|
+
username: access_key_id,
|
124
|
+
password: @agents[:config].get_secret(url: base_url, username: access_key_id)
|
120
125
|
}}
|
121
126
|
api_ak_auth=Rest.new(rest_params)
|
122
|
-
return {:
|
123
|
-
else raise
|
127
|
+
return {type: :single_object, data: api_ak_auth.read('servers')[:data]}
|
128
|
+
else raise 'INTERNAL ERROR'
|
124
129
|
end
|
125
130
|
end
|
126
131
|
|
127
132
|
def execute_action_cluster_pub
|
128
|
-
command=
|
133
|
+
command=options.get_next_command([:clouds, :list, :show])
|
129
134
|
case command
|
130
135
|
when :clouds
|
131
|
-
return {:
|
136
|
+
return {type: :single_object, data: @ats_api_pub.cloud_names, columns: ['id','name']}
|
132
137
|
when :list
|
133
|
-
return {:
|
138
|
+
return {type: :object_list, data: @ats_api_pub.all_servers, fields: ['id','cloud','region']}
|
134
139
|
when :show
|
135
|
-
|
136
|
-
if server_id.nil?
|
140
|
+
if options.get_option(:cloud,:optional) || options.get_option(:region,:optional)
|
137
141
|
server_data=server_by_cloud_region
|
138
142
|
else
|
143
|
+
server_id=instance_identifier()
|
139
144
|
server_data=@ats_api_pub.all_servers.select {|i| i['id'].eql?(server_id)}.first
|
140
|
-
raise
|
145
|
+
raise 'no such server id' if server_data.nil?
|
141
146
|
end
|
142
|
-
return {:
|
147
|
+
return {type: :single_object, data: server_data}
|
143
148
|
end
|
144
149
|
end
|
145
150
|
|
146
151
|
def ats_api_v2_auth_ibm(rest_add_headers={})
|
147
152
|
return Rest.new({
|
148
|
-
:
|
149
|
-
:
|
150
|
-
:
|
151
|
-
|
152
|
-
:
|
153
|
-
#does not work: :
|
154
|
-
:
|
155
|
-
:
|
156
|
-
|
157
|
-
|
153
|
+
base_url: AtsApi.base_url+'/v2',
|
154
|
+
headers: rest_add_headers,
|
155
|
+
auth: {
|
156
|
+
type: :oauth2,
|
157
|
+
base_url: 'https://iam.bluemix.net/identity',
|
158
|
+
#does not work: base_url: 'https://iam.cloud.ibm.com/identity',
|
159
|
+
crtype: :generic,
|
160
|
+
generic: {
|
161
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
162
|
+
response_type: 'cloud_iam',
|
163
|
+
apikey: options.get_option(:ibm_api_key,:mandatory)
|
164
|
+
}}})
|
158
165
|
end
|
159
166
|
|
160
167
|
def execute_action_api_key
|
161
|
-
command=
|
168
|
+
command=options.get_next_command([:instances, :create, :list, :show, :delete])
|
162
169
|
if [:show,:delete].include?(command)
|
163
|
-
concerned_id=
|
170
|
+
concerned_id=instance_identifier()
|
164
171
|
end
|
165
172
|
rest_add_header={}
|
166
173
|
if !command.eql?(:instances)
|
167
|
-
instance=
|
174
|
+
instance=options.get_option(:instance,:optional)
|
168
175
|
#Log.log.error("1>>#{instance}".red)
|
169
176
|
if instance.nil?
|
170
177
|
# Take the first Aspera on Cloud transfer service instance ID if not provided by user
|
@@ -179,31 +186,31 @@ module Aspera
|
|
179
186
|
when :instances
|
180
187
|
instances=ats_ibm_api.read('instances')[:data]
|
181
188
|
Log.log.warn("more instances remaining: #{instances['remaining']}") unless instances['remaining'].to_i.eql?(0)
|
182
|
-
return {:
|
189
|
+
return {type: :value_list, data: instances['data'], name: 'instance'}
|
183
190
|
when :create
|
184
|
-
create_value=
|
191
|
+
create_value=options.get_option(:value,:optional)||{}
|
185
192
|
created_key=ats_ibm_api.create('api_keys',create_value)[:data]
|
186
|
-
return {:
|
193
|
+
return {type: :single_object, data: created_key}
|
187
194
|
when :list # list known api keys in ATS (this require an api_key ...)
|
188
195
|
res=ats_ibm_api.read('api_keys',{'offset'=>0,'max_results'=>1000})
|
189
|
-
return {:
|
196
|
+
return {type: :value_list, data: res[:data]['data'], name: 'ats_id'}
|
190
197
|
when :show # show one of api_key in ATS
|
191
198
|
res=ats_ibm_api.read("api_keys/#{concerned_id}")
|
192
|
-
return {:
|
193
|
-
when :delete
|
194
|
-
|
199
|
+
return {type: :single_object, data: res[:data]}
|
200
|
+
when :delete
|
201
|
+
ats_ibm_api.delete("api_keys/#{concerned_id}")
|
195
202
|
return Main.result_status("deleted #{concerned_id}")
|
196
|
-
else raise
|
203
|
+
else raise 'INTERNAL ERROR'
|
197
204
|
end
|
198
205
|
end
|
199
206
|
|
200
|
-
ACTIONS=[
|
207
|
+
ACTIONS=[:cluster, :access_key,:api_key, :aws_trust_policy]
|
201
208
|
|
202
209
|
# called for legacy and AoC
|
203
210
|
def execute_action_gen(ats_api_arg)
|
204
211
|
actions=ACTIONS
|
205
212
|
actions.delete(:api_key) unless ats_api_arg.nil?
|
206
|
-
command=
|
213
|
+
command=options.get_next_command(actions)
|
207
214
|
@ats_api_pub_v1_cache=ats_api_arg
|
208
215
|
# keep as member variable as we may want to use the api in AoC name space
|
209
216
|
@ats_api_pub = AtsApi.new
|
@@ -215,9 +222,9 @@ module Aspera
|
|
215
222
|
when :api_key # manage credential to access ATS API
|
216
223
|
return execute_action_api_key
|
217
224
|
when :aws_trust_policy
|
218
|
-
res=ats_api_pub_v1.read('aws/trustpolicy',{:
|
219
|
-
return {:
|
220
|
-
else raise
|
225
|
+
res=ats_api_pub_v1.read('aws/trustpolicy',{region: options.get_option(:region,:mandatory)})[:data]
|
226
|
+
return {type: :single_object, data: res}
|
227
|
+
else raise 'ERROR'
|
221
228
|
end
|
222
229
|
end
|
223
230
|
|
@@ -1,21 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/rest'
|
2
3
|
|
3
4
|
module Aspera
|
4
5
|
module Cli
|
5
6
|
module Plugins
|
6
7
|
class Bss < BasicAuthPlugin
|
7
|
-
|
8
|
-
ACTIONS=[ :subscription ]
|
8
|
+
ACTIONS=[:subscription]
|
9
9
|
|
10
10
|
FIELDS={
|
11
|
-
'bssSubscriptions' => %w
|
11
|
+
'bssSubscriptions' => %w[id name termVolumeGb termMonths trial startDate endDate plan renewalType chargeAgreementNumber customerName]
|
12
12
|
}
|
13
13
|
|
14
14
|
def initialize(env)
|
15
15
|
super(env)
|
16
|
-
if env.has_key?(:bss_api)
|
17
|
-
@api_bss=env[:bss_api]
|
18
|
-
end
|
16
|
+
@api_bss=env[:bss_api] if env.has_key?(:bss_api)
|
19
17
|
end
|
20
18
|
|
21
19
|
def all_fields(name)
|
@@ -24,44 +22,44 @@ module Aspera
|
|
24
22
|
|
25
23
|
def execute_action
|
26
24
|
if @api_bss.nil?
|
27
|
-
key =
|
25
|
+
key = options.get_option(:password,:mandatory)
|
28
26
|
@api_bss=Rest.new(
|
29
27
|
base_url: 'https://dashboard.bss.asperasoft.com/platform',
|
30
28
|
headers: {cookie: "_dashboard_key=#{key}"})
|
31
29
|
end
|
32
|
-
command=
|
30
|
+
command=options.get_next_command(ACTIONS)
|
33
31
|
case command
|
34
32
|
when :subscription
|
35
|
-
command=
|
33
|
+
command=options.get_next_command([:find,:show, :instances])
|
36
34
|
object='bssSubscriptions'
|
37
35
|
case command
|
38
36
|
when :find
|
39
|
-
query =
|
40
|
-
value =
|
37
|
+
query = options.get_option(:query,:mandatory) # AOC_ORGANIZATION_QUERY AOC_USER_EMAIL
|
38
|
+
value = options.get_option(:value,:mandatory)
|
41
39
|
request={
|
42
40
|
'variables'=>{'filter'=>{'key'=>query,'value'=>value}},
|
43
41
|
'query'=>"query($filter: BssSubscriptionFilter!) {#{object}(filter: $filter) { #{all_fields('bssSubscriptions')} } }"
|
44
42
|
}
|
45
43
|
result=@api_bss.create('graphql',request)[:data]
|
46
44
|
# give fields to keep order
|
47
|
-
return {:
|
45
|
+
return {type: :object_list, data: result['data'][object],fields: FIELDS['bssSubscriptions']}
|
48
46
|
when :show
|
49
|
-
id =
|
47
|
+
id = instance_identifier()
|
50
48
|
request={
|
51
49
|
'variables'=>{'id'=>id},
|
52
50
|
'query'=>"query($id: ID!) {#{object}(id: $id) { #{all_fields('bssSubscriptions')} roleAssignments(uniqueSubjectId: true) { id subjectId } instances { id state planId serviceId ssmSubscriptionId entitlement { id } aocOrganization { id subdomainName name status tier urlId trialExpiresAt users(organizationAdmin: true) { id name email atsAdmin subscriptionAdmin } } } } }"
|
53
51
|
}
|
54
52
|
result=@api_bss.create('graphql',request)[:data]['data'][object].first
|
55
53
|
result.delete('instances')
|
56
|
-
return {:
|
54
|
+
return {type: :single_object, data: result}
|
57
55
|
when :instances
|
58
|
-
id =
|
56
|
+
id = instance_identifier()
|
59
57
|
request={
|
60
58
|
'variables'=>{'id'=>id},
|
61
59
|
'query'=>"query($id: ID!) {#{object}(id: $id) { aocOrganization { id subdomainName name status tier urlId trialExpiresAt } } } }"
|
62
60
|
}
|
63
61
|
result=@api_bss.create('graphql',request)[:data]['data'][object].first
|
64
|
-
return {:
|
62
|
+
return {type: :object_list, data: result['instances']}
|
65
63
|
end
|
66
64
|
end
|
67
65
|
end
|