aspera-cli 4.10.0 → 4.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +19 -0
- data/CHANGELOG.md +528 -0
- data/CONTRIBUTING.md +143 -0
- data/README.md +977 -589
- data/bin/ascli +4 -4
- data/bin/asession +12 -12
- data/docs/test_env.conf +29 -19
- data/examples/aoc.rb +6 -6
- data/examples/dascli +18 -16
- data/examples/faspex4.rb +15 -15
- data/examples/node.rb +12 -12
- data/examples/proxy.pac +2 -2
- data/examples/server.rb +12 -12
- data/lib/aspera/aoc.rb +344 -272
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +9 -9
- data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +16 -21
- data/lib/aspera/cli/main.rb +72 -73
- data/lib/aspera/cli/manager.rb +112 -112
- data/lib/aspera/cli/plugin.rb +68 -48
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +322 -720
- data/lib/aspera/cli/plugins/ats.rb +50 -52
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +514 -410
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +134 -136
- data/lib/aspera/cli/plugins/faspex5.rb +235 -70
- data/lib/aspera/cli/plugins/node.rb +378 -309
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
- data/lib/aspera/cli/plugins/preview.rb +129 -120
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +77 -52
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +61 -61
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +78 -74
- data/lib/aspera/cos_node.rb +31 -29
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +17 -15
- data/lib/aspera/fasp/agent_connect.rb +34 -32
- data/lib/aspera/fasp/agent_direct.rb +70 -73
- data/lib/aspera/fasp/agent_httpgw.rb +79 -74
- data/lib/aspera/fasp/agent_node.rb +26 -26
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +80 -80
- data/lib/aspera/fasp/listener.rb +2 -2
- data/lib/aspera/fasp/parameters.rb +103 -92
- data/lib/aspera/fasp/parameters.yaml +313 -214
- data/lib/aspera/fasp/resume_policy.rb +10 -10
- data/lib/aspera/fasp/transfer_spec.rb +22 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +80 -159
- data/lib/aspera/faspex_postproc.rb +77 -0
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/nagios.rb +24 -23
- data/lib/aspera/node.rb +217 -38
- data/lib/aspera/oauth.rb +78 -74
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +63 -63
- data/lib/aspera/proxy_auto_config.rb +19 -19
- data/lib/aspera/rest.rb +65 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +22 -21
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +17 -14
- data/lib/aspera/ssh.rb +15 -14
- data/lib/aspera/sync.rb +177 -62
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +13 -64
- data/lib/aspera/web_server_simple.rb +76 -0
- data.tar.gz.sig +0 -0
- metadata +11 -6
- metadata.gz.sig +0 -0
@@ -6,26 +6,26 @@ require 'aspera/ats_api'
|
|
6
6
|
module Aspera
|
7
7
|
module Cli
|
8
8
|
module Plugins
|
9
|
-
#
|
9
|
+
# Access Aspera Transfer Service
|
10
10
|
# https://52.44.83.163/docs/
|
11
11
|
# https://developer.ibm.com/aspera/docs/ats-api-reference/creating-ats-api-keys/
|
12
|
-
class Ats < Plugin
|
12
|
+
class Ats < Aspera::Cli::Plugin
|
13
13
|
def initialize(env)
|
14
14
|
super(env)
|
15
|
-
options.add_opt_simple(:ibm_api_key,'IBM API key, see https://cloud.ibm.com/iam/apikeys')
|
16
|
-
options.add_opt_simple(:instance,'ATS instance in ibm cloud')
|
17
|
-
options.add_opt_simple(:ats_key,'ATS key identifier (ats_xxx)')
|
18
|
-
options.add_opt_simple(:ats_secret,'ATS key secret')
|
19
|
-
options.add_opt_simple(:params,'Parameters access key creation (@json:)')
|
20
|
-
options.add_opt_simple(:cloud,'Cloud provider')
|
21
|
-
options.add_opt_simple(:region,'Cloud region')
|
15
|
+
options.add_opt_simple(:ibm_api_key, 'IBM API key, see https://cloud.ibm.com/iam/apikeys')
|
16
|
+
options.add_opt_simple(:instance, 'ATS instance in ibm cloud')
|
17
|
+
options.add_opt_simple(:ats_key, 'ATS key identifier (ats_xxx)')
|
18
|
+
options.add_opt_simple(:ats_secret, 'ATS key secret')
|
19
|
+
options.add_opt_simple(:params, 'Parameters access key creation (@json:)')
|
20
|
+
options.add_opt_simple(:cloud, 'Cloud provider')
|
21
|
+
options.add_opt_simple(:region, 'Cloud region')
|
22
22
|
options.parse_options!
|
23
23
|
end
|
24
24
|
|
25
25
|
def server_by_cloud_region
|
26
|
-
#
|
27
|
-
cloud = options.get_option(:cloud,is_type: :mandatory).upcase
|
28
|
-
region = options.get_option(:region,is_type: :mandatory)
|
26
|
+
# TODO: provide list ?
|
27
|
+
cloud = options.get_option(:cloud, is_type: :mandatory).upcase
|
28
|
+
region = options.get_option(:region, is_type: :mandatory)
|
29
29
|
return @ats_api_pub.read("servers/#{cloud}/#{region}")[:data]
|
30
30
|
end
|
31
31
|
|
@@ -36,27 +36,27 @@ module Aspera
|
|
36
36
|
base_url: AtsApi.base_url + '/pub/v1',
|
37
37
|
auth: {
|
38
38
|
type: :basic,
|
39
|
-
username: options.get_option(:ats_key,is_type: :mandatory),
|
40
|
-
password: options.get_option(:ats_secret,is_type: :mandatory)}
|
39
|
+
username: options.get_option(:ats_key, is_type: :mandatory),
|
40
|
+
password: options.get_option(:ats_secret, is_type: :mandatory)}
|
41
41
|
})
|
42
42
|
end
|
43
43
|
|
44
44
|
def execute_action_access_key
|
45
45
|
commands = %i[create list show modify delete node cluster entitlement]
|
46
46
|
command = options.get_next_command(commands)
|
47
|
-
# those
|
47
|
+
# those do not require access key id
|
48
48
|
access_key_id = instance_identifier unless %i[create list].include?(command)
|
49
49
|
case command
|
50
50
|
when :create
|
51
51
|
params = options.get_option(:params) || {}
|
52
52
|
server_data = nil
|
53
53
|
# if transfer_server_id not provided, get it from command line options
|
54
|
-
if !params.
|
54
|
+
if !params.key?('transfer_server_id')
|
55
55
|
server_data = server_by_cloud_region
|
56
56
|
params['transfer_server_id'] = server_data['id']
|
57
57
|
end
|
58
|
-
Log.log.debug
|
59
|
-
if params.
|
58
|
+
Log.log.debug{"using params: #{params}".bg_red.gray}
|
59
|
+
if params.key?('storage')
|
60
60
|
case params['storage']['type']
|
61
61
|
# here we need somehow to map storage type to field to get for auth end point
|
62
62
|
when 'ibm-s3'
|
@@ -67,36 +67,36 @@ module Aspera
|
|
67
67
|
else
|
68
68
|
server_data2 = @ats_api_pub.all_servers.find do |s|
|
69
69
|
s['cloud'].eql?(server_data['cloud']) &&
|
70
|
-
|
71
|
-
|
70
|
+
s['region'].eql?(server_data['region']) &&
|
71
|
+
s.key?('s3_authentication_endpoint')
|
72
72
|
end
|
73
73
|
raise "no such transfer server id: #{params['transfer_server_id']}" if server_data2.nil?
|
74
74
|
# specific one do not have s3 end point in id
|
75
75
|
params['transfer_server_id'] = server_data2['id']
|
76
76
|
end
|
77
|
-
if !params['storage'].
|
77
|
+
if !params['storage'].key?('authentication_endpoint')
|
78
78
|
params['storage']['endpoint'] = server_data2['s3_authentication_endpoint']
|
79
79
|
end
|
80
80
|
end
|
81
81
|
end
|
82
|
-
res = ats_api_pub_v1.create('access_keys',params)
|
82
|
+
res = ats_api_pub_v1.create('access_keys', params)
|
83
83
|
return {type: :single_object, data: res[:data]}
|
84
84
|
# TODO : action : modify, with "PUT"
|
85
85
|
when :list
|
86
|
-
params = options.get_option(:params) || {'offset' => 0,'max_results' => 1000}
|
87
|
-
res = ats_api_pub_v1.read('access_keys',params)
|
88
|
-
return {type: :object_list, data: res[:data]['data'], fields: ['name','id','created.at','modified.at']}
|
86
|
+
params = options.get_option(:params) || {'offset' => 0, 'max_results' => 1000}
|
87
|
+
res = ats_api_pub_v1.read('access_keys', params)
|
88
|
+
return {type: :object_list, data: res[:data]['data'], fields: ['name', 'id', 'created.at', 'modified.at']}
|
89
89
|
when :show
|
90
90
|
res = ats_api_pub_v1.read("access_keys/#{access_key_id}")
|
91
91
|
return {type: :single_object, data: res[:data]}
|
92
92
|
when :modify
|
93
|
-
params = options.get_option(:value,is_type: :mandatory)
|
93
|
+
params = options.get_option(:value, is_type: :mandatory)
|
94
94
|
params['id'] = access_key_id
|
95
|
-
ats_api_pub_v1.update("access_keys/#{access_key_id}",params)
|
95
|
+
ats_api_pub_v1.update("access_keys/#{access_key_id}", params)
|
96
96
|
return Main.result_status('modified')
|
97
97
|
when :entitlement
|
98
98
|
ak = ats_api_pub_v1.read("access_keys/#{access_key_id}")[:data]
|
99
|
-
api_bss = AoC.metering_api(ak['license']['entitlement_id'],ak['license']['customer_id'])
|
99
|
+
api_bss = AoC.metering_api(ak['license']['entitlement_id'], ak['license']['customer_id'])
|
100
100
|
return {type: :single_object, data: api_bss.read('entitlement')[:data]}
|
101
101
|
when :delete
|
102
102
|
ats_api_pub_v1.delete("access_keys/#{access_key_id}")
|
@@ -104,25 +104,25 @@ module Aspera
|
|
104
104
|
when :node
|
105
105
|
ak_data = ats_api_pub_v1.read("access_keys/#{access_key_id}")[:data]
|
106
106
|
server_data = @ats_api_pub.all_servers.find {|i| i['id'].start_with?(ak_data['transfer_server_id'])}
|
107
|
-
raise CliError,'no such server found' if server_data.nil?
|
108
|
-
|
109
|
-
api_node =
|
110
|
-
base_url:
|
107
|
+
raise CliError, 'no such server found' if server_data.nil?
|
108
|
+
node_url = server_data['transfer_setup_url']
|
109
|
+
api_node = Aspera::Node.new(params: {
|
110
|
+
base_url: node_url,
|
111
111
|
auth: {
|
112
112
|
type: :basic,
|
113
113
|
username: access_key_id,
|
114
|
-
password: @agents[:config].
|
114
|
+
password: @agents[:config].lookup_secret(url: node_url, username: access_key_id)
|
115
115
|
}})
|
116
|
-
command = options.get_next_command(Node::
|
117
|
-
return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).
|
116
|
+
command = options.get_next_command(Node::COMMANDS_GEN4)
|
117
|
+
return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).execute_command_gen4(command, ak_data['root_file_id'])
|
118
118
|
when :cluster
|
119
|
-
|
119
|
+
ats_url = ats_api_pub_v1.params[:base_url]
|
120
120
|
rest_params = {
|
121
|
-
base_url:
|
121
|
+
base_url: ats_url,
|
122
122
|
auth: {
|
123
123
|
type: :basic,
|
124
124
|
username: access_key_id,
|
125
|
-
password: @agents[:config].
|
125
|
+
password: @agents[:config].lookup_secret(url: ats_url, username: access_key_id)
|
126
126
|
}}
|
127
127
|
api_ak_auth = Rest.new(rest_params)
|
128
128
|
return {type: :single_object, data: api_ak_auth.read('servers')[:data]}
|
@@ -138,7 +138,7 @@ module Aspera
|
|
138
138
|
when :list
|
139
139
|
return {type: :object_list, data: @ats_api_pub.all_servers, fields: %w[id cloud region]}
|
140
140
|
when :show
|
141
|
-
if options.get_option(:cloud) || options.get_option(:region,is_type: :optional)
|
141
|
+
if options.get_option(:cloud) || options.get_option(:region, is_type: :optional)
|
142
142
|
server_data = server_by_cloud_region
|
143
143
|
else
|
144
144
|
server_id = instance_identifier
|
@@ -154,14 +154,14 @@ module Aspera
|
|
154
154
|
base_url: AtsApi.base_url + '/v2',
|
155
155
|
headers: rest_add_headers,
|
156
156
|
auth: {
|
157
|
-
type:
|
158
|
-
base_url:
|
159
|
-
#does not work: base_url: 'https://iam.cloud.ibm.com/identity',
|
160
|
-
|
161
|
-
generic:
|
157
|
+
type: :oauth2,
|
158
|
+
base_url: 'https://iam.bluemix.net/identity',
|
159
|
+
# does not work: base_url: 'https://iam.cloud.ibm.com/identity',
|
160
|
+
grant_method: :generic,
|
161
|
+
generic: {
|
162
162
|
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
163
163
|
response_type: 'cloud_iam',
|
164
|
-
apikey: options.get_option(:ibm_api_key,is_type: :mandatory)
|
164
|
+
apikey: options.get_option(:ibm_api_key, is_type: :mandatory)
|
165
165
|
}}})
|
166
166
|
end
|
167
167
|
|
@@ -173,27 +173,25 @@ module Aspera
|
|
173
173
|
rest_add_header = {}
|
174
174
|
if !command.eql?(:instances)
|
175
175
|
instance = options.get_option(:instance)
|
176
|
-
#Log.log.error("1>>#{instance}".red)
|
177
176
|
if instance.nil?
|
178
177
|
# Take the first Aspera on Cloud transfer service instance ID if not provided by user
|
179
178
|
instance = ats_api_v2_auth_ibm.read('instances')[:data]['data'].first
|
180
|
-
|
179
|
+
formatter.display_status("using first instance: #{instance}")
|
181
180
|
end
|
182
|
-
#Log.log.error("2>>#{instance}".red)
|
183
181
|
rest_add_header = {'X-ATS-Service-Instance-Id' => instance}
|
184
182
|
end
|
185
183
|
ats_ibm_api = ats_api_v2_auth_ibm(rest_add_header)
|
186
184
|
case command
|
187
185
|
when :instances
|
188
186
|
instances = ats_ibm_api.read('instances')[:data]
|
189
|
-
Log.log.warn
|
187
|
+
Log.log.warn{"more instances remaining: #{instances['remaining']}"} unless instances['remaining'].to_i.eql?(0)
|
190
188
|
return {type: :value_list, data: instances['data'], name: 'instance'}
|
191
189
|
when :create
|
192
190
|
create_value = options.get_option(:value) || {}
|
193
|
-
created_key = ats_ibm_api.create('api_keys',create_value)[:data]
|
191
|
+
created_key = ats_ibm_api.create('api_keys', create_value)[:data]
|
194
192
|
return {type: :single_object, data: created_key}
|
195
193
|
when :list # list known api keys in ATS (this require an api_key ...)
|
196
|
-
res = ats_ibm_api.read('api_keys',{'offset' => 0,'max_results' => 1000})
|
194
|
+
res = ats_ibm_api.read('api_keys', {'offset' => 0, 'max_results' => 1000})
|
197
195
|
return {type: :value_list, data: res[:data]['data'], name: 'ats_id'}
|
198
196
|
when :show # show one of api_key in ATS
|
199
197
|
res = ats_ibm_api.read("api_keys/#{concerned_id}")
|
@@ -223,7 +221,7 @@ module Aspera
|
|
223
221
|
when :api_key # manage credential to access ATS API
|
224
222
|
return execute_action_api_key
|
225
223
|
when :aws_trust_policy
|
226
|
-
res = ats_api_pub_v1.read('aws/trustpolicy',{region: options.get_option(:region,is_type: :mandatory)})[:data]
|
224
|
+
res = ats_api_pub_v1.read('aws/trustpolicy', {region: options.get_option(:region, is_type: :mandatory)})[:data]
|
227
225
|
return {type: :single_object, data: res}
|
228
226
|
else raise 'ERROR'
|
229
227
|
end
|
@@ -5,7 +5,7 @@ require 'aspera/rest'
|
|
5
5
|
module Aspera
|
6
6
|
module Cli
|
7
7
|
module Plugins
|
8
|
-
class Bss < BasicAuthPlugin
|
8
|
+
class Bss < Aspera::Cli::BasicAuthPlugin
|
9
9
|
ACTIONS = %i[subscription].freeze
|
10
10
|
|
11
11
|
FIELDS = {
|
@@ -14,7 +14,7 @@ module Aspera
|
|
14
14
|
|
15
15
|
def initialize(env)
|
16
16
|
super(env)
|
17
|
-
@api_bss = env[:bss_api] if env.
|
17
|
+
@api_bss = env[:bss_api] if env.key?(:bss_api)
|
18
18
|
end
|
19
19
|
|
20
20
|
def all_fields(name)
|
@@ -23,7 +23,7 @@ module Aspera
|
|
23
23
|
|
24
24
|
def execute_action
|
25
25
|
if @api_bss.nil?
|
26
|
-
key = options.get_option(:password,is_type: :mandatory)
|
26
|
+
key = options.get_option(:password, is_type: :mandatory)
|
27
27
|
@api_bss = Rest.new(
|
28
28
|
base_url: 'https://dashboard.bss.asperasoft.com/platform',
|
29
29
|
headers: {cookie: "_dashboard_key=#{key}"})
|
@@ -35,15 +35,15 @@ module Aspera
|
|
35
35
|
object = 'bssSubscriptions'
|
36
36
|
case command
|
37
37
|
when :find
|
38
|
-
query = options.get_option(:query,is_type: :mandatory) # AOC_ORGANIZATION_QUERY AOC_USER_EMAIL
|
39
|
-
value = options.get_option(:value,is_type: :mandatory)
|
38
|
+
query = options.get_option(:query, is_type: :mandatory) # AOC_ORGANIZATION_QUERY AOC_USER_EMAIL
|
39
|
+
value = options.get_option(:value, is_type: :mandatory)
|
40
40
|
request = {
|
41
|
-
'variables' => {'filter' => {'key' => query,'value' => value}},
|
41
|
+
'variables' => {'filter' => {'key' => query, 'value' => value}},
|
42
42
|
'query' => "query($filter: BssSubscriptionFilter!) {#{object}(filter: $filter) { #{all_fields('bssSubscriptions')} } }"
|
43
43
|
}
|
44
|
-
result = @api_bss.create('graphql',request)[:data]
|
44
|
+
result = @api_bss.create('graphql', request)[:data]
|
45
45
|
# give fields to keep order
|
46
|
-
return {type: :object_list, data: result['data'][object],fields: FIELDS['bssSubscriptions']}
|
46
|
+
return {type: :object_list, data: result['data'][object], fields: FIELDS['bssSubscriptions']}
|
47
47
|
when :show
|
48
48
|
id = instance_identifier
|
49
49
|
request = {
|
@@ -52,7 +52,7 @@ module Aspera
|
|
52
52
|
'instances { id state planId serviceId ssmSubscriptionId entitlement { id } aocOrganization { id subdomainName name status tier urlId trialExpiresAt '\
|
53
53
|
'users(organizationAdmin: true) { id name email atsAdmin subscriptionAdmin } } } } }'
|
54
54
|
}
|
55
|
-
result = @api_bss.create('graphql',request)[:data]['data'][object].first
|
55
|
+
result = @api_bss.create('graphql', request)[:data]['data'][object].first
|
56
56
|
result.delete('instances')
|
57
57
|
return {type: :single_object, data: result}
|
58
58
|
when :instances
|
@@ -61,7 +61,7 @@ module Aspera
|
|
61
61
|
'variables' => {'id' => id},
|
62
62
|
'query' => "query($id: ID!) {#{object}(id: $id) { aocOrganization { id subdomainName name status tier urlId trialExpiresAt } } } }"
|
63
63
|
}
|
64
|
-
result = @api_bss.create('graphql',request)[:data]['data'][object].first
|
64
|
+
result = @api_bss.create('graphql', request)[:data]['data'][object].first
|
65
65
|
return {type: :object_list, data: result['instances']}
|
66
66
|
end
|
67
67
|
end
|