aspera-cli 4.10.0 → 4.11.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +20 -0
  4. data/CHANGELOG.md +509 -0
  5. data/CONTRIBUTING.md +118 -0
  6. data/README.md +621 -378
  7. data/bin/ascli +4 -4
  8. data/bin/asession +11 -11
  9. data/docs/test_env.conf +28 -19
  10. data/examples/aoc.rb +4 -4
  11. data/examples/dascli +11 -9
  12. data/examples/faspex4.rb +8 -8
  13. data/examples/node.rb +11 -11
  14. data/examples/server.rb +9 -9
  15. data/lib/aspera/aoc.rb +273 -266
  16. data/lib/aspera/ascmd.rb +56 -54
  17. data/lib/aspera/ats_api.rb +4 -4
  18. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  19. data/lib/aspera/cli/extended_value.rb +5 -5
  20. data/lib/aspera/cli/formater.rb +64 -64
  21. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  22. data/lib/aspera/cli/listener/logger.rb +1 -1
  23. data/lib/aspera/cli/listener/progress.rb +5 -6
  24. data/lib/aspera/cli/listener/progress_multi.rb +14 -19
  25. data/lib/aspera/cli/main.rb +66 -67
  26. data/lib/aspera/cli/manager.rb +110 -110
  27. data/lib/aspera/cli/plugin.rb +54 -37
  28. data/lib/aspera/cli/plugins/alee.rb +4 -4
  29. data/lib/aspera/cli/plugins/aoc.rb +308 -669
  30. data/lib/aspera/cli/plugins/ats.rb +44 -46
  31. data/lib/aspera/cli/plugins/bss.rb +10 -10
  32. data/lib/aspera/cli/plugins/config.rb +447 -344
  33. data/lib/aspera/cli/plugins/console.rb +12 -12
  34. data/lib/aspera/cli/plugins/cos.rb +18 -20
  35. data/lib/aspera/cli/plugins/faspex.rb +110 -112
  36. data/lib/aspera/cli/plugins/faspex5.rb +67 -46
  37. data/lib/aspera/cli/plugins/node.rb +364 -288
  38. data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
  39. data/lib/aspera/cli/plugins/preview.rb +122 -114
  40. data/lib/aspera/cli/plugins/server.rb +137 -83
  41. data/lib/aspera/cli/plugins/shares.rb +30 -29
  42. data/lib/aspera/cli/plugins/sync.rb +13 -33
  43. data/lib/aspera/cli/transfer_agent.rb +57 -57
  44. data/lib/aspera/cli/version.rb +1 -1
  45. data/lib/aspera/colors.rb +3 -3
  46. data/lib/aspera/command_line_builder.rb +27 -27
  47. data/lib/aspera/cos_node.rb +22 -20
  48. data/lib/aspera/data_repository.rb +1 -1
  49. data/lib/aspera/environment.rb +30 -28
  50. data/lib/aspera/fasp/agent_base.rb +15 -15
  51. data/lib/aspera/fasp/agent_connect.rb +23 -21
  52. data/lib/aspera/fasp/agent_direct.rb +65 -67
  53. data/lib/aspera/fasp/agent_httpgw.rb +72 -68
  54. data/lib/aspera/fasp/agent_node.rb +23 -21
  55. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  56. data/lib/aspera/fasp/error.rb +3 -2
  57. data/lib/aspera/fasp/error_info.rb +11 -8
  58. data/lib/aspera/fasp/installation.rb +78 -78
  59. data/lib/aspera/fasp/listener.rb +1 -1
  60. data/lib/aspera/fasp/parameters.rb +75 -72
  61. data/lib/aspera/fasp/parameters.yaml +2 -2
  62. data/lib/aspera/fasp/resume_policy.rb +8 -8
  63. data/lib/aspera/fasp/transfer_spec.rb +35 -2
  64. data/lib/aspera/fasp/uri.rb +7 -7
  65. data/lib/aspera/faspex_gw.rb +7 -5
  66. data/lib/aspera/hash_ext.rb +3 -3
  67. data/lib/aspera/id_generator.rb +5 -5
  68. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  69. data/lib/aspera/keychain/macos_security.rb +21 -20
  70. data/lib/aspera/log.rb +7 -7
  71. data/lib/aspera/nagios.rb +19 -18
  72. data/lib/aspera/node.rb +209 -35
  73. data/lib/aspera/oauth.rb +37 -36
  74. data/lib/aspera/open_application.rb +19 -11
  75. data/lib/aspera/persistency_action_once.rb +4 -4
  76. data/lib/aspera/persistency_folder.rb +13 -13
  77. data/lib/aspera/preview/file_types.rb +8 -8
  78. data/lib/aspera/preview/generator.rb +67 -67
  79. data/lib/aspera/preview/utils.rb +27 -27
  80. data/lib/aspera/proxy_auto_config.js +41 -41
  81. data/lib/aspera/proxy_auto_config.rb +16 -16
  82. data/lib/aspera/rest.rb +56 -60
  83. data/lib/aspera/rest_call_error.rb +2 -1
  84. data/lib/aspera/rest_error_analyzer.rb +18 -17
  85. data/lib/aspera/rest_errors_aspera.rb +16 -16
  86. data/lib/aspera/secret_hider.rb +15 -13
  87. data/lib/aspera/ssh.rb +11 -10
  88. data/lib/aspera/sync.rb +158 -44
  89. data/lib/aspera/temp_file_manager.rb +2 -2
  90. data/lib/aspera/uri_reader.rb +4 -4
  91. data/lib/aspera/web_auth.rb +14 -13
  92. data.tar.gz.sig +0 -0
  93. metadata +8 -5
  94. metadata.gz.sig +0 -0
@@ -6,26 +6,26 @@ require 'aspera/ats_api'
6
6
  module Aspera
7
7
  module Cli
8
8
  module Plugins
9
- # list and download connect client versions
9
+ # Access Aspera Transfer Service
10
10
  # https://52.44.83.163/docs/
11
11
  # https://developer.ibm.com/aspera/docs/ats-api-reference/creating-ats-api-keys/
12
- class Ats < Plugin
12
+ class Ats < Aspera::Cli::Plugin
13
13
  def initialize(env)
14
14
  super(env)
15
- options.add_opt_simple(:ibm_api_key,'IBM API key, see https://cloud.ibm.com/iam/apikeys')
16
- options.add_opt_simple(:instance,'ATS instance in ibm cloud')
17
- options.add_opt_simple(:ats_key,'ATS key identifier (ats_xxx)')
18
- options.add_opt_simple(:ats_secret,'ATS key secret')
19
- options.add_opt_simple(:params,'Parameters access key creation (@json:)')
20
- options.add_opt_simple(:cloud,'Cloud provider')
21
- options.add_opt_simple(:region,'Cloud region')
15
+ options.add_opt_simple(:ibm_api_key, 'IBM API key, see https://cloud.ibm.com/iam/apikeys')
16
+ options.add_opt_simple(:instance, 'ATS instance in ibm cloud')
17
+ options.add_opt_simple(:ats_key, 'ATS key identifier (ats_xxx)')
18
+ options.add_opt_simple(:ats_secret, 'ATS key secret')
19
+ options.add_opt_simple(:params, 'Parameters access key creation (@json:)')
20
+ options.add_opt_simple(:cloud, 'Cloud provider')
21
+ options.add_opt_simple(:region, 'Cloud region')
22
22
  options.parse_options!
23
23
  end
24
24
 
25
25
  def server_by_cloud_region
26
- # todo: provide list ?
27
- cloud = options.get_option(:cloud,is_type: :mandatory).upcase
28
- region = options.get_option(:region,is_type: :mandatory)
26
+ # TODO: provide list ?
27
+ cloud = options.get_option(:cloud, is_type: :mandatory).upcase
28
+ region = options.get_option(:region, is_type: :mandatory)
29
29
  return @ats_api_pub.read("servers/#{cloud}/#{region}")[:data]
30
30
  end
31
31
 
@@ -36,8 +36,8 @@ module Aspera
36
36
  base_url: AtsApi.base_url + '/pub/v1',
37
37
  auth: {
38
38
  type: :basic,
39
- username: options.get_option(:ats_key,is_type: :mandatory),
40
- password: options.get_option(:ats_secret,is_type: :mandatory)}
39
+ username: options.get_option(:ats_key, is_type: :mandatory),
40
+ password: options.get_option(:ats_secret, is_type: :mandatory)}
41
41
  })
42
42
  end
43
43
 
@@ -51,12 +51,12 @@ module Aspera
51
51
  params = options.get_option(:params) || {}
52
52
  server_data = nil
53
53
  # if transfer_server_id not provided, get it from command line options
54
- if !params.has_key?('transfer_server_id')
54
+ if !params.key?('transfer_server_id')
55
55
  server_data = server_by_cloud_region
56
56
  params['transfer_server_id'] = server_data['id']
57
57
  end
58
- Log.log.debug("using params: #{params}".bg_red.gray)
59
- if params.has_key?('storage')
58
+ Log.log.debug{"using params: #{params}".bg_red.gray}
59
+ if params.key?('storage')
60
60
  case params['storage']['type']
61
61
  # here we need somehow to map storage type to field to get for auth end point
62
62
  when 'ibm-s3'
@@ -67,36 +67,36 @@ module Aspera
67
67
  else
68
68
  server_data2 = @ats_api_pub.all_servers.find do |s|
69
69
  s['cloud'].eql?(server_data['cloud']) &&
70
- s['region'].eql?(server_data['region']) &&
71
- s.has_key?('s3_authentication_endpoint')
70
+ s['region'].eql?(server_data['region']) &&
71
+ s.key?('s3_authentication_endpoint')
72
72
  end
73
73
  raise "no such transfer server id: #{params['transfer_server_id']}" if server_data2.nil?
74
74
  # specific one do not have s3 end point in id
75
75
  params['transfer_server_id'] = server_data2['id']
76
76
  end
77
- if !params['storage'].has_key?('authentication_endpoint')
77
+ if !params['storage'].key?('authentication_endpoint')
78
78
  params['storage']['endpoint'] = server_data2['s3_authentication_endpoint']
79
79
  end
80
80
  end
81
81
  end
82
- res = ats_api_pub_v1.create('access_keys',params)
82
+ res = ats_api_pub_v1.create('access_keys', params)
83
83
  return {type: :single_object, data: res[:data]}
84
84
  # TODO : action : modify, with "PUT"
85
85
  when :list
86
- params = options.get_option(:params) || {'offset' => 0,'max_results' => 1000}
87
- res = ats_api_pub_v1.read('access_keys',params)
88
- return {type: :object_list, data: res[:data]['data'], fields: ['name','id','created.at','modified.at']}
86
+ params = options.get_option(:params) || {'offset' => 0, 'max_results' => 1000}
87
+ res = ats_api_pub_v1.read('access_keys', params)
88
+ return {type: :object_list, data: res[:data]['data'], fields: ['name', 'id', 'created.at', 'modified.at']}
89
89
  when :show
90
90
  res = ats_api_pub_v1.read("access_keys/#{access_key_id}")
91
91
  return {type: :single_object, data: res[:data]}
92
92
  when :modify
93
- params = options.get_option(:value,is_type: :mandatory)
93
+ params = options.get_option(:value, is_type: :mandatory)
94
94
  params['id'] = access_key_id
95
- ats_api_pub_v1.update("access_keys/#{access_key_id}",params)
95
+ ats_api_pub_v1.update("access_keys/#{access_key_id}", params)
96
96
  return Main.result_status('modified')
97
97
  when :entitlement
98
98
  ak = ats_api_pub_v1.read("access_keys/#{access_key_id}")[:data]
99
- api_bss = AoC.metering_api(ak['license']['entitlement_id'],ak['license']['customer_id'])
99
+ api_bss = AoC.metering_api(ak['license']['entitlement_id'], ak['license']['customer_id'])
100
100
  return {type: :single_object, data: api_bss.read('entitlement')[:data]}
101
101
  when :delete
102
102
  ats_api_pub_v1.delete("access_keys/#{access_key_id}")
@@ -104,25 +104,25 @@ module Aspera
104
104
  when :node
105
105
  ak_data = ats_api_pub_v1.read("access_keys/#{access_key_id}")[:data]
106
106
  server_data = @ats_api_pub.all_servers.find {|i| i['id'].start_with?(ak_data['transfer_server_id'])}
107
- raise CliError,'no such server found' if server_data.nil?
108
- base_url = server_data['transfer_setup_url']
109
- api_node = Rest.new({
110
- base_url: base_url,
107
+ raise CliError, 'no such server found' if server_data.nil?
108
+ node_url = server_data['transfer_setup_url']
109
+ api_node = Aspera::Node.new(params: {
110
+ base_url: node_url,
111
111
  auth: {
112
112
  type: :basic,
113
113
  username: access_key_id,
114
- password: @agents[:config].get_secret(url: base_url,username: access_key_id)
114
+ password: @agents[:config].lookup_secret(url: node_url, username: access_key_id)
115
115
  }})
116
- command = options.get_next_command(Node::COMMON_ACTIONS)
117
- return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).execute_action(command)
116
+ command = options.get_next_command(Node::COMMANDS_GEN4)
117
+ return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).execute_command_gen4(command, ak_data['root_file_id'])
118
118
  when :cluster
119
- base_url = ats_api_pub_v1.params[:base_url]
119
+ ats_url = ats_api_pub_v1.params[:base_url]
120
120
  rest_params = {
121
- base_url: base_url,
121
+ base_url: ats_url,
122
122
  auth: {
123
123
  type: :basic,
124
124
  username: access_key_id,
125
- password: @agents[:config].get_secret(url: base_url, username: access_key_id)
125
+ password: @agents[:config].lookup_secret(url: ats_url, username: access_key_id)
126
126
  }}
127
127
  api_ak_auth = Rest.new(rest_params)
128
128
  return {type: :single_object, data: api_ak_auth.read('servers')[:data]}
@@ -138,7 +138,7 @@ module Aspera
138
138
  when :list
139
139
  return {type: :object_list, data: @ats_api_pub.all_servers, fields: %w[id cloud region]}
140
140
  when :show
141
- if options.get_option(:cloud) || options.get_option(:region,is_type: :optional)
141
+ if options.get_option(:cloud) || options.get_option(:region, is_type: :optional)
142
142
  server_data = server_by_cloud_region
143
143
  else
144
144
  server_id = instance_identifier
@@ -156,12 +156,12 @@ module Aspera
156
156
  auth: {
157
157
  type: :oauth2,
158
158
  base_url: 'https://iam.bluemix.net/identity',
159
- #does not work: base_url: 'https://iam.cloud.ibm.com/identity',
159
+ # does not work: base_url: 'https://iam.cloud.ibm.com/identity',
160
160
  crtype: :generic,
161
161
  generic: {
162
162
  grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
163
163
  response_type: 'cloud_iam',
164
- apikey: options.get_option(:ibm_api_key,is_type: :mandatory)
164
+ apikey: options.get_option(:ibm_api_key, is_type: :mandatory)
165
165
  }}})
166
166
  end
167
167
 
@@ -173,27 +173,25 @@ module Aspera
173
173
  rest_add_header = {}
174
174
  if !command.eql?(:instances)
175
175
  instance = options.get_option(:instance)
176
- #Log.log.error("1>>#{instance}".red)
177
176
  if instance.nil?
178
177
  # Take the first Aspera on Cloud transfer service instance ID if not provided by user
179
178
  instance = ats_api_v2_auth_ibm.read('instances')[:data]['data'].first
180
179
  self.format.display_status("using first instance: #{instance}")
181
180
  end
182
- #Log.log.error("2>>#{instance}".red)
183
181
  rest_add_header = {'X-ATS-Service-Instance-Id' => instance}
184
182
  end
185
183
  ats_ibm_api = ats_api_v2_auth_ibm(rest_add_header)
186
184
  case command
187
185
  when :instances
188
186
  instances = ats_ibm_api.read('instances')[:data]
189
- Log.log.warn("more instances remaining: #{instances['remaining']}") unless instances['remaining'].to_i.eql?(0)
187
+ Log.log.warn{"more instances remaining: #{instances['remaining']}"} unless instances['remaining'].to_i.eql?(0)
190
188
  return {type: :value_list, data: instances['data'], name: 'instance'}
191
189
  when :create
192
190
  create_value = options.get_option(:value) || {}
193
- created_key = ats_ibm_api.create('api_keys',create_value)[:data]
191
+ created_key = ats_ibm_api.create('api_keys', create_value)[:data]
194
192
  return {type: :single_object, data: created_key}
195
193
  when :list # list known api keys in ATS (this require an api_key ...)
196
- res = ats_ibm_api.read('api_keys',{'offset' => 0,'max_results' => 1000})
194
+ res = ats_ibm_api.read('api_keys', {'offset' => 0, 'max_results' => 1000})
197
195
  return {type: :value_list, data: res[:data]['data'], name: 'ats_id'}
198
196
  when :show # show one of api_key in ATS
199
197
  res = ats_ibm_api.read("api_keys/#{concerned_id}")
@@ -223,7 +221,7 @@ module Aspera
223
221
  when :api_key # manage credential to access ATS API
224
222
  return execute_action_api_key
225
223
  when :aws_trust_policy
226
- res = ats_api_pub_v1.read('aws/trustpolicy',{region: options.get_option(:region,is_type: :mandatory)})[:data]
224
+ res = ats_api_pub_v1.read('aws/trustpolicy', {region: options.get_option(:region, is_type: :mandatory)})[:data]
227
225
  return {type: :single_object, data: res}
228
226
  else raise 'ERROR'
229
227
  end
@@ -5,7 +5,7 @@ require 'aspera/rest'
5
5
  module Aspera
6
6
  module Cli
7
7
  module Plugins
8
- class Bss < BasicAuthPlugin
8
+ class Bss < Aspera::Cli::BasicAuthPlugin
9
9
  ACTIONS = %i[subscription].freeze
10
10
 
11
11
  FIELDS = {
@@ -14,7 +14,7 @@ module Aspera
14
14
 
15
15
  def initialize(env)
16
16
  super(env)
17
- @api_bss = env[:bss_api] if env.has_key?(:bss_api)
17
+ @api_bss = env[:bss_api] if env.key?(:bss_api)
18
18
  end
19
19
 
20
20
  def all_fields(name)
@@ -23,7 +23,7 @@ module Aspera
23
23
 
24
24
  def execute_action
25
25
  if @api_bss.nil?
26
- key = options.get_option(:password,is_type: :mandatory)
26
+ key = options.get_option(:password, is_type: :mandatory)
27
27
  @api_bss = Rest.new(
28
28
  base_url: 'https://dashboard.bss.asperasoft.com/platform',
29
29
  headers: {cookie: "_dashboard_key=#{key}"})
@@ -35,15 +35,15 @@ module Aspera
35
35
  object = 'bssSubscriptions'
36
36
  case command
37
37
  when :find
38
- query = options.get_option(:query,is_type: :mandatory) # AOC_ORGANIZATION_QUERY AOC_USER_EMAIL
39
- value = options.get_option(:value,is_type: :mandatory)
38
+ query = options.get_option(:query, is_type: :mandatory) # AOC_ORGANIZATION_QUERY AOC_USER_EMAIL
39
+ value = options.get_option(:value, is_type: :mandatory)
40
40
  request = {
41
- 'variables' => {'filter' => {'key' => query,'value' => value}},
41
+ 'variables' => {'filter' => {'key' => query, 'value' => value}},
42
42
  'query' => "query($filter: BssSubscriptionFilter!) {#{object}(filter: $filter) { #{all_fields('bssSubscriptions')} } }"
43
43
  }
44
- result = @api_bss.create('graphql',request)[:data]
44
+ result = @api_bss.create('graphql', request)[:data]
45
45
  # give fields to keep order
46
- return {type: :object_list, data: result['data'][object],fields: FIELDS['bssSubscriptions']}
46
+ return {type: :object_list, data: result['data'][object], fields: FIELDS['bssSubscriptions']}
47
47
  when :show
48
48
  id = instance_identifier
49
49
  request = {
@@ -52,7 +52,7 @@ module Aspera
52
52
  'instances { id state planId serviceId ssmSubscriptionId entitlement { id } aocOrganization { id subdomainName name status tier urlId trialExpiresAt '\
53
53
  'users(organizationAdmin: true) { id name email atsAdmin subscriptionAdmin } } } } }'
54
54
  }
55
- result = @api_bss.create('graphql',request)[:data]['data'][object].first
55
+ result = @api_bss.create('graphql', request)[:data]['data'][object].first
56
56
  result.delete('instances')
57
57
  return {type: :single_object, data: result}
58
58
  when :instances
@@ -61,7 +61,7 @@ module Aspera
61
61
  'variables' => {'id' => id},
62
62
  'query' => "query($id: ID!) {#{object}(id: $id) { aocOrganization { id subdomainName name status tier urlId trialExpiresAt } } } }"
63
63
  }
64
- result = @api_bss.create('graphql',request)[:data]['data'][object].first
64
+ result = @api_bss.create('graphql', request)[:data]['data'][object].first
65
65
  return {type: :object_list, data: result['instances']}
66
66
  end
67
67
  end