aspera-cli 4.10.0 → 4.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +19 -0
  4. data/CHANGELOG.md +528 -0
  5. data/CONTRIBUTING.md +143 -0
  6. data/README.md +977 -589
  7. data/bin/ascli +4 -4
  8. data/bin/asession +12 -12
  9. data/docs/test_env.conf +29 -19
  10. data/examples/aoc.rb +6 -6
  11. data/examples/dascli +18 -16
  12. data/examples/faspex4.rb +15 -15
  13. data/examples/node.rb +12 -12
  14. data/examples/proxy.pac +2 -2
  15. data/examples/server.rb +12 -12
  16. data/lib/aspera/aoc.rb +344 -272
  17. data/lib/aspera/ascmd.rb +56 -54
  18. data/lib/aspera/ats_api.rb +4 -4
  19. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  20. data/lib/aspera/cli/extended_value.rb +9 -9
  21. data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
  22. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  23. data/lib/aspera/cli/listener/logger.rb +1 -1
  24. data/lib/aspera/cli/listener/progress.rb +5 -6
  25. data/lib/aspera/cli/listener/progress_multi.rb +16 -21
  26. data/lib/aspera/cli/main.rb +72 -73
  27. data/lib/aspera/cli/manager.rb +112 -112
  28. data/lib/aspera/cli/plugin.rb +68 -48
  29. data/lib/aspera/cli/plugins/alee.rb +4 -4
  30. data/lib/aspera/cli/plugins/aoc.rb +322 -720
  31. data/lib/aspera/cli/plugins/ats.rb +50 -52
  32. data/lib/aspera/cli/plugins/bss.rb +10 -10
  33. data/lib/aspera/cli/plugins/config.rb +514 -410
  34. data/lib/aspera/cli/plugins/console.rb +12 -12
  35. data/lib/aspera/cli/plugins/cos.rb +18 -20
  36. data/lib/aspera/cli/plugins/faspex.rb +134 -136
  37. data/lib/aspera/cli/plugins/faspex5.rb +235 -70
  38. data/lib/aspera/cli/plugins/node.rb +378 -309
  39. data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
  40. data/lib/aspera/cli/plugins/preview.rb +129 -120
  41. data/lib/aspera/cli/plugins/server.rb +137 -83
  42. data/lib/aspera/cli/plugins/shares.rb +77 -52
  43. data/lib/aspera/cli/plugins/sync.rb +13 -33
  44. data/lib/aspera/cli/transfer_agent.rb +61 -61
  45. data/lib/aspera/cli/version.rb +2 -1
  46. data/lib/aspera/colors.rb +3 -3
  47. data/lib/aspera/command_line_builder.rb +78 -74
  48. data/lib/aspera/cos_node.rb +31 -29
  49. data/lib/aspera/data_repository.rb +1 -1
  50. data/lib/aspera/environment.rb +30 -28
  51. data/lib/aspera/fasp/agent_base.rb +17 -15
  52. data/lib/aspera/fasp/agent_connect.rb +34 -32
  53. data/lib/aspera/fasp/agent_direct.rb +70 -73
  54. data/lib/aspera/fasp/agent_httpgw.rb +79 -74
  55. data/lib/aspera/fasp/agent_node.rb +26 -26
  56. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  57. data/lib/aspera/fasp/error.rb +3 -2
  58. data/lib/aspera/fasp/error_info.rb +11 -8
  59. data/lib/aspera/fasp/installation.rb +80 -80
  60. data/lib/aspera/fasp/listener.rb +2 -2
  61. data/lib/aspera/fasp/parameters.rb +103 -92
  62. data/lib/aspera/fasp/parameters.yaml +313 -214
  63. data/lib/aspera/fasp/resume_policy.rb +10 -10
  64. data/lib/aspera/fasp/transfer_spec.rb +22 -2
  65. data/lib/aspera/fasp/uri.rb +7 -7
  66. data/lib/aspera/faspex_gw.rb +80 -159
  67. data/lib/aspera/faspex_postproc.rb +77 -0
  68. data/lib/aspera/hash_ext.rb +3 -3
  69. data/lib/aspera/id_generator.rb +5 -5
  70. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  71. data/lib/aspera/keychain/macos_security.rb +21 -20
  72. data/lib/aspera/log.rb +13 -13
  73. data/lib/aspera/nagios.rb +24 -23
  74. data/lib/aspera/node.rb +217 -38
  75. data/lib/aspera/oauth.rb +78 -74
  76. data/lib/aspera/open_application.rb +19 -11
  77. data/lib/aspera/persistency_action_once.rb +4 -4
  78. data/lib/aspera/persistency_folder.rb +13 -13
  79. data/lib/aspera/preview/file_types.rb +8 -8
  80. data/lib/aspera/preview/generator.rb +67 -67
  81. data/lib/aspera/preview/utils.rb +27 -27
  82. data/lib/aspera/proxy_auto_config.js +63 -63
  83. data/lib/aspera/proxy_auto_config.rb +19 -19
  84. data/lib/aspera/rest.rb +65 -67
  85. data/lib/aspera/rest_call_error.rb +2 -1
  86. data/lib/aspera/rest_error_analyzer.rb +22 -21
  87. data/lib/aspera/rest_errors_aspera.rb +16 -16
  88. data/lib/aspera/secret_hider.rb +17 -14
  89. data/lib/aspera/ssh.rb +15 -14
  90. data/lib/aspera/sync.rb +177 -62
  91. data/lib/aspera/temp_file_manager.rb +2 -2
  92. data/lib/aspera/uri_reader.rb +4 -4
  93. data/lib/aspera/web_auth.rb +13 -64
  94. data/lib/aspera/web_server_simple.rb +76 -0
  95. data.tar.gz.sig +0 -0
  96. metadata +11 -6
  97. metadata.gz.sig +0 -0
@@ -6,26 +6,26 @@ require 'aspera/ats_api'
6
6
  module Aspera
7
7
  module Cli
8
8
  module Plugins
9
- # list and download connect client versions
9
+ # Access Aspera Transfer Service
10
10
  # https://52.44.83.163/docs/
11
11
  # https://developer.ibm.com/aspera/docs/ats-api-reference/creating-ats-api-keys/
12
- class Ats < Plugin
12
+ class Ats < Aspera::Cli::Plugin
13
13
  def initialize(env)
14
14
  super(env)
15
- options.add_opt_simple(:ibm_api_key,'IBM API key, see https://cloud.ibm.com/iam/apikeys')
16
- options.add_opt_simple(:instance,'ATS instance in ibm cloud')
17
- options.add_opt_simple(:ats_key,'ATS key identifier (ats_xxx)')
18
- options.add_opt_simple(:ats_secret,'ATS key secret')
19
- options.add_opt_simple(:params,'Parameters access key creation (@json:)')
20
- options.add_opt_simple(:cloud,'Cloud provider')
21
- options.add_opt_simple(:region,'Cloud region')
15
+ options.add_opt_simple(:ibm_api_key, 'IBM API key, see https://cloud.ibm.com/iam/apikeys')
16
+ options.add_opt_simple(:instance, 'ATS instance in ibm cloud')
17
+ options.add_opt_simple(:ats_key, 'ATS key identifier (ats_xxx)')
18
+ options.add_opt_simple(:ats_secret, 'ATS key secret')
19
+ options.add_opt_simple(:params, 'Parameters access key creation (@json:)')
20
+ options.add_opt_simple(:cloud, 'Cloud provider')
21
+ options.add_opt_simple(:region, 'Cloud region')
22
22
  options.parse_options!
23
23
  end
24
24
 
25
25
  def server_by_cloud_region
26
- # todo: provide list ?
27
- cloud = options.get_option(:cloud,is_type: :mandatory).upcase
28
- region = options.get_option(:region,is_type: :mandatory)
26
+ # TODO: provide list ?
27
+ cloud = options.get_option(:cloud, is_type: :mandatory).upcase
28
+ region = options.get_option(:region, is_type: :mandatory)
29
29
  return @ats_api_pub.read("servers/#{cloud}/#{region}")[:data]
30
30
  end
31
31
 
@@ -36,27 +36,27 @@ module Aspera
36
36
  base_url: AtsApi.base_url + '/pub/v1',
37
37
  auth: {
38
38
  type: :basic,
39
- username: options.get_option(:ats_key,is_type: :mandatory),
40
- password: options.get_option(:ats_secret,is_type: :mandatory)}
39
+ username: options.get_option(:ats_key, is_type: :mandatory),
40
+ password: options.get_option(:ats_secret, is_type: :mandatory)}
41
41
  })
42
42
  end
43
43
 
44
44
  def execute_action_access_key
45
45
  commands = %i[create list show modify delete node cluster entitlement]
46
46
  command = options.get_next_command(commands)
47
- # those dont require access key id
47
+ # those do not require access key id
48
48
  access_key_id = instance_identifier unless %i[create list].include?(command)
49
49
  case command
50
50
  when :create
51
51
  params = options.get_option(:params) || {}
52
52
  server_data = nil
53
53
  # if transfer_server_id not provided, get it from command line options
54
- if !params.has_key?('transfer_server_id')
54
+ if !params.key?('transfer_server_id')
55
55
  server_data = server_by_cloud_region
56
56
  params['transfer_server_id'] = server_data['id']
57
57
  end
58
- Log.log.debug("using params: #{params}".bg_red.gray)
59
- if params.has_key?('storage')
58
+ Log.log.debug{"using params: #{params}".bg_red.gray}
59
+ if params.key?('storage')
60
60
  case params['storage']['type']
61
61
  # here we need somehow to map storage type to field to get for auth end point
62
62
  when 'ibm-s3'
@@ -67,36 +67,36 @@ module Aspera
67
67
  else
68
68
  server_data2 = @ats_api_pub.all_servers.find do |s|
69
69
  s['cloud'].eql?(server_data['cloud']) &&
70
- s['region'].eql?(server_data['region']) &&
71
- s.has_key?('s3_authentication_endpoint')
70
+ s['region'].eql?(server_data['region']) &&
71
+ s.key?('s3_authentication_endpoint')
72
72
  end
73
73
  raise "no such transfer server id: #{params['transfer_server_id']}" if server_data2.nil?
74
74
  # specific one do not have s3 end point in id
75
75
  params['transfer_server_id'] = server_data2['id']
76
76
  end
77
- if !params['storage'].has_key?('authentication_endpoint')
77
+ if !params['storage'].key?('authentication_endpoint')
78
78
  params['storage']['endpoint'] = server_data2['s3_authentication_endpoint']
79
79
  end
80
80
  end
81
81
  end
82
- res = ats_api_pub_v1.create('access_keys',params)
82
+ res = ats_api_pub_v1.create('access_keys', params)
83
83
  return {type: :single_object, data: res[:data]}
84
84
  # TODO : action : modify, with "PUT"
85
85
  when :list
86
- params = options.get_option(:params) || {'offset' => 0,'max_results' => 1000}
87
- res = ats_api_pub_v1.read('access_keys',params)
88
- return {type: :object_list, data: res[:data]['data'], fields: ['name','id','created.at','modified.at']}
86
+ params = options.get_option(:params) || {'offset' => 0, 'max_results' => 1000}
87
+ res = ats_api_pub_v1.read('access_keys', params)
88
+ return {type: :object_list, data: res[:data]['data'], fields: ['name', 'id', 'created.at', 'modified.at']}
89
89
  when :show
90
90
  res = ats_api_pub_v1.read("access_keys/#{access_key_id}")
91
91
  return {type: :single_object, data: res[:data]}
92
92
  when :modify
93
- params = options.get_option(:value,is_type: :mandatory)
93
+ params = options.get_option(:value, is_type: :mandatory)
94
94
  params['id'] = access_key_id
95
- ats_api_pub_v1.update("access_keys/#{access_key_id}",params)
95
+ ats_api_pub_v1.update("access_keys/#{access_key_id}", params)
96
96
  return Main.result_status('modified')
97
97
  when :entitlement
98
98
  ak = ats_api_pub_v1.read("access_keys/#{access_key_id}")[:data]
99
- api_bss = AoC.metering_api(ak['license']['entitlement_id'],ak['license']['customer_id'])
99
+ api_bss = AoC.metering_api(ak['license']['entitlement_id'], ak['license']['customer_id'])
100
100
  return {type: :single_object, data: api_bss.read('entitlement')[:data]}
101
101
  when :delete
102
102
  ats_api_pub_v1.delete("access_keys/#{access_key_id}")
@@ -104,25 +104,25 @@ module Aspera
104
104
  when :node
105
105
  ak_data = ats_api_pub_v1.read("access_keys/#{access_key_id}")[:data]
106
106
  server_data = @ats_api_pub.all_servers.find {|i| i['id'].start_with?(ak_data['transfer_server_id'])}
107
- raise CliError,'no such server found' if server_data.nil?
108
- base_url = server_data['transfer_setup_url']
109
- api_node = Rest.new({
110
- base_url: base_url,
107
+ raise CliError, 'no such server found' if server_data.nil?
108
+ node_url = server_data['transfer_setup_url']
109
+ api_node = Aspera::Node.new(params: {
110
+ base_url: node_url,
111
111
  auth: {
112
112
  type: :basic,
113
113
  username: access_key_id,
114
- password: @agents[:config].get_secret(url: base_url,username: access_key_id)
114
+ password: @agents[:config].lookup_secret(url: node_url, username: access_key_id)
115
115
  }})
116
- command = options.get_next_command(Node::COMMON_ACTIONS)
117
- return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).execute_action(command)
116
+ command = options.get_next_command(Node::COMMANDS_GEN4)
117
+ return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).execute_command_gen4(command, ak_data['root_file_id'])
118
118
  when :cluster
119
- base_url = ats_api_pub_v1.params[:base_url]
119
+ ats_url = ats_api_pub_v1.params[:base_url]
120
120
  rest_params = {
121
- base_url: base_url,
121
+ base_url: ats_url,
122
122
  auth: {
123
123
  type: :basic,
124
124
  username: access_key_id,
125
- password: @agents[:config].get_secret(url: base_url, username: access_key_id)
125
+ password: @agents[:config].lookup_secret(url: ats_url, username: access_key_id)
126
126
  }}
127
127
  api_ak_auth = Rest.new(rest_params)
128
128
  return {type: :single_object, data: api_ak_auth.read('servers')[:data]}
@@ -138,7 +138,7 @@ module Aspera
138
138
  when :list
139
139
  return {type: :object_list, data: @ats_api_pub.all_servers, fields: %w[id cloud region]}
140
140
  when :show
141
- if options.get_option(:cloud) || options.get_option(:region,is_type: :optional)
141
+ if options.get_option(:cloud) || options.get_option(:region, is_type: :optional)
142
142
  server_data = server_by_cloud_region
143
143
  else
144
144
  server_id = instance_identifier
@@ -154,14 +154,14 @@ module Aspera
154
154
  base_url: AtsApi.base_url + '/v2',
155
155
  headers: rest_add_headers,
156
156
  auth: {
157
- type: :oauth2,
158
- base_url: 'https://iam.bluemix.net/identity',
159
- #does not work: base_url: 'https://iam.cloud.ibm.com/identity',
160
- crtype: :generic,
161
- generic: {
157
+ type: :oauth2,
158
+ base_url: 'https://iam.bluemix.net/identity',
159
+ # does not work: base_url: 'https://iam.cloud.ibm.com/identity',
160
+ grant_method: :generic,
161
+ generic: {
162
162
  grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
163
163
  response_type: 'cloud_iam',
164
- apikey: options.get_option(:ibm_api_key,is_type: :mandatory)
164
+ apikey: options.get_option(:ibm_api_key, is_type: :mandatory)
165
165
  }}})
166
166
  end
167
167
 
@@ -173,27 +173,25 @@ module Aspera
173
173
  rest_add_header = {}
174
174
  if !command.eql?(:instances)
175
175
  instance = options.get_option(:instance)
176
- #Log.log.error("1>>#{instance}".red)
177
176
  if instance.nil?
178
177
  # Take the first Aspera on Cloud transfer service instance ID if not provided by user
179
178
  instance = ats_api_v2_auth_ibm.read('instances')[:data]['data'].first
180
- self.format.display_status("using first instance: #{instance}")
179
+ formatter.display_status("using first instance: #{instance}")
181
180
  end
182
- #Log.log.error("2>>#{instance}".red)
183
181
  rest_add_header = {'X-ATS-Service-Instance-Id' => instance}
184
182
  end
185
183
  ats_ibm_api = ats_api_v2_auth_ibm(rest_add_header)
186
184
  case command
187
185
  when :instances
188
186
  instances = ats_ibm_api.read('instances')[:data]
189
- Log.log.warn("more instances remaining: #{instances['remaining']}") unless instances['remaining'].to_i.eql?(0)
187
+ Log.log.warn{"more instances remaining: #{instances['remaining']}"} unless instances['remaining'].to_i.eql?(0)
190
188
  return {type: :value_list, data: instances['data'], name: 'instance'}
191
189
  when :create
192
190
  create_value = options.get_option(:value) || {}
193
- created_key = ats_ibm_api.create('api_keys',create_value)[:data]
191
+ created_key = ats_ibm_api.create('api_keys', create_value)[:data]
194
192
  return {type: :single_object, data: created_key}
195
193
  when :list # list known api keys in ATS (this require an api_key ...)
196
- res = ats_ibm_api.read('api_keys',{'offset' => 0,'max_results' => 1000})
194
+ res = ats_ibm_api.read('api_keys', {'offset' => 0, 'max_results' => 1000})
197
195
  return {type: :value_list, data: res[:data]['data'], name: 'ats_id'}
198
196
  when :show # show one of api_key in ATS
199
197
  res = ats_ibm_api.read("api_keys/#{concerned_id}")
@@ -223,7 +221,7 @@ module Aspera
223
221
  when :api_key # manage credential to access ATS API
224
222
  return execute_action_api_key
225
223
  when :aws_trust_policy
226
- res = ats_api_pub_v1.read('aws/trustpolicy',{region: options.get_option(:region,is_type: :mandatory)})[:data]
224
+ res = ats_api_pub_v1.read('aws/trustpolicy', {region: options.get_option(:region, is_type: :mandatory)})[:data]
227
225
  return {type: :single_object, data: res}
228
226
  else raise 'ERROR'
229
227
  end
@@ -5,7 +5,7 @@ require 'aspera/rest'
5
5
  module Aspera
6
6
  module Cli
7
7
  module Plugins
8
- class Bss < BasicAuthPlugin
8
+ class Bss < Aspera::Cli::BasicAuthPlugin
9
9
  ACTIONS = %i[subscription].freeze
10
10
 
11
11
  FIELDS = {
@@ -14,7 +14,7 @@ module Aspera
14
14
 
15
15
  def initialize(env)
16
16
  super(env)
17
- @api_bss = env[:bss_api] if env.has_key?(:bss_api)
17
+ @api_bss = env[:bss_api] if env.key?(:bss_api)
18
18
  end
19
19
 
20
20
  def all_fields(name)
@@ -23,7 +23,7 @@ module Aspera
23
23
 
24
24
  def execute_action
25
25
  if @api_bss.nil?
26
- key = options.get_option(:password,is_type: :mandatory)
26
+ key = options.get_option(:password, is_type: :mandatory)
27
27
  @api_bss = Rest.new(
28
28
  base_url: 'https://dashboard.bss.asperasoft.com/platform',
29
29
  headers: {cookie: "_dashboard_key=#{key}"})
@@ -35,15 +35,15 @@ module Aspera
35
35
  object = 'bssSubscriptions'
36
36
  case command
37
37
  when :find
38
- query = options.get_option(:query,is_type: :mandatory) # AOC_ORGANIZATION_QUERY AOC_USER_EMAIL
39
- value = options.get_option(:value,is_type: :mandatory)
38
+ query = options.get_option(:query, is_type: :mandatory) # AOC_ORGANIZATION_QUERY AOC_USER_EMAIL
39
+ value = options.get_option(:value, is_type: :mandatory)
40
40
  request = {
41
- 'variables' => {'filter' => {'key' => query,'value' => value}},
41
+ 'variables' => {'filter' => {'key' => query, 'value' => value}},
42
42
  'query' => "query($filter: BssSubscriptionFilter!) {#{object}(filter: $filter) { #{all_fields('bssSubscriptions')} } }"
43
43
  }
44
- result = @api_bss.create('graphql',request)[:data]
44
+ result = @api_bss.create('graphql', request)[:data]
45
45
  # give fields to keep order
46
- return {type: :object_list, data: result['data'][object],fields: FIELDS['bssSubscriptions']}
46
+ return {type: :object_list, data: result['data'][object], fields: FIELDS['bssSubscriptions']}
47
47
  when :show
48
48
  id = instance_identifier
49
49
  request = {
@@ -52,7 +52,7 @@ module Aspera
52
52
  'instances { id state planId serviceId ssmSubscriptionId entitlement { id } aocOrganization { id subdomainName name status tier urlId trialExpiresAt '\
53
53
  'users(organizationAdmin: true) { id name email atsAdmin subscriptionAdmin } } } } }'
54
54
  }
55
- result = @api_bss.create('graphql',request)[:data]['data'][object].first
55
+ result = @api_bss.create('graphql', request)[:data]['data'][object].first
56
56
  result.delete('instances')
57
57
  return {type: :single_object, data: result}
58
58
  when :instances
@@ -61,7 +61,7 @@ module Aspera
61
61
  'variables' => {'id' => id},
62
62
  'query' => "query($id: ID!) {#{object}(id: $id) { aocOrganization { id subdomainName name status tier urlId trialExpiresAt } } } }"
63
63
  }
64
- result = @api_bss.create('graphql',request)[:data]['data'][object].first
64
+ result = @api_bss.create('graphql', request)[:data]['data'][object].first
65
65
  return {type: :object_list, data: result['instances']}
66
66
  end
67
67
  end