aspera-cli 4.9.0 → 4.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +20 -0
  4. data/CHANGELOG.md +509 -0
  5. data/CONTRIBUTING.md +118 -0
  6. data/README.md +1241 -916
  7. data/bin/ascli +4 -4
  8. data/bin/asession +11 -11
  9. data/docs/test_env.conf +32 -21
  10. data/examples/aoc.rb +4 -4
  11. data/examples/dascli +16 -9
  12. data/examples/faspex4.rb +8 -8
  13. data/examples/node.rb +12 -12
  14. data/examples/server.rb +10 -10
  15. data/lib/aspera/aoc.rb +273 -266
  16. data/lib/aspera/ascmd.rb +56 -54
  17. data/lib/aspera/ats_api.rb +4 -4
  18. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  19. data/lib/aspera/cli/extended_value.rb +5 -5
  20. data/lib/aspera/cli/formater.rb +64 -64
  21. data/lib/aspera/cli/info.rb +2 -2
  22. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  23. data/lib/aspera/cli/listener/logger.rb +1 -1
  24. data/lib/aspera/cli/listener/progress.rb +5 -6
  25. data/lib/aspera/cli/listener/progress_multi.rb +14 -19
  26. data/lib/aspera/cli/main.rb +66 -67
  27. data/lib/aspera/cli/manager.rb +112 -110
  28. data/lib/aspera/cli/plugin.rb +57 -36
  29. data/lib/aspera/cli/plugins/alee.rb +4 -4
  30. data/lib/aspera/cli/plugins/aoc.rb +309 -670
  31. data/lib/aspera/cli/plugins/ats.rb +44 -46
  32. data/lib/aspera/cli/plugins/bss.rb +10 -10
  33. data/lib/aspera/cli/plugins/config.rb +497 -378
  34. data/lib/aspera/cli/plugins/console.rb +12 -12
  35. data/lib/aspera/cli/plugins/cos.rb +18 -20
  36. data/lib/aspera/cli/plugins/faspex.rb +112 -114
  37. data/lib/aspera/cli/plugins/faspex5.rb +71 -46
  38. data/lib/aspera/cli/plugins/node.rb +379 -283
  39. data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
  40. data/lib/aspera/cli/plugins/preview.rb +122 -114
  41. data/lib/aspera/cli/plugins/server.rb +137 -83
  42. data/lib/aspera/cli/plugins/shares.rb +30 -29
  43. data/lib/aspera/cli/plugins/sync.rb +13 -33
  44. data/lib/aspera/cli/transfer_agent.rb +60 -59
  45. data/lib/aspera/cli/version.rb +1 -1
  46. data/lib/aspera/colors.rb +3 -3
  47. data/lib/aspera/command_line_builder.rb +27 -27
  48. data/lib/aspera/cos_node.rb +22 -20
  49. data/lib/aspera/data_repository.rb +1 -1
  50. data/lib/aspera/environment.rb +35 -15
  51. data/lib/aspera/fasp/agent_base.rb +15 -15
  52. data/lib/aspera/fasp/agent_connect.rb +23 -21
  53. data/lib/aspera/fasp/agent_direct.rb +66 -64
  54. data/lib/aspera/fasp/agent_httpgw.rb +141 -78
  55. data/lib/aspera/fasp/agent_node.rb +23 -21
  56. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  57. data/lib/aspera/fasp/error.rb +3 -2
  58. data/lib/aspera/fasp/error_info.rb +11 -8
  59. data/lib/aspera/fasp/installation.rb +79 -79
  60. data/lib/aspera/fasp/listener.rb +1 -1
  61. data/lib/aspera/fasp/parameters.rb +86 -71
  62. data/lib/aspera/fasp/parameters.yaml +7 -4
  63. data/lib/aspera/fasp/resume_policy.rb +8 -8
  64. data/lib/aspera/fasp/transfer_spec.rb +35 -2
  65. data/lib/aspera/fasp/uri.rb +7 -7
  66. data/lib/aspera/faspex_gw.rb +7 -5
  67. data/lib/aspera/hash_ext.rb +3 -3
  68. data/lib/aspera/id_generator.rb +5 -5
  69. data/lib/aspera/keychain/encrypted_hash.rb +38 -105
  70. data/lib/aspera/keychain/macos_security.rb +128 -57
  71. data/lib/aspera/log.rb +7 -7
  72. data/lib/aspera/nagios.rb +19 -18
  73. data/lib/aspera/node.rb +209 -35
  74. data/lib/aspera/oauth.rb +37 -36
  75. data/lib/aspera/open_application.rb +19 -11
  76. data/lib/aspera/persistency_action_once.rb +4 -4
  77. data/lib/aspera/persistency_folder.rb +16 -15
  78. data/lib/aspera/preview/file_types.rb +8 -8
  79. data/lib/aspera/preview/generator.rb +67 -67
  80. data/lib/aspera/preview/utils.rb +27 -27
  81. data/lib/aspera/proxy_auto_config.js +41 -41
  82. data/lib/aspera/proxy_auto_config.rb +21 -14
  83. data/lib/aspera/rest.rb +72 -67
  84. data/lib/aspera/rest_call_error.rb +2 -1
  85. data/lib/aspera/rest_error_analyzer.rb +18 -17
  86. data/lib/aspera/rest_errors_aspera.rb +16 -16
  87. data/lib/aspera/secret_hider.rb +15 -13
  88. data/lib/aspera/ssh.rb +11 -10
  89. data/lib/aspera/sync.rb +158 -44
  90. data/lib/aspera/temp_file_manager.rb +2 -2
  91. data/lib/aspera/uri_reader.rb +4 -4
  92. data/lib/aspera/web_auth.rb +14 -13
  93. data.tar.gz.sig +0 -0
  94. metadata +11 -36
  95. metadata.gz.sig +0 -0
@@ -9,7 +9,7 @@ require 'securerandom'
9
9
  module Aspera
10
10
  module Cli
11
11
  module Plugins
12
- class Faspex5 < BasicAuthPlugin
12
+ class Faspex5 < Aspera::Cli::BasicAuthPlugin
13
13
  class << self
14
14
  def detect(base_url)
15
15
  api = Rest.new(base_url: base_url, redirect_max: 1)
@@ -21,32 +21,31 @@ module Aspera
21
21
  end
22
22
  end
23
23
 
24
- VAL_ALL = 'ALL'
25
24
  TRANSFER_CONNECT = 'connect'
26
- private_constant :VAL_ALL,:TRANSFER_CONNECT
25
+ private_constant :TRANSFER_CONNECT
27
26
 
28
27
  def initialize(env)
29
28
  super(env)
30
- options.add_opt_simple(:client_id,'OAuth client identifier')
31
- options.add_opt_simple(:client_secret,'OAuth client secret')
32
- options.add_opt_simple(:redirect_uri,'OAuth redirect URI for web authentication')
33
- options.add_opt_list(:auth,[Oauth::STD_AUTH_TYPES,:boot].flatten,'OAuth type of authentication')
34
- options.add_opt_simple(:private_key,'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
35
- options.add_opt_simple(:passphrase,'RSA private key passphrase')
36
- options.set_option(:auth,:jwt)
29
+ options.add_opt_simple(:client_id, 'OAuth client identifier')
30
+ options.add_opt_simple(:client_secret, 'OAuth client secret')
31
+ options.add_opt_simple(:redirect_uri, 'OAuth redirect URI for web authentication')
32
+ options.add_opt_list(:auth, [:boot].concat(Oauth::STD_AUTH_TYPES), 'OAuth type of authentication')
33
+ options.add_opt_simple(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
34
+ options.add_opt_simple(:passphrase, 'RSA private key passphrase')
35
+ options.set_option(:auth, :jwt)
37
36
  options.parse_options!
38
37
  end
39
38
 
40
39
  def set_api
41
- @faxpex5_api_base_url = options.get_option(:url,is_type: :mandatory).gsub(%r{/+$},'')
40
+ @faxpex5_api_base_url = options.get_option(:url, is_type: :mandatory).gsub(%r{/+$}, '')
42
41
  @faxpex5_api_auth_url = "#{@faxpex5_api_base_url}/auth"
43
42
  faxpex5_api_v5_url = "#{@faxpex5_api_base_url}/api/v5"
44
- case options.get_option(:auth,is_type: :mandatory)
43
+ case options.get_option(:auth, is_type: :mandatory)
45
44
  when :boot
46
45
  # the password here is the token copied directly from browser in developer mode
47
46
  @api_v5 = Rest.new({
48
47
  base_url: faxpex5_api_v5_url,
49
- headers: {'Authorization' => options.get_option(:password,is_type: :mandatory)}
48
+ headers: {'Authorization' => options.get_option(:password, is_type: :mandatory)}
50
49
  })
51
50
  when :web
52
51
  # opens a browser and ask user to auth using web
@@ -56,11 +55,11 @@ module Aspera
56
55
  type: :oauth2,
57
56
  base_url: @faxpex5_api_auth_url,
58
57
  crtype: :web,
59
- client_id: options.get_option(:client_id,is_type: :mandatory),
60
- web: {redirect_uri: options.get_option(:redirect_uri,is_type: :mandatory)}
58
+ client_id: options.get_option(:client_id, is_type: :mandatory),
59
+ web: {redirect_uri: options.get_option(:redirect_uri, is_type: :mandatory)}
61
60
  }})
62
61
  when :jwt
63
- app_client_id = options.get_option(:client_id,is_type: :mandatory)
62
+ app_client_id = options.get_option(:client_id, is_type: :mandatory)
64
63
  @api_v5 = Rest.new({
65
64
  base_url: faxpex5_api_v5_url,
66
65
  auth: {
@@ -72,31 +71,33 @@ module Aspera
72
71
  payload: {
73
72
  iss: app_client_id, # issuer
74
73
  aud: app_client_id, # audience TODO: ???
75
- sub: "user:#{options.get_option(:username,is_type: :mandatory)}" # subject also "client:#{app_client_id}" + auth user/pass
74
+ sub: "user:#{options.get_option(:username, is_type: :mandatory)}" # subject also "client:#{app_client_id}" + auth user/pass
76
75
  },
77
- #auth: {type: :basic, options.get_option(:username,is_type: :mandatory), options.get_option(:password,is_type: :mandatory),
78
- private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key,is_type: :mandatory),options.get_option(:passphrase)),
76
+ # auth: {type: :basic, options.get_option(:username,is_type: :mandatory), options.get_option(:password,is_type: :mandatory),
77
+ private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key, is_type: :mandatory), options.get_option(:passphrase)),
79
78
  headers: {typ: 'JWT'}
80
79
  }
81
80
  }})
82
81
  end
83
82
  end
84
83
 
85
- ACTIONS = %i[health package admin user].freeze
84
+ ACTIONS = %i[health version user bearer_token package admin].freeze
86
85
 
87
86
  def execute_action
88
87
  set_api
89
88
  command = options.get_next_command(ACTIONS)
90
89
  case command
90
+ when :version
91
+ return { type: :single_object, data: @api_v5.read('version')[:data] }
91
92
  when :health
92
93
  nagios = Nagios.new
93
94
  begin
94
- result=Rest.new(base_url: @faxpex5_api_base_url).read('health')[:data]
95
- result.each do |k,v|
96
- nagios.add_ok(k,v.to_s)
95
+ result = Rest.new(base_url: @faxpex5_api_base_url).read('health')[:data]
96
+ result.each do |k, v|
97
+ nagios.add_ok(k, v.to_s)
97
98
  end
98
99
  rescue StandardError => e
99
- nagios.add_critical('faspex api',e.to_s)
100
+ nagios.add_critical('faspex api', e.to_s)
100
101
  end
101
102
  return nagios.result
102
103
  when :user
@@ -106,10 +107,12 @@ module Aspera
106
107
  when :show
107
108
  return { type: :single_object, data: @api_v5.read('account/preferences')[:data] }
108
109
  when :modify
109
- @api_v5.update('account/preferences',options.get_next_argument('modified parameters (Hash)'))
110
+ @api_v5.update('account/preferences', options.get_next_argument('modified parameters', type: Hash))
110
111
  return Main.result_status('modified')
111
112
  end
112
113
  end
114
+ when :bearer_token
115
+ return {type: :text, data: @api_v5.oauth_token}
113
116
  when :package
114
117
  command = options.get_next_command(%i[list show send receive])
115
118
  case command
@@ -117,48 +120,70 @@ module Aspera
117
120
  parameters = options.get_option(:value)
118
121
  return {
119
122
  type: :object_list,
120
- data: @api_v5.read('packages',parameters)[:data]['packages'],
123
+ data: @api_v5.read('packages', parameters)[:data]['packages'],
121
124
  fields: %w[id title release_date total_bytes total_files created_time state]
122
125
  }
123
126
  when :show
124
127
  id = instance_identifier
125
128
  return {type: :single_object, data: @api_v5.read("packages/#{id}")[:data]}
126
129
  when :send
127
- parameters = options.get_option(:value,is_type: :mandatory)
128
- raise CliBadArgument,'value must be hash, refer to API' unless parameters.is_a?(Hash)
129
- package = @api_v5.create('packages',parameters)[:data]
130
- # TODO: option to send from remote source
131
- transfer_spec = @api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: TRANSFER_CONNECT})[:data]
130
+ parameters = options.get_option(:value, is_type: :mandatory)
131
+ raise CliBadArgument, 'value must be hash, refer to API' unless parameters.is_a?(Hash)
132
+ package = @api_v5.create('packages', parameters)[:data]
133
+ # TODO: option to send from remote source or httpgw
134
+ transfer_spec = @api_v5.call(
135
+ operation: 'POST',
136
+ subpath: "packages/#{package['id']}/transfer_spec/upload",
137
+ headers: {'Accept' => 'application/json'},
138
+ url_params: {transfer_type: TRANSFER_CONNECT},
139
+ json_params: {paths: transfer.ts_source_paths.map{|i|i['source']}}
140
+ )[:data]
132
141
  transfer_spec.delete('authentication')
133
- return Main.result_transfer(transfer.start(transfer_spec,{src: :node_gen3}))
142
+ return Main.result_transfer(transfer.start(transfer_spec))
134
143
  when :receive
135
144
  pkg_type = 'received'
136
145
  pack_id = instance_identifier
137
146
  package_ids = [pack_id]
138
147
  skip_ids_data = []
139
148
  skip_ids_persistency = nil
140
- if options.get_option(:once_only,is_type: :mandatory)
149
+ if options.get_option(:once_only, is_type: :mandatory)
141
150
  # read ids from persistency
142
151
  skip_ids_persistency = PersistencyActionOnce.new(
143
152
  manager: @agents[:persistency],
144
153
  data: skip_ids_data,
145
- id: IdGenerator.from_list(['faspex_recv',options.get_option(:url,is_type: :mandatory),options.get_option(:username,is_type: :mandatory),pkg_type]))
154
+ id: IdGenerator.from_list([
155
+ 'faspex_recv',
156
+ options.get_option(:url, is_type: :mandatory),
157
+ options.get_option(:username, is_type: :mandatory),
158
+ pkg_type]))
146
159
  end
147
- if pack_id.eql?(VAL_ALL)
160
+ if VAL_ALL.eql?(pack_id)
148
161
  # TODO: if packages have same name, they will overwrite
149
162
  parameters = options.get_option(:value)
150
- parameters ||= {'type' => 'received','subtype' => 'mypackages','limit' => 1000}
151
- raise CliBadArgument,'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
152
- package_ids = @api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
163
+ parameters ||= {'type' => 'received', 'subtype' => 'mypackages', 'limit' => 1000}
164
+ raise CliBadArgument, 'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
165
+ package_ids = @api_v5.read('packages', parameters)[:data]['packages'].map{|p|p['id']}
153
166
  package_ids.reject!{|i|skip_ids_data.include?(i)}
154
167
  end
155
168
  result_transfer = []
156
169
  package_ids.each do |pkgid|
170
+ param_file_list = {}
171
+ begin
172
+ param_file_list['paths'] = transfer.ts_source_paths.map{|i|i['source']}
173
+ rescue Aspera::Cli::CliBadArgument
174
+ # paths is optional
175
+ end
157
176
  # TODO: allow from sent as well ?
158
- transfer_spec = @api_v5.create("packages/#{pkgid}/transfer_spec/download",{transfer_type: TRANSFER_CONNECT, type: pkg_type})[:data]
177
+ transfer_spec = @api_v5.call(
178
+ operation: 'POST',
179
+ subpath: "packages/#{pkgid}/transfer_spec/download",
180
+ headers: {'Accept' => 'application/json'},
181
+ url_params: {transfer_type: TRANSFER_CONNECT, type: pkg_type},
182
+ json_params: param_file_list
183
+ )[:data]
159
184
  transfer_spec.delete('authentication')
160
- statuses = transfer.start(transfer_spec,{src: :node_gen3})
161
- result_transfer.push({'package' => pkgid,Main::STATUS_FIELD => statuses})
185
+ statuses = transfer.start(transfer_spec)
186
+ result_transfer.push({'package' => pkgid, Main::STATUS_FIELD => statuses})
162
187
  # skip only if all sessions completed
163
188
  skip_ids_data.push(pkgid) if TransferAgent.session_status(statuses).eql?(:success)
164
189
  end
@@ -172,19 +197,19 @@ module Aspera
172
197
  res_path = list_key = res_type.to_s
173
198
  case res_type
174
199
  when :metadata_profiles
175
- res_path='configuration/metadata_profiles'
176
- list_key='profiles'
200
+ res_path = 'configuration/metadata_profiles'
201
+ list_key = 'profiles'
177
202
  end
178
203
  display_fields =
179
204
  case res_type
180
- when :accounts then [:all_but,'user_profile_data_attributes']
181
- when :oauth_clients then [:all_but,'public_key']
205
+ when :accounts then [:all_but, 'user_profile_data_attributes']
206
+ when :oauth_clients then [:all_but, 'public_key']
182
207
  end
183
208
  adm_api = @api_v5
184
209
  if res_type.eql?(:oauth_clients)
185
210
  adm_api = Rest.new(@api_v5.params.merge({base_url: @faxpex5_api_auth_url}))
186
211
  end
187
- return entity_action(adm_api,res_path,item_list_key: list_key, display_fields: display_fields)
212
+ return entity_action(adm_api, res_path, item_list_key: list_key, display_fields: display_fields)
188
213
  end
189
214
  end # case command
190
215
  end # action