aspera-cli 4.5.0 → 4.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +1 -0
  3. data/README.md +1894 -1574
  4. data/bin/ascli +21 -1
  5. data/bin/asession +38 -34
  6. data/docs/test_env.conf +14 -3
  7. data/examples/aoc.rb +17 -15
  8. data/examples/dascli +26 -0
  9. data/examples/faspex4.rb +42 -35
  10. data/examples/proxy.pac +1 -1
  11. data/examples/transfer.rb +38 -37
  12. data/lib/aspera/aoc.rb +245 -205
  13. data/lib/aspera/ascmd.rb +111 -90
  14. data/lib/aspera/ats_api.rb +16 -14
  15. data/lib/aspera/cli/basic_auth_plugin.rb +19 -18
  16. data/lib/aspera/cli/extended_value.rb +50 -39
  17. data/lib/aspera/cli/formater.rb +161 -135
  18. data/lib/aspera/cli/info.rb +18 -0
  19. data/lib/aspera/cli/listener/line_dump.rb +4 -2
  20. data/lib/aspera/cli/listener/logger.rb +3 -1
  21. data/lib/aspera/cli/listener/progress.rb +20 -21
  22. data/lib/aspera/cli/listener/progress_multi.rb +29 -31
  23. data/lib/aspera/cli/main.rb +194 -183
  24. data/lib/aspera/cli/manager.rb +213 -206
  25. data/lib/aspera/cli/plugin.rb +71 -49
  26. data/lib/aspera/cli/plugins/alee.rb +8 -7
  27. data/lib/aspera/cli/plugins/aoc.rb +675 -558
  28. data/lib/aspera/cli/plugins/ats.rb +116 -109
  29. data/lib/aspera/cli/plugins/bss.rb +35 -34
  30. data/lib/aspera/cli/plugins/config.rb +722 -542
  31. data/lib/aspera/cli/plugins/console.rb +28 -22
  32. data/lib/aspera/cli/plugins/cos.rb +28 -37
  33. data/lib/aspera/cli/plugins/faspex.rb +281 -227
  34. data/lib/aspera/cli/plugins/faspex5.rb +129 -84
  35. data/lib/aspera/cli/plugins/node.rb +426 -232
  36. data/lib/aspera/cli/plugins/orchestrator.rb +106 -98
  37. data/lib/aspera/cli/plugins/preview.rb +196 -191
  38. data/lib/aspera/cli/plugins/server.rb +131 -126
  39. data/lib/aspera/cli/plugins/shares.rb +49 -36
  40. data/lib/aspera/cli/plugins/sync.rb +27 -28
  41. data/lib/aspera/cli/transfer_agent.rb +84 -79
  42. data/lib/aspera/cli/version.rb +3 -1
  43. data/lib/aspera/colors.rb +37 -28
  44. data/lib/aspera/command_line_builder.rb +84 -63
  45. data/lib/aspera/cos_node.rb +68 -34
  46. data/lib/aspera/data_repository.rb +4 -2
  47. data/lib/aspera/environment.rb +61 -46
  48. data/lib/aspera/fasp/agent_base.rb +36 -31
  49. data/lib/aspera/fasp/agent_connect.rb +44 -37
  50. data/lib/aspera/fasp/agent_direct.rb +101 -104
  51. data/lib/aspera/fasp/agent_httpgw.rb +91 -90
  52. data/lib/aspera/fasp/agent_node.rb +36 -33
  53. data/lib/aspera/fasp/agent_trsdk.rb +28 -31
  54. data/lib/aspera/fasp/error.rb +3 -1
  55. data/lib/aspera/fasp/error_info.rb +81 -54
  56. data/lib/aspera/fasp/installation.rb +171 -151
  57. data/lib/aspera/fasp/listener.rb +2 -0
  58. data/lib/aspera/fasp/parameters.rb +105 -111
  59. data/lib/aspera/fasp/parameters.yaml +305 -249
  60. data/lib/aspera/fasp/resume_policy.rb +20 -20
  61. data/lib/aspera/fasp/transfer_spec.rb +27 -0
  62. data/lib/aspera/fasp/uri.rb +31 -29
  63. data/lib/aspera/faspex_gw.rb +95 -118
  64. data/lib/aspera/hash_ext.rb +12 -13
  65. data/lib/aspera/id_generator.rb +11 -9
  66. data/lib/aspera/keychain/encrypted_hash.rb +73 -57
  67. data/lib/aspera/keychain/macos_security.rb +27 -29
  68. data/lib/aspera/log.rb +40 -39
  69. data/lib/aspera/nagios.rb +24 -22
  70. data/lib/aspera/node.rb +38 -30
  71. data/lib/aspera/oauth.rb +217 -248
  72. data/lib/aspera/open_application.rb +9 -7
  73. data/lib/aspera/persistency_action_once.rb +15 -14
  74. data/lib/aspera/persistency_folder.rb +15 -18
  75. data/lib/aspera/preview/file_types.rb +266 -270
  76. data/lib/aspera/preview/generator.rb +94 -92
  77. data/lib/aspera/preview/image_error.png +0 -0
  78. data/lib/aspera/preview/options.rb +20 -17
  79. data/lib/aspera/preview/utils.rb +99 -102
  80. data/lib/aspera/preview/video_error.png +0 -0
  81. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  82. data/lib/aspera/proxy_auto_config.rb +114 -21
  83. data/lib/aspera/rest.rb +144 -142
  84. data/lib/aspera/rest_call_error.rb +3 -2
  85. data/lib/aspera/rest_error_analyzer.rb +31 -31
  86. data/lib/aspera/rest_errors_aspera.rb +18 -16
  87. data/lib/aspera/secret_hider.rb +68 -0
  88. data/lib/aspera/ssh.rb +20 -16
  89. data/lib/aspera/sync.rb +57 -54
  90. data/lib/aspera/temp_file_manager.rb +20 -14
  91. data/lib/aspera/timer_limiter.rb +10 -8
  92. data/lib/aspera/uri_reader.rb +14 -15
  93. data/lib/aspera/web_auth.rb +85 -80
  94. data.tar.gz.sig +0 -0
  95. metadata +169 -40
  96. metadata.gz.sig +2 -0
  97. data/bin/dascli +0 -13
  98. data/docs/Makefile +0 -63
  99. data/docs/README.erb.md +0 -4221
  100. data/docs/README.md +0 -13
  101. data/docs/diagrams.txt +0 -49
  102. data/docs/doc_tools.rb +0 -58
  103. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  104. data/lib/aspera/fasp/default.rb +0 -17
@@ -1,6 +1,9 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'aspera/cli/basic_auth_plugin'
2
4
  require 'aspera/persistency_action_once'
3
5
  require 'aspera/id_generator'
6
+ require 'aspera/nagios'
4
7
  require 'securerandom'
5
8
 
6
9
  module Aspera
@@ -9,141 +12,183 @@ module Aspera
9
12
  class Faspex5 < BasicAuthPlugin
10
13
  class << self
11
14
  def detect(base_url)
12
- api=Rest.new({:base_url=>base_url})
13
- result=api.read('api/v5/configuration/ping')
14
- if result[:http].code.start_with?('2') and result[:http].body.strip.empty?
15
+ api = Rest.new(base_url: base_url, redirect_max: 1)
16
+ result = api.read('api/v5/configuration/ping')
17
+ if result[:http].code.start_with?('2') && result[:http].body.strip.empty?
15
18
  return {version: '5'}
16
19
  end
17
20
  return nil
18
21
  end
19
22
  end
20
23
 
21
- VAL_ALL='ALL'
22
- private_constant :VAL_ALL
24
+ VAL_ALL = 'ALL'
25
+ TRANSFER_CONNECT = 'connect'
26
+ private_constant :VAL_ALL,:TRANSFER_CONNECT
23
27
 
24
28
  def initialize(env)
25
29
  super(env)
26
30
  options.add_opt_simple(:client_id,'OAuth client identifier')
27
31
  options.add_opt_simple(:client_secret,'OAuth client secret')
28
- options.add_opt_simple(:redirect_uri,'OAuth redirect URI')
29
- options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'OAuth type of authentication')
30
- options.add_opt_simple(:private_key,'Oauth RSA private key PEM value for JWT (prefix file path with @val:@file:)')
32
+ options.add_opt_simple(:redirect_uri,'OAuth redirect URI for web authentication')
33
+ options.add_opt_list(:auth,[Oauth::STD_AUTH_TYPES,:boot].flatten,'OAuth type of authentication')
34
+ options.add_opt_simple(:private_key,'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
35
+ options.add_opt_simple(:passphrase,'RSA private key passphrase')
31
36
  options.set_option(:auth,:jwt)
32
37
  options.parse_options!
33
38
  end
34
39
 
35
40
  def set_api
36
- @faxpex5_api_base_url=options.get_option(:url,:mandatory)
37
- faxpex5_api_v5_url="#{@faxpex5_api_base_url}/api/v5"
38
- faxpex5_api_auth_url="#{@faxpex5_api_base_url}/auth"
39
- case options.get_option(:auth,:mandatory)
41
+ @faxpex5_api_base_url = options.get_option(:url,is_type: :mandatory).gsub(%r{/+$},'')
42
+ @faxpex5_api_auth_url = "#{@faxpex5_api_base_url}/auth"
43
+ faxpex5_api_v5_url = "#{@faxpex5_api_base_url}/api/v5"
44
+ case options.get_option(:auth,is_type: :mandatory)
40
45
  when :boot
41
46
  # the password here is the token copied directly from browser in developer mode
42
- @api_v5=Rest.new({
43
- :base_url => faxpex5_api_v5_url,
44
- :headers => {'Authorization'=>options.get_option(:password,:mandatory)},
47
+ @api_v5 = Rest.new({
48
+ base_url: faxpex5_api_v5_url,
49
+ headers: {'Authorization' => options.get_option(:password,is_type: :mandatory)}
45
50
  })
46
51
  when :web
47
52
  # opens a browser and ask user to auth using web
48
- @api_v5=Rest.new({
49
- :base_url => faxpex5_api_v5_url,
50
- :auth => {
51
- :type => :oauth2,
52
- :base_url => faxpex5_api_auth_url,
53
- :grant => :web,
54
- :state => SecureRandom.uuid,
55
- :client_id => options.get_option(:client_id,:mandatory),
56
- :redirect_uri => options.get_option(:redirect_uri,:mandatory),
53
+ @api_v5 = Rest.new({
54
+ base_url: faxpex5_api_v5_url,
55
+ auth: {
56
+ type: :oauth2,
57
+ base_url: @faxpex5_api_auth_url,
58
+ crtype: :web,
59
+ client_id: options.get_option(:client_id,is_type: :mandatory),
60
+ web: {redirect_uri: options.get_option(:redirect_uri,is_type: :mandatory)}
57
61
  }})
58
62
  when :jwt
59
- # currently Faspex 5 beta 4 only supports non-user based apis (e.g. jobs)
60
- app_client_id=options.get_option(:client_id,:mandatory)
61
- @api_v5=Rest.new({
62
- :base_url => faxpex5_api_v5_url,
63
- :auth => {
64
- :type => :oauth2,
65
- :base_url => faxpex5_api_auth_url,
66
- :grant => :jwt,
67
- :f5_username => options.get_option(:username,:mandatory),
68
- :f5_password => options.get_option(:password,:mandatory),
69
- :client_id => app_client_id,
70
- :client_secret => options.get_option(:client_secret,:mandatory),
71
- :jwt_subject => "client:#{app_client_id}", # TODO Mmmm
72
- :jwt_audience => app_client_id, # TODO Mmmm
73
- :jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
74
- :jwt_headers => {typ: 'JWT'}
63
+ app_client_id = options.get_option(:client_id,is_type: :mandatory)
64
+ @api_v5 = Rest.new({
65
+ base_url: faxpex5_api_v5_url,
66
+ auth: {
67
+ type: :oauth2,
68
+ base_url: @faxpex5_api_auth_url,
69
+ crtype: :jwt,
70
+ client_id: app_client_id,
71
+ jwt: {
72
+ payload: {
73
+ iss: app_client_id, # issuer
74
+ aud: app_client_id, # audience TODO: ???
75
+ sub: "user:#{options.get_option(:username,is_type: :mandatory)}" # subject also "client:#{app_client_id}" + auth user/pass
76
+ },
77
+ #auth: {type: :basic, options.get_option(:username,is_type: :mandatory), options.get_option(:password,is_type: :mandatory),
78
+ private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key,is_type: :mandatory),options.get_option(:passphrase)),
79
+ headers: {typ: 'JWT'}
80
+ }
75
81
  }})
76
82
  end
77
83
  end
78
84
 
79
- ACTIONS=[ :node, :package, :auth_client, :jobs ]
85
+ ACTIONS = %i[health package admin user].freeze
80
86
 
81
- #
82
87
  def execute_action
83
88
  set_api
84
- command=options.get_next_command(ACTIONS)
89
+ command = options.get_next_command(ACTIONS)
85
90
  case command
86
- when :auth_client
87
- api_auth=Rest.new(@api_v5.params.merge({base_url: "#{@faxpex5_api_base_url}/auth"}))
88
- return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
89
- when :node
90
- return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
91
- when :jobs
92
- # to test JWT
93
- return self.entity_action(@api_v5,'jobs',nil,:id,nil,true)
91
+ when :health
92
+ nagios = Nagios.new
93
+ begin
94
+ result=Rest.new(base_url: @faxpex5_api_base_url).read('health')[:data]
95
+ result.each do |k,v|
96
+ nagios.add_ok(k,v.to_s)
97
+ end
98
+ rescue StandardError => e
99
+ nagios.add_critical('faspex api',e.to_s)
100
+ end
101
+ return nagios.result
102
+ when :user
103
+ case options.get_next_command(%i[profile])
104
+ when :profile
105
+ case options.get_next_command(%i[show modify])
106
+ when :show
107
+ return { type: :single_object, data: @api_v5.read('account/preferences')[:data] }
108
+ when :modify
109
+ @api_v5.update('account/preferences',options.get_next_argument('modified parameters (Hash)'))
110
+ return Main.result_status('modified')
111
+ end
112
+ end
94
113
  when :package
95
- command=options.get_next_command([:list,:show,:send,:receive])
114
+ command = options.get_next_command(%i[list show send receive])
96
115
  case command
97
116
  when :list
98
- parameters=options.get_option(:value,:optional)
99
- return {:type => :object_list, :data=>@api_v5.read('packages',parameters)[:data]['packages']}
117
+ parameters = options.get_option(:value)
118
+ return {
119
+ type: :object_list,
120
+ data: @api_v5.read('packages',parameters)[:data]['packages'],
121
+ fields: %w[id title release_date total_bytes total_files created_time state]
122
+ }
100
123
  when :show
101
- id=instance_identifier()
102
- return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
124
+ id = instance_identifier
125
+ return {type: :single_object, data: @api_v5.read("packages/#{id}")[:data]}
103
126
  when :send
104
- parameters=options.get_option(:value,:mandatory)
127
+ parameters = options.get_option(:value,is_type: :mandatory)
105
128
  raise CliBadArgument,'value must be hash, refer to API' unless parameters.is_a?(Hash)
106
- package=@api_v5.create('packages',parameters)[:data]
107
- transfer_spec=@api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
129
+ package = @api_v5.create('packages',parameters)[:data]
130
+ # TODO: option to send from remote source
131
+ transfer_spec = @api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: TRANSFER_CONNECT})[:data]
108
132
  transfer_spec.delete('authentication')
109
- return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
133
+ return Main.result_transfer(transfer.start(transfer_spec,{src: :node_gen3}))
110
134
  when :receive
111
- pkg_type='received'
112
- pack_id=instance_identifier()
113
- package_ids=[pack_id]
114
- skip_ids_data=[]
115
- skip_ids_persistency=nil
116
- if options.get_option(:once_only,:mandatory)
135
+ pkg_type = 'received'
136
+ pack_id = instance_identifier
137
+ package_ids = [pack_id]
138
+ skip_ids_data = []
139
+ skip_ids_persistency = nil
140
+ if options.get_option(:once_only,is_type: :mandatory)
117
141
  # read ids from persistency
118
- skip_ids_persistency=PersistencyActionOnce.new(
119
- manager: @agents[:persistency],
120
- data: skip_ids_data,
121
- id: IdGenerator.from_list(['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type]))
142
+ skip_ids_persistency = PersistencyActionOnce.new(
143
+ manager: @agents[:persistency],
144
+ data: skip_ids_data,
145
+ id: IdGenerator.from_list(['faspex_recv',options.get_option(:url,is_type: :mandatory),options.get_option(:username,is_type: :mandatory),pkg_type]))
122
146
  end
123
147
  if pack_id.eql?(VAL_ALL)
124
148
  # TODO: if packages have same name, they will overwrite
125
- parameters=options.get_option(:value,:optional)
126
- parameters||={'type'=>'received','subtype'=>'mypackages','limit'=>1000}
149
+ parameters = options.get_option(:value)
150
+ parameters ||= {'type' => 'received','subtype' => 'mypackages','limit' => 1000}
127
151
  raise CliBadArgument,'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
128
- package_ids=@api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
129
- package_ids.select!{|i|!skip_ids_data.include?(i)}
152
+ package_ids = @api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
153
+ package_ids.reject!{|i|skip_ids_data.include?(i)}
130
154
  end
131
- result_transfer=[]
132
- package_ids.each do |id|
155
+ result_transfer = []
156
+ package_ids.each do |pkgid|
133
157
  # TODO: allow from sent as well ?
134
- transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
158
+ transfer_spec = @api_v5.create("packages/#{pkgid}/transfer_spec/download",{transfer_type: TRANSFER_CONNECT, type: pkg_type})[:data]
135
159
  transfer_spec.delete('authentication')
136
- statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
137
- result_transfer.push({'package'=>id,Main::STATUS_FIELD=>statuses})
160
+ statuses = transfer.start(transfer_spec,{src: :node_gen3})
161
+ result_transfer.push({'package' => pkgid,Main::STATUS_FIELD => statuses})
138
162
  # skip only if all sessions completed
139
- skip_ids_data.push(id) if TransferAgent.session_status(statuses).eql?(:success)
163
+ skip_ids_data.push(pkgid) if TransferAgent.session_status(statuses).eql?(:success)
140
164
  end
141
- skip_ids_persistency.save unless skip_ids_persistency.nil?
165
+ skip_ids_persistency&.save
142
166
  return Main.result_transfer_multiple(result_transfer)
167
+ end # case package
168
+ when :admin
169
+ case options.get_next_command([:resource])
170
+ when :resource
171
+ res_type = options.get_next_command(%i[accounts contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs metadata_profiles])
172
+ res_path = list_key = res_type.to_s
173
+ case res_type
174
+ when :metadata_profiles
175
+ res_path='configuration/metadata_profiles'
176
+ list_key='profiles'
177
+ end
178
+ display_fields =
179
+ case res_type
180
+ when :accounts then [:all_but,'user_profile_data_attributes']
181
+ when :oauth_clients then [:all_but,'public_key']
182
+ end
183
+ adm_api = @api_v5
184
+ if res_type.eql?(:oauth_clients)
185
+ adm_api = Rest.new(@api_v5.params.merge({base_url: @faxpex5_api_auth_url}))
186
+ end
187
+ return entity_action(adm_api,res_path,item_list_key: list_key, display_fields: display_fields)
143
188
  end
144
- end
145
- end
146
- end
189
+ end # case command
190
+ end # action
191
+ end # Faspex5
147
192
  end # Plugins
148
193
  end # Cli
149
194
  end # Aspera