aspera-cli 4.16.0 → 4.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +50 -19
  4. data/CONTRIBUTING.md +3 -1
  5. data/README.md +965 -793
  6. data/bin/asession +29 -21
  7. data/lib/aspera/{fasp/agent_alpha.rb → agent/alpha.rb} +26 -25
  8. data/lib/aspera/{fasp/agent_base.rb → agent/base.rb} +15 -12
  9. data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
  10. data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +49 -53
  11. data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +20 -19
  12. data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +20 -33
  13. data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb} +11 -11
  14. data/lib/aspera/api/aoc.rb +586 -0
  15. data/lib/aspera/api/ats.rb +46 -0
  16. data/lib/aspera/api/cos_node.rb +95 -0
  17. data/lib/aspera/api/node.rb +344 -0
  18. data/lib/aspera/ascmd.rb +46 -10
  19. data/lib/aspera/{fasp → ascp}/installation.rb +5 -5
  20. data/lib/aspera/{fasp → ascp}/management.rb +3 -8
  21. data/lib/aspera/{fasp → ascp}/products.rb +1 -1
  22. data/lib/aspera/assert.rb +30 -30
  23. data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
  24. data/lib/aspera/cli/extended_value.rb +1 -1
  25. data/lib/aspera/cli/formatter.rb +13 -13
  26. data/lib/aspera/cli/hints.rb +5 -5
  27. data/lib/aspera/cli/main.rb +35 -28
  28. data/lib/aspera/cli/manager.rb +25 -24
  29. data/lib/aspera/cli/plugin.rb +22 -15
  30. data/lib/aspera/cli/plugin_factory.rb +61 -0
  31. data/lib/aspera/cli/plugins/alee.rb +7 -7
  32. data/lib/aspera/cli/plugins/aoc.rb +83 -77
  33. data/lib/aspera/cli/plugins/ats.rb +32 -33
  34. data/lib/aspera/cli/plugins/bss.rb +3 -4
  35. data/lib/aspera/cli/plugins/config.rb +169 -186
  36. data/lib/aspera/cli/plugins/console.rb +8 -6
  37. data/lib/aspera/cli/plugins/cos.rb +19 -18
  38. data/lib/aspera/cli/plugins/faspex.rb +61 -54
  39. data/lib/aspera/cli/plugins/faspex5.rb +150 -103
  40. data/lib/aspera/cli/plugins/node.rb +68 -73
  41. data/lib/aspera/cli/plugins/orchestrator.rb +34 -44
  42. data/lib/aspera/cli/plugins/preview.rb +31 -31
  43. data/lib/aspera/cli/plugins/server.rb +31 -33
  44. data/lib/aspera/cli/plugins/shares.rb +13 -11
  45. data/lib/aspera/cli/sync_actions.rb +8 -8
  46. data/lib/aspera/cli/transfer_agent.rb +32 -19
  47. data/lib/aspera/cli/transfer_progress.rb +1 -1
  48. data/lib/aspera/cli/version.rb +1 -1
  49. data/lib/aspera/colors.rb +5 -0
  50. data/lib/aspera/command_line_builder.rb +14 -14
  51. data/lib/aspera/coverage.rb +1 -2
  52. data/lib/aspera/data_repository.rb +1 -1
  53. data/lib/aspera/environment.rb +2 -3
  54. data/lib/aspera/faspex_gw.rb +5 -6
  55. data/lib/aspera/faspex_postproc.rb +1 -1
  56. data/lib/aspera/id_generator.rb +2 -2
  57. data/lib/aspera/json_rpc.rb +5 -5
  58. data/lib/aspera/keychain/encrypted_hash.rb +6 -6
  59. data/lib/aspera/keychain/macos_security.rb +27 -22
  60. data/lib/aspera/log.rb +2 -2
  61. data/lib/aspera/nagios.rb +3 -3
  62. data/lib/aspera/node_simulator.rb +5 -6
  63. data/lib/aspera/oauth/base.rb +143 -0
  64. data/lib/aspera/oauth/factory.rb +124 -0
  65. data/lib/aspera/oauth/generic.rb +34 -0
  66. data/lib/aspera/oauth/jwt.rb +51 -0
  67. data/lib/aspera/oauth/url_json.rb +31 -0
  68. data/lib/aspera/oauth/web.rb +50 -0
  69. data/lib/aspera/oauth.rb +5 -331
  70. data/lib/aspera/open_application.rb +7 -7
  71. data/lib/aspera/persistency_action_once.rb +4 -4
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/generator.rb +5 -5
  74. data/lib/aspera/preview/terminal.rb +3 -2
  75. data/lib/aspera/preview/utils.rb +3 -3
  76. data/lib/aspera/proxy_auto_config.rb +4 -4
  77. data/lib/aspera/rest.rb +175 -144
  78. data/lib/aspera/rest_errors_aspera.rb +3 -3
  79. data/lib/aspera/resumer.rb +77 -0
  80. data/lib/aspera/ssh.rb +6 -1
  81. data/lib/aspera/{fasp → transfer}/error.rb +3 -3
  82. data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
  83. data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
  84. data/lib/aspera/{fasp → transfer}/parameters.rb +58 -89
  85. data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +18 -16
  86. data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
  87. data/lib/aspera/{fasp → transfer}/sync.rb +32 -32
  88. data/lib/aspera/{fasp → transfer}/uri.rb +9 -8
  89. data/lib/aspera/web_server_simple.rb +11 -3
  90. data.tar.gz.sig +0 -0
  91. metadata +36 -63
  92. metadata.gz.sig +0 -0
  93. data/lib/aspera/aoc.rb +0 -601
  94. data/lib/aspera/ats_api.rb +0 -47
  95. data/lib/aspera/cos_node.rb +0 -94
  96. data/lib/aspera/fasp/resume_policy.rb +0 -79
  97. data/lib/aspera/node.rb +0 -339
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'singleton'
4
+ require 'aspera/log'
5
+ require 'aspera/assert'
6
+
7
+ module Aspera
8
+ # implements a simple resume policy
9
+ class Resumer
10
+ # list of supported parameters and default values
11
+ DEFAULTS = {
12
+ iter_max: 7,
13
+ sleep_initial: 2,
14
+ sleep_factor: 2,
15
+ sleep_max: 60
16
+ }.freeze
17
+
18
+ # @param params see DEFAULTS
19
+ def initialize(params=nil)
20
+ @parameters = DEFAULTS.dup
21
+ if !params.nil?
22
+ Aspera.assert_type(params, Hash)
23
+ params.each do |k, v|
24
+ Aspera.assert_values(k, DEFAULTS.keys){'resume parameter'}
25
+ Aspera.assert_type(v, Integer){k}
26
+ @parameters[k] = v
27
+ end
28
+ end
29
+ Log.log.debug{"resume params=#{@parameters}"}
30
+ end
31
+
32
+ # calls block a number of times (resumes) until success or limit reached
33
+ # this is re-entrant, one resumer can handle multiple transfers in //
34
+ def execute_with_resume
35
+ Aspera.assert(block_given?)
36
+ # maximum of retry
37
+ remaining_resumes = @parameters[:iter_max]
38
+ sleep_seconds = @parameters[:sleep_initial]
39
+ Log.log.debug{"retries=#{remaining_resumes}"}
40
+ # try to send the file until ascp is successful
41
+ loop do
42
+ Log.log.debug('transfer starting')
43
+ begin
44
+ # call provided block
45
+ yield
46
+ # exit retry loop if success
47
+ break
48
+ rescue Transfer::Error => e
49
+ Log.log.warn{"An error occurred during transfer: #{e.message}"}
50
+ # failure in ascp
51
+ if e.retryable?
52
+ # exit if we exceed the max number of retry
53
+ raise Transfer::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
54
+ else
55
+ # give one chance only to non retryable errors
56
+ unless remaining_resumes.eql?(@parameters[:iter_max])
57
+ Log.log.error('non-retryable error'.red.blink)
58
+ raise e
59
+ end
60
+ end
61
+ end
62
+
63
+ # take this retry in account
64
+ remaining_resumes -= 1
65
+ Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
66
+
67
+ # wait a bit before retrying, maybe network condition will be better
68
+ sleep(sleep_seconds)
69
+
70
+ # increase retry period
71
+ sleep_seconds *= @parameters[:sleep_factor]
72
+ # cap value
73
+ sleep_seconds = @parameters[:sleep_max] if sleep_seconds > @parameters[:sleep_max]
74
+ end # loop
75
+ end
76
+ end
77
+ end
data/lib/aspera/ssh.rb CHANGED
@@ -3,7 +3,7 @@
3
3
  require 'net/ssh'
4
4
 
5
5
  if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
6
- # HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually cause problems
6
+ # HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
7
7
  old_verbose = $VERBOSE
8
8
  $VERBOSE = nil
9
9
  begin
@@ -14,6 +14,11 @@ if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
14
14
  $VERBOSE = old_verbose
15
15
  end
16
16
 
17
+ if RUBY_ENGINE == 'jruby' && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')
18
+ Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a =~ /^ecd(sa|h)-sha2/ } }
19
+ Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t =~ /^ecd(sa|h)-sha2/ }
20
+ end
21
+
17
22
  module Aspera
18
23
  # A simple wrapper around Net::SSH
19
24
  # executes one command and get its result from stdout
@@ -1,9 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/fasp/error_info'
3
+ require 'aspera/transfer/error_info'
4
4
 
5
5
  module Aspera
6
- module Fasp
6
+ module Transfer
7
7
  # error raised if transfer fails
8
8
  class Error < StandardError
9
9
  attr_reader :err_code
@@ -14,7 +14,7 @@ module Aspera
14
14
  end
15
15
 
16
16
  def info
17
- r = Fasp::ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
17
+ r = ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
18
18
  return r.merge({i: @err_code})
19
19
  end
20
20
 
@@ -3,7 +3,7 @@
3
3
  # cspell:words mnemo PROTO RCVR NOLIC PMTU BRTT VLINK BWMEAS SSEAR FIPS
4
4
 
5
5
  module Aspera
6
- module Fasp
6
+ module Transfer
7
7
  # from https://www.google.com/search?q=FASP+error+codes
8
8
  # Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
9
9
  # rubocop:disable Layout/MultilineHashKeyLineBreaks
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Aspera
4
- module Fasp
4
+ module Transfer
5
5
  # generates a pseudo file stream
6
6
  class FauxFile
7
7
  # marker for faux file
@@ -4,19 +4,19 @@ require 'aspera/log'
4
4
  require 'aspera/assert'
5
5
  require 'aspera/command_line_builder'
6
6
  require 'aspera/temp_file_manager'
7
- require 'aspera/fasp/error'
8
- require 'aspera/fasp/installation'
7
+ require 'aspera/transfer/error'
8
+ require 'aspera/transfer/spec'
9
+ require 'aspera/ascp/installation'
9
10
  require 'aspera/cli/formatter'
10
11
  require 'aspera/rest'
11
12
  require 'securerandom'
12
13
  require 'base64'
13
14
  require 'json'
14
- require 'yaml'
15
15
  require 'fileutils'
16
16
  require 'openssl'
17
17
 
18
18
  module Aspera
19
- module Fasp
19
+ module Transfer
20
20
  # translate transfer specification to ascp parameter list
21
21
  class Parameters
22
22
  # Agents shown in manual for parameters (sub list)
@@ -25,7 +25,7 @@ module Aspera
25
25
  SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
26
26
  FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
27
27
  # options that can be provided to the constructor, and then in @options
28
- SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore quiet trusted_certs].freeze
28
+ SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore_cb quiet trusted_certs].freeze
29
29
 
30
30
  private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS, :SUPPORTED_OPTIONS
31
31
 
@@ -34,22 +34,12 @@ module Aspera
34
34
  # because of garbage collection takes any file there
35
35
  # this could be refined, as , for instance, on macos, temp folder is already user specific
36
36
  @file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists') # cspell:disable-line
37
- @param_description_cache = nil
38
- # @return normalized description of transfer spec parameters, direct from yaml
39
- def description
40
- if @param_description_cache.nil?
41
- # config file in same folder with same name as this source
42
- description_from_yaml = YAML.load_file("#{__FILE__[0..-3]}yaml")
43
- @param_description_cache = Aspera::CommandLineBuilder.normalize_description(description_from_yaml)
44
- end
45
- return @param_description_cache
46
- end
47
37
 
48
38
  # @param to_text [bool] replace HTML entities with text equivalent
49
39
  # @return a table suitable to display in manual
50
40
  def man_table
51
41
  result = []
52
- description.each do |name, options|
42
+ Spec::DESCRIPTION.each do |name, options|
53
43
  param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
54
44
  # add flags for supported agents in doc
55
45
  SUPPORTED_AGENTS.each do |a|
@@ -85,13 +75,7 @@ module Aspera
85
75
  param[:description] = param[:description].gsub('&sol;', '\\')
86
76
  result.push(param)
87
77
  end
88
- return result.sort do |a, b|
89
- if a[:name].start_with?('EX_').eql?(b[:name].start_with?('EX_'))
90
- a[:name] <=> b[:name]
91
- else
92
- b[:name] <=> a[:name]
93
- end
94
- end
78
+ return result.sort_by { |a| a[:name] }
95
79
  end
96
80
 
97
81
  # special encoding methods used in YAML (key: :convert)
@@ -104,13 +88,9 @@ module Aspera
104
88
  def convert_base64(v); Base64.strict_encode64(v); end
105
89
 
106
90
  # file list is provided directly with ascp arguments
107
- def ts_has_ascp_file_list(ts, ascp_args)
108
- # it can also be option transfer_info
109
- ascp_args = ascp_args['ascp_args'] if ascp_args.is_a?(Hash)
110
- (ts['EX_ascp_args'].is_a?(Array) && ts['EX_ascp_args'].any?{|i|FILE_LIST_OPTIONS.include?(i)}) ||
111
- (ascp_args.is_a?(Array) && ascp_args.any?{|i|FILE_LIST_OPTIONS.include?(i)}) ||
112
- ts.key?('EX_file_list') ||
113
- ts.key?('EX_file_pair_list')
91
+ # @param ascp_args [Array,NilClass] ascp arguments
92
+ def ascp_args_file_list?(ascp_args)
93
+ ascp_args&.any?{|i|FILE_LIST_OPTIONS.include?(i)}
114
94
  end
115
95
 
116
96
  # temp file list files are created here
@@ -127,67 +107,56 @@ module Aspera
127
107
 
128
108
  # @param options [Hash] key: :wss: bool, :ascp_args: array of strings
129
109
  def initialize(job_spec, options)
130
- assert_type(job_spec, Hash)
131
- assert_type(options, Hash)
110
+ Aspera.assert_type(job_spec, Hash)
111
+ Aspera.assert_type(options, Hash)
132
112
  @job_spec = job_spec
133
113
  # check necessary options
134
114
  missing_options = SUPPORTED_OPTIONS - options.keys
135
- assert(missing_options.empty?){"missing options: #{missing_options.join(', ')}"}
115
+ Aspera.assert(missing_options.empty?){"missing options: #{missing_options.join(', ')}"}
136
116
  @options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
137
117
  Log.log.debug{Log.dump(:parameters_options, @options)}
138
118
  Log.log.debug{Log.dump(:dismiss_options, options.keys - SUPPORTED_OPTIONS)}
139
- assert_type(@options[:ascp_args], Array){'ascp_args'}
140
- assert(@options[:ascp_args].all?(String)){'ascp arguments must Strings'}
141
- @builder = Aspera::CommandLineBuilder.new(@job_spec, self.class.description)
119
+ Aspera.assert_type(@options[:ascp_args], Array){'ascp_args'}
120
+ Aspera.assert(@options[:ascp_args].all?(String)){'ascp arguments must Strings'}
121
+ @builder = CommandLineBuilder.new(@job_spec, Spec::DESCRIPTION)
142
122
  end
143
123
 
144
124
  # either place source files on command line, or add file list file
145
125
  def process_file_list
146
- # is the file list provided through EX_ parameters?
147
- ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @options[:ascp_args])
126
+ # is the file list provided through ascp parameters?
127
+ ascp_file_list_provided = self.class.ascp_args_file_list?(@options[:ascp_args])
148
128
  # set if paths is mandatory in ts
149
129
  @builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided # cspell:words keepalive
150
130
  # get paths in transfer spec (after setting if it is mandatory)
151
131
  ts_paths_array = @builder.read_param('paths')
152
- if ascp_file_list_provided && !ts_paths_array.nil?
153
- raise 'file list provided both in transfer spec and ascp file list. Remove one of them.'
154
- end
155
- # option 1: EX_file_list
156
- file_list_file = @builder.read_param('EX_file_list')
157
- if file_list_file.nil?
158
- # option 2: EX_file_pair_list
159
- file_list_file = @builder.read_param('EX_file_pair_list')
160
- if !file_list_file.nil?
161
- option = '--file-pair-list'
162
- elsif !ts_paths_array.nil?
163
- # option 3: in TS, it is an array
164
- if self.class.file_list_folder.nil?
165
- # not safe for special characters ? (maybe not, depends on OS)
166
- Log.log.debug('placing source file list on command line (no file list file)')
167
- @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
132
+ file_list_option = nil
133
+ # transfer spec contains paths ?
134
+ if !ts_paths_array.nil?
135
+ Aspera.assert(!ascp_file_list_provided){'file list provided both in transfer spec and ascp file list. Remove one of them.'}
136
+ Aspera.assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
137
+ is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
138
+ raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
139
+ if self.class.file_list_folder.nil?
140
+ Aspera.assert(!is_pair_list){'file pair list is not supported when file list folder is not set'}
141
+ # not safe for special characters ? (maybe not, depends on OS)
142
+ Log.log.debug('placing source file list on command line (no file list file)')
143
+ @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
144
+ else
145
+ # safer option: generate a file list file if there is storage defined for it
146
+ if is_pair_list
147
+ file_list_option = '--file-pair-list'
148
+ lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']) }
168
149
  else
169
- assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
170
- is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
171
- raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
172
- # safer option: generate a file list file if there is storage defined for it
173
- # if there is one destination in paths, then use file-pair-list
174
- if is_pair_list
175
- option = '--file-pair-list'
176
- lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'] || e['source']) }
177
- else
178
- option = '--file-list'
179
- lines = ts_paths_array.map{|i|i['source']}
180
- end
181
- file_list_file = Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
182
- Log.log.debug{Log.dump(:file_list, lines)}
183
- File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
184
- Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
150
+ file_list_option = '--file-list'
151
+ lines = ts_paths_array.map{|i|i['source']}
185
152
  end
153
+ file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
154
+ Log.log.debug{Log.dump(:file_list, lines)}
155
+ File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
156
+ Log.log.debug{"#{file_list_option}=\n#{File.read(file_list_file)}".red}
186
157
  end
187
- else
188
- option = '--file-list'
189
158
  end
190
- @builder.add_command_line_options(["#{option}=#{file_list_file}"]) unless option.nil?
159
+ @builder.add_command_line_options(["#{file_list_option}=#{file_list_file}"]) unless file_list_option.nil?
191
160
  end
192
161
 
193
162
  def remote_certificates
@@ -200,31 +169,32 @@ module Aspera
200
169
  # This will need to be cleaned up in aspera core
201
170
  @job_spec['ssh_port'] = @builder.read_param('wss_port')
202
171
  @job_spec.delete('fasp_port')
203
- @job_spec.delete('EX_ssh_key_paths')
204
172
  @job_spec.delete('sshfp')
173
+ # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
174
+ certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs].is_a?(Array)
205
175
  # ignore cert for wss ?
206
- if @options[:check_ignore]&.call(@job_spec['remote_host'], @job_spec['wss_port'])
176
+ if @options[:check_ignore_cb]&.call(@job_spec['remote_host'], @job_spec['wss_port'])
207
177
  wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
208
178
  wss_url = "https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}"
209
- File.write(wss_cert_file, Rest.remote_certificates(wss_url))
210
- certificates_to_use.push(wss_cert_file)
179
+ File.write(wss_cert_file, Rest.remote_certificate_chain(wss_url))
180
+ # place in front, as more priority
181
+ certificates_to_use.unshift(wss_cert_file)
211
182
  end
212
- # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
213
- certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs]
183
+ # when wss is used, only first `-i` is used... Hum...
184
+ certificates_to_use = [certificates_to_use.first] unless certificates_to_use.empty?
214
185
  else
215
186
  # remove unused parameter (avoid warning)
216
187
  @job_spec.delete('wss_port')
217
188
  # add SSH bypass keys when authentication is token and no auth is provided
218
189
  if @job_spec.key?('token') && !@job_spec.key?('remote_password')
219
- # @job_spec['remote_password'] = Installation.instance.ssh_cert_uuid # not used: no passphrase
220
- certificates_to_use.concat(Installation.instance.aspera_token_ssh_key_paths)
190
+ # @job_spec['remote_password'] = Ascp::Installation.instance.ssh_cert_uuid # not used: no passphrase
191
+ certificates_to_use.concat(Ascp::Installation.instance.aspera_token_ssh_key_paths)
221
192
  end
222
193
  end
223
194
  return certificates_to_use
224
195
  end
225
196
 
226
197
  # translate transfer spec to env vars and command line arguments for ascp
227
- # NOTE: parameters starting with "EX_" (extended) are not standard
228
198
  def ascp_args
229
199
  env_args = {
230
200
  args: [],
@@ -236,11 +206,11 @@ module Aspera
236
206
  @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
237
207
 
238
208
  # notify multi-session was already used, anyway it was deleted by agent direct
239
- assert(!@builder.read_param('multi_session'))
209
+ Aspera.assert(!@builder.read_param('multi_session'))
240
210
 
241
211
  # add ssh or wss certificates
242
- remote_certificates.each do |cert|
243
- Log.log.trace1{"adding certificate: #{cert}"}
212
+ # (reverse, to keep order, as we unshift)
213
+ remote_certificates.reverse_each do |cert|
244
214
  env_args[:args].unshift('-i', cert)
245
215
  end
246
216
 
@@ -248,7 +218,7 @@ module Aspera
248
218
  @builder.process_params
249
219
 
250
220
  base64_destination = false
251
- # symbol must be index of Installation.paths
221
+ # symbol must be index of Ascp::Installation.paths
252
222
  if @builder.read_param('use_ascp4')
253
223
  env_args[:ascp_version] = :ascp4
254
224
  else
@@ -258,7 +228,6 @@ module Aspera
258
228
  # destination will be base64 encoded, put this before source path arguments
259
229
  @builder.add_command_line_options(['--dest64']) if base64_destination
260
230
  # optional arguments, at the end to override previous ones (to allow override)
261
- @builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
262
231
  @builder.add_command_line_options(@options[:ascp_args])
263
232
  # get list of source files to transfer and build arg for ascp
264
233
  process_file_list
@@ -272,8 +241,8 @@ module Aspera
272
241
  env_args[:args].unshift('-q') if @options[:quiet]
273
242
  # add fallback cert and key as arguments if needed
274
243
  if ['1', 1, true, 'force'].include?(@job_spec['http_fallback'])
275
- env_args[:args].unshift('-Y', Installation.instance.path(:fallback_private_key))
276
- env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
244
+ env_args[:args].unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
245
+ env_args[:args].unshift('-I', Ascp::Installation.instance.path(:fallback_certificate))
277
246
  end
278
247
  Log.log.debug{"ascp args: #{env_args}"}
279
248
  return env_args
@@ -1,12 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/fasp/parameters'
3
+ require 'aspera/command_line_builder'
4
4
  require 'aspera/assert'
5
+ require 'yaml'
5
6
 
6
7
  module Aspera
7
- module Fasp
8
+ module Transfer
8
9
  # parameters for Transfer Spec
9
- class TransferSpec
10
+ class Spec
10
11
  # default transfer username for access key based transfers
11
12
  ACCESS_KEY_TRANSFER_USER = 'xfer'
12
13
  SSH_PORT = 33_001
@@ -18,35 +19,36 @@ module Aspera
18
19
  }.freeze
19
20
  # reserved tag for Aspera
20
21
  TAG_RESERVED = 'aspera'
21
- # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
22
- Aspera::Fasp::Parameters.description.each do |name, description|
23
- next unless description[:enum].is_a?(Array)
24
- TransferSpec.const_set("#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
25
- description[:enum].each do |enum|
26
- TransferSpec.const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
27
- end
28
- end
29
22
  class << self
30
23
  def action_to_direction(tspec, command)
31
- assert_type(tspec, Hash){'transfer spec'}
24
+ Aspera.assert_type(tspec, Hash){'transfer spec'}
32
25
  tspec['direction'] = case command.to_sym
33
26
  when :upload then DIRECTION_SEND
34
27
  when :download then DIRECTION_RECEIVE
35
- else error_unexpected_value(command.to_sym)
28
+ else Aspera.error_unexpected_value(command.to_sym)
36
29
  end
37
30
  return tspec
38
31
  end
39
32
 
40
33
  def action(tspec)
41
- assert_type(tspec, Hash){'transfer spec'}
42
- assert_values(tspec['direction'], [DIRECTION_SEND, DIRECTION_RECEIVE]){'direction'}
34
+ Aspera.assert_type(tspec, Hash){'transfer spec'}
35
+ Aspera.assert_values(tspec['direction'], [DIRECTION_SEND, DIRECTION_RECEIVE]){'direction'}
43
36
  case tspec['direction']
44
37
  when DIRECTION_SEND then :upload
45
38
  when DIRECTION_RECEIVE then :download
46
- else error_unexpected_value(tspec['direction'])
39
+ else Aspera.error_unexpected_value(tspec['direction'])
47
40
  end
48
41
  end
49
42
  end
43
+ DESCRIPTION = CommandLineBuilder.normalize_description(YAML.load_file("#{__FILE__[0..-3]}yaml"))
44
+ # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
45
+ DESCRIPTION.each do |name, description|
46
+ next unless description[:enum].is_a?(Array)
47
+ const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
48
+ description[:enum].each do |enum|
49
+ const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
50
+ end
51
+ end
50
52
  end
51
53
  end
52
54
  end
@@ -27,7 +27,7 @@ cipher:
27
27
  :cli:
28
28
  :type: :opt_with_arg
29
29
  :switch: "-c"
30
- :convert: Aspera::Fasp::Parameters.convert_remove_hyphen
30
+ :convert: Aspera::Transfer::Parameters.convert_remove_hyphen
31
31
  content_protection:
32
32
  :desc: Enable client-side encryption at rest. (CSEAR, content protection)
33
33
  :enum:
@@ -424,7 +424,7 @@ tags:
424
424
  :cli:
425
425
  :type: :opt_with_arg
426
426
  :switch: "--tags64"
427
- :convert: Aspera::Fasp::Parameters.convert_json64
427
+ :convert: Aspera::Transfer::Parameters.convert_json64
428
428
  target_rate_cap_kbps:
429
429
  :desc: Returned by upload/download_setup node API.
430
430
  :accepted_types: :int
@@ -479,7 +479,7 @@ source_root:
479
479
  :cli:
480
480
  :type: :opt_with_arg
481
481
  :switch: "--source-prefix64"
482
- :convert: Aspera::Fasp::Parameters.convert_base64
482
+ :convert: Aspera::Transfer::Parameters.convert_base64
483
483
  min_rate_cap_kbps:
484
484
  :desc: |-
485
485
  The highest minimum rate that an incoming transfer can request, in kilobits per second.
@@ -540,7 +540,7 @@ src_base:
540
540
  :cli:
541
541
  :type: :opt_with_arg
542
542
  :switch: "--src-base64"
543
- :convert: Aspera::Fasp::Parameters.convert_base64
543
+ :convert: Aspera::Transfer::Parameters.convert_base64
544
544
  preserve_acls:
545
545
  :desc: "Preserve access control lists."
546
546
  :enum:
@@ -610,34 +610,6 @@ remove_empty_source_directory:
610
610
  - :sdk
611
611
  :cli:
612
612
  :type: :opt_without_arg
613
- EX_at_rest_password:
614
- :desc: Content protection password
615
- :deprecation: "(4.13) Use standard spec parameter: content_protection_password"
616
- :agents:
617
- - :direct
618
- :cli:
619
- :type: :envvar
620
- :variable: ASPERA_SCP_FILEPASS
621
- EX_proxy_password:
622
- :desc: |-
623
- Password used for Aspera proxy server authentication.
624
- May be overridden by password in URL provided in parameter: proxy.
625
-
626
-
627
- :deprecation: (4.14) Use env var ASPERA_PROXY_PASS
628
- :agents:
629
- - :direct
630
- :cli:
631
- :type: :envvar
632
- :variable: ASPERA_PROXY_PASS
633
- EX_license_text:
634
- :desc: "License file text override.\nBy default ascp looks for license file near executable."
635
- :deprecation: (4.14) Use env var ASPERA_SCP_LICENSE
636
- :agents:
637
- - :direct
638
- :cli:
639
- :type: :envvar
640
- :variable: ASPERA_SCP_LICENSE
641
613
  keepalive:
642
614
  :desc: The session is running in persistent session mode.
643
615
  :agents:
@@ -663,49 +635,6 @@ sshfp:
663
635
  :cli:
664
636
  :type: :opt_with_arg
665
637
  :switch: "--check-sshfp"
666
- EX_http_proxy_url:
667
- :desc: Specify the proxy server address used by HTTP Fallback
668
- :deprecation: (4.14) TODO, use proxy option ?
669
- :agents:
670
- - :direct
671
- :cli:
672
- :type: :opt_with_arg
673
- :switch: "-x"
674
- EX_ssh_key_paths:
675
- :desc: Use public key authentication for SSH and specify the private key file paths
676
- :deprecation: (4.14) Use option transfer_info.ascp_args
677
- :accepted_types: :array
678
- :agents:
679
- - :direct
680
- :cli:
681
- :type: :opt_with_arg
682
- :switch: "-i"
683
- EX_http_transfer_jpeg:
684
- :desc: HTTP transfers as JPEG file
685
- :deprecation: (4.14) Use option transfer_info.ascp_args
686
- :accepted_types: :int
687
- :default: '0'
688
- :agents:
689
- - :direct
690
- :cli:
691
- :type: :opt_with_arg
692
- :switch: "-j"
693
- EX_no_read:
694
- :desc: no read source
695
- :deprecation: (4.14) Use option transfer_info.ascp_args
696
- :agents:
697
- - :direct
698
- :cli:
699
- :type: :opt_without_arg
700
- :switch: "--no-read"
701
- EX_no_write:
702
- :desc: no write on destination
703
- :deprecation: (4.14) Use option transfer_info.ascp_args
704
- :agents:
705
- - :direct
706
- :cli:
707
- :type: :opt_without_arg
708
- :switch: "--no-write"
709
638
  target_rate_percentage:
710
639
  :desc: "TODO: remove ?"
711
640
  :cli:
@@ -754,30 +683,6 @@ obfuscate_file_names:
754
683
  - :httpgw
755
684
  :cli:
756
685
  :type: :ignore
757
- EX_file_list:
758
- :desc: source file list
759
- :deprecation: (4.14) Use command line file list, or option transfer_info.ascp_args
760
- :agents:
761
- - :direct
762
- :cli:
763
- :type: :special
764
- :switch: "--file-list"
765
- EX_file_pair_list:
766
- :desc: source file pair list
767
- :deprecation: (4.14) Use command line file pair list, or option transfer_info.ascp_args
768
- :agents:
769
- - :direct
770
- :cli:
771
- :type: :special
772
- :switch: "--file-pair-list"
773
- EX_ascp_args:
774
- :desc: Add native command line arguments to ascp
775
- :deprecation: (4.13) Use option transfer_info.ascp_args
776
- :accepted_types: :array
777
- :agents:
778
- - :direct
779
- :cli:
780
- :type: :special
781
686
  wss_enabled:
782
687
  :desc: Server has Web Socket service enabled
783
688
  :accepted_types: :bool