aspera-cli 4.16.0 → 4.17.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +50 -19
  4. data/CONTRIBUTING.md +3 -1
  5. data/README.md +965 -793
  6. data/bin/asession +29 -21
  7. data/lib/aspera/{fasp/agent_alpha.rb → agent/alpha.rb} +26 -25
  8. data/lib/aspera/{fasp/agent_base.rb → agent/base.rb} +15 -12
  9. data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
  10. data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +49 -53
  11. data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +20 -19
  12. data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +20 -33
  13. data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb} +11 -11
  14. data/lib/aspera/api/aoc.rb +586 -0
  15. data/lib/aspera/api/ats.rb +46 -0
  16. data/lib/aspera/api/cos_node.rb +95 -0
  17. data/lib/aspera/api/node.rb +344 -0
  18. data/lib/aspera/ascmd.rb +46 -10
  19. data/lib/aspera/{fasp → ascp}/installation.rb +5 -5
  20. data/lib/aspera/{fasp → ascp}/management.rb +3 -8
  21. data/lib/aspera/{fasp → ascp}/products.rb +1 -1
  22. data/lib/aspera/assert.rb +30 -30
  23. data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
  24. data/lib/aspera/cli/extended_value.rb +1 -1
  25. data/lib/aspera/cli/formatter.rb +13 -13
  26. data/lib/aspera/cli/hints.rb +5 -5
  27. data/lib/aspera/cli/main.rb +35 -28
  28. data/lib/aspera/cli/manager.rb +25 -24
  29. data/lib/aspera/cli/plugin.rb +22 -15
  30. data/lib/aspera/cli/plugin_factory.rb +61 -0
  31. data/lib/aspera/cli/plugins/alee.rb +7 -7
  32. data/lib/aspera/cli/plugins/aoc.rb +83 -77
  33. data/lib/aspera/cli/plugins/ats.rb +32 -33
  34. data/lib/aspera/cli/plugins/bss.rb +3 -4
  35. data/lib/aspera/cli/plugins/config.rb +169 -186
  36. data/lib/aspera/cli/plugins/console.rb +8 -6
  37. data/lib/aspera/cli/plugins/cos.rb +19 -18
  38. data/lib/aspera/cli/plugins/faspex.rb +61 -54
  39. data/lib/aspera/cli/plugins/faspex5.rb +150 -103
  40. data/lib/aspera/cli/plugins/node.rb +68 -73
  41. data/lib/aspera/cli/plugins/orchestrator.rb +34 -44
  42. data/lib/aspera/cli/plugins/preview.rb +31 -31
  43. data/lib/aspera/cli/plugins/server.rb +31 -33
  44. data/lib/aspera/cli/plugins/shares.rb +13 -11
  45. data/lib/aspera/cli/sync_actions.rb +8 -8
  46. data/lib/aspera/cli/transfer_agent.rb +32 -19
  47. data/lib/aspera/cli/transfer_progress.rb +1 -1
  48. data/lib/aspera/cli/version.rb +1 -1
  49. data/lib/aspera/colors.rb +5 -0
  50. data/lib/aspera/command_line_builder.rb +14 -14
  51. data/lib/aspera/coverage.rb +1 -2
  52. data/lib/aspera/data_repository.rb +1 -1
  53. data/lib/aspera/environment.rb +2 -3
  54. data/lib/aspera/faspex_gw.rb +5 -6
  55. data/lib/aspera/faspex_postproc.rb +1 -1
  56. data/lib/aspera/id_generator.rb +2 -2
  57. data/lib/aspera/json_rpc.rb +5 -5
  58. data/lib/aspera/keychain/encrypted_hash.rb +6 -6
  59. data/lib/aspera/keychain/macos_security.rb +27 -22
  60. data/lib/aspera/log.rb +2 -2
  61. data/lib/aspera/nagios.rb +3 -3
  62. data/lib/aspera/node_simulator.rb +5 -6
  63. data/lib/aspera/oauth/base.rb +143 -0
  64. data/lib/aspera/oauth/factory.rb +124 -0
  65. data/lib/aspera/oauth/generic.rb +34 -0
  66. data/lib/aspera/oauth/jwt.rb +51 -0
  67. data/lib/aspera/oauth/url_json.rb +31 -0
  68. data/lib/aspera/oauth/web.rb +50 -0
  69. data/lib/aspera/oauth.rb +5 -331
  70. data/lib/aspera/open_application.rb +7 -7
  71. data/lib/aspera/persistency_action_once.rb +4 -4
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/generator.rb +5 -5
  74. data/lib/aspera/preview/terminal.rb +3 -2
  75. data/lib/aspera/preview/utils.rb +3 -3
  76. data/lib/aspera/proxy_auto_config.rb +4 -4
  77. data/lib/aspera/rest.rb +175 -144
  78. data/lib/aspera/rest_errors_aspera.rb +3 -3
  79. data/lib/aspera/resumer.rb +77 -0
  80. data/lib/aspera/ssh.rb +6 -1
  81. data/lib/aspera/{fasp → transfer}/error.rb +3 -3
  82. data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
  83. data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
  84. data/lib/aspera/{fasp → transfer}/parameters.rb +58 -89
  85. data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +18 -16
  86. data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
  87. data/lib/aspera/{fasp → transfer}/sync.rb +32 -32
  88. data/lib/aspera/{fasp → transfer}/uri.rb +9 -8
  89. data/lib/aspera/web_server_simple.rb +11 -3
  90. data.tar.gz.sig +0 -0
  91. metadata +36 -63
  92. metadata.gz.sig +0 -0
  93. data/lib/aspera/aoc.rb +0 -601
  94. data/lib/aspera/ats_api.rb +0 -47
  95. data/lib/aspera/cos_node.rb +0 -94
  96. data/lib/aspera/fasp/resume_policy.rb +0 -79
  97. data/lib/aspera/node.rb +0 -339
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'singleton'
4
+ require 'aspera/log'
5
+ require 'aspera/assert'
6
+
7
+ module Aspera
8
+ # implements a simple resume policy
9
+ class Resumer
10
+ # list of supported parameters and default values
11
+ DEFAULTS = {
12
+ iter_max: 7,
13
+ sleep_initial: 2,
14
+ sleep_factor: 2,
15
+ sleep_max: 60
16
+ }.freeze
17
+
18
+ # @param params see DEFAULTS
19
+ def initialize(params=nil)
20
+ @parameters = DEFAULTS.dup
21
+ if !params.nil?
22
+ Aspera.assert_type(params, Hash)
23
+ params.each do |k, v|
24
+ Aspera.assert_values(k, DEFAULTS.keys){'resume parameter'}
25
+ Aspera.assert_type(v, Integer){k}
26
+ @parameters[k] = v
27
+ end
28
+ end
29
+ Log.log.debug{"resume params=#{@parameters}"}
30
+ end
31
+
32
+ # calls block a number of times (resumes) until success or limit reached
33
+ # this is re-entrant, one resumer can handle multiple transfers in //
34
+ def execute_with_resume
35
+ Aspera.assert(block_given?)
36
+ # maximum of retry
37
+ remaining_resumes = @parameters[:iter_max]
38
+ sleep_seconds = @parameters[:sleep_initial]
39
+ Log.log.debug{"retries=#{remaining_resumes}"}
40
+ # try to send the file until ascp is successful
41
+ loop do
42
+ Log.log.debug('transfer starting')
43
+ begin
44
+ # call provided block
45
+ yield
46
+ # exit retry loop if success
47
+ break
48
+ rescue Transfer::Error => e
49
+ Log.log.warn{"An error occurred during transfer: #{e.message}"}
50
+ # failure in ascp
51
+ if e.retryable?
52
+ # exit if we exceed the max number of retry
53
+ raise Transfer::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
54
+ else
55
+ # give one chance only to non retryable errors
56
+ unless remaining_resumes.eql?(@parameters[:iter_max])
57
+ Log.log.error('non-retryable error'.red.blink)
58
+ raise e
59
+ end
60
+ end
61
+ end
62
+
63
+ # take this retry in account
64
+ remaining_resumes -= 1
65
+ Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
66
+
67
+ # wait a bit before retrying, maybe network condition will be better
68
+ sleep(sleep_seconds)
69
+
70
+ # increase retry period
71
+ sleep_seconds *= @parameters[:sleep_factor]
72
+ # cap value
73
+ sleep_seconds = @parameters[:sleep_max] if sleep_seconds > @parameters[:sleep_max]
74
+ end # loop
75
+ end
76
+ end
77
+ end
data/lib/aspera/ssh.rb CHANGED
@@ -3,7 +3,7 @@
3
3
  require 'net/ssh'
4
4
 
5
5
  if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
6
- # HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually cause problems
6
+ # HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
7
7
  old_verbose = $VERBOSE
8
8
  $VERBOSE = nil
9
9
  begin
@@ -14,6 +14,11 @@ if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
14
14
  $VERBOSE = old_verbose
15
15
  end
16
16
 
17
+ if RUBY_ENGINE == 'jruby' && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')
18
+ Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a =~ /^ecd(sa|h)-sha2/ } }
19
+ Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t =~ /^ecd(sa|h)-sha2/ }
20
+ end
21
+
17
22
  module Aspera
18
23
  # A simple wrapper around Net::SSH
19
24
  # executes one command and get its result from stdout
@@ -1,9 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/fasp/error_info'
3
+ require 'aspera/transfer/error_info'
4
4
 
5
5
  module Aspera
6
- module Fasp
6
+ module Transfer
7
7
  # error raised if transfer fails
8
8
  class Error < StandardError
9
9
  attr_reader :err_code
@@ -14,7 +14,7 @@ module Aspera
14
14
  end
15
15
 
16
16
  def info
17
- r = Fasp::ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
17
+ r = ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
18
18
  return r.merge({i: @err_code})
19
19
  end
20
20
 
@@ -3,7 +3,7 @@
3
3
  # cspell:words mnemo PROTO RCVR NOLIC PMTU BRTT VLINK BWMEAS SSEAR FIPS
4
4
 
5
5
  module Aspera
6
- module Fasp
6
+ module Transfer
7
7
  # from https://www.google.com/search?q=FASP+error+codes
8
8
  # Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
9
9
  # rubocop:disable Layout/MultilineHashKeyLineBreaks
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Aspera
4
- module Fasp
4
+ module Transfer
5
5
  # generates a pseudo file stream
6
6
  class FauxFile
7
7
  # marker for faux file
@@ -4,19 +4,19 @@ require 'aspera/log'
4
4
  require 'aspera/assert'
5
5
  require 'aspera/command_line_builder'
6
6
  require 'aspera/temp_file_manager'
7
- require 'aspera/fasp/error'
8
- require 'aspera/fasp/installation'
7
+ require 'aspera/transfer/error'
8
+ require 'aspera/transfer/spec'
9
+ require 'aspera/ascp/installation'
9
10
  require 'aspera/cli/formatter'
10
11
  require 'aspera/rest'
11
12
  require 'securerandom'
12
13
  require 'base64'
13
14
  require 'json'
14
- require 'yaml'
15
15
  require 'fileutils'
16
16
  require 'openssl'
17
17
 
18
18
  module Aspera
19
- module Fasp
19
+ module Transfer
20
20
  # translate transfer specification to ascp parameter list
21
21
  class Parameters
22
22
  # Agents shown in manual for parameters (sub list)
@@ -25,7 +25,7 @@ module Aspera
25
25
  SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
26
26
  FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
27
27
  # options that can be provided to the constructor, and then in @options
28
- SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore quiet trusted_certs].freeze
28
+ SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore_cb quiet trusted_certs].freeze
29
29
 
30
30
  private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS, :SUPPORTED_OPTIONS
31
31
 
@@ -34,22 +34,12 @@ module Aspera
34
34
  # because of garbage collection takes any file there
35
35
  # this could be refined, as , for instance, on macos, temp folder is already user specific
36
36
  @file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists') # cspell:disable-line
37
- @param_description_cache = nil
38
- # @return normalized description of transfer spec parameters, direct from yaml
39
- def description
40
- if @param_description_cache.nil?
41
- # config file in same folder with same name as this source
42
- description_from_yaml = YAML.load_file("#{__FILE__[0..-3]}yaml")
43
- @param_description_cache = Aspera::CommandLineBuilder.normalize_description(description_from_yaml)
44
- end
45
- return @param_description_cache
46
- end
47
37
 
48
38
  # @param to_text [bool] replace HTML entities with text equivalent
49
39
  # @return a table suitable to display in manual
50
40
  def man_table
51
41
  result = []
52
- description.each do |name, options|
42
+ Spec::DESCRIPTION.each do |name, options|
53
43
  param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
54
44
  # add flags for supported agents in doc
55
45
  SUPPORTED_AGENTS.each do |a|
@@ -85,13 +75,7 @@ module Aspera
85
75
  param[:description] = param[:description].gsub('&sol;', '\\')
86
76
  result.push(param)
87
77
  end
88
- return result.sort do |a, b|
89
- if a[:name].start_with?('EX_').eql?(b[:name].start_with?('EX_'))
90
- a[:name] <=> b[:name]
91
- else
92
- b[:name] <=> a[:name]
93
- end
94
- end
78
+ return result.sort_by { |a| a[:name] }
95
79
  end
96
80
 
97
81
  # special encoding methods used in YAML (key: :convert)
@@ -104,13 +88,9 @@ module Aspera
104
88
  def convert_base64(v); Base64.strict_encode64(v); end
105
89
 
106
90
  # file list is provided directly with ascp arguments
107
- def ts_has_ascp_file_list(ts, ascp_args)
108
- # it can also be option transfer_info
109
- ascp_args = ascp_args['ascp_args'] if ascp_args.is_a?(Hash)
110
- (ts['EX_ascp_args'].is_a?(Array) && ts['EX_ascp_args'].any?{|i|FILE_LIST_OPTIONS.include?(i)}) ||
111
- (ascp_args.is_a?(Array) && ascp_args.any?{|i|FILE_LIST_OPTIONS.include?(i)}) ||
112
- ts.key?('EX_file_list') ||
113
- ts.key?('EX_file_pair_list')
91
+ # @param ascp_args [Array,NilClass] ascp arguments
92
+ def ascp_args_file_list?(ascp_args)
93
+ ascp_args&.any?{|i|FILE_LIST_OPTIONS.include?(i)}
114
94
  end
115
95
 
116
96
  # temp file list files are created here
@@ -127,67 +107,56 @@ module Aspera
127
107
 
128
108
  # @param options [Hash] key: :wss: bool, :ascp_args: array of strings
129
109
  def initialize(job_spec, options)
130
- assert_type(job_spec, Hash)
131
- assert_type(options, Hash)
110
+ Aspera.assert_type(job_spec, Hash)
111
+ Aspera.assert_type(options, Hash)
132
112
  @job_spec = job_spec
133
113
  # check necessary options
134
114
  missing_options = SUPPORTED_OPTIONS - options.keys
135
- assert(missing_options.empty?){"missing options: #{missing_options.join(', ')}"}
115
+ Aspera.assert(missing_options.empty?){"missing options: #{missing_options.join(', ')}"}
136
116
  @options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
137
117
  Log.log.debug{Log.dump(:parameters_options, @options)}
138
118
  Log.log.debug{Log.dump(:dismiss_options, options.keys - SUPPORTED_OPTIONS)}
139
- assert_type(@options[:ascp_args], Array){'ascp_args'}
140
- assert(@options[:ascp_args].all?(String)){'ascp arguments must Strings'}
141
- @builder = Aspera::CommandLineBuilder.new(@job_spec, self.class.description)
119
+ Aspera.assert_type(@options[:ascp_args], Array){'ascp_args'}
120
+ Aspera.assert(@options[:ascp_args].all?(String)){'ascp arguments must Strings'}
121
+ @builder = CommandLineBuilder.new(@job_spec, Spec::DESCRIPTION)
142
122
  end
143
123
 
144
124
  # either place source files on command line, or add file list file
145
125
  def process_file_list
146
- # is the file list provided through EX_ parameters?
147
- ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @options[:ascp_args])
126
+ # is the file list provided through ascp parameters?
127
+ ascp_file_list_provided = self.class.ascp_args_file_list?(@options[:ascp_args])
148
128
  # set if paths is mandatory in ts
149
129
  @builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided # cspell:words keepalive
150
130
  # get paths in transfer spec (after setting if it is mandatory)
151
131
  ts_paths_array = @builder.read_param('paths')
152
- if ascp_file_list_provided && !ts_paths_array.nil?
153
- raise 'file list provided both in transfer spec and ascp file list. Remove one of them.'
154
- end
155
- # option 1: EX_file_list
156
- file_list_file = @builder.read_param('EX_file_list')
157
- if file_list_file.nil?
158
- # option 2: EX_file_pair_list
159
- file_list_file = @builder.read_param('EX_file_pair_list')
160
- if !file_list_file.nil?
161
- option = '--file-pair-list'
162
- elsif !ts_paths_array.nil?
163
- # option 3: in TS, it is an array
164
- if self.class.file_list_folder.nil?
165
- # not safe for special characters ? (maybe not, depends on OS)
166
- Log.log.debug('placing source file list on command line (no file list file)')
167
- @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
132
+ file_list_option = nil
133
+ # transfer spec contains paths ?
134
+ if !ts_paths_array.nil?
135
+ Aspera.assert(!ascp_file_list_provided){'file list provided both in transfer spec and ascp file list. Remove one of them.'}
136
+ Aspera.assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
137
+ is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
138
+ raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
139
+ if self.class.file_list_folder.nil?
140
+ Aspera.assert(!is_pair_list){'file pair list is not supported when file list folder is not set'}
141
+ # not safe for special characters ? (maybe not, depends on OS)
142
+ Log.log.debug('placing source file list on command line (no file list file)')
143
+ @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
144
+ else
145
+ # safer option: generate a file list file if there is storage defined for it
146
+ if is_pair_list
147
+ file_list_option = '--file-pair-list'
148
+ lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']) }
168
149
  else
169
- assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
170
- is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
171
- raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
172
- # safer option: generate a file list file if there is storage defined for it
173
- # if there is one destination in paths, then use file-pair-list
174
- if is_pair_list
175
- option = '--file-pair-list'
176
- lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'] || e['source']) }
177
- else
178
- option = '--file-list'
179
- lines = ts_paths_array.map{|i|i['source']}
180
- end
181
- file_list_file = Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
182
- Log.log.debug{Log.dump(:file_list, lines)}
183
- File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
184
- Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
150
+ file_list_option = '--file-list'
151
+ lines = ts_paths_array.map{|i|i['source']}
185
152
  end
153
+ file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
154
+ Log.log.debug{Log.dump(:file_list, lines)}
155
+ File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
156
+ Log.log.debug{"#{file_list_option}=\n#{File.read(file_list_file)}".red}
186
157
  end
187
- else
188
- option = '--file-list'
189
158
  end
190
- @builder.add_command_line_options(["#{option}=#{file_list_file}"]) unless option.nil?
159
+ @builder.add_command_line_options(["#{file_list_option}=#{file_list_file}"]) unless file_list_option.nil?
191
160
  end
192
161
 
193
162
  def remote_certificates
@@ -200,31 +169,32 @@ module Aspera
200
169
  # This will need to be cleaned up in aspera core
201
170
  @job_spec['ssh_port'] = @builder.read_param('wss_port')
202
171
  @job_spec.delete('fasp_port')
203
- @job_spec.delete('EX_ssh_key_paths')
204
172
  @job_spec.delete('sshfp')
173
+ # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
174
+ certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs].is_a?(Array)
205
175
  # ignore cert for wss ?
206
- if @options[:check_ignore]&.call(@job_spec['remote_host'], @job_spec['wss_port'])
176
+ if @options[:check_ignore_cb]&.call(@job_spec['remote_host'], @job_spec['wss_port'])
207
177
  wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
208
178
  wss_url = "https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}"
209
- File.write(wss_cert_file, Rest.remote_certificates(wss_url))
210
- certificates_to_use.push(wss_cert_file)
179
+ File.write(wss_cert_file, Rest.remote_certificate_chain(wss_url))
180
+ # place in front, as more priority
181
+ certificates_to_use.unshift(wss_cert_file)
211
182
  end
212
- # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
213
- certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs]
183
+ # when wss is used, only first `-i` is used... Hum...
184
+ certificates_to_use = [certificates_to_use.first] unless certificates_to_use.empty?
214
185
  else
215
186
  # remove unused parameter (avoid warning)
216
187
  @job_spec.delete('wss_port')
217
188
  # add SSH bypass keys when authentication is token and no auth is provided
218
189
  if @job_spec.key?('token') && !@job_spec.key?('remote_password')
219
- # @job_spec['remote_password'] = Installation.instance.ssh_cert_uuid # not used: no passphrase
220
- certificates_to_use.concat(Installation.instance.aspera_token_ssh_key_paths)
190
+ # @job_spec['remote_password'] = Ascp::Installation.instance.ssh_cert_uuid # not used: no passphrase
191
+ certificates_to_use.concat(Ascp::Installation.instance.aspera_token_ssh_key_paths)
221
192
  end
222
193
  end
223
194
  return certificates_to_use
224
195
  end
225
196
 
226
197
  # translate transfer spec to env vars and command line arguments for ascp
227
- # NOTE: parameters starting with "EX_" (extended) are not standard
228
198
  def ascp_args
229
199
  env_args = {
230
200
  args: [],
@@ -236,11 +206,11 @@ module Aspera
236
206
  @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
237
207
 
238
208
  # notify multi-session was already used, anyway it was deleted by agent direct
239
- assert(!@builder.read_param('multi_session'))
209
+ Aspera.assert(!@builder.read_param('multi_session'))
240
210
 
241
211
  # add ssh or wss certificates
242
- remote_certificates.each do |cert|
243
- Log.log.trace1{"adding certificate: #{cert}"}
212
+ # (reverse, to keep order, as we unshift)
213
+ remote_certificates.reverse_each do |cert|
244
214
  env_args[:args].unshift('-i', cert)
245
215
  end
246
216
 
@@ -248,7 +218,7 @@ module Aspera
248
218
  @builder.process_params
249
219
 
250
220
  base64_destination = false
251
- # symbol must be index of Installation.paths
221
+ # symbol must be index of Ascp::Installation.paths
252
222
  if @builder.read_param('use_ascp4')
253
223
  env_args[:ascp_version] = :ascp4
254
224
  else
@@ -258,7 +228,6 @@ module Aspera
258
228
  # destination will be base64 encoded, put this before source path arguments
259
229
  @builder.add_command_line_options(['--dest64']) if base64_destination
260
230
  # optional arguments, at the end to override previous ones (to allow override)
261
- @builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
262
231
  @builder.add_command_line_options(@options[:ascp_args])
263
232
  # get list of source files to transfer and build arg for ascp
264
233
  process_file_list
@@ -272,8 +241,8 @@ module Aspera
272
241
  env_args[:args].unshift('-q') if @options[:quiet]
273
242
  # add fallback cert and key as arguments if needed
274
243
  if ['1', 1, true, 'force'].include?(@job_spec['http_fallback'])
275
- env_args[:args].unshift('-Y', Installation.instance.path(:fallback_private_key))
276
- env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
244
+ env_args[:args].unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
245
+ env_args[:args].unshift('-I', Ascp::Installation.instance.path(:fallback_certificate))
277
246
  end
278
247
  Log.log.debug{"ascp args: #{env_args}"}
279
248
  return env_args
@@ -1,12 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/fasp/parameters'
3
+ require 'aspera/command_line_builder'
4
4
  require 'aspera/assert'
5
+ require 'yaml'
5
6
 
6
7
  module Aspera
7
- module Fasp
8
+ module Transfer
8
9
  # parameters for Transfer Spec
9
- class TransferSpec
10
+ class Spec
10
11
  # default transfer username for access key based transfers
11
12
  ACCESS_KEY_TRANSFER_USER = 'xfer'
12
13
  SSH_PORT = 33_001
@@ -18,35 +19,36 @@ module Aspera
18
19
  }.freeze
19
20
  # reserved tag for Aspera
20
21
  TAG_RESERVED = 'aspera'
21
- # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
22
- Aspera::Fasp::Parameters.description.each do |name, description|
23
- next unless description[:enum].is_a?(Array)
24
- TransferSpec.const_set("#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
25
- description[:enum].each do |enum|
26
- TransferSpec.const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
27
- end
28
- end
29
22
  class << self
30
23
  def action_to_direction(tspec, command)
31
- assert_type(tspec, Hash){'transfer spec'}
24
+ Aspera.assert_type(tspec, Hash){'transfer spec'}
32
25
  tspec['direction'] = case command.to_sym
33
26
  when :upload then DIRECTION_SEND
34
27
  when :download then DIRECTION_RECEIVE
35
- else error_unexpected_value(command.to_sym)
28
+ else Aspera.error_unexpected_value(command.to_sym)
36
29
  end
37
30
  return tspec
38
31
  end
39
32
 
40
33
  def action(tspec)
41
- assert_type(tspec, Hash){'transfer spec'}
42
- assert_values(tspec['direction'], [DIRECTION_SEND, DIRECTION_RECEIVE]){'direction'}
34
+ Aspera.assert_type(tspec, Hash){'transfer spec'}
35
+ Aspera.assert_values(tspec['direction'], [DIRECTION_SEND, DIRECTION_RECEIVE]){'direction'}
43
36
  case tspec['direction']
44
37
  when DIRECTION_SEND then :upload
45
38
  when DIRECTION_RECEIVE then :download
46
- else error_unexpected_value(tspec['direction'])
39
+ else Aspera.error_unexpected_value(tspec['direction'])
47
40
  end
48
41
  end
49
42
  end
43
+ DESCRIPTION = CommandLineBuilder.normalize_description(YAML.load_file("#{__FILE__[0..-3]}yaml"))
44
+ # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
45
+ DESCRIPTION.each do |name, description|
46
+ next unless description[:enum].is_a?(Array)
47
+ const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
48
+ description[:enum].each do |enum|
49
+ const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
50
+ end
51
+ end
50
52
  end
51
53
  end
52
54
  end
@@ -27,7 +27,7 @@ cipher:
27
27
  :cli:
28
28
  :type: :opt_with_arg
29
29
  :switch: "-c"
30
- :convert: Aspera::Fasp::Parameters.convert_remove_hyphen
30
+ :convert: Aspera::Transfer::Parameters.convert_remove_hyphen
31
31
  content_protection:
32
32
  :desc: Enable client-side encryption at rest. (CSEAR, content protection)
33
33
  :enum:
@@ -424,7 +424,7 @@ tags:
424
424
  :cli:
425
425
  :type: :opt_with_arg
426
426
  :switch: "--tags64"
427
- :convert: Aspera::Fasp::Parameters.convert_json64
427
+ :convert: Aspera::Transfer::Parameters.convert_json64
428
428
  target_rate_cap_kbps:
429
429
  :desc: Returned by upload/download_setup node API.
430
430
  :accepted_types: :int
@@ -479,7 +479,7 @@ source_root:
479
479
  :cli:
480
480
  :type: :opt_with_arg
481
481
  :switch: "--source-prefix64"
482
- :convert: Aspera::Fasp::Parameters.convert_base64
482
+ :convert: Aspera::Transfer::Parameters.convert_base64
483
483
  min_rate_cap_kbps:
484
484
  :desc: |-
485
485
  The highest minimum rate that an incoming transfer can request, in kilobits per second.
@@ -540,7 +540,7 @@ src_base:
540
540
  :cli:
541
541
  :type: :opt_with_arg
542
542
  :switch: "--src-base64"
543
- :convert: Aspera::Fasp::Parameters.convert_base64
543
+ :convert: Aspera::Transfer::Parameters.convert_base64
544
544
  preserve_acls:
545
545
  :desc: "Preserve access control lists."
546
546
  :enum:
@@ -610,34 +610,6 @@ remove_empty_source_directory:
610
610
  - :sdk
611
611
  :cli:
612
612
  :type: :opt_without_arg
613
- EX_at_rest_password:
614
- :desc: Content protection password
615
- :deprecation: "(4.13) Use standard spec parameter: content_protection_password"
616
- :agents:
617
- - :direct
618
- :cli:
619
- :type: :envvar
620
- :variable: ASPERA_SCP_FILEPASS
621
- EX_proxy_password:
622
- :desc: |-
623
- Password used for Aspera proxy server authentication.
624
- May be overridden by password in URL provided in parameter: proxy.
625
-
626
-
627
- :deprecation: (4.14) Use env var ASPERA_PROXY_PASS
628
- :agents:
629
- - :direct
630
- :cli:
631
- :type: :envvar
632
- :variable: ASPERA_PROXY_PASS
633
- EX_license_text:
634
- :desc: "License file text override.\nBy default ascp looks for license file near executable."
635
- :deprecation: (4.14) Use env var ASPERA_SCP_LICENSE
636
- :agents:
637
- - :direct
638
- :cli:
639
- :type: :envvar
640
- :variable: ASPERA_SCP_LICENSE
641
613
  keepalive:
642
614
  :desc: The session is running in persistent session mode.
643
615
  :agents:
@@ -663,49 +635,6 @@ sshfp:
663
635
  :cli:
664
636
  :type: :opt_with_arg
665
637
  :switch: "--check-sshfp"
666
- EX_http_proxy_url:
667
- :desc: Specify the proxy server address used by HTTP Fallback
668
- :deprecation: (4.14) TODO, use proxy option ?
669
- :agents:
670
- - :direct
671
- :cli:
672
- :type: :opt_with_arg
673
- :switch: "-x"
674
- EX_ssh_key_paths:
675
- :desc: Use public key authentication for SSH and specify the private key file paths
676
- :deprecation: (4.14) Use option transfer_info.ascp_args
677
- :accepted_types: :array
678
- :agents:
679
- - :direct
680
- :cli:
681
- :type: :opt_with_arg
682
- :switch: "-i"
683
- EX_http_transfer_jpeg:
684
- :desc: HTTP transfers as JPEG file
685
- :deprecation: (4.14) Use option transfer_info.ascp_args
686
- :accepted_types: :int
687
- :default: '0'
688
- :agents:
689
- - :direct
690
- :cli:
691
- :type: :opt_with_arg
692
- :switch: "-j"
693
- EX_no_read:
694
- :desc: no read source
695
- :deprecation: (4.14) Use option transfer_info.ascp_args
696
- :agents:
697
- - :direct
698
- :cli:
699
- :type: :opt_without_arg
700
- :switch: "--no-read"
701
- EX_no_write:
702
- :desc: no write on destination
703
- :deprecation: (4.14) Use option transfer_info.ascp_args
704
- :agents:
705
- - :direct
706
- :cli:
707
- :type: :opt_without_arg
708
- :switch: "--no-write"
709
638
  target_rate_percentage:
710
639
  :desc: "TODO: remove ?"
711
640
  :cli:
@@ -754,30 +683,6 @@ obfuscate_file_names:
754
683
  - :httpgw
755
684
  :cli:
756
685
  :type: :ignore
757
- EX_file_list:
758
- :desc: source file list
759
- :deprecation: (4.14) Use command line file list, or option transfer_info.ascp_args
760
- :agents:
761
- - :direct
762
- :cli:
763
- :type: :special
764
- :switch: "--file-list"
765
- EX_file_pair_list:
766
- :desc: source file pair list
767
- :deprecation: (4.14) Use command line file pair list, or option transfer_info.ascp_args
768
- :agents:
769
- - :direct
770
- :cli:
771
- :type: :special
772
- :switch: "--file-pair-list"
773
- EX_ascp_args:
774
- :desc: Add native command line arguments to ascp
775
- :deprecation: (4.13) Use option transfer_info.ascp_args
776
- :accepted_types: :array
777
- :agents:
778
- - :direct
779
- :cli:
780
- :type: :special
781
686
  wss_enabled:
782
687
  :desc: Server has Web Socket service enabled
783
688
  :accepted_types: :bool