aspera-cli 4.21.2 → 4.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +402 -374
  5. data/CONTRIBUTING.md +6 -10
  6. data/README.md +1018 -687
  7. data/lib/aspera/agent/base.rb +9 -5
  8. data/lib/aspera/agent/connect.rb +30 -28
  9. data/lib/aspera/agent/desktop.rb +29 -25
  10. data/lib/aspera/agent/direct.rb +137 -125
  11. data/lib/aspera/agent/httpgw.rb +22 -26
  12. data/lib/aspera/agent/node.rb +14 -11
  13. data/lib/aspera/agent/transferd.rb +6 -2
  14. data/lib/aspera/api/aoc.rb +15 -18
  15. data/lib/aspera/api/cos_node.rb +1 -1
  16. data/lib/aspera/api/httpgw.rb +15 -7
  17. data/lib/aspera/api/node.rb +6 -4
  18. data/lib/aspera/ascmd.rb +17 -9
  19. data/lib/aspera/ascp/installation.rb +21 -19
  20. data/lib/aspera/ascp/management.rb +1 -1
  21. data/lib/aspera/assert.rb +14 -5
  22. data/lib/aspera/cli/error.rb +2 -2
  23. data/lib/aspera/cli/extended_value.rb +38 -19
  24. data/lib/aspera/cli/formatter.rb +48 -48
  25. data/lib/aspera/cli/hints.rb +10 -2
  26. data/lib/aspera/cli/main.rb +190 -168
  27. data/lib/aspera/cli/manager.rb +16 -16
  28. data/lib/aspera/cli/plugin.rb +24 -21
  29. data/lib/aspera/cli/plugin_factory.rb +1 -1
  30. data/lib/aspera/cli/plugins/alee.rb +1 -1
  31. data/lib/aspera/cli/plugins/aoc.rb +173 -126
  32. data/lib/aspera/cli/plugins/ats.rb +19 -17
  33. data/lib/aspera/cli/plugins/config.rb +87 -98
  34. data/lib/aspera/cli/plugins/console.rb +5 -3
  35. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  36. data/lib/aspera/cli/plugins/faspex5.rb +104 -80
  37. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  38. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  39. data/lib/aspera/cli/plugins/node.rb +336 -205
  40. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  41. data/lib/aspera/cli/plugins/preview.rb +3 -3
  42. data/lib/aspera/cli/plugins/server.rb +7 -6
  43. data/lib/aspera/cli/plugins/shares.rb +5 -5
  44. data/lib/aspera/cli/sync_actions.rb +19 -18
  45. data/lib/aspera/cli/transfer_agent.rb +11 -15
  46. data/lib/aspera/cli/transfer_progress.rb +2 -2
  47. data/lib/aspera/cli/version.rb +1 -1
  48. data/lib/aspera/command_line_builder.rb +116 -95
  49. data/lib/aspera/coverage.rb +4 -3
  50. data/lib/aspera/data_repository.rb +1 -0
  51. data/lib/aspera/environment.rb +7 -6
  52. data/lib/aspera/faspex_gw.rb +14 -14
  53. data/lib/aspera/faspex_postproc.rb +7 -6
  54. data/lib/aspera/hash_ext.rb +2 -2
  55. data/lib/aspera/json_rpc.rb +1 -1
  56. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  57. data/lib/aspera/keychain/factory.rb +41 -0
  58. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  59. data/lib/aspera/keychain/macos_security.rb +19 -11
  60. data/lib/aspera/log.rb +29 -34
  61. data/lib/aspera/nagios.rb +6 -6
  62. data/lib/aspera/node_simulator.rb +8 -8
  63. data/lib/aspera/oauth/base.rb +10 -6
  64. data/lib/aspera/oauth/factory.rb +6 -6
  65. data/lib/aspera/oauth/url_json.rb +6 -6
  66. data/lib/aspera/persistency_action_once.rb +6 -4
  67. data/lib/aspera/persistency_folder.rb +2 -2
  68. data/lib/aspera/preview/file_types.rb +40 -33
  69. data/lib/aspera/preview/generator.rb +1 -1
  70. data/lib/aspera/preview/options.rb +16 -16
  71. data/lib/aspera/preview/terminal.rb +3 -3
  72. data/lib/aspera/preview/utils.rb +11 -13
  73. data/lib/aspera/products/connect.rb +2 -1
  74. data/lib/aspera/products/desktop.rb +1 -1
  75. data/lib/aspera/products/transferd.rb +1 -1
  76. data/lib/aspera/proxy_auto_config.rb +2 -2
  77. data/lib/aspera/rest.rb +70 -50
  78. data/lib/aspera/rest_error_analyzer.rb +1 -0
  79. data/lib/aspera/rest_errors_aspera.rb +1 -1
  80. data/lib/aspera/secret_hider.rb +5 -5
  81. data/lib/aspera/ssh.rb +5 -5
  82. data/lib/aspera/temp_file_manager.rb +1 -0
  83. data/lib/aspera/timer_limiter.rb +7 -5
  84. data/lib/aspera/transfer/async_conf.schema.yaml +716 -0
  85. data/lib/aspera/transfer/convert.rb +29 -0
  86. data/lib/aspera/transfer/error_info.rb +66 -66
  87. data/lib/aspera/transfer/parameters.rb +13 -68
  88. data/lib/aspera/transfer/spec.rb +5 -6
  89. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  90. data/lib/aspera/transfer/spec_doc.rb +62 -0
  91. data/lib/aspera/transfer/sync.rb +37 -76
  92. data/lib/aspera/transfer/sync_instance.schema.yaml +20 -0
  93. data/lib/aspera/transfer/sync_session.schema.yaml +86 -0
  94. data/lib/aspera/transfer/uri.rb +6 -6
  95. data/lib/aspera/uri_reader.rb +1 -1
  96. data/lib/aspera/web_auth.rb +1 -1
  97. data/lib/aspera/web_server_simple.rb +53 -44
  98. data.tar.gz.sig +0 -0
  99. metadata +38 -7
  100. metadata.gz.sig +0 -0
  101. data/examples/build_package.sh +0 -28
  102. data/examples/dascli +0 -30
  103. data/examples/get_proto_file.rb +0 -8
  104. data/examples/proxy.pac +0 -60
  105. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/agent/base'
4
+
5
+ module Aspera
6
+ module Transfer
7
+ # translate transfer specification to ascp parameter list
8
+ class SpecDoc
9
+ class << self
10
+ # first letter of agent name symbol
11
+ def agent_to_short(agent_sym)
12
+ agent_sym.to_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
13
+ end
14
+
15
+ # @columns formatter [Cli::Formatter] formatter to use, methods: special_format, check_row
16
+ # @columns &block modify parameter info if needed
17
+ # @return a table suitable to display in manual
18
+ def man_table(formatter, cli: true)
19
+ col_local = agent_to_short(:direct)
20
+ Spec::SCHEMA['properties'].filter_map do |name, properties|
21
+ # manual table
22
+ columns = {
23
+ name: name,
24
+ type: properties['type'],
25
+ description: []
26
+ }
27
+ # replace "back solidus" HTML entity with its text value and split lines
28
+ columns[:description].concat(properties['description'].gsub('&bsol;', '\\').split("\n")) if properties.key?('description')
29
+ columns[:description].unshift("DEPRECATED: #{properties['x-deprecation']}") if properties.key?('x-deprecation')
30
+ # add flags for supported agents in doc
31
+ AGENT_LIST.each do |agent_info|
32
+ columns[agent_info.last] = Cli::Formatter.tick(properties['x-agents'].nil? || properties['x-agents'].include?(agent_info.first.to_s))
33
+ end
34
+ columns[col_local] = Cli::Formatter.tick(true) if properties['x-cli-option']
35
+ # only keep lines that are usable in supported agents
36
+ next false if AGENT_LIST.map(&:last).inject(true){ |memory, agent_short_sym| memory && columns[agent_short_sym].empty?}
37
+ columns[:description].push("Allowed values: #{properties['enum'].join(', ')}") if properties.key?('enum')
38
+ cli_option =
39
+ if properties['x-cli-switch']
40
+ properties['x-cli-option']
41
+ elsif properties['x-cli-special']
42
+ formatter.special_format('special')
43
+ elsif properties['x-cli-option']
44
+ arg_type = properties.key?('enum') ? '{enum}' : "{#{[properties['type']].flatten.join('|')}}"
45
+ conversion_tag = properties.key?('x-cli-convert') ? '(conversion)' : ''
46
+ sep = properties['x-cli-option'].start_with?('--') ? '=' : ' '
47
+ "#{properties['x-cli-option']}#{sep}#{conversion_tag}#{arg_type}"
48
+ end
49
+ cli_option = 'env:' + properties['x-cli-envvar'] if properties.key?('x-cli-envvar')
50
+ columns[:description].push("(#{cli_option})") if cli && !cli_option.to_s.empty?
51
+ formatter.check_row(columns)
52
+ end.sort_by{ |i| i[:name]}
53
+ end
54
+ end
55
+ # Agents shown in manual for parameters (sub list)
56
+ AGENT_LIST = Agent::Base.agent_list.map do |agent_sym|
57
+ [agent_sym, agent_sym.to_s.capitalize, agent_to_short(agent_sym)]
58
+ end.sort_by(&:last).freeze
59
+ TABLE_COLUMNS = (%i[name type] + AGENT_LIST.map(&:last) + %i[description]).freeze
60
+ end
61
+ end
62
+ end
@@ -5,6 +5,7 @@
5
5
  require 'aspera/command_line_builder'
6
6
  require 'aspera/ascp/installation'
7
7
  require 'aspera/agent/direct'
8
+ require 'aspera/transfer/convert'
8
9
  require 'aspera/log'
9
10
  require 'aspera/assert'
10
11
  require 'json'
@@ -16,62 +17,18 @@ module Aspera
16
17
  module Transfer
17
18
  # builds command line arg for async and execute it
18
19
  module Sync
19
- # sync direction, default is push
20
+ # sync direction
20
21
  DIRECTIONS = %i[push pull bidi].freeze
22
+ # default direction for sync
23
+ DEFAULT_DIRECTION = :push
21
24
  # JSON for async instance command line options
22
- CMDLINE_PARAMS_INSTANCE =
23
- {
24
- 'alt_logdir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
25
- 'watchd' => { cli: { type: :opt_with_arg}, accepted_types: :string},
26
- 'apply_local_docroot' => { cli: { type: :opt_without_arg}},
27
- 'quiet' => { cli: { type: :opt_without_arg}},
28
- 'ws_connect' => { cli: { type: :opt_without_arg}}
29
- }.freeze
30
-
31
- # map sync session parameters to transfer spec: sync -> ts, true if same
32
- CMDLINE_PARAMS_SESSION =
33
- {
34
- 'name' => { cli: { type: :opt_with_arg}, accepted_types: :string},
35
- 'local_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
36
- 'remote_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
37
- 'local_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
38
- 'remote_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
39
- 'host' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_host},
40
- 'user' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_user},
41
- 'private_key_paths' => { cli: { type: :opt_with_arg, switch: '--private-key-path'}, accepted_types: :array},
42
- 'direction' => { cli: { type: :opt_with_arg}, accepted_types: :string},
43
- 'checksum' => { cli: { type: :opt_with_arg}, accepted_types: :string},
44
- 'tags' => { cli: { type: :opt_with_arg, switch: '--tags64', convert: 'Aspera::Transfer::Parameters.convert_json64'},
45
- accepted_types: :hash, ts: true},
46
- 'tcp_port' => { cli: { type: :opt_with_arg}, accepted_types: :int, ts: :ssh_port},
47
- 'rate_policy' => { cli: { type: :opt_with_arg}, accepted_types: :string},
48
- 'target_rate' => { cli: { type: :opt_with_arg}, accepted_types: :string},
49
- 'cooloff' => { cli: { type: :opt_with_arg}, accepted_types: :int},
50
- 'pending_max' => { cli: { type: :opt_with_arg}, accepted_types: :int},
51
- 'scan_intensity' => { cli: { type: :opt_with_arg}, accepted_types: :string},
52
- 'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Transfer::Parameters.convert_remove_hyphen'},
53
- accepted_types: :string, ts: true},
54
- 'transfer_threads' => { cli: { type: :opt_with_arg}, accepted_types: :int},
55
- 'preserve_time' => { cli: { type: :opt_without_arg}, ts: :preserve_times},
56
- 'preserve_access_time' => { cli: { type: :opt_without_arg}, ts: nil},
57
- 'preserve_modification_time' => { cli: { type: :opt_without_arg}, ts: nil},
58
- 'preserve_uid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_uid},
59
- 'preserve_gid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_gid},
60
- 'create_dir' => { cli: { type: :opt_without_arg}, ts: true},
61
- 'reset' => { cli: { type: :opt_without_arg}},
62
- # NOTE: only one env var, but multiple sessions... could be a problem
63
- 'remote_password' => { cli: { type: :envvar, variable: 'ASPERA_SCP_PASS'}, ts: true},
64
- 'cookie' => { cli: { type: :envvar, variable: 'ASPERA_SCP_COOKIE'}, ts: true},
65
- 'token' => { cli: { type: :envvar, variable: 'ASPERA_SCP_TOKEN'}, ts: true},
66
- 'license' => { cli: { type: :envvar, variable: 'ASPERA_SCP_LICENSE'}}
67
- }.freeze
68
-
69
- CommandLineBuilder.normalize_description(CMDLINE_PARAMS_INSTANCE)
70
- CommandLineBuilder.normalize_description(CMDLINE_PARAMS_SESSION)
25
+ INSTANCE_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'instance')
26
+ SESSION_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'session')
71
27
 
72
28
  CMDLINE_PARAMS_KEYS = %w[instance sessions].freeze
73
29
 
74
30
  # Translation of transfer spec parameters to async v2 API (asyncs)
31
+ # TODO: complete this list with all transfer spec parameters or include in async json schema with x- parameters
75
32
  TSPEC_TO_ASYNC_CONF = {
76
33
  'remote_host' => 'remote.host',
77
34
  'remote_user' => 'remote.user',
@@ -86,10 +43,10 @@ module Aspera
86
43
 
87
44
  ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
88
45
 
89
- private_constant :CMDLINE_PARAMS_INSTANCE, :CMDLINE_PARAMS_SESSION, :CMDLINE_PARAMS_KEYS, :TSPEC_TO_ASYNC_CONF, :ASYNC_ADMIN_EXECUTABLE
46
+ private_constant :INSTANCE_SCHEMA, :SESSION_SCHEMA, :CMDLINE_PARAMS_KEYS, :TSPEC_TO_ASYNC_CONF, :ASYNC_ADMIN_EXECUTABLE
90
47
 
91
48
  class << self
92
- # Set remote_dir in sync parameters based on transfer spec
49
+ # Set `remote_dir` in sync parameters based on transfer spec
93
50
  # @param params [Hash] sync parameters, old or new format
94
51
  # @param remote_dir_key [String] key to update in above hash
95
52
  # @param transfer_spec [Hash] transfer spec
@@ -100,7 +57,8 @@ module Aspera
100
57
  elsif transfer_spec['cookie']&.start_with?('aspera.shares2')
101
58
  # TODO : something more generic, independent of Shares
102
59
  # in Shares, the actual folder on remote end is not always the same as the name of the share
103
- actual_remote = transfer_spec['paths']&.first&.[]('source')
60
+ remote_key = transfer_spec['direction'].eql?('send') ? 'destination' : 'source'
61
+ actual_remote = transfer_spec['paths']&.first&.[](remote_key)
104
62
  sync_params[remote_dir_key] = actual_remote if actual_remote
105
63
  end
106
64
  nil
@@ -132,14 +90,19 @@ module Aspera
132
90
  return certificates_to_use
133
91
  end
134
92
 
93
+ # Get symbol of sync direction, defaulting to :push
94
+ # @param params [Hash] sync parameters, old or new format
95
+ # @return [Symbol] direction symbol, one of :push, :pull, :bidi
96
+ def direction_sym(params)
97
+ (params['direction'] || DEFAULT_DIRECTION).to_sym
98
+ end
99
+
135
100
  # @param sync_params [Hash] sync parameters, old or new format
136
- # @param block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
137
- def start(
138
- sync_params,
139
- &block
140
- )
101
+ # @param &block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
102
+ def start(sync_params)
141
103
  Log.log.debug{Log.dump(:sync_params_initial, sync_params)}
142
104
  Aspera.assert_type(sync_params, Hash)
105
+ Aspera.assert(%w[local sessions].any?{ |k| sync_params.key?(k)}){'At least one of `local` or `sessions` must be present in async parameters'}
143
106
  env_args = {
144
107
  args: [],
145
108
  env: {}
@@ -151,8 +114,8 @@ module Aspera
151
114
  Aspera.assert_type(remote, Hash){'remote'}
152
115
  Aspera.assert_type(remote['path'], String){'remote path'}
153
116
  # get transfer spec if possible, and feed back to new structure
154
- if block
155
- transfer_spec = yield((sync_params['direction'] || 'push').to_sym, sync_params['local']['path'], remote['path'])
117
+ if block_given?
118
+ transfer_spec = yield(direction_sym(sync_params), sync_params['local']['path'], remote['path'])
156
119
  # translate transfer spec to async parameters
157
120
  TSPEC_TO_ASYNC_CONF.each do |ts_param, sy_path|
158
121
  next unless transfer_spec.key?(ts_param)
@@ -172,24 +135,25 @@ module Aspera
172
135
  end
173
136
  # '--exclusive-mgmt-port=12345', '--arg-err-path=-',
174
137
  env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"]
175
- Log.log.debug{Log.dump(:sync_params_enriched, sync_params)}
138
+ Log.log.debug{Log.dump(:sync_conf, sync_params)}
176
139
  agent = Agent::Direct.new
177
140
  agent.start_and_monitor_process(session: {}, name: :async, **env_args)
178
- elsif sync_params.key?('sessions')
141
+ else
142
+ # key 'sessions' is present
179
143
  # ascli JSON format (cmdline)
180
144
  raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
181
145
  sync_params.keys.push('instance').uniq.sort.eql?(CMDLINE_PARAMS_KEYS)
182
146
  Aspera.assert_type(sync_params['sessions'], Array)
183
147
  Aspera.assert_type(sync_params['sessions'].first, Hash)
184
- if block
148
+ if block_given?
185
149
  sync_params['sessions'].each do |session|
186
150
  Aspera.assert_type(session['local_dir'], String){'local_dir'}
187
151
  Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
188
- transfer_spec = yield((session['direction'] || 'push').to_sym, session['local_dir'], session['remote_dir'])
189
- CMDLINE_PARAMS_SESSION.each do |async_param, behavior|
190
- if behavior.key?(:ts)
191
- tspec_param = behavior[:ts].is_a?(TrueClass) ? async_param : behavior[:ts].to_s
192
- session[async_param] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
152
+ transfer_spec = yield(direction_sym(session), session['local_dir'], session['remote_dir'])
153
+ SESSION_SCHEMA['properties'].each do |name, properties|
154
+ if properties.key?('x-tspec')
155
+ tspec_param = properties['x-tspec'].is_a?(TrueClass) ? name : properties['x-tspec'].to_s
156
+ session[name] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
193
157
  end
194
158
  end
195
159
  session['private_key_paths'] = Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa) if transfer_spec.key?('token')
@@ -198,21 +162,18 @@ module Aspera
198
162
  end
199
163
  if sync_params.key?('instance')
200
164
  Aspera.assert_type(sync_params['instance'], Hash)
201
- instance_builder = CommandLineBuilder.new(sync_params['instance'], CMDLINE_PARAMS_INSTANCE)
165
+ instance_builder = CommandLineBuilder.new(sync_params['instance'], INSTANCE_SCHEMA, Convert)
202
166
  instance_builder.process_params
203
167
  instance_builder.add_env_args(env_args)
204
168
  end
205
-
206
169
  sync_params['sessions'].each do |session_params|
207
170
  Aspera.assert_type(session_params, Hash)
208
- Aspera.assert(session_params.key?('name')){'session must contain at least name'}
209
- session_builder = CommandLineBuilder.new(session_params, CMDLINE_PARAMS_SESSION)
171
+ Aspera.assert(session_params.key?('name')){'session must contain at least: name'}
172
+ session_builder = CommandLineBuilder.new(session_params, SESSION_SCHEMA, Convert)
210
173
  session_builder.process_params
211
174
  session_builder.add_env_args(env_args)
212
175
  end
213
176
  Environment.secure_execute(exec: Ascp::Installation.instance.path(:async), **env_args)
214
- else
215
- raise 'At least one of `local` or `sessions` must be present in async parameters'
216
177
  end
217
178
  return nil
218
179
  end
@@ -248,8 +209,8 @@ module Aspera
248
209
  raise 'Missing either local_db_dir or local.path'
249
210
  end
250
211
  elsif sync_params.key?('sessions')
251
- session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{|s|s['name'].eql?(session_name)}
252
- raise "Session #{session_name} not found in #{sync_params['sessions'].map{|s|s['name']}.join(',')}" if session.nil?
212
+ session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{ |s| s['name'].eql?(session_name)}
213
+ raise "Session #{session_name} not found in #{sync_params['sessions'].map{ |s| s['name']}.join(',')}" if session.nil?
253
214
  raise 'Missing session name' if session['name'].nil?
254
215
  arguments.push("--name=#{session['name']}")
255
216
  if session.key?('local_db_dir')
@@ -0,0 +1,20 @@
1
+ $schema: https://json-schema.org/draft/2020-12/schema
2
+ $id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/transfer/sync_instance.schema.yaml
3
+ $comment: >-
4
+ `x-` fields documented in command_line_builder.rb
5
+ This spec is specific to ascli.
6
+ The native async conf format is now preferred.
7
+ title: SyncInstanceSpec
8
+ description: Global parameters for async.
9
+ type: object
10
+ properties:
11
+ alt_logdir:
12
+ type: string
13
+ watchd:
14
+ type: string
15
+ apply_local_docroot:
16
+ x-cli-switch: true
17
+ quiet:
18
+ x-cli-switch: true
19
+ ws_connect:
20
+ x-cli-switch: true
@@ -0,0 +1,86 @@
1
+ $schema: https://json-schema.org/draft/2020-12/schema
2
+ $id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/transfer/sync_instance.schema.yaml
3
+ $comment: >-
4
+ `x-` fields documented in command_line_builder.rb
5
+ This spec is specific to ascli.
6
+ The native async conf format is now preferred.
7
+ title: SyncSessionSpec
8
+ description: Sync session parameters for async.
9
+ type: object
10
+ properties:
11
+ name:
12
+ type: string
13
+ local_dir:
14
+ type: string
15
+ remote_dir:
16
+ type: string
17
+ local_db_dir:
18
+ type: string
19
+ remote_db_dir:
20
+ type: string
21
+ host:
22
+ type: string
23
+ x-tspec: remote_host
24
+ user:
25
+ type: string
26
+ x-tspec: remote_user
27
+ private_key_paths:
28
+ type: array
29
+ x-cli-option: "--private-key-path"
30
+ direction:
31
+ type: string
32
+ checksum:
33
+ type: string
34
+ tags:
35
+ type: object
36
+ x-cli-option: "--tags64"
37
+ x-cli-convert: json64
38
+ x-tspec: true
39
+ tcp_port:
40
+ type: integer
41
+ x-tspec: ssh_port
42
+ rate_policy:
43
+ type: string
44
+ target_rate:
45
+ type: string
46
+ cooloff:
47
+ type: integer
48
+ pending_max:
49
+ type: integer
50
+ scan_intensity:
51
+ type: string
52
+ cipher:
53
+ type: string
54
+ x-cli-convert: remove_hyphen
55
+ x-tspec: true
56
+ transfer_threads:
57
+ type: integer
58
+ preserve_time:
59
+ x-cli-switch: true
60
+ x-tspec: preserve_times
61
+ preserve_access_time:
62
+ x-cli-switch: true
63
+ preserve_modification_time:
64
+ x-cli-switch: true
65
+ preserve_uid:
66
+ x-cli-switch: true
67
+ x-tspec: preserve_file_owner_uid
68
+ preserve_gid:
69
+ x-cli-switch: true
70
+ x-tspec: preserve_file_owner_gid
71
+ create_dir:
72
+ x-cli-switch: true
73
+ x-tspec: true
74
+ reset:
75
+ x-cli-switch: true
76
+ remote_password:
77
+ x-cli-envvar: ASPERA_SCP_PASS
78
+ x-tspec: true
79
+ cookie:
80
+ x-cli-envvar: ASPERA_SCP_COOKIE
81
+ x-tspec: true
82
+ token:
83
+ x-cli-envvar: ASPERA_SCP_TOKEN
84
+ x-tspec: true
85
+ license:
86
+ x-cli-envvar: ASPERA_SCP_LICENSE
@@ -4,7 +4,7 @@
4
4
 
5
5
  require 'aspera/log'
6
6
  require 'aspera/rest'
7
- require 'aspera/command_line_builder'
7
+ require 'aspera/transfer/convert'
8
8
 
9
9
  module Aspera
10
10
  module Transfer
@@ -24,7 +24,7 @@ module Aspera
24
24
  result_ts['ssh_port'] = @fasp_uri.port
25
25
  result_ts['paths'] = [{'source' => URI.decode_www_form_component(@fasp_uri.path)}]
26
26
  # faspex 4 does not encode trailing base64 padding, fix that to be able to decode properly
27
- fixed_query = @fasp_uri.query.gsub(/(=+)$/){|trail_equals|'%3D' * trail_equals.length}
27
+ fixed_query = @fasp_uri.query.gsub(/(=+)$/){ |trail_equals| '%3D' * trail_equals.length}
28
28
 
29
29
  Rest.query_to_h(fixed_query).each do |name, value|
30
30
  case name
@@ -39,10 +39,10 @@ module Aspera
39
39
  when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
40
40
  when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub('--', '-')
41
41
  when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
42
- when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
43
- when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
44
- when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
45
- when 'lockminrate' then result_ts['lock_min_rate'] = CommandLineBuilder.yes_to_true(value)
42
+ when 'createpath' then result_ts['create_dir'] = Convert.yes_to_true(value)
43
+ when 'fallback' then result_ts['http_fallback'] = Convert.yes_to_true(value)
44
+ when 'lockpolicy' then result_ts['lock_rate_policy'] = Convert.yes_to_true(value)
45
+ when 'lockminrate' then result_ts['lock_min_rate'] = Convert.yes_to_true(value)
46
46
  when 'auth' then Log.log.debug{"ignoring #{name}=#{value}"} # Not used (yes/no)
47
47
  when 'v' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch -- Not used (shall be 2)
48
48
  when 'protect' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch -- TODO: what is this ?
@@ -22,7 +22,7 @@ module Aspera
22
22
  raise 'URL shall have a path, check syntax' if local_file_path.nil?
23
23
  local_file_path = File.expand_path(local_file_path.gsub(%r{^/}, '')) if %r{^/(~|.|..)/}.match?(local_file_path)
24
24
  return File.read(local_file_path)
25
- else Aspera.error_unexpected_value(uri.scheme) {"scheme for [#{uri_to_read}]"}
25
+ else Aspera.error_unexpected_value(uri.scheme){"scheme for [#{uri_to_read}]"}
26
26
  end
27
27
  end
28
28
 
@@ -182,7 +182,7 @@ module Aspera
182
182
  # last argument (self) is provided to constructor of servlet
183
183
  mount(@expected_path, WebAuthServlet, self)
184
184
  # server runs in thread
185
- Thread.new { start }
185
+ Thread.new{start}
186
186
  end
187
187
 
188
188
  # Called by web server thread on received request
@@ -10,19 +10,23 @@ require 'openssl'
10
10
  module Aspera
11
11
  # Simple WEBrick server with HTTPS support
12
12
  class WebServerSimple < WEBrick::HTTPServer
13
- CERT_PARAMETERS = %i[key cert chain pkcs12].freeze
13
+ PARAMS = %i[cert key chain].freeze
14
+ DEFAULT_URL = 'http://localhost:8080'
14
15
  GENERIC_ISSUER = '/C=FR/O=Test/OU=Test/CN=Test'
15
16
  ONE_YEAR_SECONDS = 365 * 24 * 60 * 60
17
+ PKCS12_EXT = %w[p12 pfx].map{ |i| ".#{i}"}.freeze
18
+ CLOCK_SKEW_OFFSET_SEC = 5
16
19
 
17
- private_constant :CERT_PARAMETERS, :GENERIC_ISSUER, :ONE_YEAR_SECONDS
20
+ private_constant :GENERIC_ISSUER, :ONE_YEAR_SECONDS, :PKCS12_EXT, :CLOCK_SKEW_OFFSET_SEC
18
21
 
19
22
  class << self
20
- # Fill and self sign provided certificate
21
- def fill_self_signed_cert(cert, key, digest = 'SHA256')
23
+ # Generate or fill and self sign certificate
24
+ def self_signed_cert(private_key, digest: 'SHA256')
25
+ cert = OpenSSL::X509::Certificate.new
22
26
  cert.subject = cert.issuer = OpenSSL::X509::Name.parse(GENERIC_ISSUER)
23
- cert.not_before = cert.not_after = Time.now
24
- cert.not_after += ONE_YEAR_SECONDS
25
- cert.public_key = key.public_key
27
+ cert.not_before = Time.now - CLOCK_SKEW_OFFSET_SEC
28
+ cert.not_after = cert.not_before + ONE_YEAR_SECONDS
29
+ cert.public_key = private_key.public_key
26
30
  cert.serial = 0x0
27
31
  cert.version = 2
28
32
  ef = OpenSSL::X509::ExtensionFactory.new
@@ -34,68 +38,73 @@ module Aspera
34
38
  # ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
35
39
  ]
36
40
  cert.add_extension(ef.create_extension('authorityKeyIdentifier', 'keyid:always,issuer:always'))
37
- cert.sign(key, OpenSSL::Digest.new(digest))
41
+ cert.sign(private_key, OpenSSL::Digest.new(digest))
42
+ cert
43
+ end
44
+
45
+ # @return a list of Certificates from chain file
46
+ def read_chain_file(chain)
47
+ File.read(chain).scan(/-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----/m).map{ |i| OpenSSL::X509::Certificate.new(i)}
38
48
  end
39
49
  end
40
50
 
41
- # @param uri [URI]
42
- def initialize(uri, certificate: nil)
43
- @url = uri
51
+ # @param url [URI] Local address where server will listen (use scheme, host and port only)
52
+ # @param cert [String] Path to certificate file, either with extension .p12 or .pfx, else assumed PEM
53
+ # @param key [String] Path to key file (PEM) or passphrase (pkcs12)
54
+ # @param chain [String] Path to certificate chain file (PEM only)
55
+ def initialize(uri, cert: nil, key: nil, chain: nil)
56
+ Aspera.assert_type(uri, URI)
57
+ @uri = uri
44
58
  # see https://www.rubydoc.info/stdlib/webrick/WEBrick/Config
45
59
  webrick_options = {
46
- BindAddress: uri.host,
47
- Port: uri.port,
60
+ BindAddress: @uri.host,
61
+ Port: @uri.port,
48
62
  Logger: Log.log,
49
63
  AccessLog: [[self, WEBrick::AccessLog::COMMON_LOG_FORMAT]] # replace default access log to call local method "<<" below
50
64
  }
51
- case uri.scheme
65
+ case @uri.scheme
52
66
  when 'http'
53
67
  Log.log.debug('HTTP mode')
54
68
  when 'https'
55
69
  webrick_options[:SSLEnable] = true
56
- if certificate.nil?
70
+ if cert.nil? && key.nil?
57
71
  webrick_options[:SSLCertName] = [['CN', WEBrick::Utils.getservername]]
72
+ elsif cert && PKCS12_EXT.include?(File.extname(cert).downcase)
73
+ # PKCS12
74
+ Log.log.debug('Using PKCS12 certificate')
75
+ raise 'PKCS12 requires a key (password)' if key.nil?
76
+ pkcs12 = OpenSSL::PKCS12.new(File.read(cert), key)
77
+ webrick_options[:SSLCertificate] = pkcs12.certificate
78
+ webrick_options[:SSLPrivateKey] = pkcs12.key
79
+ webrick_options[:SSLExtraChainCert] = pkcs12.ca_certs
58
80
  else
59
- Aspera.assert_type(certificate, Hash)
60
- certificate = certificate.symbolize_keys
61
- raise "unexpected key in certificate config: only: #{CERT_PARAMETERS.join(', ')}" if certificate.keys.any?{|key|!CERT_PARAMETERS.include?(key)}
62
- if certificate.key?(:pkcs12)
63
- Log.log.debug('Using PKCS12 certificate')
64
- raise 'pkcs12 requires a key (password)' unless certificate.key?(:key)
65
- pkcs12 = OpenSSL::PKCS12.new(File.read(certificate[:pkcs12]), certificate[:key])
66
- webrick_options[:SSLCertificate] = pkcs12.certificate
67
- webrick_options[:SSLPrivateKey] = pkcs12.key
68
- webrick_options[:SSLExtraChainCert] = pkcs12.ca_certs
81
+ Log.log.debug('Using PEM certificate')
82
+ webrick_options[:SSLPrivateKey] = if key.nil?
83
+ OpenSSL::PKey::RSA.new(4096)
84
+ else
85
+ OpenSSL::PKey::RSA.new(File.read(key))
86
+ end
87
+ webrick_options[:SSLCertificate] = if cert.nil?
88
+ self.class.self_signed_cert(webrick_options[:SSLPrivateKey])
69
89
  else
70
- Log.log.debug('Using PEM certificate')
71
- webrick_options[:SSLPrivateKey] = if certificate.key?(:key)
72
- OpenSSL::PKey::RSA.new(File.read(certificate[:key]))
73
- else
74
- OpenSSL::PKey::RSA.new(4096)
75
- end
76
- if certificate.key?(:cert)
77
- webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read(certificate[:cert]))
78
- else
79
- webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new
80
- self.class.fill_self_signed_cert(webrick_options[:SSLCertificate], webrick_options[:SSLPrivateKey])
81
- end
82
- if certificate.key?(:chain)
83
- webrick_options[:SSLExtraChainCert] = [OpenSSL::X509::Certificate.new(File.read(certificate[:chain]))]
84
- end
90
+ OpenSSL::X509::Certificate.new(File.read(cert))
85
91
  end
92
+ webrick_options[:SSLExtraChainCert] = read_chain_file(chain) unless chain.nil?
93
+ raise 'key and cert do not match' unless webrick_options[:SSLCertificate].public_key.to_der == webrick_options[:SSLPrivateKey].public_key.to_der
86
94
  end
87
95
  end
88
96
  # call constructor of parent class, but capture STDERR
89
97
  # self signed certificate generates characters on STDERR
90
98
  # see create_self_signed_cert in webrick/ssl.rb
91
- Log.capture_stderr { super(webrick_options) }
99
+ Log.capture_stderr{super(webrick_options)}
92
100
  end
93
101
 
94
102
  # blocking
95
103
  def start
96
- Log.log.info{"Listening on #{@url}"}
97
- # kill -HUP for graceful shutdown
98
- Kernel.trap('HUP') { shutdown }
104
+ Log.log.info{"Listening on #{@uri}"}
105
+ # kill (-TERM) for graceful shutdown
106
+ handler = proc{shutdown}
107
+ %i{INT TERM}.each{ |sig| trap(sig, &handler)}
99
108
  super
100
109
  end
101
110
 
data.tar.gz.sig CHANGED
Binary file