aspera-cli 4.21.1 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +52 -22
  5. data/CONTRIBUTING.md +69 -148
  6. data/README.md +929 -668
  7. data/bin/ascli +5 -14
  8. data/bin/asession +1 -3
  9. data/examples/get_proto_file.rb +4 -3
  10. data/examples/proxy.pac +20 -20
  11. data/lib/aspera/agent/base.rb +11 -5
  12. data/lib/aspera/agent/connect.rb +30 -28
  13. data/lib/aspera/agent/{alpha.rb → desktop.rb} +35 -31
  14. data/lib/aspera/agent/direct.rb +141 -121
  15. data/lib/aspera/agent/httpgw.rb +22 -26
  16. data/lib/aspera/agent/node.rb +14 -11
  17. data/lib/aspera/agent/transferd.rb +30 -19
  18. data/lib/aspera/api/alee.rb +1 -1
  19. data/lib/aspera/api/aoc.rb +6 -6
  20. data/lib/aspera/api/cos_node.rb +2 -2
  21. data/lib/aspera/api/httpgw.rb +7 -3
  22. data/lib/aspera/api/node.rb +10 -8
  23. data/lib/aspera/ascmd.rb +3 -3
  24. data/lib/aspera/ascp/installation.rb +53 -72
  25. data/lib/aspera/ascp/management.rb +1 -1
  26. data/lib/aspera/assert.rb +11 -2
  27. data/lib/aspera/cli/error.rb +2 -2
  28. data/lib/aspera/cli/extended_value.rb +46 -21
  29. data/lib/aspera/cli/formatter.rb +55 -48
  30. data/lib/aspera/cli/hints.rb +1 -1
  31. data/lib/aspera/cli/info.rb +1 -0
  32. data/lib/aspera/cli/main.rb +192 -170
  33. data/lib/aspera/cli/manager.rb +18 -18
  34. data/lib/aspera/cli/plugin.rb +23 -20
  35. data/lib/aspera/cli/plugin_factory.rb +1 -1
  36. data/lib/aspera/cli/plugins/alee.rb +1 -1
  37. data/lib/aspera/cli/plugins/aoc.rb +247 -159
  38. data/lib/aspera/cli/plugins/ats.rb +19 -17
  39. data/lib/aspera/cli/plugins/config.rb +76 -113
  40. data/lib/aspera/cli/plugins/console.rb +5 -3
  41. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  42. data/lib/aspera/cli/plugins/faspex5.rb +111 -84
  43. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  44. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  45. data/lib/aspera/cli/plugins/node.rb +312 -182
  46. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  47. data/lib/aspera/cli/plugins/preview.rb +3 -3
  48. data/lib/aspera/cli/plugins/server.rb +6 -6
  49. data/lib/aspera/cli/plugins/shares.rb +5 -5
  50. data/lib/aspera/cli/sync_actions.rb +19 -18
  51. data/lib/aspera/cli/transfer_agent.rb +5 -5
  52. data/lib/aspera/cli/transfer_progress.rb +2 -2
  53. data/lib/aspera/cli/version.rb +1 -1
  54. data/lib/aspera/command_line_builder.rb +116 -95
  55. data/lib/aspera/coverage.rb +8 -5
  56. data/lib/aspera/environment.rb +26 -17
  57. data/lib/aspera/faspex_gw.rb +14 -14
  58. data/lib/aspera/faspex_postproc.rb +10 -11
  59. data/lib/aspera/hash_ext.rb +4 -14
  60. data/lib/aspera/json_rpc.rb +1 -1
  61. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  62. data/lib/aspera/keychain/factory.rb +41 -0
  63. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  64. data/lib/aspera/keychain/macos_security.rb +19 -11
  65. data/lib/aspera/log.rb +28 -34
  66. data/lib/aspera/nagios.rb +6 -6
  67. data/lib/aspera/node_simulator.rb +8 -8
  68. data/lib/aspera/oauth/base.rb +14 -7
  69. data/lib/aspera/oauth/factory.rb +5 -6
  70. data/lib/aspera/oauth/url_json.rb +6 -6
  71. data/lib/aspera/persistency_action_once.rb +6 -4
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/generator.rb +13 -10
  74. data/lib/aspera/preview/options.rb +16 -16
  75. data/lib/aspera/preview/terminal.rb +4 -4
  76. data/lib/aspera/preview/utils.rb +15 -17
  77. data/lib/aspera/products/connect.rb +35 -1
  78. data/lib/aspera/products/{alpha.rb → desktop.rb} +3 -3
  79. data/lib/aspera/products/transferd.rb +9 -2
  80. data/lib/aspera/proxy_auto_config.rb +2 -2
  81. data/lib/aspera/rest.rb +56 -47
  82. data/lib/aspera/rest_errors_aspera.rb +1 -1
  83. data/lib/aspera/secret_hider.rb +12 -5
  84. data/lib/aspera/ssh.rb +4 -4
  85. data/lib/aspera/temp_file_manager.rb +5 -4
  86. data/lib/aspera/transfer/convert.rb +29 -0
  87. data/lib/aspera/transfer/error_info.rb +66 -66
  88. data/lib/aspera/transfer/parameters.rb +13 -68
  89. data/lib/aspera/transfer/spec.rb +5 -6
  90. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  91. data/lib/aspera/transfer/spec_doc.rb +62 -0
  92. data/lib/aspera/transfer/sync.rb +23 -72
  93. data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
  94. data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
  95. data/lib/aspera/transfer/uri.rb +6 -6
  96. data/lib/aspera/uri_reader.rb +18 -1
  97. data/lib/aspera/web_auth.rb +1 -1
  98. data/lib/aspera/web_server_simple.rb +53 -44
  99. data.tar.gz.sig +0 -0
  100. metadata +28 -165
  101. metadata.gz.sig +0 -0
  102. data/examples/build_exec +0 -74
  103. data/examples/build_exec_rubyc +0 -40
  104. data/examples/build_package.sh +0 -28
  105. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/agent/base'
4
+
5
+ module Aspera
6
+ module Transfer
7
+ # translate transfer specification to ascp parameter list
8
+ class SpecDoc
9
+ class << self
10
+ # first letter of agent name symbol
11
+ def agent_to_short(agent_sym)
12
+ agent_sym.to_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
13
+ end
14
+
15
+ # @columns formatter [Cli::Formatter] formatter to use, methods: special_format, check_row
16
+ # @columns &block modify parameter info if needed
17
+ # @return a table suitable to display in manual
18
+ def man_table(formatter, cli: true)
19
+ col_local = agent_to_short(:direct)
20
+ Spec::SCHEMA['properties'].filter_map do |name, properties|
21
+ # manual table
22
+ columns = {
23
+ name: name,
24
+ type: properties['type'],
25
+ description: []
26
+ }
27
+ # replace "back solidus" HTML entity with its text value and split lines
28
+ columns[:description].concat(properties['description'].gsub('&bsol;', '\\').split("\n")) if properties.key?('description')
29
+ columns[:description].unshift("DEPRECATED: #{properties['x-deprecation']}") if properties.key?('x-deprecation')
30
+ # add flags for supported agents in doc
31
+ AGENT_LIST.each do |agent_info|
32
+ columns[agent_info.last] = Cli::Formatter.tick(properties['x-agents'].nil? || properties['x-agents'].include?(agent_info.first.to_s))
33
+ end
34
+ columns[col_local] = Cli::Formatter.tick(true) if properties['x-cli-option']
35
+ # only keep lines that are usable in supported agents
36
+ next false if AGENT_LIST.map(&:last).inject(true){ |memory, agent_short_sym| memory && columns[agent_short_sym].empty?}
37
+ columns[:description].push("Allowed values: #{properties['enum'].join(', ')}") if properties.key?('enum')
38
+ cli_option =
39
+ if properties['x-cli-switch']
40
+ properties['x-cli-option']
41
+ elsif properties['x-cli-special']
42
+ formatter.special_format('special')
43
+ elsif properties['x-cli-option']
44
+ arg_type = properties.key?('enum') ? '{enum}' : "{#{[properties['type']].flatten.join('|')}}"
45
+ conversion_tag = properties.key?('x-cli-convert') ? '(conversion)' : ''
46
+ sep = properties['x-cli-option'].start_with?('--') ? '=' : ' '
47
+ "#{properties['x-cli-option']}#{sep}#{conversion_tag}#{arg_type}"
48
+ end
49
+ cli_option = 'env:' + properties['x-cli-envvar'] if properties.key?('x-cli-envvar')
50
+ columns[:description].push("(#{cli_option})") if cli && !cli_option.to_s.empty?
51
+ formatter.check_row(columns)
52
+ end.sort_by{ |i| i[:name]}
53
+ end
54
+ end
55
+ # Agents shown in manual for parameters (sub list)
56
+ AGENT_LIST = Agent::Base.agent_list.map do |agent_sym|
57
+ [agent_sym, agent_sym.to_s.capitalize, agent_to_short(agent_sym)]
58
+ end.sort_by(&:last).freeze
59
+ TABLE_COLUMNS = (%i[name type] + AGENT_LIST.map(&:last) + %i[description]).freeze
60
+ end
61
+ end
62
+ end
@@ -5,6 +5,7 @@
5
5
  require 'aspera/command_line_builder'
6
6
  require 'aspera/ascp/installation'
7
7
  require 'aspera/agent/direct'
8
+ require 'aspera/transfer/convert'
8
9
  require 'aspera/log'
9
10
  require 'aspera/assert'
10
11
  require 'json'
@@ -19,55 +20,8 @@ module Aspera
19
20
  # sync direction, default is push
20
21
  DIRECTIONS = %i[push pull bidi].freeze
21
22
  # JSON for async instance command line options
22
- CMDLINE_PARAMS_INSTANCE =
23
- {
24
- 'alt_logdir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
25
- 'watchd' => { cli: { type: :opt_with_arg}, accepted_types: :string},
26
- 'apply_local_docroot' => { cli: { type: :opt_without_arg}},
27
- 'quiet' => { cli: { type: :opt_without_arg}},
28
- 'ws_connect' => { cli: { type: :opt_without_arg}}
29
- }.freeze
30
-
31
- # map sync session parameters to transfer spec: sync -> ts, true if same
32
- CMDLINE_PARAMS_SESSION =
33
- {
34
- 'name' => { cli: { type: :opt_with_arg}, accepted_types: :string},
35
- 'local_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
36
- 'remote_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
37
- 'local_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
38
- 'remote_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
39
- 'host' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_host},
40
- 'user' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_user},
41
- 'private_key_paths' => { cli: { type: :opt_with_arg, switch: '--private-key-path'}, accepted_types: :array},
42
- 'direction' => { cli: { type: :opt_with_arg}, accepted_types: :string},
43
- 'checksum' => { cli: { type: :opt_with_arg}, accepted_types: :string},
44
- 'tags' => { cli: { type: :opt_with_arg, switch: '--tags64', convert: 'Aspera::Transfer::Parameters.convert_json64'},
45
- accepted_types: :hash, ts: true},
46
- 'tcp_port' => { cli: { type: :opt_with_arg}, accepted_types: :int, ts: :ssh_port},
47
- 'rate_policy' => { cli: { type: :opt_with_arg}, accepted_types: :string},
48
- 'target_rate' => { cli: { type: :opt_with_arg}, accepted_types: :string},
49
- 'cooloff' => { cli: { type: :opt_with_arg}, accepted_types: :int},
50
- 'pending_max' => { cli: { type: :opt_with_arg}, accepted_types: :int},
51
- 'scan_intensity' => { cli: { type: :opt_with_arg}, accepted_types: :string},
52
- 'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Transfer::Parameters.convert_remove_hyphen'},
53
- accepted_types: :string, ts: true},
54
- 'transfer_threads' => { cli: { type: :opt_with_arg}, accepted_types: :int},
55
- 'preserve_time' => { cli: { type: :opt_without_arg}, ts: :preserve_times},
56
- 'preserve_access_time' => { cli: { type: :opt_without_arg}, ts: nil},
57
- 'preserve_modification_time' => { cli: { type: :opt_without_arg}, ts: nil},
58
- 'preserve_uid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_uid},
59
- 'preserve_gid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_gid},
60
- 'create_dir' => { cli: { type: :opt_without_arg}, ts: true},
61
- 'reset' => { cli: { type: :opt_without_arg}},
62
- # NOTE: only one env var, but multiple sessions... could be a problem
63
- 'remote_password' => { cli: { type: :envvar, variable: 'ASPERA_SCP_PASS'}, ts: true},
64
- 'cookie' => { cli: { type: :envvar, variable: 'ASPERA_SCP_COOKIE'}, ts: true},
65
- 'token' => { cli: { type: :envvar, variable: 'ASPERA_SCP_TOKEN'}, ts: true},
66
- 'license' => { cli: { type: :envvar, variable: 'ASPERA_SCP_LICENSE'}}
67
- }.freeze
68
-
69
- CommandLineBuilder.normalize_description(CMDLINE_PARAMS_INSTANCE)
70
- CommandLineBuilder.normalize_description(CMDLINE_PARAMS_SESSION)
23
+ INSTANCE_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'instance')
24
+ SESSION_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'session')
71
25
 
72
26
  CMDLINE_PARAMS_KEYS = %w[instance sessions].freeze
73
27
 
@@ -86,10 +40,10 @@ module Aspera
86
40
 
87
41
  ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
88
42
 
89
- private_constant :CMDLINE_PARAMS_INSTANCE, :CMDLINE_PARAMS_SESSION, :CMDLINE_PARAMS_KEYS, :TSPEC_TO_ASYNC_CONF, :ASYNC_ADMIN_EXECUTABLE
43
+ private_constant :INSTANCE_SCHEMA, :SESSION_SCHEMA, :CMDLINE_PARAMS_KEYS, :TSPEC_TO_ASYNC_CONF, :ASYNC_ADMIN_EXECUTABLE
90
44
 
91
45
  class << self
92
- # Set remote_dir in sync parameters based on transfer spec
46
+ # Set `remote_dir` in sync parameters based on transfer spec
93
47
  # @param params [Hash] sync parameters, old or new format
94
48
  # @param remote_dir_key [String] key to update in above hash
95
49
  # @param transfer_spec [Hash] transfer spec
@@ -100,7 +54,8 @@ module Aspera
100
54
  elsif transfer_spec['cookie']&.start_with?('aspera.shares2')
101
55
  # TODO : something more generic, independent of Shares
102
56
  # in Shares, the actual folder on remote end is not always the same as the name of the share
103
- actual_remote = transfer_spec['paths']&.first&.[]('source')
57
+ remote_key = transfer_spec['direction'].eql?('send') ? 'destination' : 'source'
58
+ actual_remote = transfer_spec['paths']&.first&.[](remote_key)
104
59
  sync_params[remote_dir_key] = actual_remote if actual_remote
105
60
  end
106
61
  nil
@@ -133,13 +88,11 @@ module Aspera
133
88
  end
134
89
 
135
90
  # @param sync_params [Hash] sync parameters, old or new format
136
- # @param block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
137
- def start(
138
- sync_params,
139
- &block
140
- )
91
+ # @param &block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
92
+ def start(sync_params)
141
93
  Log.log.debug{Log.dump(:sync_params_initial, sync_params)}
142
94
  Aspera.assert_type(sync_params, Hash)
95
+ Aspera.assert(%w[local sessions].any?{ |k| sync_params.key?(k)}){'At least one of `local` or `sessions` must be present in async parameters'}
143
96
  env_args = {
144
97
  args: [],
145
98
  env: {}
@@ -151,7 +104,7 @@ module Aspera
151
104
  Aspera.assert_type(remote, Hash){'remote'}
152
105
  Aspera.assert_type(remote['path'], String){'remote path'}
153
106
  # get transfer spec if possible, and feed back to new structure
154
- if block
107
+ if block_given?
155
108
  transfer_spec = yield((sync_params['direction'] || 'push').to_sym, sync_params['local']['path'], remote['path'])
156
109
  # translate transfer spec to async parameters
157
110
  TSPEC_TO_ASYNC_CONF.each do |ts_param, sy_path|
@@ -172,24 +125,25 @@ module Aspera
172
125
  end
173
126
  # '--exclusive-mgmt-port=12345', '--arg-err-path=-',
174
127
  env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"]
175
- Log.log.debug{Log.dump(:sync_params_enriched, sync_params)}
128
+ Log.log.debug{Log.dump(:sync_conf, sync_params)}
176
129
  agent = Agent::Direct.new
177
130
  agent.start_and_monitor_process(session: {}, name: :async, **env_args)
178
- elsif sync_params.key?('sessions')
131
+ else
132
+ # key 'sessions' is present
179
133
  # ascli JSON format (cmdline)
180
134
  raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
181
135
  sync_params.keys.push('instance').uniq.sort.eql?(CMDLINE_PARAMS_KEYS)
182
136
  Aspera.assert_type(sync_params['sessions'], Array)
183
137
  Aspera.assert_type(sync_params['sessions'].first, Hash)
184
- if block
138
+ if block_given?
185
139
  sync_params['sessions'].each do |session|
186
140
  Aspera.assert_type(session['local_dir'], String){'local_dir'}
187
141
  Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
188
142
  transfer_spec = yield((session['direction'] || 'push').to_sym, session['local_dir'], session['remote_dir'])
189
- CMDLINE_PARAMS_SESSION.each do |async_param, behavior|
190
- if behavior.key?(:ts)
191
- tspec_param = behavior[:ts].is_a?(TrueClass) ? async_param : behavior[:ts].to_s
192
- session[async_param] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
143
+ SESSION_SCHEMA['properties'].each do |name, properties|
144
+ if properties.key?('x-tspec')
145
+ tspec_param = properties['x-tspec'].is_a?(TrueClass) ? name : properties['x-tspec'].to_s
146
+ session[name] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
193
147
  end
194
148
  end
195
149
  session['private_key_paths'] = Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa) if transfer_spec.key?('token')
@@ -198,21 +152,18 @@ module Aspera
198
152
  end
199
153
  if sync_params.key?('instance')
200
154
  Aspera.assert_type(sync_params['instance'], Hash)
201
- instance_builder = CommandLineBuilder.new(sync_params['instance'], CMDLINE_PARAMS_INSTANCE)
155
+ instance_builder = CommandLineBuilder.new(sync_params['instance'], INSTANCE_SCHEMA, Convert)
202
156
  instance_builder.process_params
203
157
  instance_builder.add_env_args(env_args)
204
158
  end
205
-
206
159
  sync_params['sessions'].each do |session_params|
207
160
  Aspera.assert_type(session_params, Hash)
208
161
  Aspera.assert(session_params.key?('name')){'session must contain at least name'}
209
- session_builder = CommandLineBuilder.new(session_params, CMDLINE_PARAMS_SESSION)
162
+ session_builder = CommandLineBuilder.new(session_params, SESSION_SCHEMA, Convert)
210
163
  session_builder.process_params
211
164
  session_builder.add_env_args(env_args)
212
165
  end
213
166
  Environment.secure_execute(exec: Ascp::Installation.instance.path(:async), **env_args)
214
- else
215
- raise 'At least one of `local` or `sessions` must be present in async parameters'
216
167
  end
217
168
  return nil
218
169
  end
@@ -248,8 +199,8 @@ module Aspera
248
199
  raise 'Missing either local_db_dir or local.path'
249
200
  end
250
201
  elsif sync_params.key?('sessions')
251
- session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{|s|s['name'].eql?(session_name)}
252
- raise "Session #{session_name} not found in #{sync_params['sessions'].map{|s|s['name']}.join(',')}" if session.nil?
202
+ session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{ |s| s['name'].eql?(session_name)}
203
+ raise "Session #{session_name} not found in #{sync_params['sessions'].map{ |s| s['name']}.join(',')}" if session.nil?
253
204
  raise 'Missing session name' if session['name'].nil?
254
205
  arguments.push("--name=#{session['name']}")
255
206
  if session.key?('local_db_dir')
@@ -0,0 +1,13 @@
1
+ title: SyncInstanceSpec
2
+ type: object
3
+ properties:
4
+ alt_logdir:
5
+ type: string
6
+ watchd:
7
+ type: string
8
+ apply_local_docroot:
9
+ x-cli-switch: true
10
+ quiet:
11
+ x-cli-switch: true
12
+ ws_connect:
13
+ x-cli-switch: true
@@ -0,0 +1,79 @@
1
+ title: SyncSessionSpec
2
+ type: object
3
+ properties:
4
+ name:
5
+ type: string
6
+ local_dir:
7
+ type: string
8
+ remote_dir:
9
+ type: string
10
+ local_db_dir:
11
+ type: string
12
+ remote_db_dir:
13
+ type: string
14
+ host:
15
+ type: string
16
+ x-tspec: remote_host
17
+ user:
18
+ type: string
19
+ x-tspec: remote_user
20
+ private_key_paths:
21
+ type: array
22
+ x-cli-option: "--private-key-path"
23
+ direction:
24
+ type: string
25
+ checksum:
26
+ type: string
27
+ tags:
28
+ type: object
29
+ x-cli-option: "--tags64"
30
+ x-cli-convert: json64
31
+ x-tspec: true
32
+ tcp_port:
33
+ type: integer
34
+ x-tspec: ssh_port
35
+ rate_policy:
36
+ type: string
37
+ target_rate:
38
+ type: string
39
+ cooloff:
40
+ type: integer
41
+ pending_max:
42
+ type: integer
43
+ scan_intensity:
44
+ type: string
45
+ cipher:
46
+ type: string
47
+ x-cli-convert: remove_hyphen
48
+ x-tspec: true
49
+ transfer_threads:
50
+ type: integer
51
+ preserve_time:
52
+ x-cli-switch: true
53
+ x-tspec: preserve_times
54
+ preserve_access_time:
55
+ x-cli-switch: true
56
+ preserve_modification_time:
57
+ x-cli-switch: true
58
+ preserve_uid:
59
+ x-cli-switch: true
60
+ x-tspec: preserve_file_owner_uid
61
+ preserve_gid:
62
+ x-cli-switch: true
63
+ x-tspec: preserve_file_owner_gid
64
+ create_dir:
65
+ x-cli-switch: true
66
+ x-tspec: true
67
+ reset:
68
+ x-cli-switch: true
69
+ remote_password:
70
+ x-cli-envvar: ASPERA_SCP_PASS
71
+ x-tspec: true
72
+ cookie:
73
+ x-cli-envvar: ASPERA_SCP_COOKIE
74
+ x-tspec: true
75
+ token:
76
+ x-cli-envvar: ASPERA_SCP_TOKEN
77
+ x-tspec: true
78
+ license:
79
+ x-cli-envvar: ASPERA_SCP_LICENSE
@@ -4,7 +4,7 @@
4
4
 
5
5
  require 'aspera/log'
6
6
  require 'aspera/rest'
7
- require 'aspera/command_line_builder'
7
+ require 'aspera/transfer/convert'
8
8
 
9
9
  module Aspera
10
10
  module Transfer
@@ -24,7 +24,7 @@ module Aspera
24
24
  result_ts['ssh_port'] = @fasp_uri.port
25
25
  result_ts['paths'] = [{'source' => URI.decode_www_form_component(@fasp_uri.path)}]
26
26
  # faspex 4 does not encode trailing base64 padding, fix that to be able to decode properly
27
- fixed_query = @fasp_uri.query.gsub(/(=+)$/){|trail_equals|'%3D' * trail_equals.length}
27
+ fixed_query = @fasp_uri.query.gsub(/(=+)$/){ |trail_equals| '%3D' * trail_equals.length}
28
28
 
29
29
  Rest.query_to_h(fixed_query).each do |name, value|
30
30
  case name
@@ -39,10 +39,10 @@ module Aspera
39
39
  when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
40
40
  when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub('--', '-')
41
41
  when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
42
- when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
43
- when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
44
- when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
45
- when 'lockminrate' then result_ts['lock_min_rate'] = CommandLineBuilder.yes_to_true(value)
42
+ when 'createpath' then result_ts['create_dir'] = Convert.yes_to_true(value)
43
+ when 'fallback' then result_ts['http_fallback'] = Convert.yes_to_true(value)
44
+ when 'lockpolicy' then result_ts['lock_rate_policy'] = Convert.yes_to_true(value)
45
+ when 'lockminrate' then result_ts['lock_min_rate'] = Convert.yes_to_true(value)
46
46
  when 'auth' then Log.log.debug{"ignoring #{name}=#{value}"} # Not used (yes/no)
47
47
  when 'v' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch -- Not used (shall be 2)
48
48
  when 'protect' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch -- TODO: what is this ?
@@ -1,11 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'uri'
4
+ require 'aspera/assert'
4
5
  require 'aspera/rest'
6
+ require 'aspera/temp_file_manager'
5
7
 
6
8
  module Aspera
7
9
  # read some content from some URI, support file: , http: and https: schemes
8
10
  module UriReader
11
+ FILE_SCHEME_PREFIX = 'file:///'
12
+ private_constant :FILE_SCHEME_PREFIX
9
13
  class << self
10
14
  # read some content from some URI, support file: , http: and https: schemes
11
15
  def read(uri_to_read)
@@ -18,8 +22,21 @@ module Aspera
18
22
  raise 'URL shall have a path, check syntax' if local_file_path.nil?
19
23
  local_file_path = File.expand_path(local_file_path.gsub(%r{^/}, '')) if %r{^/(~|.|..)/}.match?(local_file_path)
20
24
  return File.read(local_file_path)
25
+ else Aspera.error_unexpected_value(uri.scheme){"scheme for [#{uri_to_read}]"}
26
+ end
27
+ end
28
+
29
+ # @return path to file with content at URL
30
+ def read_as_file(url)
31
+ if url.start_with?('file:')
32
+ # require specific file scheme: the path part is "relative", or absolute if there are 4 slash
33
+ raise "use format: #{FILE_SCHEME_PREFIX}<path>" unless url.start_with?(FILE_SCHEME_PREFIX)
34
+ return File.expand_path(url[FILE_SCHEME_PREFIX.length..-1])
21
35
  else
22
- raise "unknown scheme: [#{uri.scheme}] for [#{uri_to_read}]"
36
+ # autodelete on exit
37
+ sdk_archive_path = TempFileManager.instance.new_file_path_global(suffix: File.basename(url))
38
+ Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to_file: sdk_archive_path)
39
+ return sdk_archive_path
23
40
  end
24
41
  end
25
42
  end
@@ -182,7 +182,7 @@ module Aspera
182
182
  # last argument (self) is provided to constructor of servlet
183
183
  mount(@expected_path, WebAuthServlet, self)
184
184
  # server runs in thread
185
- Thread.new { start }
185
+ Thread.new{start}
186
186
  end
187
187
 
188
188
  # Called by web server thread on received request
@@ -10,19 +10,23 @@ require 'openssl'
10
10
  module Aspera
11
11
  # Simple WEBrick server with HTTPS support
12
12
  class WebServerSimple < WEBrick::HTTPServer
13
- CERT_PARAMETERS = %i[key cert chain pkcs12].freeze
13
+ PARAMS = %i[cert key chain].freeze
14
+ DEFAULT_URL = 'http://localhost:8080'
14
15
  GENERIC_ISSUER = '/C=FR/O=Test/OU=Test/CN=Test'
15
16
  ONE_YEAR_SECONDS = 365 * 24 * 60 * 60
17
+ PKCS12_EXT = %w[p12 pfx].map{ |i| ".#{i}"}.freeze
18
+ CLOCK_SKEW_OFFSET_SEC = 5
16
19
 
17
- private_constant :CERT_PARAMETERS, :GENERIC_ISSUER, :ONE_YEAR_SECONDS
20
+ private_constant :GENERIC_ISSUER, :ONE_YEAR_SECONDS, :PKCS12_EXT, :CLOCK_SKEW_OFFSET_SEC
18
21
 
19
22
  class << self
20
- # Fill and self sign provided certificate
21
- def fill_self_signed_cert(cert, key, digest = 'SHA256')
23
+ # Generate or fill and self sign certificate
24
+ def self_signed_cert(private_key, digest: 'SHA256')
25
+ cert = OpenSSL::X509::Certificate.new
22
26
  cert.subject = cert.issuer = OpenSSL::X509::Name.parse(GENERIC_ISSUER)
23
- cert.not_before = cert.not_after = Time.now
24
- cert.not_after += ONE_YEAR_SECONDS
25
- cert.public_key = key.public_key
27
+ cert.not_before = Time.now - CLOCK_SKEW_OFFSET_SEC
28
+ cert.not_after = cert.not_before + ONE_YEAR_SECONDS
29
+ cert.public_key = private_key.public_key
26
30
  cert.serial = 0x0
27
31
  cert.version = 2
28
32
  ef = OpenSSL::X509::ExtensionFactory.new
@@ -34,68 +38,73 @@ module Aspera
34
38
  # ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
35
39
  ]
36
40
  cert.add_extension(ef.create_extension('authorityKeyIdentifier', 'keyid:always,issuer:always'))
37
- cert.sign(key, OpenSSL::Digest.new(digest))
41
+ cert.sign(private_key, OpenSSL::Digest.new(digest))
42
+ cert
43
+ end
44
+
45
+ # @return a list of Certificates from chain file
46
+ def read_chain_file(chain)
47
+ File.read(chain).scan(/-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----/m).map{ |i| OpenSSL::X509::Certificate.new(i)}
38
48
  end
39
49
  end
40
50
 
41
- # @param uri [URI]
42
- def initialize(uri, certificate: nil)
43
- @url = uri
51
+ # @param url [URI] Local address where server will listen (use scheme, host and port only)
52
+ # @param cert [String] Path to certificate file, either with extension .p12 or .pfx, else assumed PEM
53
+ # @param key [String] Path to key file (PEM) or passphrase (pkcs12)
54
+ # @param chain [String] Path to certificate chain file (PEM only)
55
+ def initialize(uri, cert: nil, key: nil, chain: nil)
56
+ Aspera.assert_type(uri, URI)
57
+ @uri = uri
44
58
  # see https://www.rubydoc.info/stdlib/webrick/WEBrick/Config
45
59
  webrick_options = {
46
- BindAddress: uri.host,
47
- Port: uri.port,
60
+ BindAddress: @uri.host,
61
+ Port: @uri.port,
48
62
  Logger: Log.log,
49
63
  AccessLog: [[self, WEBrick::AccessLog::COMMON_LOG_FORMAT]] # replace default access log to call local method "<<" below
50
64
  }
51
- case uri.scheme
65
+ case @uri.scheme
52
66
  when 'http'
53
67
  Log.log.debug('HTTP mode')
54
68
  when 'https'
55
69
  webrick_options[:SSLEnable] = true
56
- if certificate.nil?
70
+ if cert.nil? && key.nil?
57
71
  webrick_options[:SSLCertName] = [['CN', WEBrick::Utils.getservername]]
72
+ elsif cert && PKCS12_EXT.include?(File.extname(cert).downcase)
73
+ # PKCS12
74
+ Log.log.debug('Using PKCS12 certificate')
75
+ raise 'PKCS12 requires a key (password)' if key.nil?
76
+ pkcs12 = OpenSSL::PKCS12.new(File.read(cert), key)
77
+ webrick_options[:SSLCertificate] = pkcs12.certificate
78
+ webrick_options[:SSLPrivateKey] = pkcs12.key
79
+ webrick_options[:SSLExtraChainCert] = pkcs12.ca_certs
58
80
  else
59
- Aspera.assert_type(certificate, Hash)
60
- certificate = certificate.symbolize_keys
61
- raise "unexpected key in certificate config: only: #{CERT_PARAMETERS.join(', ')}" if certificate.keys.any?{|key|!CERT_PARAMETERS.include?(key)}
62
- if certificate.key?(:pkcs12)
63
- Log.log.debug('Using PKCS12 certificate')
64
- raise 'pkcs12 requires a key (password)' unless certificate.key?(:key)
65
- pkcs12 = OpenSSL::PKCS12.new(File.read(certificate[:pkcs12]), certificate[:key])
66
- webrick_options[:SSLCertificate] = pkcs12.certificate
67
- webrick_options[:SSLPrivateKey] = pkcs12.key
68
- webrick_options[:SSLExtraChainCert] = pkcs12.ca_certs
81
+ Log.log.debug('Using PEM certificate')
82
+ webrick_options[:SSLPrivateKey] = if key.nil?
83
+ OpenSSL::PKey::RSA.new(4096)
84
+ else
85
+ OpenSSL::PKey::RSA.new(File.read(key))
86
+ end
87
+ webrick_options[:SSLCertificate] = if cert.nil?
88
+ self.class.self_signed_cert(webrick_options[:SSLPrivateKey])
69
89
  else
70
- Log.log.debug('Using PEM certificate')
71
- webrick_options[:SSLPrivateKey] = if certificate.key?(:key)
72
- OpenSSL::PKey::RSA.new(File.read(certificate[:key]))
73
- else
74
- OpenSSL::PKey::RSA.new(4096)
75
- end
76
- if certificate.key?(:cert)
77
- webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read(certificate[:cert]))
78
- else
79
- webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new
80
- self.class.fill_self_signed_cert(webrick_options[:SSLCertificate], webrick_options[:SSLPrivateKey])
81
- end
82
- if certificate.key?(:chain)
83
- webrick_options[:SSLExtraChainCert] = [OpenSSL::X509::Certificate.new(File.read(certificate[:chain]))]
84
- end
90
+ OpenSSL::X509::Certificate.new(File.read(cert))
85
91
  end
92
+ webrick_options[:SSLExtraChainCert] = read_chain_file(chain) unless chain.nil?
93
+ raise 'key and cert do not match' unless webrick_options[:SSLCertificate].public_key.to_der == webrick_options[:SSLPrivateKey].public_key.to_der
86
94
  end
87
95
  end
88
96
  # call constructor of parent class, but capture STDERR
89
97
  # self signed certificate generates characters on STDERR
90
98
  # see create_self_signed_cert in webrick/ssl.rb
91
- Log.capture_stderr { super(webrick_options) }
99
+ Log.capture_stderr{super(webrick_options)}
92
100
  end
93
101
 
94
102
  # blocking
95
103
  def start
96
- Log.log.info{"Listening on #{@url}"}
97
- # kill -HUP for graceful shutdown
98
- Kernel.trap('HUP') { shutdown }
104
+ Log.log.info{"Listening on #{@uri}"}
105
+ # kill (-TERM) for graceful shutdown
106
+ handler = proc{shutdown}
107
+ %i{INT TERM}.each{ |sig| trap(sig, &handler)}
99
108
  super
100
109
  end
101
110
 
data.tar.gz.sig CHANGED
Binary file