aspera-cli 4.15.0 → 4.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +29 -3
  4. data/CHANGELOG.md +292 -228
  5. data/CONTRIBUTING.md +69 -18
  6. data/README.md +1102 -952
  7. data/bin/ascli +13 -31
  8. data/bin/asession +3 -1
  9. data/examples/dascli +2 -2
  10. data/lib/aspera/aoc.rb +28 -33
  11. data/lib/aspera/ascmd.rb +3 -6
  12. data/lib/aspera/assert.rb +45 -0
  13. data/lib/aspera/cli/extended_value.rb +5 -5
  14. data/lib/aspera/cli/formatter.rb +26 -13
  15. data/lib/aspera/cli/hints.rb +4 -3
  16. data/lib/aspera/cli/main.rb +16 -3
  17. data/lib/aspera/cli/manager.rb +45 -36
  18. data/lib/aspera/cli/plugin.rb +20 -13
  19. data/lib/aspera/cli/plugins/aoc.rb +103 -73
  20. data/lib/aspera/cli/plugins/ats.rb +4 -3
  21. data/lib/aspera/cli/plugins/config.rb +114 -119
  22. data/lib/aspera/cli/plugins/cos.rb +2 -2
  23. data/lib/aspera/cli/plugins/faspex.rb +23 -19
  24. data/lib/aspera/cli/plugins/faspex5.rb +75 -43
  25. data/lib/aspera/cli/plugins/node.rb +28 -15
  26. data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
  27. data/lib/aspera/cli/plugins/preview.rb +9 -7
  28. data/lib/aspera/cli/plugins/server.rb +6 -3
  29. data/lib/aspera/cli/plugins/shares.rb +30 -26
  30. data/lib/aspera/cli/sync_actions.rb +9 -9
  31. data/lib/aspera/cli/transfer_agent.rb +21 -14
  32. data/lib/aspera/cli/transfer_progress.rb +2 -3
  33. data/lib/aspera/cli/version.rb +1 -1
  34. data/lib/aspera/command_line_builder.rb +13 -11
  35. data/lib/aspera/cos_node.rb +3 -2
  36. data/lib/aspera/coverage.rb +22 -0
  37. data/lib/aspera/data_repository.rb +33 -2
  38. data/lib/aspera/environment.rb +4 -2
  39. data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
  40. data/lib/aspera/fasp/agent_base.rb +17 -7
  41. data/lib/aspera/fasp/agent_direct.rb +88 -84
  42. data/lib/aspera/fasp/agent_httpgw.rb +4 -3
  43. data/lib/aspera/fasp/agent_node.rb +3 -2
  44. data/lib/aspera/fasp/agent_trsdk.rb +79 -37
  45. data/lib/aspera/fasp/installation.rb +51 -12
  46. data/lib/aspera/fasp/management.rb +11 -6
  47. data/lib/aspera/fasp/parameters.rb +53 -47
  48. data/lib/aspera/fasp/resume_policy.rb +7 -5
  49. data/lib/aspera/fasp/sync.rb +273 -0
  50. data/lib/aspera/fasp/transfer_spec.rb +10 -8
  51. data/lib/aspera/fasp/uri.rb +2 -2
  52. data/lib/aspera/faspex_gw.rb +11 -8
  53. data/lib/aspera/faspex_postproc.rb +6 -5
  54. data/lib/aspera/id_generator.rb +3 -1
  55. data/lib/aspera/json_rpc.rb +10 -8
  56. data/lib/aspera/keychain/encrypted_hash.rb +46 -11
  57. data/lib/aspera/keychain/macos_security.rb +15 -13
  58. data/lib/aspera/log.rb +4 -3
  59. data/lib/aspera/nagios.rb +7 -2
  60. data/lib/aspera/node.rb +17 -16
  61. data/lib/aspera/node_simulator.rb +214 -0
  62. data/lib/aspera/oauth.rb +22 -19
  63. data/lib/aspera/persistency_action_once.rb +13 -14
  64. data/lib/aspera/persistency_folder.rb +3 -2
  65. data/lib/aspera/preview/file_types.rb +53 -267
  66. data/lib/aspera/preview/generator.rb +7 -5
  67. data/lib/aspera/preview/terminal.rb +14 -5
  68. data/lib/aspera/preview/utils.rb +8 -7
  69. data/lib/aspera/proxy_auto_config.rb +6 -3
  70. data/lib/aspera/rest.rb +29 -13
  71. data/lib/aspera/rest_error_analyzer.rb +1 -0
  72. data/lib/aspera/rest_errors_aspera.rb +2 -0
  73. data/lib/aspera/secret_hider.rb +5 -2
  74. data/lib/aspera/ssh.rb +10 -8
  75. data/lib/aspera/temp_file_manager.rb +1 -1
  76. data/lib/aspera/web_server_simple.rb +2 -1
  77. data.tar.gz.sig +0 -0
  78. metadata +96 -45
  79. metadata.gz.sig +0 -0
  80. data/lib/aspera/sync.rb +0 -219
data/lib/aspera/sync.rb DELETED
@@ -1,219 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # cspell:words logdir bidi watchd cooloff asyncadmin
4
-
5
- require 'aspera/command_line_builder'
6
- require 'aspera/fasp/installation'
7
- require 'aspera/log'
8
- require 'json'
9
- require 'base64'
10
- require 'open3'
11
- require 'English'
12
-
13
- module Aspera
14
- # builds command line arg for async
15
- module Sync
16
- # sync direction, default is push
17
- DIRECTIONS = %i[push pull bidi].freeze
18
- PARAMS_VX_INSTANCE =
19
- {
20
- 'alt_logdir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
21
- 'watchd' => { cli: { type: :opt_with_arg}, accepted_types: :string},
22
- 'apply_local_docroot' => { cli: { type: :opt_without_arg}},
23
- 'quiet' => { cli: { type: :opt_without_arg}},
24
- 'ws_connect' => { cli: { type: :opt_without_arg}}
25
- }.freeze
26
-
27
- # map sync session parameters to transfer spec: sync -> ts, true if same
28
- PARAMS_VX_SESSION =
29
- {
30
- 'name' => { cli: { type: :opt_with_arg}, accepted_types: :string},
31
- 'local_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
32
- 'remote_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
33
- 'local_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
34
- 'remote_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
35
- 'host' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_host},
36
- 'user' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_user},
37
- 'private_key_paths' => { cli: { type: :opt_with_arg, switch: '--private-key-path'}, accepted_types: :array},
38
- 'direction' => { cli: { type: :opt_with_arg}, accepted_types: :string},
39
- 'checksum' => { cli: { type: :opt_with_arg}, accepted_types: :string},
40
- 'tags' => { cli: { type: :opt_with_arg, switch: '--tags64', convert: 'Aspera::Fasp::Parameters.convert_json64'},
41
- accepted_types: :hash, ts: true},
42
- 'tcp_port' => { cli: { type: :opt_with_arg}, accepted_types: :int, ts: :ssh_port},
43
- 'rate_policy' => { cli: { type: :opt_with_arg}, accepted_types: :string},
44
- 'target_rate' => { cli: { type: :opt_with_arg}, accepted_types: :string},
45
- 'cooloff' => { cli: { type: :opt_with_arg}, accepted_types: :int},
46
- 'pending_max' => { cli: { type: :opt_with_arg}, accepted_types: :int},
47
- 'scan_intensity' => { cli: { type: :opt_with_arg}, accepted_types: :string},
48
- 'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Fasp::Parameters.convert_remove_hyphen'}, accepted_types: :string, ts: true},
49
- 'transfer_threads' => { cli: { type: :opt_with_arg}, accepted_types: :int},
50
- 'preserve_time' => { cli: { type: :opt_without_arg}, ts: :preserve_times},
51
- 'preserve_access_time' => { cli: { type: :opt_without_arg}, ts: nil},
52
- 'preserve_modification_time' => { cli: { type: :opt_without_arg}, ts: nil},
53
- 'preserve_uid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_uid},
54
- 'preserve_gid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_gid},
55
- 'create_dir' => { cli: { type: :opt_without_arg}, ts: true},
56
- 'reset' => { cli: { type: :opt_without_arg}},
57
- # NOTE: only one env var, but multiple sessions... could be a problem
58
- 'remote_password' => { cli: { type: :envvar, variable: 'ASPERA_SCP_PASS'}, ts: true},
59
- 'cookie' => { cli: { type: :envvar, variable: 'ASPERA_SCP_COOKIE'}, ts: true},
60
- 'token' => { cli: { type: :envvar, variable: 'ASPERA_SCP_TOKEN'}, ts: true},
61
- 'license' => { cli: { type: :envvar, variable: 'ASPERA_SCP_LICENSE'}}
62
- }.freeze
63
-
64
- Aspera::CommandLineBuilder.normalize_description(PARAMS_VX_INSTANCE)
65
- Aspera::CommandLineBuilder.normalize_description(PARAMS_VX_SESSION)
66
-
67
- PARAMS_VX_KEYS = %w[instance sessions].freeze
68
-
69
- # new API
70
- TS_TO_PARAMS_V2 = {
71
- 'remote_host' => 'remote.host',
72
- 'remote_user' => 'remote.user',
73
- 'remote_password' => 'remote.pass',
74
- 'sshfp' => 'remote.fingerprint',
75
- 'ssh_port' => 'remote.port',
76
- 'wss_port' => 'remote.ws_port',
77
- 'proxy' => 'remote.proxy',
78
- 'token' => 'remote.token',
79
- 'tags' => 'tags'
80
- }.freeze
81
-
82
- ASYNC_EXECUTABLE = 'async'
83
- ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
84
-
85
- private_constant :PARAMS_VX_INSTANCE, :PARAMS_VX_SESSION, :PARAMS_VX_KEYS, :TS_TO_PARAMS_V2, :ASYNC_EXECUTABLE, :ASYNC_ADMIN_EXECUTABLE
86
-
87
- class << self
88
- # Set remote_dir in sync parameters based on transfer spec
89
- # @param params [Hash] sync parameters, old or new format
90
- # @param remote_dir_key [String] key to update in above hash
91
- # @param transfer_spec [Hash] transfer spec
92
- def update_remote_dir(sync_params, remote_dir_key, transfer_spec)
93
- if transfer_spec.dig(*%w[tags aspera node file_id])
94
- # in AoC, use gen4
95
- sync_params[remote_dir_key] = '/'
96
- elsif transfer_spec['cookie']&.start_with?('aspera.shares2')
97
- # TODO : something more generic, independent of Shares
98
- # in Shares, the actual folder on remote end is not always the same as the name of the share
99
- actual_remote = transfer_spec['paths']&.first&.[]('source')
100
- sync_params[remote_dir_key] = actual_remote if actual_remote
101
- end
102
- nil
103
- end
104
-
105
- # @param sync_params [Hash] sync parameters, old or new format
106
- # @param block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
107
- def start(sync_params, &block)
108
- raise 'Internal Error: sync_params parameter must be Hash' unless sync_params.is_a?(Hash)
109
- env_args = {
110
- args: [],
111
- env: {}
112
- }
113
- if sync_params.key?('local')
114
- # async native JSON format (v2)
115
- raise StandardError, 'remote must be Hash' unless sync_params['remote'].is_a?(Hash)
116
- if block
117
- transfer_spec = yield((sync_params['direction'] || 'push').to_sym, sync_params['local']['path'], sync_params['remote']['path'])
118
- # async native JSON format
119
- raise StandardError, 'sync parameter "local" must be Hash' unless sync_params['local'].is_a?(Hash)
120
- TS_TO_PARAMS_V2.each do |ts_param, sy_path|
121
- next unless transfer_spec.key?(ts_param)
122
- sy_dig = sy_path.split('.')
123
- param = sy_dig.pop
124
- hash = sy_dig.empty? ? sync_params : sync_params[sy_dig.first]
125
- hash = sync_params[sy_dig.first] = {} if hash.nil?
126
- hash[param] = transfer_spec[ts_param]
127
- end
128
- sync_params['remote']['connect_mode'] ||= sync_params['remote'].key?('ws_port') ? 'ws' : 'ssh'
129
- sync_params['remote']['private_key_paths'] ||= Fasp::Installation.instance.aspera_token_ssh_key_paths if transfer_spec.key?('token')
130
- update_remote_dir(sync_params['remote'], 'path', transfer_spec)
131
- end
132
- env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"]
133
- elsif sync_params.key?('sessions')
134
- # ascli JSON format (v1)
135
- if block
136
- sync_params['sessions'].each do |session|
137
- transfer_spec = yield((session['direction'] || 'push').to_sym, session['local_dir'], session['remote_dir'])
138
- PARAMS_VX_SESSION.each do |async_param, behavior|
139
- if behavior.key?(:ts)
140
- tspec_param = behavior[:ts].is_a?(TrueClass) ? async_param : behavior[:ts].to_s
141
- session[async_param] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
142
- end
143
- end
144
- session['private_key_paths'] = Fasp::Installation.instance.aspera_token_ssh_key_paths if transfer_spec.key?('token')
145
- update_remote_dir(session, 'remote_dir', transfer_spec)
146
- end
147
- end
148
- raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
149
- sync_params.keys.push('instance').uniq.sort.eql?(PARAMS_VX_KEYS)
150
- raise StandardError, 'sessions key must be Array' unless sync_params['sessions'].is_a?(Array)
151
- raise StandardError, 'sessions key requires at least one Hash' unless sync_params['sessions'].first.is_a?(Hash)
152
-
153
- if sync_params.key?('instance')
154
- raise StandardError, 'instance key must be Hash' unless sync_params['instance'].is_a?(Hash)
155
- instance_builder = Aspera::CommandLineBuilder.new(sync_params['instance'], PARAMS_VX_INSTANCE)
156
- instance_builder.process_params
157
- instance_builder.add_env_args(env_args)
158
- end
159
-
160
- sync_params['sessions'].each do |session_params|
161
- raise StandardError, 'sessions must contain hashes' unless session_params.is_a?(Hash)
162
- raise StandardError, 'session must contain at least name' unless session_params.key?('name')
163
- session_builder = Aspera::CommandLineBuilder.new(session_params, PARAMS_VX_SESSION)
164
- session_builder.process_params
165
- session_builder.add_env_args(env_args)
166
- end
167
- else
168
- raise 'At least one of `local` or `sessions` must be present in async parameters'
169
- end
170
- Log.log.debug{Log.dump(:sync_params, sync_params)}
171
-
172
- Log.log.debug{"execute: #{env_args[:env].map{|k, v| "#{k}=\"#{v}\""}.join(' ')} \"#{ASYNC_EXECUTABLE}\" \"#{env_args[:args].join('" "')}\""}
173
- res = system(env_args[:env], [ASYNC_EXECUTABLE, ASYNC_EXECUTABLE], *env_args[:args])
174
- Log.log.debug{"result=#{res}"}
175
- case res
176
- when true then return nil
177
- when false then raise "failed: #{$CHILD_STATUS}"
178
- when nil then raise "not started: #{$CHILD_STATUS}"
179
- else raise 'internal error: unspecified case'
180
- end
181
- end
182
-
183
- def admin_status(sync_params, session_name)
184
- command_line = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
185
- if sync_params.key?('local')
186
- raise 'Missing session name' if sync_params['name'].nil?
187
- raise 'Session not found' unless session_name.nil? || session_name.eql?(sync_params['name'])
188
- command_line.push("--name=#{sync_params['name']}")
189
- if sync_params.key?('local_db_dir')
190
- command_line.push("--local-db-dir=#{sync_params['local_db_dir']}")
191
- elsif sync_params.dig('local', 'path')
192
- command_line.push("--local-dir=#{sync_params.dig('local', 'path')}")
193
- else
194
- raise 'Missing either local_db_dir or local.path'
195
- end
196
- elsif sync_params.key?('sessions')
197
- session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{|s|s['name'].eql?(session_name)}
198
- raise "Session #{session_name} not found in #{sync_params['sessions'].map{|s|s['name']}.join(',')}" if session.nil?
199
- raise 'Missing session name' if session['name'].nil?
200
- command_line.push("--name=#{session['name']}")
201
- if session.key?('local_db_dir')
202
- command_line.push("--local-db-dir=#{session['local_db_dir']}")
203
- elsif session.key?('local_dir')
204
- command_line.push("--local-dir=#{session['local_dir']}")
205
- else
206
- raise 'Missing either local_db_dir or local_dir'
207
- end
208
- else
209
- raise 'At least one of `local` or `sessions` must be present in async parameters'
210
- end
211
- Log.log.debug{"execute: #{command_line.join(' ')}"}
212
- stdout, stderr, status = Open3.capture3(*command_line)
213
- Log.log.debug{"status=#{status}, stderr=#{stderr}"}
214
- raise "Sync failed: #{status.exitstatus} : #{stderr}" unless status.success?
215
- return stdout.split("\n").each_with_object({}){|l, m|i = l.split(':', 2); m[i.first.lstrip] = i.last.lstrip} # rubocop:disable Style/Semicolon
216
- end
217
- end
218
- end # end Sync
219
- end # end Aspera