aspera-cli 4.26.2 → 4.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +310 -21
  4. data/CONTRIBUTING.md +99 -14
  5. data/TODO.md +50 -0
  6. data/bin/ascli +3 -3
  7. data/docs/README.md +10977 -0
  8. data/docs/test-mcp-with-ai.md +202 -0
  9. data/lib/aspera/agent/base.rb +29 -7
  10. data/lib/aspera/agent/connect.rb +58 -21
  11. data/lib/aspera/agent/desktop.rb +53 -18
  12. data/lib/aspera/agent/direct.rb +71 -40
  13. data/lib/aspera/agent/factory.rb +3 -3
  14. data/lib/aspera/agent/httpgw.rb +55 -14
  15. data/lib/aspera/agent/node.rb +34 -2
  16. data/lib/aspera/agent/transferd.rb +48 -12
  17. data/lib/aspera/api/alee.rb +1 -1
  18. data/lib/aspera/api/aoc.rb +42 -44
  19. data/lib/aspera/api/cos_node.rb +10 -9
  20. data/lib/aspera/api/faspex.rb +16 -15
  21. data/lib/aspera/api/httpgw.rb +23 -23
  22. data/lib/aspera/api/node.rb +51 -70
  23. data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
  24. data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
  25. data/lib/aspera/ascmd.rb +32 -30
  26. data/lib/aspera/ascp/installation.rb +75 -69
  27. data/lib/aspera/ascp/management.rb +7 -8
  28. data/lib/aspera/assert.rb +12 -13
  29. data/lib/aspera/cli/ascp_actions.rb +155 -0
  30. data/lib/aspera/cli/async_transfer_store.rb +125 -0
  31. data/lib/aspera/cli/bootstrapper.rb +197 -0
  32. data/lib/aspera/cli/command_registry.rb +145 -0
  33. data/lib/aspera/cli/command_spec.rb +141 -0
  34. data/lib/aspera/cli/context.rb +19 -6
  35. data/lib/aspera/cli/error.rb +16 -0
  36. data/lib/aspera/cli/extended_value.rb +27 -26
  37. data/lib/aspera/cli/formatter.rb +95 -59
  38. data/lib/aspera/cli/gem_checker.rb +65 -0
  39. data/lib/aspera/cli/hints.rb +3 -3
  40. data/lib/aspera/cli/http.rb +74 -30
  41. data/lib/aspera/cli/info.rb +2 -0
  42. data/lib/aspera/cli/mailer.rb +97 -0
  43. data/lib/aspera/cli/mcp_tool.rb +225 -0
  44. data/lib/aspera/cli/option_declarator.rb +77 -0
  45. data/lib/aspera/cli/options.schema.yaml +634 -7
  46. data/lib/aspera/cli/parser.rb +1217 -0
  47. data/lib/aspera/cli/plugins/alee.rb +20 -22
  48. data/lib/aspera/cli/plugins/aoc.rb +1178 -881
  49. data/lib/aspera/cli/plugins/ats.rb +205 -162
  50. data/lib/aspera/cli/plugins/base.rb +703 -175
  51. data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
  52. data/lib/aspera/cli/plugins/config.rb +434 -757
  53. data/lib/aspera/cli/plugins/console.rb +107 -65
  54. data/lib/aspera/cli/plugins/cos.rb +46 -33
  55. data/lib/aspera/cli/plugins/factory.rb +6 -6
  56. data/lib/aspera/cli/plugins/faspex5.rb +627 -388
  57. data/lib/aspera/cli/plugins/faspio.rb +54 -51
  58. data/lib/aspera/cli/plugins/httpgw.rb +18 -25
  59. data/lib/aspera/cli/plugins/mcp.rb +279 -0
  60. data/lib/aspera/cli/plugins/node.rb +988 -840
  61. data/lib/aspera/cli/plugins/oauth.rb +7 -10
  62. data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
  63. data/lib/aspera/cli/plugins/preview.rb +195 -157
  64. data/lib/aspera/cli/plugins/server.rb +141 -91
  65. data/lib/aspera/cli/plugins/shares.rb +343 -110
  66. data/lib/aspera/cli/preset_actions.rb +159 -0
  67. data/lib/aspera/cli/preset_manager.rb +82 -39
  68. data/lib/aspera/cli/result.rb +78 -26
  69. data/lib/aspera/cli/runner.rb +323 -153
  70. data/lib/aspera/cli/secret_finder.rb +40 -0
  71. data/lib/aspera/cli/special_values.rb +1 -0
  72. data/lib/aspera/cli/sync_actions.rb +89 -64
  73. data/lib/aspera/cli/terminal_formatter.rb +3 -3
  74. data/lib/aspera/cli/transfer_actions.rb +90 -0
  75. data/lib/aspera/cli/transfer_agent.rb +129 -64
  76. data/lib/aspera/cli/transfer_progress.rb +9 -9
  77. data/lib/aspera/cli/vault_manager.rb +79 -0
  78. data/lib/aspera/cli/version.rb +1 -1
  79. data/lib/aspera/cli/wizard.rb +28 -30
  80. data/lib/aspera/colors.rb +3 -3
  81. data/lib/aspera/command_line_builder.rb +34 -34
  82. data/lib/aspera/command_line_converter.rb +1 -1
  83. data/lib/aspera/coverage.rb +1 -2
  84. data/lib/aspera/data_repository.rb +2 -1
  85. data/lib/aspera/dot_container.rb +12 -12
  86. data/lib/aspera/environment.rb +30 -23
  87. data/lib/aspera/exec_spec.rb +13 -0
  88. data/lib/aspera/faspex_gw.rb +5 -5
  89. data/lib/aspera/faspex_postproc.rb +16 -10
  90. data/lib/aspera/graphql.rb +37 -0
  91. data/lib/aspera/hash_ext.rb +8 -2
  92. data/lib/aspera/json_rpc/client.rb +62 -0
  93. data/lib/aspera/json_rpc/version.rb +7 -0
  94. data/lib/aspera/keychain/base.rb +10 -3
  95. data/lib/aspera/keychain/encrypted_hash.rb +92 -17
  96. data/lib/aspera/keychain/factory.rb +20 -8
  97. data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
  98. data/lib/aspera/keychain/macos_security.rb +23 -25
  99. data/lib/aspera/keychain/one_password_api.rb +86 -0
  100. data/lib/aspera/keychain/one_password_base.rb +34 -0
  101. data/lib/aspera/keychain/one_password_cli.rb +98 -0
  102. data/lib/aspera/link_header.rb +82 -0
  103. data/lib/aspera/log.rb +29 -15
  104. data/lib/aspera/markdown.rb +90 -7
  105. data/lib/aspera/nagios.rb +8 -8
  106. data/lib/aspera/node_simulator.rb +35 -27
  107. data/lib/aspera/oauth/base.rb +9 -10
  108. data/lib/aspera/oauth/boot.rb +5 -5
  109. data/lib/aspera/oauth/factory.rb +12 -10
  110. data/lib/aspera/oauth/jwt.rb +9 -11
  111. data/lib/aspera/oauth/web.rb +6 -6
  112. data/lib/aspera/persistency_action_once.rb +12 -12
  113. data/lib/aspera/persistency_folder.rb +18 -11
  114. data/lib/aspera/preview/file_types.rb +5 -5
  115. data/lib/aspera/preview/generator.rb +52 -49
  116. data/lib/aspera/preview/options.rb +3 -2
  117. data/lib/aspera/preview/terminal.rb +5 -5
  118. data/lib/aspera/preview/utils.rb +66 -19
  119. data/lib/aspera/products/connect.rb +2 -2
  120. data/lib/aspera/products/desktop.rb +1 -1
  121. data/lib/aspera/products/other.rb +3 -3
  122. data/lib/aspera/products/transferd.rb +4 -4
  123. data/lib/aspera/proxy_auto_config.rb +15 -14
  124. data/lib/aspera/rest.rb +159 -151
  125. data/lib/aspera/rest_error_analyzer.rb +6 -6
  126. data/lib/aspera/rest_errors_aspera.rb +0 -10
  127. data/lib/aspera/rest_list.rb +11 -10
  128. data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
  129. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
  130. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
  131. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
  132. data/lib/aspera/schema/documentation.rb +65 -28
  133. data/lib/aspera/schema/reader.rb +136 -13
  134. data/lib/aspera/schema/registry.rb +60 -8
  135. data/lib/aspera/secret_hider.rb +12 -7
  136. data/lib/aspera/ssh.rb +64 -31
  137. data/lib/aspera/ssl.rb +5 -5
  138. data/lib/aspera/sync/conf.schema.yaml +2 -2
  139. data/lib/aspera/sync/database.rb +2 -2
  140. data/lib/aspera/sync/operations.rb +22 -25
  141. data/lib/aspera/temp_file_manager.rb +25 -5
  142. data/lib/aspera/timer_limiter.rb +1 -1
  143. data/lib/aspera/transfer/faux_file.rb +24 -11
  144. data/lib/aspera/transfer/parameters.rb +36 -34
  145. data/lib/aspera/transfer/result.rb +74 -0
  146. data/lib/aspera/transfer/resumer.rb +10 -10
  147. data/lib/aspera/transfer/spec.rb +18 -1
  148. data/lib/aspera/transfer/spec.schema.yaml +12 -3
  149. data/lib/aspera/uri_reader.rb +52 -15
  150. data/lib/aspera/web_auth.rb +8 -7
  151. data/lib/aspera/web_server_simple.rb +15 -12
  152. data/lib/aspera/yaml.rb +5 -4
  153. data.tar.gz.sig +0 -0
  154. metadata +34 -6
  155. metadata.gz.sig +0 -0
  156. data/lib/aspera/cli/manager.rb +0 -766
  157. data/lib/aspera/cli/plugins/faspex.rb +0 -539
  158. data/lib/aspera/json_rpc.rb +0 -52
  159. data/lib/aspera/transfer/uri.rb +0 -56
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'aspera/log'
4
4
  require 'aspera/assert'
5
+ require 'aspera/exec_spec'
5
6
  require 'aspera/command_line_builder'
6
7
  require 'aspera/temp_file_manager'
7
8
  require 'aspera/transfer/error'
@@ -35,19 +36,22 @@ module Aspera
35
36
  # because of garbage collection takes any file there
36
37
  # this could be refined, as, for example, on macos, temp folder is already user specific
37
38
  def file_list_folder
38
- @file_list_folder ||= TempFileManager.instance.new_file_path_global('asession_filelists')
39
+ @file_list_folder ||= TempFileManager.instance.new_dir_path_global('asession_filelists')
39
40
  end
40
41
 
41
- # File list is provided directly with `ascp` arguments
42
- # @columns ascp_args [Array,NilClass] `ascp` arguments
42
+ # Check whether a file list option is already present in the `ascp` argument list
43
+ # @param ascp_args [Array, nil] `ascp` arguments
44
+ # @return [Boolean] true if a file list option is present
43
45
  def ascp_args_file_list?(ascp_args)
44
- ascp_args&.any?{ |i| FILE_LIST_OPTIONS.include?(i)}
46
+ ascp_args&.any? { |i| FILE_LIST_OPTIONS.include?(i) }
45
47
  end
46
48
  end
47
49
 
48
50
  # @param job_spec [Hash] Transfer specification
49
51
  # @param ascp_args [Array] Other `ascp` args
52
+ # @param wss [Boolean] `true`: if both SSH and wss in ts: prefer wss
50
53
  # @param quiet [Boolean] Remove `ascp` progress bar if `true`
54
+ # @param file_list [Boolean] `true`: provide source list via a temp file, `false`: place directly on command line
51
55
  # @param client_ssh_key [:rsa,:dsa] Type of Aspera Client SSH key to use.
52
56
  # @param trusted_certs [Array<String>] List of path to trusted certificates stores when using WSS
53
57
  # @param check_ignore_cb [Proc] Callback to check if WSS connection shall ignore certificate validity
@@ -56,6 +60,7 @@ module Aspera
56
60
  ascp_args: nil,
57
61
  wss: true,
58
62
  quiet: true,
63
+ file_list: true,
59
64
  trusted_certs: nil,
60
65
  client_ssh_key: nil,
61
66
  check_ignore_cb: nil
@@ -63,18 +68,19 @@ module Aspera
63
68
  @job_spec = job_spec
64
69
  Aspera.assert_type(@job_spec, Hash)
65
70
  @ascp_args = ascp_args.nil? ? [] : ascp_args
66
- Aspera.assert_array_all(@ascp_args, String){'ascp_args'}
71
+ Aspera.assert_array_all(@ascp_args, String) { 'ascp_args' }
67
72
  @wss = wss
68
73
  @quiet = quiet
74
+ @file_list = file_list
69
75
  @trusted_certs = trusted_certs.nil? ? [] : trusted_certs
70
- Aspera.assert_type(@trusted_certs, Array){'trusted_certs'}
76
+ Aspera.assert_type(@trusted_certs, Array) { 'trusted_certs' }
71
77
  @client_ssh_key = client_ssh_key.nil? ? :rsa : client_ssh_key.to_sym
72
78
  Aspera.assert_values(@client_ssh_key, Ascp::Installation::CLIENT_SSH_KEY_OPTIONS)
73
79
  @check_ignore_cb = check_ignore_cb
74
80
  @builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, CommandLineConverter)
75
81
  end
76
82
 
77
- # either place source files on command line, or add file list file
83
+ # Place source files on command line or in a temp file list, controlled by @file_list and file_list_folder
78
84
  def process_file_list
79
85
  # is the file list provided through ascp parameters?
80
86
  ascp_file_list_provided = self.class.ascp_args_file_list?(@ascp_args)
@@ -86,33 +92,33 @@ module Aspera
86
92
  # transfer spec contains paths ?
87
93
  if !ts_paths_array.nil?
88
94
  Aspera.assert(!ascp_file_list_provided, 'file list provided both in transfer spec and ascp file list. Remove one of them.')
89
- Aspera.assert(ts_paths_array.all?{ |i| i.key?('source')}, "All elements of paths must have a 'source' key")
90
- is_pair_list = ts_paths_array.any?{ |i| i.key?('destination')}
91
- raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{ |i| i.key?('destination')}
92
- if self.class.file_list_folder.nil?
95
+ Aspera.assert(ts_paths_array.all? { |i| i.key?('source') }, "All elements of paths must have a 'source' key")
96
+ is_pair_list = ts_paths_array.any? { |i| i.key?('destination') }
97
+ Aspera.assert(!(is_pair_list && !ts_paths_array.all? { |i| i.key?('destination') })) { "All elements of paths must be consistent with 'destination' key" }
98
+ if !@file_list || self.class.file_list_folder.nil?
93
99
  Aspera.assert(!is_pair_list, 'file pair list is not supported when file list folder is not set')
94
100
  # not safe for special characters ? (maybe not, depends on OS)
95
101
  Log.log.debug('placing source file list on command line (no file list file)')
96
- @builder.add_command_line_options(ts_paths_array.map{ |i| i['source']})
102
+ @builder.add_command_line_options(ts_paths_array.map { |i| i['source'] })
97
103
  else
98
104
  # safer option: generate a file list file if there is storage defined for it
99
105
  if is_pair_list
100
106
  file_list_option = '--file-pair-list'
101
- lines = ts_paths_array.each_with_object([]){ |e, m| m.push(e['source'], e['destination'])}
107
+ lines = ts_paths_array.each_with_object([]) { |e, m| m.push(e['source'], e['destination']) }
102
108
  else
103
109
  file_list_option = '--file-list'
104
- lines = ts_paths_array.map{ |i| i['source']}
110
+ lines = ts_paths_array.map { |i| i['source'] }
105
111
  end
106
112
  file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
107
113
  Log.dump(:file_list, lines)
108
114
  File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
109
- Log.log.debug{"#{file_list_option}=\n#{File.read(file_list_file)}".red}
115
+ Log.log.debug { "#{file_list_option}=\n#{File.read(file_list_file)}".red }
110
116
  end
111
117
  end
112
118
  @builder.add_command_line_options("#{file_list_option}=#{file_list_file}") unless file_list_option.nil?
113
119
  end
114
120
 
115
- # @return the list of certificates (option `-i`) to use when token/ssh or wss are used
121
+ # @return [Array<String>] the list of certificates (option `-i`) to use when token/ssh or wss are used
116
122
  def remote_certificates
117
123
  certificates_to_use = []
118
124
  # use web socket secure for session ?
@@ -150,11 +156,7 @@ module Aspera
150
156
 
151
157
  # Translate transfer spec to env vars and command line arguments for `ascp`
152
158
  def ascp_args
153
- env_args = {
154
- args: [],
155
- env: {},
156
- name: :ascp
157
- }
159
+ exec_spec = Aspera::ExecSpec.new(exec: :ascp)
158
160
 
159
161
  # Special cases
160
162
  @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
@@ -165,16 +167,16 @@ module Aspera
165
167
  # Add ssh or wss certificates
166
168
  # (reverse, to keep order, as we unshift)
167
169
  remote_certificates&.reverse_each do |cert|
168
- env_args[:args].unshift('-i', cert)
170
+ exec_spec.args.unshift('-i', cert)
169
171
  end
170
172
 
171
173
  case (delete_source = @builder.read_param('delete_source'))
172
174
  when true
173
- DELETE_EQUIV.each{ |i| @job_spec[i] = true}
175
+ DELETE_EQUIV.each { |i| @job_spec[i] = true }
174
176
  when false
175
- DELETE_EQUIV.each{ |i| @job_spec.delete(i)}
177
+ DELETE_EQUIV.each { |i| @job_spec.delete(i) }
176
178
  when nil
177
- else Aspera.error_unexpected_value(delete_source){'delete_source'}
179
+ else Aspera.error_unexpected_value(delete_source) { 'delete_source' }
178
180
  end
179
181
 
180
182
  # process parameters as specified in table
@@ -183,9 +185,9 @@ module Aspera
183
185
  base64_destination = false
184
186
  # symbol must be index of Ascp::Installation.paths
185
187
  if @builder.read_param('use_ascp4')
186
- env_args[:name] = :ascp4
188
+ exec_spec.exec = :ascp4
187
189
  else
188
- env_args[:name] = :ascp
190
+ exec_spec.exec = :ascp
189
191
  base64_destination = true
190
192
  end
191
193
  # destination will be base64 encoded, put this before source path arguments
@@ -200,17 +202,17 @@ module Aspera
200
202
  destination_folder = Base64.strict_encode64(destination_folder) if base64_destination
201
203
  # destination MUST be last command line argument to ascp
202
204
  @builder.add_command_line_options(destination_folder)
203
- @builder.add_env_args(env_args)
204
- env_args[:args].unshift('-q') if @quiet
205
+ @builder.add_to_exec_spec(exec_spec)
206
+ exec_spec.args.unshift('-q') if @quiet
205
207
  # add fallback cert and key as arguments if needed
206
208
  if HTTP_FALLBACK_ACTIVATION_VALUES.include?(@job_spec['http_fallback'])
207
- env_args[:args].unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
208
- env_args[:args].unshift('-I', Ascp::Installation.instance.path(:fallback_certificate))
209
+ exec_spec.args.unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
210
+ exec_spec.args.unshift('-I', Ascp::Installation.instance.path(:fallback_certificate))
209
211
  end
210
212
  # disable redis in client, only for ascp, this makes ascp4 fail
211
- env_args[:env]['ASPERA_TEST_REDIS_DISABLE'] = 'true' if env_args[:name].eql?(:ascp)
212
- Log.log.debug{"ascp args: #{env_args}"}
213
- return env_args
213
+ exec_spec.env['ASPERA_TEST_REDIS_DISABLE'] = 'true' if exec_spec.exec.eql?(:ascp)
214
+ Log.log.debug { "ascp args: #{exec_spec}" }
215
+ return exec_spec
214
216
  end
215
217
  DELETE_EQUIV = %w[remove_after_transfer remove_empty_directories remove_empty_source_directory]
216
218
  # `ascp` options to provide a file list
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aspera
4
+ module Transfer
5
+ # Typed result of a transfer operation.
6
+ # Replaces the ad-hoc Array polymorphism previously returned by agents.
7
+ #
8
+ # Agents return one of the three concrete sub-classes:
9
+ # Transfer::Result::Success — all sessions completed successfully
10
+ # Transfer::Result::Error — at least one session failed
11
+ # Transfer::Result::Async — transfer submitted to a daemon; not yet complete
12
+ class Result
13
+ # All sessions completed successfully.
14
+ class Success < Result
15
+ def to_s
16
+ 'success'
17
+ end
18
+ end
19
+
20
+ # At least one session failed.
21
+ class Error < Result
22
+ # @return [StandardError]
23
+ attr_reader :exception
24
+
25
+ def initialize(exception)
26
+ super()
27
+ @exception = exception
28
+ end
29
+
30
+ def to_s
31
+ "error: #{@exception.message}"
32
+ end
33
+ end
34
+
35
+ # Transfer submitted to an external daemon; status not yet known.
36
+ class Async < Result
37
+ # @return [String] UUID assigned to this background job
38
+ attr_reader :job_id
39
+ # @return [String] initial status string (e.g. 'running')
40
+ attr_reader :status
41
+
42
+ def initialize(job_id:, status: 'running')
43
+ super()
44
+ @job_id = job_id
45
+ @status = status
46
+ end
47
+
48
+ # Convenience: export as a plain Hash for serialization / display
49
+ def to_h
50
+ {'job_id' => @job_id, 'status' => @status}
51
+ end
52
+
53
+ def to_s
54
+ "async job_id=#{@job_id} status=#{@status}"
55
+ end
56
+ end
57
+
58
+ # Factory helpers
59
+ class << self
60
+ def success
61
+ Success.new
62
+ end
63
+
64
+ def error(exception)
65
+ Error.new(exception)
66
+ end
67
+
68
+ def async(job_id:, status: 'running')
69
+ Async.new(job_id: job_id, status: status)
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
@@ -12,20 +12,20 @@ module Aspera
12
12
  # @param iter_max [Integer] Maximum number of executions
13
13
  # @param sleep_initial [Integer] Initial wait to re-execute
14
14
  # @param sleep_factor [Integer] Multiplier
15
- # @param sleep_max. [Integer] Max iterations
15
+ # @param sleep_max [Integer] Max iterations
16
16
  def initialize(
17
17
  iter_max: 7,
18
18
  sleep_initial: 2,
19
19
  sleep_factor: 2,
20
20
  sleep_max: 60
21
21
  )
22
- Aspera.assert_type(iter_max, Integer){k}
22
+ Aspera.assert_type(iter_max, Integer) { 'iter_max' }
23
23
  @iter_max = iter_max
24
- Aspera.assert_type(sleep_initial, Integer){k}
24
+ Aspera.assert_type(sleep_initial, Integer) { 'sleep_initial' }
25
25
  @sleep_initial = sleep_initial
26
- Aspera.assert_type(sleep_factor, Integer){k}
26
+ Aspera.assert_type(sleep_factor, Integer) { 'sleep_factor' }
27
27
  @sleep_factor = sleep_factor
28
- Aspera.assert_type(sleep_max, Integer){k}
28
+ Aspera.assert_type(sleep_max, Integer) { 'sleep_max' }
29
29
  @sleep_max = sleep_max
30
30
  end
31
31
 
@@ -38,7 +38,7 @@ module Aspera
38
38
  # maximum of retry
39
39
  remaining_resumes = @iter_max
40
40
  sleep_seconds = @sleep_initial
41
- Log.log.debug{"retries=#{remaining_resumes}"}
41
+ Log.log.debug { "retries=#{remaining_resumes}" }
42
42
  # try to send the file until ascp is successful
43
43
  loop do
44
44
  Log.log.debug('Starting task execution')
@@ -48,17 +48,17 @@ module Aspera
48
48
  # Exit retry loop if success
49
49
  break
50
50
  rescue Error => e
51
- Log.log.warn{"An error occurred during task: #{e.message}"}
52
- Log.log.debug{"Retryable ? #{e.retryable?}"}
51
+ Log.log.warn { "An error occurred during task: #{e.message}" }
52
+ Log.log.debug { "Retryable ? #{e.retryable?}" }
53
53
  # do not retry non-retryable
54
54
  raise unless e.retryable?
55
55
  # exit if we exceed the max number of retry
56
- raise Error, "Maximum number of retry reached: #{@iter_max}" if remaining_resumes <= 0
56
+ Aspera.assert(remaining_resumes > 0, type: Error) { "Maximum number of retry reached: #{@iter_max}" }
57
57
  end
58
58
 
59
59
  # take this retry in account
60
60
  remaining_resumes -= 1
61
- Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
61
+ Log.log.warn { "Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})" }
62
62
 
63
63
  # wait a bit before retrying, maybe network condition will be better
64
64
  sleep(sleep_seconds)
@@ -34,7 +34,13 @@ module Aspera
34
34
  'sparse_csum' => 'sparse_checksum',
35
35
  'full_csum' => 'full_checksum'
36
36
  }
37
- private_constant :POLICY_FIX
37
+ # Multipliers for target_rate suffix (result is in kbps)
38
+ RATE_SUFFIX_KBPS = {
39
+ 'k' => 1,
40
+ 'm' => 1_000,
41
+ 'g' => 1_000_000
42
+ }.freeze
43
+ private_constant :POLICY_FIX, :RATE_SUFFIX_KBPS
38
44
  # translate upload/download to send/receive
39
45
  def transfer_type_to_direction(transfer_type)
40
46
  XFER_TYPE_TO_DIR.fetch(transfer_type)
@@ -45,6 +51,17 @@ module Aspera
45
51
  XFER_DIR_TO_TYPE.fetch(direction)
46
52
  end
47
53
 
54
+ # Parse a human-readable rate string (as accepted by ascp -l) into an integer kbps value.
55
+ # Accepted: plain integer (kbps), or integer + suffix k/K (kbps), m/M (x1000 kbps), g/G (x1000000 kbps).
56
+ # @param value [String] e.g. "100m", "500000", "1g"
57
+ # @return [Integer] value in kbps
58
+ def rate_string_to_kbps(value)
59
+ m = value.to_s.strip.match(/\A(\d+)([kKmMgG])?\z/)
60
+ Aspera.assert(m) { "Invalid rate value: #{value.inspect}. Expected integer with optional suffix k/K, m/M or g/G." }
61
+ multiplier = m[2] ? RATE_SUFFIX_KBPS.fetch(m[2].downcase) : 1
62
+ return m[1].to_i * multiplier
63
+ end
64
+
48
65
  def fix_transferd_resume_policy(transfer_spec)
49
66
  # Fix discrepancy in transfer spec
50
67
  transfer_spec['resume_policy'] = POLICY_FIX[transfer_spec['resume_policy']] if transfer_spec.key?('resume_policy')
@@ -377,6 +377,7 @@ properties:
377
377
  description: Specifies whether to precalculate the job size.
378
378
  type: boolean
379
379
  x-cli-switch: true
380
+ x-cli-false: --precalculate-disable
380
381
  preserve_access_time:
381
382
  description: >-
382
383
  Preserve the source-file access timestamps at the destination.
@@ -711,11 +712,11 @@ properties:
711
712
  Refer to HSTS manual for more details.
712
713
 
713
714
  - `follow`: Follow a symbolic link and transfer the contents of the linked file or directory when the link target is in the user's docroot.
714
-
715
+
715
716
  - `copy`: Copy only the symbolic link. If a file with the same name exists at the destination, the symbolic link does not replace the file.
716
-
717
+
717
718
  - `skip`: Skip-symbolic links. The link or the file to which it points are not transferred.
718
-
719
+
719
720
  - `copy+force`: Copy only the symbolic link. If a file with the same name exists at the destination, the symbolic link replaces the file. If the file of the same name at the destination is a symbolic link to a directory, it is not replaced.
720
721
  type: string
721
722
  enum:
@@ -753,6 +754,14 @@ properties:
753
754
  description: Specifies desired speed for the transfer.
754
755
  type: integer
755
756
  x-cli-option: -l
757
+ target_rate:
758
+ description: >-
759
+ Pseudo-parameter: desired transfer rate as a human-readable string with an optional unit suffix.
760
+ Converted to `target_rate_kbps` before the transfer is started, overriding it if both are present.
761
+ Accepted formats: plain integer (kbps), or integer followed by `k`/`K` (kbps), `m`/`M` (x1000 kbps), `g`/`G` (x1000000 kbps).
762
+ Examples: `100000`, `100000k`, `100m`, `1g`.
763
+ type: string
764
+ x-cli-special: true
756
765
  title:
757
766
  description: Title of the transfer.
758
767
  type: string
@@ -7,31 +7,66 @@ require 'aspera/rest'
7
7
  require 'aspera/temp_file_manager'
8
8
 
9
9
  module Aspera
10
- # Read some content from some URI, support file: , http: and https: schemes
10
+ # Read content from a URI; supported schemes: file:, http:, https:, data:
11
+ #
12
+ # == file: URL convention
13
+ # Two equivalent forms are accepted:
14
+ #
15
+ # file:///relative/path -> relative path "relative/path"
16
+ # file:////absolute/path -> absolute path "/absolute/path"
17
+ #
18
+ # A shorter form is also accepted (no authority component):
19
+ #
20
+ # file:relative/path -> relative path "relative/path"
21
+ # file:/absolute/path -> absolute path "/absolute/path"
22
+ #
23
+ # The short form is consistent with RFC 8089 (file:///<path> = absolute,
24
+ # file:<path> = relative). Both forms are handled identically by this module.
25
+ # The canonical form built by {file_url} uses the +file:///+ prefix.
11
26
  module UriReader
12
27
  SCHEME_FILE = 'file'
13
28
  SCHEME_FILE_PFX1 = "#{SCHEME_FILE}:"
29
+ # Canonical prefix for file: URLs (no host, three slashes).
30
+ # What follows is the literal path: relative or absolute (starting with a second slash).
14
31
  SCHEME_FILE_PFX2 = "#{SCHEME_FILE_PFX1}///"
15
32
  private_constant :SCHEME_FILE, :SCHEME_FILE_PFX1, :SCHEME_FILE_PFX2
16
33
  class << self
17
- # @return [Boolean] true if the URL is a file:// URL
34
+ # @return [Boolean] true if +url+ uses the file: scheme recognised by this module
18
35
  def file?(url)
19
- url.start_with?(SCHEME_FILE_PFX2)
36
+ url.start_with?(SCHEME_FILE_PFX1)
20
37
  end
21
38
 
22
- # @return [String] a file:// URL for the given path
39
+ # Build a file: URL from +path+.
40
+ # A relative path yields +file:///path+; an absolute path yields +file:////path+.
41
+ # @param path [String] relative or absolute file-system path
42
+ # @return [String] corresponding file: URL
23
43
  def file_url(path)
24
44
  return "#{SCHEME_FILE_PFX2}#{path}"
25
45
  end
26
46
 
27
- # @return [String] the path of a file:// URL
47
+ # Extract the file-system path from a file: URL.
48
+ # Accepts both the canonical +file:///+ form and the short +file:+ form.
49
+ # Returns the literal path (relative or absolute) without working-directory expansion.
50
+ # @param url [String] a file: URL (canonical or short form)
51
+ # @return [String] the literal path encoded in the URL
28
52
  def file_path(url)
29
- Aspera.assert(file?(url)){"use format: #{file_url('<path>')}"}
30
- File.expand_path(url.delete_prefix(SCHEME_FILE_PFX2))
53
+ Aspera.assert(file?(url)) { "use format: #{file_url('<path>')}" }
54
+ # Strip canonical prefix "file:///" first (covers relative and absolute canonical forms).
55
+ # If absent, strip only the short "file:" prefix.
56
+ return url.start_with?(SCHEME_FILE_PFX2) ? url.delete_prefix(SCHEME_FILE_PFX2) : url.delete_prefix(SCHEME_FILE_PFX1)
31
57
  end
32
58
 
33
- # Read some content from some URI, support file: , http: and https: schemes
59
+ # Read content from a URI and return it as a String.
60
+ # Supported schemes: +http+, +https+, +data+, +file+, and bare paths (no scheme).
61
+ # For file: URLs the path is extracted via {file_path} to respect the module convention
62
+ # (see module-level documentation). Ruby's URI parser is not used for file: URLs because
63
+ # it interprets the three-slash prefix differently (it always produces an absolute path).
64
+ # Bare paths (no scheme) are passed to File.read directly; leading +/~/+, +/./+, +/../+
65
+ # are expanded via +File.expand_path+ after stripping the synthetic leading slash added by
66
+ # URI.
34
67
  def read(uri_to_read)
68
+ # Handle file: URLs directly to honour the file:///relative vs file:////absolute convention.
69
+ return File.read(file_path(uri_to_read)) if file?(uri_to_read)
35
70
  uri = URI.parse(uri_to_read)
36
71
  case uri.scheme
37
72
  when 'http', 'https'
@@ -43,20 +78,22 @@ module Aspera
43
78
  else
44
79
  URI.decode_www_form_component(encoded_data)
45
80
  end
46
- when SCHEME_FILE, NilClass
81
+ when NilClass
47
82
  local_file_path = uri.path
48
- raise Error, 'URL shall have a path, check syntax' if local_file_path.nil?
83
+ Aspera.assert(!local_file_path.nil?, type: Error) { 'URL shall have a path, check syntax' }
49
84
  local_file_path = File.expand_path(local_file_path.gsub(%r{^/}, '')) if %r{^/(~|.|..)/}.match?(local_file_path)
50
85
  return File.read(local_file_path)
51
- else Aspera.error_unexpected_value(uri.scheme){"scheme for [#{uri_to_read}]"}
86
+ else Aspera.error_unexpected_value(uri.scheme) { "scheme for [#{uri_to_read}]" }
52
87
  end
53
88
  end
54
89
 
55
- # @return Path to file with content at URL
90
+ # Return the local file-system path for the content at +url+, downloading to a temp file if needed.
91
+ # For file: URLs the path is extracted directly (no download).
92
+ # For data: and http(s): URLs the content is written to a temporary file and its path is returned.
93
+ # @return [String] local path to a file containing the URL content
56
94
  def read_as_file(url)
57
95
  if url.start_with?(SCHEME_FILE_PFX1)
58
- # for file scheme, return directly the path
59
- # require specific file scheme: the path part is "relative", or absolute if there are 4 slash
96
+ # file: scheme: extract the literal path encoded in the URL (relative or absolute).
60
97
  return file_path(url)
61
98
  elsif url.start_with?('data:')
62
99
  # download to temp file
@@ -68,7 +105,7 @@ module Aspera
68
105
  # download to temp file
69
106
  # auto-delete on exit
70
107
  temp_file = TempFileManager.instance.new_file_path_global(suffix: File.basename(url))
71
- Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to_file: temp_file)
108
+ Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to: temp_file)
72
109
  return temp_file
73
110
  end
74
111
  end
@@ -2,12 +2,13 @@
2
2
 
3
3
  require 'aspera/web_server_simple'
4
4
  require 'aspera/assert'
5
+ require 'erb'
5
6
 
6
7
  module Aspera
7
8
  # servlet called on callback: it records the callback request
8
9
  class WebAuthServlet < WEBrick::HTTPServlet::AbstractServlet
9
- # @param server [WEBrick::HTTPServer]
10
- # @param web_auth [WebAuth]
10
+ # @param server [WEBrick::HTTPServer] the HTTP server instance
11
+ # @param web_auth [WebAuth] the WebAuth instance to record the callback
11
12
  def initialize(server, web_auth)
12
13
  Log.log.debug('WebAuthServlet initialize')
13
14
  super(server)
@@ -15,8 +16,8 @@ module Aspera
15
16
  end
16
17
 
17
18
  def service(request, response)
18
- Log.log.debug{"received request from browser #{request.request_method} #{request.path}"}
19
- Aspera.assert_values(request.request_method, ['GET'], type: WEBrick::HTTPStatus::MethodNotAllowed){'HTTP verb'}
19
+ Log.log.debug { "received request from browser #{request.request_method} #{request.path}" }
20
+ Aspera.assert_values(request.request_method, ['GET'], type: WEBrick::HTTPStatus::MethodNotAllowed) { 'HTTP verb' }
20
21
  additional_info = @web_auth.signal_request(request)
21
22
  response.status = 200
22
23
  response.content_type = 'text/html'
@@ -91,7 +92,7 @@ module Aspera
91
92
  <body>
92
93
  <h1>Thank You!</h1>
93
94
  <p>You can close this window.</p>
94
- <p>#{additional_info}</p>
95
+ <p>#{ERB::Util.html_escape(additional_info) unless additional_info.nil?}</p>
95
96
 
96
97
  <!-- JavaScript to generate IBM logos -->
97
98
  <script>
@@ -182,7 +183,7 @@ module Aspera
182
183
  # last argument (self) is provided to constructor of servlet
183
184
  mount(@expected_path, WebAuthServlet, self)
184
185
  # server runs in thread
185
- Thread.new{start}
186
+ Thread.new { start }
186
187
  end
187
188
 
188
189
  # Called by web server thread on received request
@@ -201,7 +202,7 @@ module Aspera
201
202
  # @return [Hash] the query
202
203
  def received_request
203
204
  # wait for signal from thread
204
- @mutex.synchronize{@cond.wait(@mutex)}
205
+ @mutex.synchronize { @cond.wait(@mutex) }
205
206
  # tell server thread to stop
206
207
  shutdown
207
208
  return @query
@@ -1,11 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'webrick'
4
- require 'webrick/https'
5
4
  require 'aspera/log'
6
5
  require 'aspera/assert'
7
6
  require 'aspera/hash_ext'
8
- require 'openssl'
9
7
 
10
8
  module Aspera
11
9
  # Simple WEBrick server with HTTPS support
@@ -14,7 +12,7 @@ module Aspera
14
12
  DEFAULT_URL = 'http://localhost:8080'
15
13
  GENERIC_ISSUER = '/C=FR/O=Test/OU=Test/CN=Test'
16
14
  ONE_YEAR_SECONDS = 365 * 24 * 60 * 60
17
- PKCS12_EXT = %w[p12 pfx].map{ |i| ".#{i}"}.freeze
15
+ PKCS12_EXT = %w[p12 pfx].map { |i| ".#{i}" }.freeze
18
16
  CLOCK_SKEW_OFFSET_SEC = 5
19
17
 
20
18
  private_constant :GENERIC_ISSUER, :ONE_YEAR_SECONDS, :PKCS12_EXT, :CLOCK_SKEW_OFFSET_SEC
@@ -42,9 +40,9 @@ module Aspera
42
40
  cert
43
41
  end
44
42
 
45
- # @return a list of Certificates from chain file
43
+ # @return [Array<OpenSSL::X509::Certificate>] list of Certificates from chain file
46
44
  def read_chain_file(chain)
47
- File.read(chain).scan(/-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----/m).map{ |i| OpenSSL::X509::Certificate.new(i)}
45
+ File.read(chain).scan(/-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----/m).map { |i| OpenSSL::X509::Certificate.new(i) }
48
46
  end
49
47
  end
50
48
 
@@ -66,13 +64,18 @@ module Aspera
66
64
  when 'http'
67
65
  Log.log.debug('HTTP mode')
68
66
  when 'https'
67
+ # Required lazily: webrick/ssl.rb reads OpenSSL::OPENSSL_VERSION at load time,
68
+ # a constant CosmoRuby's OpenSSL shim does not define, so requiring it
69
+ # unconditionally at file load would break any command in a Cosmo build
70
+ # even when no HTTPS server is ever started.
71
+ require 'webrick/https'
69
72
  webrick_options[:SSLEnable] = true
70
73
  if cert.nil? && key.nil?
71
74
  webrick_options[:SSLCertName] = [['CN', WEBrick::Utils.getservername]]
72
75
  elsif cert && PKCS12_EXT.include?(File.extname(cert).downcase)
73
76
  # PKCS12
74
77
  Log.log.debug('Using PKCS12 certificate')
75
- raise Error, 'PKCS12 requires a key (password)' if key.nil?
78
+ Aspera.assert(!key.nil?, type: Error) { 'PKCS12 requires a key (password)' }
76
79
  pkcs12 = OpenSSL::PKCS12.new(File.read(cert), key)
77
80
  webrick_options[:SSLCertificate] = pkcs12.certificate
78
81
  webrick_options[:SSLPrivateKey] = pkcs12.key
@@ -90,27 +93,27 @@ module Aspera
90
93
  OpenSSL::X509::Certificate.new(File.read(cert))
91
94
  end
92
95
  webrick_options[:SSLExtraChainCert] = read_chain_file(chain) unless chain.nil?
93
- raise Error, 'key and cert do not match' unless webrick_options[:SSLCertificate].public_key.to_der == webrick_options[:SSLPrivateKey].public_key.to_der
96
+ Aspera.assert(webrick_options[:SSLCertificate].public_key.to_der == webrick_options[:SSLPrivateKey].public_key.to_der, type: Error) { 'key and cert do not match' }
94
97
  end
95
98
  end
96
99
  # call constructor of parent class, but capture STDERR
97
100
  # self signed certificate generates characters on STDERR
98
101
  # see create_self_signed_cert in webrick/ssl.rb
99
- Log.capture_stderr{super(webrick_options)}
102
+ Log.capture_stderr { super(webrick_options) }
100
103
  end
101
104
 
102
105
  # blocking
103
106
  def start
104
- Log.log.info{"Listening on #{@uri}"}
107
+ Log.log.info { "Listening on #{@uri}" }
105
108
  # kill (-TERM) for graceful shutdown
106
- handler = proc{shutdown}
107
- %i{INT TERM}.each{ |sig| trap(sig, &handler)}
109
+ handler = proc { shutdown }
110
+ %i{INT TERM}.each { |sig| trap(sig, &handler) }
108
111
  super
109
112
  end
110
113
 
111
114
  # log web server access ( option AccessLog )
112
115
  def <<(access_log)
113
- Log.log.debug{"webrick log #{access_log.chomp}"}
116
+ Log.log.debug { "webrick log #{access_log.chomp}" }
114
117
  end
115
118
  end
116
119
  end