aspera-cli 4.26.2 → 4.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +310 -21
  4. data/CONTRIBUTING.md +99 -14
  5. data/TODO.md +50 -0
  6. data/bin/ascli +3 -3
  7. data/docs/README.md +10977 -0
  8. data/docs/test-mcp-with-ai.md +202 -0
  9. data/lib/aspera/agent/base.rb +29 -7
  10. data/lib/aspera/agent/connect.rb +58 -21
  11. data/lib/aspera/agent/desktop.rb +53 -18
  12. data/lib/aspera/agent/direct.rb +71 -40
  13. data/lib/aspera/agent/factory.rb +3 -3
  14. data/lib/aspera/agent/httpgw.rb +55 -14
  15. data/lib/aspera/agent/node.rb +34 -2
  16. data/lib/aspera/agent/transferd.rb +48 -12
  17. data/lib/aspera/api/alee.rb +1 -1
  18. data/lib/aspera/api/aoc.rb +42 -44
  19. data/lib/aspera/api/cos_node.rb +10 -9
  20. data/lib/aspera/api/faspex.rb +16 -15
  21. data/lib/aspera/api/httpgw.rb +23 -23
  22. data/lib/aspera/api/node.rb +51 -70
  23. data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
  24. data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
  25. data/lib/aspera/ascmd.rb +32 -30
  26. data/lib/aspera/ascp/installation.rb +75 -69
  27. data/lib/aspera/ascp/management.rb +7 -8
  28. data/lib/aspera/assert.rb +12 -13
  29. data/lib/aspera/cli/ascp_actions.rb +155 -0
  30. data/lib/aspera/cli/async_transfer_store.rb +125 -0
  31. data/lib/aspera/cli/bootstrapper.rb +197 -0
  32. data/lib/aspera/cli/command_registry.rb +145 -0
  33. data/lib/aspera/cli/command_spec.rb +141 -0
  34. data/lib/aspera/cli/context.rb +19 -6
  35. data/lib/aspera/cli/error.rb +16 -0
  36. data/lib/aspera/cli/extended_value.rb +27 -26
  37. data/lib/aspera/cli/formatter.rb +95 -59
  38. data/lib/aspera/cli/gem_checker.rb +65 -0
  39. data/lib/aspera/cli/hints.rb +3 -3
  40. data/lib/aspera/cli/http.rb +74 -30
  41. data/lib/aspera/cli/info.rb +2 -0
  42. data/lib/aspera/cli/mailer.rb +97 -0
  43. data/lib/aspera/cli/mcp_tool.rb +225 -0
  44. data/lib/aspera/cli/option_declarator.rb +77 -0
  45. data/lib/aspera/cli/options.schema.yaml +634 -7
  46. data/lib/aspera/cli/parser.rb +1217 -0
  47. data/lib/aspera/cli/plugins/alee.rb +20 -22
  48. data/lib/aspera/cli/plugins/aoc.rb +1178 -881
  49. data/lib/aspera/cli/plugins/ats.rb +205 -162
  50. data/lib/aspera/cli/plugins/base.rb +703 -175
  51. data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
  52. data/lib/aspera/cli/plugins/config.rb +434 -757
  53. data/lib/aspera/cli/plugins/console.rb +107 -65
  54. data/lib/aspera/cli/plugins/cos.rb +46 -33
  55. data/lib/aspera/cli/plugins/factory.rb +6 -6
  56. data/lib/aspera/cli/plugins/faspex5.rb +627 -388
  57. data/lib/aspera/cli/plugins/faspio.rb +54 -51
  58. data/lib/aspera/cli/plugins/httpgw.rb +18 -25
  59. data/lib/aspera/cli/plugins/mcp.rb +279 -0
  60. data/lib/aspera/cli/plugins/node.rb +988 -840
  61. data/lib/aspera/cli/plugins/oauth.rb +7 -10
  62. data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
  63. data/lib/aspera/cli/plugins/preview.rb +195 -157
  64. data/lib/aspera/cli/plugins/server.rb +141 -91
  65. data/lib/aspera/cli/plugins/shares.rb +343 -110
  66. data/lib/aspera/cli/preset_actions.rb +159 -0
  67. data/lib/aspera/cli/preset_manager.rb +82 -39
  68. data/lib/aspera/cli/result.rb +78 -26
  69. data/lib/aspera/cli/runner.rb +323 -153
  70. data/lib/aspera/cli/secret_finder.rb +40 -0
  71. data/lib/aspera/cli/special_values.rb +1 -0
  72. data/lib/aspera/cli/sync_actions.rb +89 -64
  73. data/lib/aspera/cli/terminal_formatter.rb +3 -3
  74. data/lib/aspera/cli/transfer_actions.rb +90 -0
  75. data/lib/aspera/cli/transfer_agent.rb +129 -64
  76. data/lib/aspera/cli/transfer_progress.rb +9 -9
  77. data/lib/aspera/cli/vault_manager.rb +79 -0
  78. data/lib/aspera/cli/version.rb +1 -1
  79. data/lib/aspera/cli/wizard.rb +28 -30
  80. data/lib/aspera/colors.rb +3 -3
  81. data/lib/aspera/command_line_builder.rb +34 -34
  82. data/lib/aspera/command_line_converter.rb +1 -1
  83. data/lib/aspera/coverage.rb +1 -2
  84. data/lib/aspera/data_repository.rb +2 -1
  85. data/lib/aspera/dot_container.rb +12 -12
  86. data/lib/aspera/environment.rb +30 -23
  87. data/lib/aspera/exec_spec.rb +13 -0
  88. data/lib/aspera/faspex_gw.rb +5 -5
  89. data/lib/aspera/faspex_postproc.rb +16 -10
  90. data/lib/aspera/graphql.rb +37 -0
  91. data/lib/aspera/hash_ext.rb +8 -2
  92. data/lib/aspera/json_rpc/client.rb +62 -0
  93. data/lib/aspera/json_rpc/version.rb +7 -0
  94. data/lib/aspera/keychain/base.rb +10 -3
  95. data/lib/aspera/keychain/encrypted_hash.rb +92 -17
  96. data/lib/aspera/keychain/factory.rb +20 -8
  97. data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
  98. data/lib/aspera/keychain/macos_security.rb +23 -25
  99. data/lib/aspera/keychain/one_password_api.rb +86 -0
  100. data/lib/aspera/keychain/one_password_base.rb +34 -0
  101. data/lib/aspera/keychain/one_password_cli.rb +98 -0
  102. data/lib/aspera/link_header.rb +82 -0
  103. data/lib/aspera/log.rb +29 -15
  104. data/lib/aspera/markdown.rb +90 -7
  105. data/lib/aspera/nagios.rb +8 -8
  106. data/lib/aspera/node_simulator.rb +35 -27
  107. data/lib/aspera/oauth/base.rb +9 -10
  108. data/lib/aspera/oauth/boot.rb +5 -5
  109. data/lib/aspera/oauth/factory.rb +12 -10
  110. data/lib/aspera/oauth/jwt.rb +9 -11
  111. data/lib/aspera/oauth/web.rb +6 -6
  112. data/lib/aspera/persistency_action_once.rb +12 -12
  113. data/lib/aspera/persistency_folder.rb +18 -11
  114. data/lib/aspera/preview/file_types.rb +5 -5
  115. data/lib/aspera/preview/generator.rb +52 -49
  116. data/lib/aspera/preview/options.rb +3 -2
  117. data/lib/aspera/preview/terminal.rb +5 -5
  118. data/lib/aspera/preview/utils.rb +66 -19
  119. data/lib/aspera/products/connect.rb +2 -2
  120. data/lib/aspera/products/desktop.rb +1 -1
  121. data/lib/aspera/products/other.rb +3 -3
  122. data/lib/aspera/products/transferd.rb +4 -4
  123. data/lib/aspera/proxy_auto_config.rb +15 -14
  124. data/lib/aspera/rest.rb +159 -151
  125. data/lib/aspera/rest_error_analyzer.rb +6 -6
  126. data/lib/aspera/rest_errors_aspera.rb +0 -10
  127. data/lib/aspera/rest_list.rb +11 -10
  128. data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
  129. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
  130. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
  131. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
  132. data/lib/aspera/schema/documentation.rb +65 -28
  133. data/lib/aspera/schema/reader.rb +136 -13
  134. data/lib/aspera/schema/registry.rb +60 -8
  135. data/lib/aspera/secret_hider.rb +12 -7
  136. data/lib/aspera/ssh.rb +64 -31
  137. data/lib/aspera/ssl.rb +5 -5
  138. data/lib/aspera/sync/conf.schema.yaml +2 -2
  139. data/lib/aspera/sync/database.rb +2 -2
  140. data/lib/aspera/sync/operations.rb +22 -25
  141. data/lib/aspera/temp_file_manager.rb +25 -5
  142. data/lib/aspera/timer_limiter.rb +1 -1
  143. data/lib/aspera/transfer/faux_file.rb +24 -11
  144. data/lib/aspera/transfer/parameters.rb +36 -34
  145. data/lib/aspera/transfer/result.rb +74 -0
  146. data/lib/aspera/transfer/resumer.rb +10 -10
  147. data/lib/aspera/transfer/spec.rb +18 -1
  148. data/lib/aspera/transfer/spec.schema.yaml +12 -3
  149. data/lib/aspera/uri_reader.rb +52 -15
  150. data/lib/aspera/web_auth.rb +8 -7
  151. data/lib/aspera/web_server_simple.rb +15 -12
  152. data/lib/aspera/yaml.rb +5 -4
  153. data.tar.gz.sig +0 -0
  154. metadata +34 -6
  155. metadata.gz.sig +0 -0
  156. data/lib/aspera/cli/manager.rb +0 -766
  157. data/lib/aspera/cli/plugins/faspex.rb +0 -539
  158. data/lib/aspera/json_rpc.rb +0 -52
  159. data/lib/aspera/transfer/uri.rb +0 -56
data/lib/aspera/ssh.rb CHANGED
@@ -8,77 +8,110 @@ module Aspera
8
8
  # A simple wrapper around Net::SSH
9
9
  # executes one command and get its result from stdout
10
10
  class Ssh
11
+ # Raised when the remote command fails or cannot be started
11
12
  class Error < Aspera::Error
12
13
  end
14
+ # Regexp matching ecdsa/ecdh-sha2 algorithm names (JRuby workaround)
15
+ EXCLUDE_ECDSHA2 = /^ecd(sa|h)-sha2/
16
+
13
17
  class << self
14
- # HACK: disable some key type
18
+ # Removes ed25519 keys from the default identity list used by Net::SSH.
19
+ # Called when the `ed25519` gem is absent or explicitly disabled via
20
+ # `ASCLI_ENABLE_ED25519=false`.
21
+ # @return [void]
15
22
  def disable_ed25519_keys
16
23
  Log.log.debug('Disabling SSH ed25519 user keys')
17
24
  old_verbose = $VERBOSE
18
25
  $VERBOSE = nil
19
26
  Net::SSH::Authentication::Session.class_eval do
20
27
  define_method(:default_keys) do
21
- %w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map{"~/#{_1}/id_#{_2}"}.freeze
28
+ %w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map { "~/#{_1}/id_#{_2}" }.freeze
22
29
  end
23
30
  private(:default_keys)
24
- end rescue nil
31
+ end
25
32
  $VERBOSE = old_verbose
26
33
  end
27
34
 
28
- # HACK: disable some algorithms
35
+ # Returns Net::SSH option overrides that exclude all ecdsa/ecdh-sha2 algorithms.
36
+ # Merge the result into `ssh_options` passed to {Ssh#initialize} to avoid
37
+ # mutating Net::SSH internal constants.
38
+ # @return [Hash{Symbol => Array<String>}] `:host_key` and `:kex` filtered lists
39
+ def no_ecd_sha2_options
40
+ Log.log.debug('Building SSH options without ecdsa/ecdh-sha2')
41
+ {
42
+ host_key: Net::SSH::Transport::Algorithms::ALGORITHMS[:host_key].reject { |a| a.match?(EXCLUDE_ECDSHA2) },
43
+ kex: Net::SSH::Transport::Algorithms::ALGORITHMS[:kex].reject { |a| a.match?(EXCLUDE_ECDSHA2) }
44
+ }
45
+ end
46
+
47
+ # Mutates Net::SSH internal algorithm lists to remove ecdsa/ecdh-sha2 entries globally.
48
+ # Kept for backwards compatibility and JRuby usage; prefer {no_ecd_sha2_options} for new code.
49
+ # @return [void]
29
50
  def disable_ecd_sha2_algorithms
30
- Log.log.debug('Disabling SSH ecdsa')
31
- Net::SSH::Transport::Algorithms::ALGORITHMS.each_value{ |a| a.reject!{ |a| a =~ /^ecd(sa|h)-sha2/}}
32
- Net::SSH::KnownHosts::SUPPORTED_TYPE.reject!{ |t| t =~ /^ecd(sa|h)-sha2/}
51
+ Log.log.debug('Disabling SSH ecdsa (global)')
52
+ Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a.match?(EXCLUDE_ECDSHA2) } }
53
+ Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t.match?(EXCLUDE_ECDSHA2) }
33
54
  end
34
55
  end
35
- # ssh_options: same as Net::SSH.start
36
- # see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
56
+ # @param host [String] remote server address
57
+ # @param username [String] SSH user name
58
+ # @param ssh_options [Hash{Symbol => Object}] options forwarded to `Net::SSH.start`
59
+ # (see {https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start Net::SSH.start}).
60
+ # Defaults are injected for `:logger`, `:verbose`, and `:use_agent` if absent.
37
61
  def initialize(host, username, ssh_options)
38
62
  Aspera.assert_type(host, String)
39
63
  Aspera.assert_type(username, String)
40
64
  Aspera.assert_type(ssh_options, Hash)
65
+ Aspera.assert_hash_all(ssh_options, Symbol, nil)
41
66
  @host = host
42
67
  @username = username
43
68
  @ssh_options = ssh_options.dup
44
69
  @ssh_options[:logger] = Log.log unless @ssh_options.key?(:logger)
45
70
  @ssh_options[:verbose] = :warn unless @ssh_options.key?(:verbose)
71
+ # @ssh_options[:verbose] = :debug
46
72
  @ssh_options[:use_agent] = false unless @ssh_options.key?(:use_agent)
47
- Log.log.debug{"ssh:#{@username}@#{@host}"}
73
+ Log.log.debug { "ssh:#{@username}@#{@host}" }
48
74
  Log.dump(:ssh_options, @ssh_options)
49
75
  end
50
76
 
51
- # Anything on stderr raises an exception
52
- def execute(cmd, input: nil, exception: false)
77
+ # Executes a single command on the remote host over a new SSH session.
78
+ # @param cmd [String] shell command to execute remotely
79
+ # @param input [String, nil] data written to the command's stdin, or `nil`
80
+ # @return [String] concatenated stdout of the remote command
81
+ # @raise [Error] if the channel cannot be opened or the remote command exits with a non-zero status
82
+ def execute(cmd, input: nil)
53
83
  Aspera.assert_type(cmd, String)
54
- Log.log.debug{"cmd=#{cmd}"}
84
+ Log.log.debug { "cmd=#{cmd}" }
85
+ # @type response [Array<String>]
55
86
  response = []
87
+ # @type error [Array<String>]
56
88
  error = []
89
+ exit_code = nil
90
+ # @param session [Net::SSH::Connection::Session]
57
91
  Net::SSH.start(@host, @username, @ssh_options) do |session|
58
- ssh_channel = session.open_channel do |channel|
59
- # prepare stdout processing
60
- channel.on_data{ |_chan, data| response.push(data)}
61
- # prepare stderr processing, stderr if type = 1
62
- channel.on_extended_data do |_chan, _type, data|
63
- error.push(data)
64
- end
65
- channel.on_request('exit-status') do |_ch, data|
92
+ # @param channel [Net::SSH::Connection::Channel]
93
+ session.open_channel do |channel|
94
+ # Register stdout/stderr before exec so no data is missed (e.g. ForceCommand errors)
95
+ channel.on_data { |_ch, data| response.push(data) }
96
+ channel.on_extended_data { |_ch, type, data| error.push(data) if type.eql?(1) }
97
+ # @param data [Net::SSH::Buffer]
98
+ channel.on_request('exit-status') do |_channel, data|
99
+ Log.dump(:data, data, level: :trace1)
66
100
  exit_code = data.read_long
67
- next if exit_code.zero?
68
- error_message = "#{cmd}: exit #{exit_code}, #{error.join.chomp}"
69
- raise Error, error_message if exception
70
- # Happens when windows user hasn't logged in and created home account.
71
- error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
72
- Log.log.debug(error_message)
73
101
  end
74
102
  # send command to SSH channel (execute) cspell: disable-next-line
75
- channel.send('cexe'.reverse, cmd){ |_ch, _success| channel.send_data(input) unless input.nil?}
103
+ channel.send('cexe'.reverse, cmd) do |_ch, success|
104
+ raise Error, "could not execute command: #{cmd}" unless success
105
+ channel.send_data(input) unless input.nil?
106
+ end
76
107
  end
77
- # wait for channel to finish (command exit)
78
- ssh_channel.wait
79
- # main SSH session loop
108
+ # wait for channel to finish and flush the session
80
109
  session.loop
81
110
  end
111
+ error_text = error.join
112
+ hint = error_text.include?('Could not chdir to home directory') ? "\nHint: home not created in Windows?" : ''
113
+ raise Error, "#{cmd}: exit #{exit_code}, #{error_text.chomp}#{hint}" if exit_code&.nonzero?
114
+ Log.log.error { "#{error_text}#{hint}" } unless error_text.empty?
82
115
  # response as single string
83
116
  return response.join
84
117
  end
data/lib/aspera/ssl.rb CHANGED
@@ -12,16 +12,16 @@ module Aspera
12
12
  @extra_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
13
13
  attr_reader :extra_options
14
14
 
15
- def option_list=(v)
16
- Aspera.assert_type(v, Array){'ssl_options'}
17
- v.each do |opt|
18
- Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
15
+ def option_list=(options)
16
+ Aspera.assert_type(options, Array) { 'ssl_options' }
17
+ options.each do |opt|
18
+ Aspera.assert_type(opt, String, Integer) { 'Expected String or Integer in ssl_options' }
19
19
  case opt
20
20
  when Integer
21
21
  @extra_options = opt
22
22
  when String
23
23
  name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
24
- raise Cli::BadArgument, "Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}" if !OpenSSL::SSL.const_defined?(name)
24
+ Aspera.assert(OpenSSL::SSL.const_defined?(name), type: Cli::BadArgument) { "Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map { |c| c.to_s.sub(/^OP_/, '') }.join(', ')}" }
25
25
  if opt.start_with?('-')
26
26
  @extra_options &= ~OpenSSL::SSL.const_get(name)
27
27
  else
@@ -637,7 +637,7 @@ properties:
637
637
  Partial transfers may exist if communication disruptions caused the underlying ascp processes to terminate early.
638
638
  Note that transfer resumption can only happen if the `reset` option is disabled.
639
639
  If an async session starts with `reset` enabled and resume enabled, transfers interrupted during that
640
- session will be resumeable, but only if async is then restarted with 'reset' disabled.
640
+ session will be resumable, but only if async is then restarted with 'reset' disabled.
641
641
  type: object
642
642
  properties:
643
643
  enabled:
@@ -654,7 +654,7 @@ properties:
654
654
  Sets the age limit in days for temporary files that will be preserved
655
655
  on cleanup (usually at async's start and stop) for potential transfer resume.
656
656
  Temp files older than the given value will be removed
657
- regardless of whether they might be resumeable.
657
+ regardless of whether they might be resumable.
658
658
  type: integer
659
659
  default: "5"
660
660
  x-cli-option: --resume-age-days
@@ -25,7 +25,7 @@ class SqLite3Wrapper
25
25
  end
26
26
  else
27
27
  def execute(sql)
28
- db = SQLite3::Database.new(@db_path).tap{ |d| d.results_as_hash = true}
28
+ db = SQLite3::Database.new(@db_path).tap { |d| d.results_as_hash = true }
29
29
  begin
30
30
  normalize_rows(db.execute(sql))
31
31
  ensure
@@ -46,7 +46,7 @@ class SqLite3Wrapper
46
46
  private
47
47
 
48
48
  def normalize_rows(rows)
49
- rows.map{ |r| r.transform_keys(&:to_s)}
49
+ rows.map { |r| r.transform_keys(&:to_s) }
50
50
  end
51
51
  end
52
52
 
@@ -4,6 +4,7 @@
4
4
 
5
5
  require 'aspera/ascp/installation'
6
6
  require 'aspera/agent/direct'
7
+ require 'aspera/exec_spec'
7
8
  require 'aspera/command_line_converter'
8
9
  require 'aspera/command_line_builder'
9
10
  require 'aspera/log'
@@ -89,17 +90,14 @@ module Aspera
89
90
  def start(sync_info, opt_ts = nil)
90
91
  Log.dump(:sync_params_initial, sync_info)
91
92
  Aspera.assert_type(sync_info, Hash)
92
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
93
- env_args = {
94
- args: [],
95
- env: {}
96
- }
93
+ Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
94
+ exec_spec = ExecSpec.new(exec: :async)
97
95
  if sync_info.key?('local')
98
96
  # `conf` format
99
- Aspera.assert_type(sync_info['local'], Hash){'local'}
97
+ Aspera.assert_type(sync_info['local'], Hash) { 'local' }
100
98
  remote = sync_info['remote']
101
- Aspera.assert_type(remote, Hash){'remote'}
102
- Aspera.assert_type(remote['path'], String){'remote path'}
99
+ Aspera.assert_type(remote, Hash) { 'remote' }
100
+ Aspera.assert_type(remote['path'], String) { 'remote path' }
103
101
  # get transfer spec if possible, and feed back to new structure
104
102
  if block_given?
105
103
  transfer_spec = yield(direction_sym(sync_info), sync_info['local']['path'], remote['path'])
@@ -115,10 +113,10 @@ module Aspera
115
113
  remote['private_key_paths'].concat(add_certificates)
116
114
  end
117
115
  # '--exclusive-mgmt-port=12345', '--arg-err-path=-',
118
- env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_info))}"]
116
+ exec_spec.args = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_info))}"]
119
117
  Log.dump(:sync_conf, sync_info)
120
118
  agent = Agent::Direct.new
121
- agent.start_and_monitor_process(session: {}, name: :async, **env_args)
119
+ agent.start_and_monitor_process(session: {}, exec_spec: exec_spec)
122
120
  else
123
121
  # `args` format
124
122
  raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
@@ -127,8 +125,8 @@ module Aspera
127
125
  Aspera.assert_type(sync_info['sessions'].first, Hash)
128
126
  if block_given?
129
127
  sync_info['sessions'].each do |session|
130
- Aspera.assert_type(session['local_dir'], String){'local_dir'}
131
- Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
128
+ Aspera.assert_type(session['local_dir'], String) { 'local_dir' }
129
+ Aspera.assert_type(session['remote_dir'], String) { 'remote_dir' }
132
130
  transfer_spec = yield(direction_sym(session), session['local_dir'], session['remote_dir'])
133
131
  Log.dump(:auth_ts, transfer_spec)
134
132
  transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
@@ -141,23 +139,23 @@ module Aspera
141
139
  Aspera.assert_type(sync_info['instance'], Hash)
142
140
  instance_builder = CommandLineBuilder.new(sync_info['instance'], ARGS_INSTANCE_SCHEMA, CommandLineConverter)
143
141
  instance_builder.process_params
144
- instance_builder.add_env_args(env_args)
142
+ instance_builder.add_to_exec_spec(exec_spec)
145
143
  end
146
144
  sync_info['sessions'].each do |session_params|
147
145
  Aspera.assert_type(session_params, Hash)
148
146
  Aspera.assert(session_params.key?('name'), 'session must contain at least: name')
149
147
  session_builder = CommandLineBuilder.new(session_params, ARGS_SESSION_SCHEMA, CommandLineConverter)
150
148
  session_builder.process_params
151
- session_builder.add_env_args(env_args)
149
+ session_builder.add_to_exec_spec(exec_spec)
152
150
  end
153
- Environment.secure_execute(Ascp::Installation.instance.path(:async), *env_args[:args], env: env_args[:env])
151
+ Environment.secure_execute(Ascp::Installation.instance.path(:async), *exec_spec.args, env: exec_spec.env)
154
152
  end
155
153
  return
156
154
  end
157
155
 
158
156
  # Parse output of asyncadmin
159
157
  def parse_status(stdout)
160
- Log.log.trace1{"stdout=#{stdout}"}
158
+ Log.log.trace1 { "stdout=#{stdout}" }
161
159
  result = {}
162
160
  ids = nil
163
161
  stdout.split("\n").each do |line|
@@ -177,7 +175,7 @@ module Aspera
177
175
  # @param sync_info [Hash] sync parameters in conf or args format
178
176
  # @return [Hash] parsed output of asyncadmin
179
177
  def admin_status(sync_info)
180
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
178
+ Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
181
179
  arguments = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
182
180
  if sync_info.key?('local')
183
181
  # `conf` format
@@ -209,14 +207,14 @@ module Aspera
209
207
  # @param sync_info [Hash] sync parameters in conf or args format
210
208
  # @return [String, nil] Path to "local DB dir", i.e. folder that contains folders that contain `snap.db`
211
209
  def local_db_folder(sync_info)
212
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
210
+ Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
213
211
  if sync_info.key?('local')
214
212
  # `conf` format
215
213
  if sync_info.key?('local_db_dir')
216
214
  return sync_info['local_db_dir']
217
215
  elsif (local_path = sync_info.dig('local', 'path'))
218
216
  return local_path
219
- elsif exception
217
+ else
220
218
  raise Error, 'Missing either local_db_dir or local.path'
221
219
  end
222
220
  else
@@ -226,15 +224,14 @@ module Aspera
226
224
  return session['local_db_dir']
227
225
  elsif session.key?('local_dir')
228
226
  return session['local_dir']
229
- elsif exception
227
+ else
230
228
  raise Error, 'Missing either local_db_dir or local_dir'
231
229
  end
232
230
  end
233
- nil
234
231
  end
235
232
 
236
233
  def session_name(sync_info)
237
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
234
+ Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
238
235
  if sync_info.key?('local')
239
236
  # `conf` format
240
237
  return sync_info['name']
@@ -246,7 +243,7 @@ module Aspera
246
243
 
247
244
  def session_db_file(sync_info)
248
245
  db_file = File.join(local_db_folder(sync_info), PRIVATE_FOLDER, session_name(sync_info), ASYNC_DB)
249
- Aspera.assert(File.exist?(db_file)){"Database file #{db_file} does not exist"}
246
+ Aspera.assert(File.exist?(db_file)) { "Database file #{db_file} does not exist" }
250
247
  db_file
251
248
  end
252
249
 
@@ -287,7 +284,7 @@ module Aspera
287
284
  # @param option [String] Option to search
288
285
  # @return [Array,nil] with path/schema for that option
289
286
  def find_option(schema, path, option)
290
- if %w[x-cli-option x-cli-short].any?{ |i| schema[i].eql?(option)}
287
+ if %w[x-cli-option x-cli-short].any? { |i| schema[i].eql?(option) }
291
288
  Log.log.debug('Special') if schema['x-cli-special']
292
289
  return [path, schema]
293
290
  end
@@ -325,7 +322,7 @@ module Aspera
325
322
  end
326
323
  end
327
324
  path, props = find_option(CONF_SCHEMA, [], option)
328
- raise "Option not found: #{option}" if path.nil?
325
+ Aspera.assert(!path.nil?) { "Option not found: #{option}" }
329
326
  last_key = path.pop
330
327
  # navigate in the current result to insert the value
331
328
  current = result
@@ -3,6 +3,7 @@
3
3
  require 'singleton'
4
4
  require 'fileutils'
5
5
  require 'etc'
6
+ require 'tempfile'
6
7
 
7
8
  module Aspera
8
9
  # create a temp file name for a given folder
@@ -40,7 +41,7 @@ module Aspera
40
41
  def delete_file(filepath)
41
42
  File.delete(filepath) if @cleanup_on_exit
42
43
  rescue => e
43
- Log.log.warn{"Problem deleting file: #{filepath}: #{e.message}"}
44
+ Log.log.warn { "Problem deleting file: #{filepath}: #{e.message}" }
44
45
  end
45
46
 
46
47
  # Call this on process exit
@@ -51,11 +52,16 @@ module Aspera
51
52
  @created_files = []
52
53
  end
53
54
 
54
- # Ensure that provided folder exists, or create it, generate a unique filename
55
- # @return path to that unique file
55
+ # Ensure that provided folder exists, or create it, atomically create and reserve a unique file.
56
+ # The file is created immediately (O_EXCL) to prevent TOCTOU race conditions.
57
+ # @return [String] path to that unique file
56
58
  def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
57
59
  FileUtils.mkdir_p(temp_folder)
58
- new_file = File.join(temp_folder, [prefix, SecureRandom.uuid, suffix].compact.join('-'))
60
+ basename = [prefix || 'aspera', SecureRandom.uuid].join('-')
61
+ # Tempfile.new creates the file atomically with O_EXCL; close keeps it on disk
62
+ tmp = Tempfile.new([basename, suffix ? "-#{suffix}" : ''], temp_folder)
63
+ tmp.close
64
+ new_file = tmp.path
59
65
  @created_files.push(new_file)
60
66
  new_file
61
67
  end
@@ -72,6 +78,20 @@ module Aspera
72
78
  new_file_path_in_folder(@global_temp, prefix: prefix, suffix: suffix)
73
79
  end
74
80
 
81
+ # Generate a unique directory path inside the global temp folder (does NOT create it).
82
+ # Unlike new_file_path_global, the caller is expected to use the path as a directory.
83
+ # @return [String] path to a unique (not yet existing) directory
84
+ def new_dir_path_global(prefix = nil)
85
+ username =
86
+ begin
87
+ Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
88
+ rescue StandardError
89
+ 'unknown_user'
90
+ end
91
+ dir_name = [prefix, username, SecureRandom.uuid].compact.join('-')
92
+ File.join(@global_temp, dir_name)
93
+ end
94
+
75
95
  # Garbage collect undeleted files
76
96
  def cleanup_expired(temp_folder)
77
97
  Dir.entries(temp_folder).each do |name|
@@ -79,7 +99,7 @@ module Aspera
79
99
  age_sec = (Time.now - File.stat(file_path).mtime).to_i
80
100
  # check age of file, delete too old
81
101
  if File.file?(file_path) && (age_sec > FILE_LIST_AGE_MAX_SEC)
82
- Log.log.debug{"garbage collecting #{name}"}
102
+ Log.log.debug { "garbage collecting #{name}" }
83
103
  delete_file(file_path)
84
104
  end
85
105
  end
@@ -3,7 +3,7 @@
3
3
  module Aspera
4
4
  # trigger returns true only if the delay has passed since the last trigger
5
5
  class TimerLimiter
6
- # @param delay in seconds (float)
6
+ # @param delay [Float] delay in seconds
7
7
  def initialize(delay)
8
8
  @delay = delay
9
9
  @last_trigger_time = nil
@@ -11,42 +11,55 @@ module Aspera
11
11
  SIZE_UNITS = %w[k m g t p e].freeze
12
12
  private_constant :SCHEME, :PREFIX, :SIZE_UNITS
13
13
  class << self
14
- # @return nil if not a faux: scheme, else a FauxFile instance
14
+ # Parse a faux URI and return a `FauxFile` instance, or `nil` if the URI does not use the faux scheme.
15
+ # URI format: `faux:///<path>?<size>`, where `<size>` is a decimal integer with an optional
16
+ # case-insensitive unit suffix: `k`, `m`, `g`, `t`, `p`, `e` (powers of 1024).
17
+ # When no suffix is given, the size is interpreted as raw bytes.
18
+ # Examples: `faux:///file.bin?10` (10 bytes), `faux:///file.bin?10m` (10 MiB)
19
+ # @param name [String] source file name, possibly a faux URI
20
+ # @return [FauxFile, nil] `nil` if not a faux scheme, else a `FauxFile` instance
15
21
  def create(name)
16
22
  return unless name.start_with?(PREFIX)
17
23
  name_params = name.delete_prefix(PREFIX).split('?', 2)
18
- raise Error, 'Format: #{PREFIX}<file path>?<size>' unless name_params.length.eql?(2)
19
- raise Error, "Format: <integer>[#{SIZE_UNITS.join(',')}]" unless (m = name_params[1].downcase.match(/^(\d+)([#{SIZE_UNITS.join('')}])$/))
20
- size = m[1].to_i
21
- suffix = m[2]
22
- SIZE_UNITS.each do |s|
23
- size *= 1024
24
- break if s.eql?(suffix)
25
- end
24
+ Aspera.assert(name_params.length.eql?(2), type: Error) { "Format: #{PREFIX}<file path>?<size>" }
25
+ m = name_params[1].downcase.match(/^(\d+)([#{SIZE_UNITS.join('')}]?)$/)
26
+ Aspera.assert(m, type: Error) { "Format: <integer>[#{SIZE_UNITS.join(',')}]" }
27
+ size = m[2].empty? ? m[1].to_i : m[1].to_i * (1024**(SIZE_UNITS.index(m[2]) + 1))
26
28
  return FauxFile.new(name_params[0], size)
27
29
  end
28
30
  end
31
+ # @return [String] virtual file path
32
+ # @return [Integer] total size in bytes
29
33
  attr_reader :path, :size
30
34
 
35
+ # @param path [String] virtual file path (from the faux URI)
36
+ # @param size [Integer] total number of bytes to produce
31
37
  def initialize(path, size)
32
38
  @path = path
33
39
  @size = size
34
40
  @offset = 0
35
- # we cache large chunks, anyway most of them will be the same size
41
+ # cache chunks by size so repeated reads of the same chunk length reuse the same buffer
36
42
  @chunk_by_size = {}
37
43
  end
38
44
 
45
+ # Read up to `chunk_size` bytes from the stream and advance the internal offset.
46
+ # Returns `nil` when the stream is exhausted.
47
+ # @param chunk_size [Integer] maximum number of bytes to read
48
+ # @return [String, nil] null-byte string of the bytes actually read, or `nil` at EOF
39
49
  def read(chunk_size)
40
50
  return if eof?
41
51
  bytes_to_read = [chunk_size, @size - @offset].min
42
52
  @offset += bytes_to_read
43
- @chunk_by_size[bytes_to_read] = "\x00" * bytes_to_read unless @chunk_by_size.key?(bytes_to_read)
53
+ @chunk_by_size[bytes_to_read] ||= "\x00" * bytes_to_read
44
54
  return @chunk_by_size[bytes_to_read]
45
55
  end
46
56
 
57
+ # No-op: required to satisfy the IO-like interface used by transfer agents.
58
+ # @return [nil]
47
59
  def close
48
60
  end
49
61
 
62
+ # @return [Boolean] true when all bytes have been read
50
63
  def eof?
51
64
  return @offset >= @size
52
65
  end