aspera-cli 4.26.1 → 4.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +300 -22
  4. data/CONTRIBUTING.md +100 -12
  5. data/bin/ascli +5 -5
  6. data/docs/README.md +10757 -0
  7. data/docs/test-mcp-with-ai.md +202 -0
  8. data/lib/aspera/agent/base.rb +20 -6
  9. data/lib/aspera/agent/connect.rb +53 -16
  10. data/lib/aspera/agent/desktop.rb +52 -17
  11. data/lib/aspera/agent/direct.rb +27 -25
  12. data/lib/aspera/agent/factory.rb +4 -0
  13. data/lib/aspera/agent/httpgw.rb +2 -2
  14. data/lib/aspera/agent/node.rb +34 -2
  15. data/lib/aspera/agent/transferd.rb +39 -3
  16. data/lib/aspera/api/alee.rb +1 -1
  17. data/lib/aspera/api/aoc.rb +56 -51
  18. data/lib/aspera/api/cos_node.rb +2 -2
  19. data/lib/aspera/api/faspex.rb +22 -20
  20. data/lib/aspera/api/httpgw.rb +4 -4
  21. data/lib/aspera/api/node.rb +33 -30
  22. data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
  23. data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
  24. data/lib/aspera/ascmd.rb +25 -23
  25. data/lib/aspera/ascp/installation.rb +43 -40
  26. data/lib/aspera/ascp/management.rb +2 -3
  27. data/lib/aspera/cli/ascp_actions.rb +155 -0
  28. data/lib/aspera/cli/async_transfer_store.rb +81 -0
  29. data/lib/aspera/cli/bootstrapper.rb +197 -0
  30. data/lib/aspera/cli/command_registry.rb +145 -0
  31. data/lib/aspera/cli/command_spec.rb +138 -0
  32. data/lib/aspera/cli/context.rb +71 -0
  33. data/lib/aspera/cli/error.rb +16 -0
  34. data/lib/aspera/cli/extended_value.rb +5 -3
  35. data/lib/aspera/cli/formatter.rb +122 -209
  36. data/lib/aspera/cli/gem_checker.rb +65 -0
  37. data/lib/aspera/cli/hints.rb +2 -2
  38. data/lib/aspera/cli/http.rb +218 -0
  39. data/lib/aspera/cli/info.rb +2 -0
  40. data/lib/aspera/cli/mailer.rb +97 -0
  41. data/lib/aspera/cli/mcp_tool.rb +198 -0
  42. data/lib/aspera/cli/options.schema.yaml +364 -7
  43. data/lib/aspera/cli/{manager.rb → parser.rb} +337 -144
  44. data/lib/aspera/cli/plugins/alee.rb +20 -22
  45. data/lib/aspera/cli/plugins/aoc.rb +1173 -849
  46. data/lib/aspera/cli/plugins/ats.rb +200 -161
  47. data/lib/aspera/cli/plugins/base.rb +470 -107
  48. data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
  49. data/lib/aspera/cli/plugins/config.rb +434 -1032
  50. data/lib/aspera/cli/plugins/console.rb +106 -64
  51. data/lib/aspera/cli/plugins/cos.rb +44 -32
  52. data/lib/aspera/cli/plugins/factory.rb +7 -4
  53. data/lib/aspera/cli/plugins/faspex.rb +296 -259
  54. data/lib/aspera/cli/plugins/faspex5.rb +592 -344
  55. data/lib/aspera/cli/plugins/faspio.rb +49 -51
  56. data/lib/aspera/cli/plugins/httpgw.rb +18 -25
  57. data/lib/aspera/cli/plugins/mcp.rb +279 -0
  58. data/lib/aspera/cli/plugins/node.rb +1001 -797
  59. data/lib/aspera/cli/plugins/oauth.rb +7 -10
  60. data/lib/aspera/cli/plugins/orchestrator.rb +111 -139
  61. data/lib/aspera/cli/plugins/preview.rb +183 -142
  62. data/lib/aspera/cli/plugins/server.rb +125 -80
  63. data/lib/aspera/cli/plugins/shares.rb +301 -107
  64. data/lib/aspera/cli/preset_actions.rb +139 -0
  65. data/lib/aspera/cli/preset_manager.rb +236 -0
  66. data/lib/aspera/cli/result.rb +360 -0
  67. data/lib/aspera/cli/runner.rb +473 -0
  68. data/lib/aspera/cli/secret_finder.rb +40 -0
  69. data/lib/aspera/cli/special_values.rb +1 -0
  70. data/lib/aspera/cli/sync_actions.rb +84 -56
  71. data/lib/aspera/cli/terminal_formatter.rb +65 -0
  72. data/lib/aspera/cli/transfer_actions.rb +83 -0
  73. data/lib/aspera/cli/transfer_agent.rb +116 -51
  74. data/lib/aspera/cli/transfer_progress.rb +9 -9
  75. data/lib/aspera/cli/vault_manager.rb +57 -0
  76. data/lib/aspera/cli/version.rb +1 -1
  77. data/lib/aspera/cli/wizard.rb +26 -26
  78. data/lib/aspera/command_line_builder.rb +23 -23
  79. data/lib/aspera/data_repository.rb +6 -1
  80. data/lib/aspera/dot_container.rb +5 -5
  81. data/lib/aspera/environment.rb +39 -13
  82. data/lib/aspera/exec_spec.rb +13 -0
  83. data/lib/aspera/faspex_gw.rb +2 -2
  84. data/lib/aspera/faspex_postproc.rb +4 -3
  85. data/lib/aspera/graphql.rb +35 -0
  86. data/lib/aspera/hash_ext.rb +6 -0
  87. data/lib/aspera/json_rpc/client.rb +62 -0
  88. data/lib/aspera/json_rpc/version.rb +7 -0
  89. data/lib/aspera/keychain/base.rb +1 -1
  90. data/lib/aspera/keychain/encrypted_hash.rb +2 -2
  91. data/lib/aspera/keychain/factory.rb +4 -4
  92. data/lib/aspera/keychain/macos_security.rb +4 -4
  93. data/lib/aspera/link_header.rb +82 -0
  94. data/lib/aspera/log.rb +23 -5
  95. data/lib/aspera/markdown.rb +85 -2
  96. data/lib/aspera/nagios.rb +2 -2
  97. data/lib/aspera/node_simulator.rb +3 -1
  98. data/lib/aspera/oauth/base.rb +10 -10
  99. data/lib/aspera/oauth/boot.rb +43 -0
  100. data/lib/aspera/oauth/factory.rb +41 -15
  101. data/lib/aspera/oauth/jwt.rb +4 -4
  102. data/lib/aspera/oauth/web.rb +4 -4
  103. data/lib/aspera/oauth.rb +1 -0
  104. data/lib/aspera/persistency_action_once.rb +10 -10
  105. data/lib/aspera/persistency_folder.rb +10 -3
  106. data/lib/aspera/preview/file_types.rb +7 -3
  107. data/lib/aspera/preview/generator.rb +3 -3
  108. data/lib/aspera/preview/utils.rb +1 -1
  109. data/lib/aspera/products/connect.rb +4 -1
  110. data/lib/aspera/products/other.rb +1 -1
  111. data/lib/aspera/products/transferd.rb +3 -3
  112. data/lib/aspera/proxy_auto_config.rb +10 -9
  113. data/lib/aspera/rest.rb +18 -40
  114. data/lib/aspera/rest_error_analyzer.rb +7 -3
  115. data/lib/aspera/rest_errors_aspera.rb +0 -10
  116. data/lib/aspera/rest_list.rb +13 -6
  117. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +14 -4
  118. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +2730 -648
  119. data/lib/aspera/schema/async_tables.yaml +361 -0
  120. data/lib/aspera/schema/documentation.rb +57 -26
  121. data/lib/aspera/schema/reader.rb +60 -11
  122. data/lib/aspera/schema/registry.rb +50 -10
  123. data/lib/aspera/secret_hider.rb +11 -2
  124. data/lib/aspera/ssl.rb +4 -4
  125. data/lib/aspera/sync/conf.schema.yaml +2 -2
  126. data/lib/aspera/sync/database.rb +14 -10
  127. data/lib/aspera/sync/operations.rb +15 -18
  128. data/lib/aspera/temp_file_manager.rb +5 -1
  129. data/lib/aspera/timer_limiter.rb +1 -1
  130. data/lib/aspera/transfer/faux_file.rb +24 -11
  131. data/lib/aspera/transfer/parameters.rb +27 -25
  132. data/lib/aspera/transfer/result.rb +74 -0
  133. data/lib/aspera/transfer/resumer.rb +7 -7
  134. data/lib/aspera/transfer/spec.rb +18 -1
  135. data/lib/aspera/transfer/spec.schema.yaml +26 -8
  136. data/lib/aspera/transfer/uri.rb +1 -1
  137. data/lib/aspera/uri_reader.rb +2 -2
  138. data/lib/aspera/web_auth.rb +2 -2
  139. data/lib/aspera/web_server_simple.rb +8 -5
  140. data/lib/aspera/yaml.rb +2 -1
  141. data.tar.gz.sig +0 -0
  142. metadata +45 -16
  143. metadata.gz.sig +0 -0
  144. data/lib/aspera/cli/main.rb +0 -480
  145. data/lib/aspera/json_rpc.rb +0 -51
@@ -7,6 +7,9 @@ require 'aspera/schema/reader'
7
7
  module Aspera
8
8
  # base class for plugins modules
9
9
  module Schema
10
+ # @!method self.instance
11
+ # Returns the singleton instance of Registry
12
+ # @return [Registry] the singleton instance
10
13
  class Registry
11
14
  include Singleton
12
15
 
@@ -15,18 +18,32 @@ module Aspera
15
18
  LOCATIONS.key?(sym)
16
19
  end
17
20
 
21
+ # Get path to request body schema, no check if it exists
22
+ # @param component [String] registry key (e.g. 'faspex', 'aoc')
23
+ # @param endpoint [String] endpoint path without leading slash (e.g. 'packages.post')
24
+ # @return [String] schema path usable in schema: keyword
18
25
  def req_body(component, endpoint)
19
26
  "#{component}:paths./#{endpoint}.requestBody.content.application/json.schema"
20
27
  end
28
+
29
+ # Get path to query parameters for a GET endpoint
30
+ # @param component [String] registry key (e.g. 'faspex', 'aoc')
31
+ # @param endpoint [String] resource path without leading slash (e.g. 'packages')
32
+ # @param method [String] HTTP method (default: 'get')
33
+ # @return [String] schema path usable in query_schema: keyword
34
+ def query_params(component, endpoint, method: 'get')
35
+ "#{component}:paths./#{endpoint}.#{method}#{QUERY_PARAMS_SUFFIX}"
36
+ end
21
37
  end
22
38
 
23
39
  LOCATIONS = {
24
- spec: 'aspera/transfer/spec.schema.yaml',
25
- args: 'aspera/sync/args.schema.yaml',
26
- conf: 'aspera/sync/conf.schema.yaml',
27
- opts: 'aspera/cli/options.schema.yaml',
28
- aoc: 'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
29
- faspex: 'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml'
40
+ spec: 'aspera/transfer/spec.schema.yaml',
41
+ args: 'aspera/sync/args.schema.yaml',
42
+ conf: 'aspera/sync/conf.schema.yaml',
43
+ opts: 'aspera/cli/options.schema.yaml',
44
+ aoc: 'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
45
+ faspex: 'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml',
46
+ async_tables: 'aspera/schema/async_tables.yaml'
30
47
  }
31
48
 
32
49
  OPTIONS = 'opts'
@@ -35,18 +52,35 @@ module Aspera
35
52
  SYNC_ARGS = 'args'
36
53
  AOC = 'aoc'
37
54
  FASPEX = 'faspex'
38
- TRANSFER_INFO = "#{OPTIONS}:components.schemas.TransferInfo"
55
+ ASYNC_TABLES = 'async_tables'
56
+ LOG_OPTIONS = "#{OPTIONS}:components.schemas.LogOptions"
57
+ DIRECT_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.DirectAgentOptions"
58
+ NODE_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.NodeAgentOptions"
59
+ HTTPGW_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.HttpgwAgentOptions"
60
+ TRANSFERD_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.TransferdAgentOptions"
61
+ TRANSFER_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.TransferAgentOptions"
62
+ SMTP_OPTIONS = "#{OPTIONS}:components.schemas.SmtpOptions"
63
+ HTTP_OPTIONS = "#{OPTIONS}:components.schemas.HttpOptions"
64
+ VAULT_OPTIONS = "#{OPTIONS}:components.schemas.VaultOptions"
65
+ IMAGE_OPTIONS = "#{OPTIONS}:components.schemas.ImageOptions"
66
+ PACKAGE_FOLDER_OPTIONS = "#{OPTIONS}:components.schemas.PackageFolderOptions"
39
67
 
40
68
  REQ_BODY = '.requestBody.content.application/json.schema'
69
+ # Suffix appended to a dotted path to signal query-param extraction in reader()
70
+ QUERY_PARAMS_SUFFIX = '.parameters'
41
71
 
42
72
  def initialize
43
73
  @cache = {}
44
74
  @main_folder = File.expand_path('../..', __dir__)
45
75
  end
46
76
 
47
- # Read schema from file or from cache
48
- # @param name_path [String] one of the keys in LOCATIONS, with optional :<path> suffix
49
- # @return [Reader] schema
77
+ # Read schema from file or from cache.
78
+ # When name_path ends with QUERY_PARAMS_SUFFIX, the OAS `parameters` array at that path
79
+ # is synthesised into an object schema via Reader.from_query_params instead of navigating
80
+ # into the tree.
81
+ # @param name_path [String] registry key with optional colon-separated dotted path suffix,
82
+ # e.g. "faspex:paths./packages.get.parameters" or "faspex:paths./packages.post.requestBody..."
83
+ # @return [Reader] schema reader
50
84
  def reader(name_path)
51
85
  name, path = name_path.split(':', 2)
52
86
  sym = name.to_sym
@@ -54,6 +88,12 @@ module Aspera
54
88
  spec_file = File.join(@main_folder, LOCATIONS[sym])
55
89
  @cache[sym] = Yaml.safe_load(File.read(spec_file)) if spec_file.end_with?('.yaml') && !@cache.key?(sym)
56
90
  @cache[sym] = JSON.parse(File.read(spec_file)) if spec_file.end_with?('.json') && !@cache.key?(sym)
91
+ # Query-params path: strip the suffix, navigate to the operation node, extract parameters
92
+ if path&.end_with?(QUERY_PARAMS_SUFFIX)
93
+ parent_path = path.delete_suffix(QUERY_PARAMS_SUFFIX)
94
+ node = @cache[sym].dig(*parent_path.split('.'))
95
+ return Reader.from_query_params(node&.fetch('parameters', []) || [])
96
+ end
57
97
  reader = Reader.new(@cache[sym])
58
98
  return reader unless path
59
99
  reader.dig(*path.split('.'))
@@ -6,6 +6,10 @@ require 'singleton'
6
6
 
7
7
  module Aspera
8
8
  # remove secret from logs and output
9
+ #
10
+ # @!method self.instance
11
+ # Returns the singleton instance of SecretHider
12
+ # @return [SecretHider] the singleton instance
9
13
  class SecretHider
10
14
  include Singleton
11
15
 
@@ -41,7 +45,7 @@ module Aspera
41
45
  private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :KEY_FALSE_POSITIVES, :REGEX_LOG_REPLACES
42
46
  attr_accessor :log_secrets
43
47
 
44
- # @return new log formatter that hides secrets
48
+ # @return [Proc] new log formatter that hides secrets
45
49
  def log_formatter(original_formatter)
46
50
  original_formatter ||= Logger::Formatter.new
47
51
  # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
@@ -55,11 +59,16 @@ module Aspera
55
59
  end
56
60
  end
57
61
 
62
+ # Replace private key content in a string with the hidden password placeholder
63
+ # @param value [String] Input string possibly containing a private key
64
+ # @return [String] String with private key replaced by placeholder
58
65
  def hide_secrets_in_string(value)
59
66
  return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
60
67
  end
61
68
 
62
- # @return true if the key denotes a secret
69
+ # @param keyword [String, Symbol] Key name to check
70
+ # @param value [String] Associated value (must be a String to be a secret)
71
+ # @return [Boolean] true if the key denotes a secret
63
72
  def secret?(keyword, value)
64
73
  keyword = keyword.to_s if keyword.is_a?(Symbol)
65
74
  # only Strings can be secrets, not booleans, or hash, arrays
data/lib/aspera/ssl.rb CHANGED
@@ -12,16 +12,16 @@ module Aspera
12
12
  @extra_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
13
13
  attr_reader :extra_options
14
14
 
15
- def option_list=(v)
16
- Aspera.assert_type(v, Array){'ssl_options'}
17
- v.each do |opt|
15
+ def option_list=(options)
16
+ Aspera.assert_type(options, Array){'ssl_options'}
17
+ options.each do |opt|
18
18
  Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
19
19
  case opt
20
20
  when Integer
21
21
  @extra_options = opt
22
22
  when String
23
23
  name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
24
- raise Cli::BadArgument, "Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}" if !OpenSSL::SSL.const_defined?(name)
24
+ Aspera.assert(OpenSSL::SSL.const_defined?(name), type: Cli::BadArgument){"Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}"}
25
25
  if opt.start_with?('-')
26
26
  @extra_options &= ~OpenSSL::SSL.const_get(name)
27
27
  else
@@ -637,7 +637,7 @@ properties:
637
637
  Partial transfers may exist if communication disruptions caused the underlying ascp processes to terminate early.
638
638
  Note that transfer resumption can only happen if the `reset` option is disabled.
639
639
  If an async session starts with `reset` enabled and resume enabled, transfers interrupted during that
640
- session will be resumeable, but only if async is then restarted with 'reset' disabled.
640
+ session will be resumable, but only if async is then restarted with 'reset' disabled.
641
641
  type: object
642
642
  properties:
643
643
  enabled:
@@ -654,7 +654,7 @@ properties:
654
654
  Sets the age limit in days for temporary files that will be preserved
655
655
  on cleanup (usually at async's start and stop) for potential transfer resume.
656
656
  Temp files older than the given value will be removed
657
- regardless of whether they might be resumeable.
657
+ regardless of whether they might be resumable.
658
658
  type: integer
659
659
  default: "5"
660
660
  x-cli-option: --resume-age-days
@@ -35,12 +35,12 @@ class SqLite3Wrapper
35
35
  end
36
36
 
37
37
  # The table contains a single row
38
- def single_table(table_name)
39
- execute("SELECT * FROM #{table_name} LIMIT 1").first
38
+ def single_table(table_name, sql_suffix = nil)
39
+ execute(["SELECT * FROM #{table_name}", sql_suffix, 'LIMIT 1'].compact.join(' ')).first
40
40
  end
41
41
 
42
- def full_table(table_name)
43
- execute("SELECT * FROM #{table_name}")
42
+ def full_table(table_name, sql_suffix = nil)
43
+ execute(["SELECT * FROM #{table_name}", sql_suffix].compact.join(' '))
44
44
  end
45
45
 
46
46
  private
@@ -68,16 +68,20 @@ module Aspera
68
68
  end
69
69
  end
70
70
 
71
- def meta
72
- @db.single_table('sync_snapmeta_table')
71
+ def meta(sql_suffix = nil)
72
+ @db.single_table('sync_snapmeta_table', sql_suffix)
73
73
  end
74
74
 
75
- def counters
76
- @db.single_table('sync_snap_counters_table')
75
+ def counters(sql_suffix = nil)
76
+ @db.single_table('sync_snap_counters_table', sql_suffix)
77
77
  end
78
78
 
79
- def file_info
80
- @db.full_table('sync_snapdb_table')
79
+ def file_info(sql_suffix = nil)
80
+ @db.full_table('sync_snapdb_table', sql_suffix)
81
+ end
82
+
83
+ def execute(sql)
84
+ @db.execute(sql)
81
85
  end
82
86
  end
83
87
  end
@@ -4,6 +4,7 @@
4
4
 
5
5
  require 'aspera/ascp/installation'
6
6
  require 'aspera/agent/direct'
7
+ require 'aspera/exec_spec'
7
8
  require 'aspera/command_line_converter'
8
9
  require 'aspera/command_line_builder'
9
10
  require 'aspera/log'
@@ -89,11 +90,8 @@ module Aspera
89
90
  def start(sync_info, opt_ts = nil)
90
91
  Log.dump(:sync_params_initial, sync_info)
91
92
  Aspera.assert_type(sync_info, Hash)
92
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
93
- env_args = {
94
- args: [],
95
- env: {}
96
- }
93
+ Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
94
+ exec_spec = ExecSpec.new(exec: :async)
97
95
  if sync_info.key?('local')
98
96
  # `conf` format
99
97
  Aspera.assert_type(sync_info['local'], Hash){'local'}
@@ -115,10 +113,10 @@ module Aspera
115
113
  remote['private_key_paths'].concat(add_certificates)
116
114
  end
117
115
  # '--exclusive-mgmt-port=12345', '--arg-err-path=-',
118
- env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_info))}"]
116
+ exec_spec.args = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_info))}"]
119
117
  Log.dump(:sync_conf, sync_info)
120
118
  agent = Agent::Direct.new
121
- agent.start_and_monitor_process(session: {}, name: :async, **env_args)
119
+ agent.start_and_monitor_process(session: {}, exec_spec: exec_spec)
122
120
  else
123
121
  # `args` format
124
122
  raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
@@ -141,16 +139,16 @@ module Aspera
141
139
  Aspera.assert_type(sync_info['instance'], Hash)
142
140
  instance_builder = CommandLineBuilder.new(sync_info['instance'], ARGS_INSTANCE_SCHEMA, CommandLineConverter)
143
141
  instance_builder.process_params
144
- instance_builder.add_env_args(env_args)
142
+ instance_builder.add_to_exec_spec(exec_spec)
145
143
  end
146
144
  sync_info['sessions'].each do |session_params|
147
145
  Aspera.assert_type(session_params, Hash)
148
- Aspera.assert(session_params.key?('name')){'session must contain at least: name'}
146
+ Aspera.assert(session_params.key?('name'), 'session must contain at least: name')
149
147
  session_builder = CommandLineBuilder.new(session_params, ARGS_SESSION_SCHEMA, CommandLineConverter)
150
148
  session_builder.process_params
151
- session_builder.add_env_args(env_args)
149
+ session_builder.add_to_exec_spec(exec_spec)
152
150
  end
153
- Environment.secure_execute(Ascp::Installation.instance.path(:async), *env_args[:args], env: env_args[:env])
151
+ Environment.secure_execute(Ascp::Installation.instance.path(:async), *exec_spec.args, env: exec_spec.env)
154
152
  end
155
153
  return
156
154
  end
@@ -177,7 +175,7 @@ module Aspera
177
175
  # @param sync_info [Hash] sync parameters in conf or args format
178
176
  # @return [Hash] parsed output of asyncadmin
179
177
  def admin_status(sync_info)
180
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
178
+ Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
181
179
  arguments = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
182
180
  if sync_info.key?('local')
183
181
  # `conf` format
@@ -209,14 +207,14 @@ module Aspera
209
207
  # @param sync_info [Hash] sync parameters in conf or args format
210
208
  # @return [String, nil] Path to "local DB dir", i.e. folder that contains folders that contain `snap.db`
211
209
  def local_db_folder(sync_info)
212
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
210
+ Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
213
211
  if sync_info.key?('local')
214
212
  # `conf` format
215
213
  if sync_info.key?('local_db_dir')
216
214
  return sync_info['local_db_dir']
217
215
  elsif (local_path = sync_info.dig('local', 'path'))
218
216
  return local_path
219
- elsif exception
217
+ else
220
218
  raise Error, 'Missing either local_db_dir or local.path'
221
219
  end
222
220
  else
@@ -226,15 +224,14 @@ module Aspera
226
224
  return session['local_db_dir']
227
225
  elsif session.key?('local_dir')
228
226
  return session['local_dir']
229
- elsif exception
227
+ else
230
228
  raise Error, 'Missing either local_db_dir or local_dir'
231
229
  end
232
230
  end
233
- nil
234
231
  end
235
232
 
236
233
  def session_name(sync_info)
237
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
234
+ Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
238
235
  if sync_info.key?('local')
239
236
  # `conf` format
240
237
  return sync_info['name']
@@ -325,7 +322,7 @@ module Aspera
325
322
  end
326
323
  end
327
324
  path, props = find_option(CONF_SCHEMA, [], option)
328
- raise "Option not found: #{option}" if path.nil?
325
+ Aspera.assert(!path.nil?){"Option not found: #{option}"}
329
326
  last_key = path.pop
330
327
  # navigate in the current result to insert the value
331
328
  current = result
@@ -7,6 +7,10 @@ require 'etc'
7
7
  module Aspera
8
8
  # create a temp file name for a given folder
9
9
  # files can be deleted on process exit by calling cleanup
10
+ #
11
+ # @!method self.instance
12
+ # Returns the singleton instance of TempFileManager
13
+ # @return [TempFileManager] the singleton instance
10
14
  class TempFileManager
11
15
  include Singleton
12
16
 
@@ -48,7 +52,7 @@ module Aspera
48
52
  end
49
53
 
50
54
  # Ensure that provided folder exists, or create it, generate a unique filename
51
- # @return path to that unique file
55
+ # @return [String] path to that unique file
52
56
  def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
53
57
  FileUtils.mkdir_p(temp_folder)
54
58
  new_file = File.join(temp_folder, [prefix, SecureRandom.uuid, suffix].compact.join('-'))
@@ -3,7 +3,7 @@
3
3
  module Aspera
4
4
  # trigger returns true only if the delay has passed since the last trigger
5
5
  class TimerLimiter
6
- # @param delay in seconds (float)
6
+ # @param delay [Float] delay in seconds
7
7
  def initialize(delay)
8
8
  @delay = delay
9
9
  @last_trigger_time = nil
@@ -11,42 +11,55 @@ module Aspera
11
11
  SIZE_UNITS = %w[k m g t p e].freeze
12
12
  private_constant :SCHEME, :PREFIX, :SIZE_UNITS
13
13
  class << self
14
- # @return nil if not a faux: scheme, else a FauxFile instance
14
+ # Parse a faux URI and return a `FauxFile` instance, or `nil` if the URI does not use the faux scheme.
15
+ # URI format: `faux:///<path>?<size>`, where `<size>` is a decimal integer with an optional
16
+ # case-insensitive unit suffix: `k`, `m`, `g`, `t`, `p`, `e` (powers of 1024).
17
+ # When no suffix is given, the size is interpreted as raw bytes.
18
+ # Examples: `faux:///file.bin?10` (10 bytes), `faux:///file.bin?10m` (10 MiB)
19
+ # @param name [String] source file name, possibly a faux URI
20
+ # @return [FauxFile, nil] `nil` if not a faux scheme, else a `FauxFile` instance
15
21
  def create(name)
16
22
  return unless name.start_with?(PREFIX)
17
23
  name_params = name.delete_prefix(PREFIX).split('?', 2)
18
- raise Error, 'Format: #{PREFIX}<file path>?<size>' unless name_params.length.eql?(2)
19
- raise Error, "Format: <integer>[#{SIZE_UNITS.join(',')}]" unless (m = name_params[1].downcase.match(/^(\d+)([#{SIZE_UNITS.join('')}])$/))
20
- size = m[1].to_i
21
- suffix = m[2]
22
- SIZE_UNITS.each do |s|
23
- size *= 1024
24
- break if s.eql?(suffix)
25
- end
24
+ Aspera.assert(name_params.length.eql?(2), type: Error){"Format: #{PREFIX}<file path>?<size>"}
25
+ m = name_params[1].downcase.match(/^(\d+)([#{SIZE_UNITS.join('')}]?)$/)
26
+ Aspera.assert(m, type: Error){"Format: <integer>[#{SIZE_UNITS.join(',')}]"}
27
+ size = m[2].empty? ? m[1].to_i : m[1].to_i * (1024**(SIZE_UNITS.index(m[2]) + 1))
26
28
  return FauxFile.new(name_params[0], size)
27
29
  end
28
30
  end
31
+ # @return [String] virtual file path
32
+ # @return [Integer] total size in bytes
29
33
  attr_reader :path, :size
30
34
 
35
+ # @param path [String] virtual file path (from the faux URI)
36
+ # @param size [Integer] total number of bytes to produce
31
37
  def initialize(path, size)
32
38
  @path = path
33
39
  @size = size
34
40
  @offset = 0
35
- # we cache large chunks, anyway most of them will be the same size
41
+ # cache chunks by size so repeated reads of the same chunk length reuse the same buffer
36
42
  @chunk_by_size = {}
37
43
  end
38
44
 
45
+ # Read up to `chunk_size` bytes from the stream and advance the internal offset.
46
+ # Returns `nil` when the stream is exhausted.
47
+ # @param chunk_size [Integer] maximum number of bytes to read
48
+ # @return [String, nil] null-byte string of the bytes actually read, or `nil` at EOF
39
49
  def read(chunk_size)
40
50
  return if eof?
41
51
  bytes_to_read = [chunk_size, @size - @offset].min
42
52
  @offset += bytes_to_read
43
- @chunk_by_size[bytes_to_read] = "\x00" * bytes_to_read unless @chunk_by_size.key?(bytes_to_read)
53
+ @chunk_by_size[bytes_to_read] ||= "\x00" * bytes_to_read
44
54
  return @chunk_by_size[bytes_to_read]
45
55
  end
46
56
 
57
+ # No-op: required to satisfy the IO-like interface used by transfer agents.
58
+ # @return [nil]
47
59
  def close
48
60
  end
49
61
 
62
+ # @return [Boolean] true when all bytes have been read
50
63
  def eof?
51
64
  return @offset >= @size
52
65
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'aspera/log'
4
4
  require 'aspera/assert'
5
+ require 'aspera/exec_spec'
5
6
  require 'aspera/command_line_builder'
6
7
  require 'aspera/temp_file_manager'
7
8
  require 'aspera/transfer/error'
@@ -38,8 +39,9 @@ module Aspera
38
39
  @file_list_folder ||= TempFileManager.instance.new_file_path_global('asession_filelists')
39
40
  end
40
41
 
41
- # File list is provided directly with `ascp` arguments
42
- # @columns ascp_args [Array,NilClass] `ascp` arguments
42
+ # Check whether a file list option is already present in the `ascp` argument list
43
+ # @param ascp_args [Array, nil] `ascp` arguments
44
+ # @return [Boolean] true if a file list option is present
43
45
  def ascp_args_file_list?(ascp_args)
44
46
  ascp_args&.any?{ |i| FILE_LIST_OPTIONS.include?(i)}
45
47
  end
@@ -47,7 +49,9 @@ module Aspera
47
49
 
48
50
  # @param job_spec [Hash] Transfer specification
49
51
  # @param ascp_args [Array] Other `ascp` args
52
+ # @param wss [Boolean] `true`: if both SSH and wss in ts: prefer wss
50
53
  # @param quiet [Boolean] Remove `ascp` progress bar if `true`
54
+ # @param file_list [Boolean] `true`: provide source list via a temp file, `false`: place directly on command line
51
55
  # @param client_ssh_key [:rsa,:dsa] Type of Aspera Client SSH key to use.
52
56
  # @param trusted_certs [Array<String>] List of path to trusted certificates stores when using WSS
53
57
  # @param check_ignore_cb [Proc] Callback to check if WSS connection shall ignore certificate validity
@@ -56,6 +60,7 @@ module Aspera
56
60
  ascp_args: nil,
57
61
  wss: true,
58
62
  quiet: true,
63
+ file_list: true,
59
64
  trusted_certs: nil,
60
65
  client_ssh_key: nil,
61
66
  check_ignore_cb: nil
@@ -66,6 +71,7 @@ module Aspera
66
71
  Aspera.assert_array_all(@ascp_args, String){'ascp_args'}
67
72
  @wss = wss
68
73
  @quiet = quiet
74
+ @file_list = file_list
69
75
  @trusted_certs = trusted_certs.nil? ? [] : trusted_certs
70
76
  Aspera.assert_type(@trusted_certs, Array){'trusted_certs'}
71
77
  @client_ssh_key = client_ssh_key.nil? ? :rsa : client_ssh_key.to_sym
@@ -74,7 +80,7 @@ module Aspera
74
80
  @builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, CommandLineConverter)
75
81
  end
76
82
 
77
- # either place source files on command line, or add file list file
83
+ # Place source files on command line or in a temp file list, controlled by @file_list and file_list_folder
78
84
  def process_file_list
79
85
  # is the file list provided through ascp parameters?
80
86
  ascp_file_list_provided = self.class.ascp_args_file_list?(@ascp_args)
@@ -85,12 +91,12 @@ module Aspera
85
91
  file_list_option = nil
86
92
  # transfer spec contains paths ?
87
93
  if !ts_paths_array.nil?
88
- Aspera.assert(!ascp_file_list_provided){'file list provided both in transfer spec and ascp file list. Remove one of them.'}
89
- Aspera.assert(ts_paths_array.all?{ |i| i.key?('source')}){"All elements of paths must have a 'source' key"}
94
+ Aspera.assert(!ascp_file_list_provided, 'file list provided both in transfer spec and ascp file list. Remove one of them.')
95
+ Aspera.assert(ts_paths_array.all?{ |i| i.key?('source')}, "All elements of paths must have a 'source' key")
90
96
  is_pair_list = ts_paths_array.any?{ |i| i.key?('destination')}
91
- raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{ |i| i.key?('destination')}
92
- if self.class.file_list_folder.nil?
93
- Aspera.assert(!is_pair_list){'file pair list is not supported when file list folder is not set'}
97
+ Aspera.assert(!(is_pair_list && !ts_paths_array.all?{ |i| i.key?('destination')})){"All elements of paths must be consistent with 'destination' key"}
98
+ if !@file_list || self.class.file_list_folder.nil?
99
+ Aspera.assert(!is_pair_list, 'file pair list is not supported when file list folder is not set')
94
100
  # not safe for special characters ? (maybe not, depends on OS)
95
101
  Log.log.debug('placing source file list on command line (no file list file)')
96
102
  @builder.add_command_line_options(ts_paths_array.map{ |i| i['source']})
@@ -112,7 +118,7 @@ module Aspera
112
118
  @builder.add_command_line_options("#{file_list_option}=#{file_list_file}") unless file_list_option.nil?
113
119
  end
114
120
 
115
- # @return the list of certificates (option `-i`) to use when token/ssh or wss are used
121
+ # @return [Array<String>] the list of certificates (option `-i`) to use when token/ssh or wss are used
116
122
  def remote_certificates
117
123
  certificates_to_use = []
118
124
  # use web socket secure for session ?
@@ -150,22 +156,18 @@ module Aspera
150
156
 
151
157
  # Translate transfer spec to env vars and command line arguments for `ascp`
152
158
  def ascp_args
153
- env_args = {
154
- args: [],
155
- env: {},
156
- name: :ascp
157
- }
159
+ exec_spec = Aspera::ExecSpec.new(exec: :ascp)
158
160
 
159
161
  # Special cases
160
162
  @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
161
163
 
162
164
  # Notify multi-session was already used, anyway it was deleted by agent direct
163
- Aspera.assert(!@builder.read_param('multi_session'))
165
+ Aspera.assert(!@builder.read_param('multi_session'), 'multi_session already consumed, must not be set here')
164
166
 
165
167
  # Add ssh or wss certificates
166
168
  # (reverse, to keep order, as we unshift)
167
169
  remote_certificates&.reverse_each do |cert|
168
- env_args[:args].unshift('-i', cert)
170
+ exec_spec.args.unshift('-i', cert)
169
171
  end
170
172
 
171
173
  case (delete_source = @builder.read_param('delete_source'))
@@ -183,9 +185,9 @@ module Aspera
183
185
  base64_destination = false
184
186
  # symbol must be index of Ascp::Installation.paths
185
187
  if @builder.read_param('use_ascp4')
186
- env_args[:name] = :ascp4
188
+ exec_spec.exec = :ascp4
187
189
  else
188
- env_args[:name] = :ascp
190
+ exec_spec.exec = :ascp
189
191
  base64_destination = true
190
192
  end
191
193
  # destination will be base64 encoded, put this before source path arguments
@@ -200,17 +202,17 @@ module Aspera
200
202
  destination_folder = Base64.strict_encode64(destination_folder) if base64_destination
201
203
  # destination MUST be last command line argument to ascp
202
204
  @builder.add_command_line_options(destination_folder)
203
- @builder.add_env_args(env_args)
204
- env_args[:args].unshift('-q') if @quiet
205
+ @builder.add_to_exec_spec(exec_spec)
206
+ exec_spec.args.unshift('-q') if @quiet
205
207
  # add fallback cert and key as arguments if needed
206
208
  if HTTP_FALLBACK_ACTIVATION_VALUES.include?(@job_spec['http_fallback'])
207
- env_args[:args].unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
208
- env_args[:args].unshift('-I', Ascp::Installation.instance.path(:fallback_certificate))
209
+ exec_spec.args.unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
210
+ exec_spec.args.unshift('-I', Ascp::Installation.instance.path(:fallback_certificate))
209
211
  end
210
212
  # disable redis in client, only for ascp, this makes ascp4 fail
211
- env_args[:env]['ASPERA_TEST_REDIS_DISABLE'] = 'true' if env_args[:name].eql?(:ascp)
212
- Log.log.debug{"ascp args: #{env_args}"}
213
- return env_args
213
+ exec_spec.env['ASPERA_TEST_REDIS_DISABLE'] = 'true' if exec_spec.exec.eql?(:ascp)
214
+ Log.log.debug{"ascp args: #{exec_spec}"}
215
+ return exec_spec
214
216
  end
215
217
  DELETE_EQUIV = %w[remove_after_transfer remove_empty_directories remove_empty_source_directory]
216
218
  # `ascp` options to provide a file list
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aspera
4
+ module Transfer
5
+ # Typed result of a transfer operation.
6
+ # Replaces the ad-hoc Array polymorphism previously returned by agents.
7
+ #
8
+ # Agents return one of the three concrete sub-classes:
9
+ # Transfer::Result::Success — all sessions completed successfully
10
+ # Transfer::Result::Error — at least one session failed
11
+ # Transfer::Result::Async — transfer submitted to a daemon; not yet complete
12
+ class Result
13
+ # All sessions completed successfully.
14
+ class Success < Result
15
+ def to_s
16
+ 'success'
17
+ end
18
+ end
19
+
20
+ # At least one session failed.
21
+ class Error < Result
22
+ # @return [StandardError]
23
+ attr_reader :exception
24
+
25
+ def initialize(exception)
26
+ super()
27
+ @exception = exception
28
+ end
29
+
30
+ def to_s
31
+ "error: #{@exception.message}"
32
+ end
33
+ end
34
+
35
+ # Transfer submitted to an external daemon; status not yet known.
36
+ class Async < Result
37
+ # @return [String] UUID assigned to this background job
38
+ attr_reader :job_id
39
+ # @return [String] initial status string (e.g. 'running')
40
+ attr_reader :status
41
+
42
+ def initialize(job_id:, status: 'running')
43
+ super()
44
+ @job_id = job_id
45
+ @status = status
46
+ end
47
+
48
+ # Convenience: export as a plain Hash for serialization / display
49
+ def to_h
50
+ {'job_id' => @job_id, 'status' => @status}
51
+ end
52
+
53
+ def to_s
54
+ "async job_id=#{@job_id} status=#{@status}"
55
+ end
56
+ end
57
+
58
+ # Factory helpers
59
+ class << self
60
+ def success
61
+ Success.new
62
+ end
63
+
64
+ def error(exception)
65
+ Error.new(exception)
66
+ end
67
+
68
+ def async(job_id:, status: 'running')
69
+ Async.new(job_id: job_id, status: status)
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end