aspera-cli 4.26.2 → 4.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +281 -21
  4. data/CONTRIBUTING.md +98 -7
  5. data/bin/ascli +3 -3
  6. data/docs/README.md +10757 -0
  7. data/docs/test-mcp-with-ai.md +202 -0
  8. data/lib/aspera/agent/base.rb +20 -6
  9. data/lib/aspera/agent/connect.rb +53 -16
  10. data/lib/aspera/agent/desktop.rb +52 -17
  11. data/lib/aspera/agent/direct.rb +26 -24
  12. data/lib/aspera/agent/httpgw.rb +1 -1
  13. data/lib/aspera/agent/node.rb +32 -0
  14. data/lib/aspera/agent/transferd.rb +39 -3
  15. data/lib/aspera/api/alee.rb +1 -1
  16. data/lib/aspera/api/aoc.rb +14 -16
  17. data/lib/aspera/api/cos_node.rb +1 -1
  18. data/lib/aspera/api/faspex.rb +15 -14
  19. data/lib/aspera/api/httpgw.rb +4 -4
  20. data/lib/aspera/api/node.rb +26 -23
  21. data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
  22. data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
  23. data/lib/aspera/ascmd.rb +21 -19
  24. data/lib/aspera/ascp/installation.rb +37 -38
  25. data/lib/aspera/ascp/management.rb +2 -3
  26. data/lib/aspera/cli/ascp_actions.rb +155 -0
  27. data/lib/aspera/cli/async_transfer_store.rb +81 -0
  28. data/lib/aspera/cli/bootstrapper.rb +197 -0
  29. data/lib/aspera/cli/command_registry.rb +145 -0
  30. data/lib/aspera/cli/command_spec.rb +138 -0
  31. data/lib/aspera/cli/context.rb +16 -3
  32. data/lib/aspera/cli/error.rb +16 -0
  33. data/lib/aspera/cli/extended_value.rb +2 -1
  34. data/lib/aspera/cli/formatter.rb +41 -27
  35. data/lib/aspera/cli/gem_checker.rb +65 -0
  36. data/lib/aspera/cli/hints.rb +2 -2
  37. data/lib/aspera/cli/http.rb +70 -19
  38. data/lib/aspera/cli/info.rb +2 -0
  39. data/lib/aspera/cli/mailer.rb +97 -0
  40. data/lib/aspera/cli/mcp_tool.rb +198 -0
  41. data/lib/aspera/cli/options.schema.yaml +364 -7
  42. data/lib/aspera/cli/{manager.rb → parser.rb} +334 -142
  43. data/lib/aspera/cli/plugins/alee.rb +20 -22
  44. data/lib/aspera/cli/plugins/aoc.rb +1166 -853
  45. data/lib/aspera/cli/plugins/ats.rb +200 -161
  46. data/lib/aspera/cli/plugins/base.rb +457 -100
  47. data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
  48. data/lib/aspera/cli/plugins/config.rb +428 -754
  49. data/lib/aspera/cli/plugins/console.rb +106 -64
  50. data/lib/aspera/cli/plugins/cos.rb +44 -32
  51. data/lib/aspera/cli/plugins/factory.rb +4 -4
  52. data/lib/aspera/cli/plugins/faspex.rb +294 -257
  53. data/lib/aspera/cli/plugins/faspex5.rb +585 -357
  54. data/lib/aspera/cli/plugins/faspio.rb +49 -51
  55. data/lib/aspera/cli/plugins/httpgw.rb +18 -25
  56. data/lib/aspera/cli/plugins/mcp.rb +279 -0
  57. data/lib/aspera/cli/plugins/node.rb +998 -794
  58. data/lib/aspera/cli/plugins/oauth.rb +7 -10
  59. data/lib/aspera/cli/plugins/orchestrator.rb +111 -134
  60. data/lib/aspera/cli/plugins/preview.rb +180 -139
  61. data/lib/aspera/cli/plugins/server.rb +124 -79
  62. data/lib/aspera/cli/plugins/shares.rb +301 -107
  63. data/lib/aspera/cli/preset_actions.rb +139 -0
  64. data/lib/aspera/cli/preset_manager.rb +13 -12
  65. data/lib/aspera/cli/result.rb +61 -11
  66. data/lib/aspera/cli/runner.rb +275 -141
  67. data/lib/aspera/cli/secret_finder.rb +40 -0
  68. data/lib/aspera/cli/special_values.rb +1 -0
  69. data/lib/aspera/cli/sync_actions.rb +83 -59
  70. data/lib/aspera/cli/terminal_formatter.rb +1 -1
  71. data/lib/aspera/cli/transfer_actions.rb +83 -0
  72. data/lib/aspera/cli/transfer_agent.rb +116 -51
  73. data/lib/aspera/cli/transfer_progress.rb +3 -3
  74. data/lib/aspera/cli/vault_manager.rb +57 -0
  75. data/lib/aspera/cli/version.rb +1 -1
  76. data/lib/aspera/cli/wizard.rb +21 -20
  77. data/lib/aspera/command_line_builder.rb +22 -22
  78. data/lib/aspera/data_repository.rb +2 -1
  79. data/lib/aspera/dot_container.rb +4 -4
  80. data/lib/aspera/environment.rb +18 -11
  81. data/lib/aspera/exec_spec.rb +13 -0
  82. data/lib/aspera/faspex_gw.rb +1 -1
  83. data/lib/aspera/faspex_postproc.rb +4 -3
  84. data/lib/aspera/graphql.rb +35 -0
  85. data/lib/aspera/hash_ext.rb +6 -0
  86. data/lib/aspera/json_rpc/client.rb +62 -0
  87. data/lib/aspera/json_rpc/version.rb +7 -0
  88. data/lib/aspera/keychain/encrypted_hash.rb +1 -1
  89. data/lib/aspera/keychain/factory.rb +3 -3
  90. data/lib/aspera/keychain/macos_security.rb +3 -3
  91. data/lib/aspera/link_header.rb +82 -0
  92. data/lib/aspera/log.rb +17 -3
  93. data/lib/aspera/markdown.rb +85 -2
  94. data/lib/aspera/node_simulator.rb +3 -1
  95. data/lib/aspera/oauth/base.rb +5 -5
  96. data/lib/aspera/oauth/boot.rb +2 -2
  97. data/lib/aspera/oauth/factory.rb +7 -5
  98. data/lib/aspera/oauth/jwt.rb +4 -4
  99. data/lib/aspera/oauth/web.rb +2 -2
  100. data/lib/aspera/persistency_action_once.rb +8 -8
  101. data/lib/aspera/persistency_folder.rb +10 -3
  102. data/lib/aspera/preview/file_types.rb +3 -3
  103. data/lib/aspera/preview/generator.rb +3 -3
  104. data/lib/aspera/preview/utils.rb +1 -1
  105. data/lib/aspera/products/connect.rb +1 -1
  106. data/lib/aspera/products/other.rb +1 -1
  107. data/lib/aspera/products/transferd.rb +1 -1
  108. data/lib/aspera/proxy_auto_config.rb +7 -6
  109. data/lib/aspera/rest.rb +10 -36
  110. data/lib/aspera/rest_error_analyzer.rb +3 -3
  111. data/lib/aspera/rest_errors_aspera.rb +0 -10
  112. data/lib/aspera/rest_list.rb +3 -3
  113. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
  114. data/lib/aspera/schema/documentation.rb +57 -26
  115. data/lib/aspera/schema/reader.rb +59 -10
  116. data/lib/aspera/schema/registry.rb +39 -6
  117. data/lib/aspera/secret_hider.rb +7 -2
  118. data/lib/aspera/ssl.rb +4 -4
  119. data/lib/aspera/sync/conf.schema.yaml +2 -2
  120. data/lib/aspera/sync/operations.rb +10 -13
  121. data/lib/aspera/temp_file_manager.rb +1 -1
  122. data/lib/aspera/timer_limiter.rb +1 -1
  123. data/lib/aspera/transfer/faux_file.rb +24 -11
  124. data/lib/aspera/transfer/parameters.rb +23 -21
  125. data/lib/aspera/transfer/result.rb +74 -0
  126. data/lib/aspera/transfer/resumer.rb +6 -6
  127. data/lib/aspera/transfer/spec.rb +18 -1
  128. data/lib/aspera/transfer/spec.schema.yaml +12 -3
  129. data/lib/aspera/uri_reader.rb +2 -2
  130. data/lib/aspera/web_auth.rb +2 -2
  131. data/lib/aspera/web_server_simple.rb +8 -5
  132. data/lib/aspera/yaml.rb +2 -1
  133. data.tar.gz.sig +0 -0
  134. metadata +25 -3
  135. metadata.gz.sig +0 -0
  136. data/lib/aspera/json_rpc.rb +0 -52
@@ -7,6 +7,15 @@ module Aspera
7
7
  module Schema
8
8
  # Generate documentation from Schema, for Transfer Spec, or async Conf spec
9
9
  class Documentation
10
+ # Map JSON Schema type names to user-friendly display names
11
+ JSON_TYPE_TO_DOC = {
12
+ 'string' => 'String',
13
+ 'integer' => 'Integer',
14
+ 'number' => 'Number',
15
+ 'boolean' => 'Bool',
16
+ 'array' => 'Array',
17
+ 'object' => 'Hash'
18
+ }.freeze
10
19
  # @param formatter [Cli::Formatter] Formatter instance with methods: markdown_text, tick, check_row
11
20
  # @param schema [Reader]
12
21
  # @param include_option [Boolean] `true`: include CLI options (switches, env vars) in descriptions
@@ -18,25 +27,27 @@ module Aspera
18
27
  @include_option = include_option
19
28
  @agent_columns = agent_columns
20
29
  @code_highlight = code_highlight
21
- @columns = %i[name type description]
22
- @columns.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short]}.sort) if @agent_columns
23
- # @type [Array<Hash<Symbol,String>>]
24
- @rows = []
30
+ @columns = %w[name type description]
31
+ @columns.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short].to_s}.sort) if @agent_columns
32
+ # Sections: each entry is {header: row_or_nil, rows: []}
33
+ # A flat schema produces a single section with no header.
34
+ @sections = [{header: nil, rows: []}]
25
35
  end
26
36
 
27
37
  def rows
28
- @rows.sort_by{ |i| i[:name]}
38
+ @sections.flat_map do |section|
39
+ sorted = section[:rows].sort_by{ |i| i['name']}
40
+ section[:header] ? [section[:header]] + sorted : sorted
41
+ end
29
42
  end
30
43
 
31
44
  # @return [Array<String>]
32
- def columns
33
- @columns.map(&:to_s)
34
- end
45
+ attr_reader :columns
35
46
 
36
- # First row is the titles
47
+ # First row is the titles (for Markdown table generation)
37
48
  # @return [Array<Array<String>>]
38
49
  def table
39
- [@columns.map(&:to_s)] + @rows.sort_by{ |i| i[:name]}.map{ |row| @columns.map{ |field| row[field]}}
50
+ [@columns] + rows.map{ |row| @columns.map{ |field| row[field]}}
40
51
  end
41
52
 
42
53
  # Generate a documentation table from a JSON schema for transfer specifications
@@ -49,37 +60,57 @@ module Aspera
49
60
  def build(schema = nil)
50
61
  code = @code_highlight ? ->(c){"`#{c}`"} : ->(c){c}
51
62
  schema ||= @schema
52
- schema.each_property do |property_schema, _name, property_full_name|
63
+ render_title = ->(title){title.gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}}
64
+ on_variant = ->(variant_reader, discriminant_property, discriminant_value) do
65
+ title = variant_reader.current['title'] || variant_reader.current['description']
66
+ header =
67
+ if discriminant_property && discriminant_value
68
+ desc = render_title.call("`#{discriminant_value}`")
69
+ desc += ": #{render_title.call(title)}" if title
70
+ @formatter.check_row({
71
+ 'name' => render_title.call("**#{discriminant_property}**"),
72
+ 'type' => code.call(JSON_TYPE_TO_DOC['string']),
73
+ 'description' => desc
74
+ })
75
+ elsif title
76
+ @formatter.check_row({'name' => "**#{render_title.call(title)}**", 'type' => '&nbsp;', 'description' => '&nbsp;'})
77
+ end
78
+ @sections.push({header: header, rows: []})
79
+ end
80
+ schema.each_property(on_variant: on_variant) do |property_schema, _name, property_full_name|
53
81
  node = property_schema.current
54
82
  # Manual table
83
+ item_type = JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])
84
+ item_type = "#{item_type}[#{JSON_TYPE_TO_DOC.fetch(node.dig('items', 'type'), node.dig('items', 'type'))}]" if node['type'].eql?('array') && node.dig('items', 'type').is_a?(String)
55
85
  item = {
56
- name: code.call(property_full_name),
57
- type: code.call(node['type']),
58
- description: []
86
+ 'name' => code.call(property_full_name),
87
+ 'type' => code.call(item_type),
88
+ 'description' => []
59
89
  }
60
90
  # Render Markdown formatting and split lines
61
- item[:description] =
91
+ item['description'] =
62
92
  node['description']
63
93
  .gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}
64
94
  .split("\n") if node.key?('description')
65
- item[:description].unshift("DEPRECATED: #{node['x-deprecation']}") if node.key?('x-deprecation')
95
+ item['description'].unshift("DEPRECATED: #{node['x-deprecation']}") if node.key?('x-deprecation')
66
96
  # Add flags for supported agents in doc
67
97
  agents = []
68
98
  Agent::Factory::ALL.each_key do |sym|
69
99
  agents.push(sym) if node['x-agents'].nil? || node['x-agents'].include?(sym.to_s)
70
100
  end
71
- Aspera.assert(agents.include?(:direct)){"#{name}: x-cli-option requires agent direct (or nil)"} if node['x-cli-option']
101
+ Aspera.assert(agents.include?(:direct)){"#{property_full_name}: x-cli-option requires agent direct (or nil)"} if node['x-cli-option']
72
102
  if @agent_columns
73
103
  Agent::Factory::ALL.each do |sym, names|
74
- item[names[:short]] = @formatter.tick(agents.include?(sym))
104
+ item[names[:short].to_s] = @formatter.tick(agents.include?(sym))
75
105
  end
76
106
  else
77
- item[:description].push("(#{agents.map{ |i| Agent::Factory::ALL[i][:short].to_s.upcase}.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
107
+ item['description'].push("(#{agents.map{ |i| Agent::Factory::ALL[i][:short].to_s.upcase}.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
78
108
  end
79
109
  # Only keep lines that are usable in supported agents
80
- next false if agents.empty?
81
- item[:description].push("Allowed values: #{node['enum'].map{ |v| @formatter.markdown_text("`#{v}`")}.join(', ')}.") if node.key?('enum')
82
- item[:description].push("Default: #{code.call(node['default'])}.") if node.key?('default')
110
+ next if agents.empty?
111
+ item['description'].push("Allowed values: #{node['enum'].map{ |v| @formatter.markdown_text("`#{v}`")}.join(', ')}.") if node.key?('enum')
112
+ item['description'].push("Default: #{@formatter.markdown_text("`#{node['default']}`")}.") if node.key?('default')
113
+ item['description'].push("Example: #{@formatter.markdown_text("`#{node['example']}`")}.") if node.key?('example')
83
114
  if @include_option
84
115
  envvar_prefix = ''
85
116
  cli_option =
@@ -87,18 +118,18 @@ module Aspera
87
118
  envvar_prefix = 'env:'
88
119
  node['x-cli-envvar']
89
120
  elsif node['x-cli-switch']
90
- node['x-cli-option']
121
+ false_part = node.key?('x-cli-false') ? " / #{node['x-cli-false']}" : ''
122
+ "#{node['x-cli-option']}#{false_part}"
91
123
  elsif node['x-cli-option']
92
124
  arg_type = node.key?('enum') ? '{enum}' : "{#{[node['type']].flatten.join('|')}}"
93
- # conversion_tag = node['x-cli-convert']
94
125
  conversion_tag = node.key?('x-cli-convert') ? 'conversion' : nil
95
126
  sep = node['x-cli-option'].start_with?('--') ? '=' : ' '
96
127
  "#{node['x-cli-option']}#{sep}#{"(#{conversion_tag})" if conversion_tag}#{arg_type}"
97
128
  end
98
129
  short = node.key?('x-cli-short') ? "(#{node['x-cli-short']})" : nil
99
- item[:description].push("(#{'special:' if node['x-cli-special']}#{envvar_prefix}#{@formatter.markdown_text("`#{cli_option}`")})#{short}") if cli_option
130
+ item['description'].push("(#{'special:' if node['x-cli-special']}#{envvar_prefix}#{@formatter.markdown_text("`#{cli_option}`")})#{short}") if cli_option
100
131
  end
101
- @rows.push(@formatter.check_row(item))
132
+ @sections.last[:rows].push(@formatter.check_row(item))
102
133
  end
103
134
  self
104
135
  end
@@ -7,11 +7,34 @@ module Aspera
7
7
  class Reader
8
8
  attr_reader :current
9
9
 
10
+ class << self
11
+ # Build a synthetic Reader from an OAS `parameters` array (entries with `in: query`).
12
+ # Produces a JSON Schema object whose `properties` map each query param name to its schema,
13
+ # with the OAS-level `description` and `required` merged in.
14
+ # @param params [Array<Hash>] raw OAS parameter objects (may contain path/header params too)
15
+ # @return [Reader]
16
+ def from_query_params(params)
17
+ properties = {}
18
+ required_names = []
19
+ params.each do |param|
20
+ next unless param['in'] == 'query'
21
+ name = param['name']
22
+ prop = (param['schema'] || {}).dup
23
+ prop['description'] = param['description'] if param['description'] && !prop.key?('description')
24
+ properties[name] = prop
25
+ required_names << name if param['required']
26
+ end
27
+ synthetic = {'type' => 'object', 'properties' => properties}
28
+ synthetic['required'] = required_names unless required_names.empty?
29
+ new(synthetic)
30
+ end
31
+ end
32
+
10
33
  # Shortcut to access current value at path
11
- # @param x [String] path element
34
+ # @param key [String] path element
12
35
  # @return [Hash, Array, String, Integer] current value at path
13
- def [](x)
14
- @current[x]
36
+ def [](key)
37
+ @current[key]
15
38
  end
16
39
 
17
40
  # Find sub path relative to current
@@ -31,6 +54,12 @@ module Aspera
31
54
  Reader.new(@root, current)
32
55
  end
33
56
 
57
+ # Resolve a $ref string to a Reader
58
+ def resolve_ref(ref)
59
+ Aspera.assert(ref.start_with?('#/')){"schema $ref must start with '#/': #{ref}"}
60
+ Reader.new(@root, @root.dig(*ref[2..].split('/')))
61
+ end
62
+
34
63
  # Read schema from file or from cache
35
64
  # @param root [Hash] root schema
36
65
  # @param current [Hash, nil] current position in
@@ -40,15 +69,35 @@ module Aspera
40
69
  @current = current || root
41
70
  end
42
71
 
43
- # Recursively traverse schema properties with a block
44
- # Handles nested objects and arrays automatically
45
- # @param prefix [String] Prefix for property names (e.g., 'parent.child.')
72
+ # Recursively traverse schema properties with a block.
73
+ # If the current node has `oneOf`, each variant is traversed in turn and
74
+ # `on_variant` is called (if given) before each variant's properties.
75
+ # @param prefix [String] Prefix for property names (e.g., 'parent.child.')
76
+ # @param on_variant [Proc, nil] Called with the variant Reader before its properties
46
77
  # @yield [property_schema, name, full_name] Yields property info to block
47
- # @yieldparam property_schema [Reader] Schema reader for this property (use .current to get node hash)
78
+ # @yieldparam property_schema [Reader] Schema reader for this property
48
79
  # @yieldparam name [String] Property name
49
80
  # @yieldparam full_name [String] Full property name with prefix
50
81
  # @return [nil]
51
- def each_property(prefix = '', &block)
82
+ def each_property(prefix = '', on_variant: nil, &block)
83
+ if @current.key?('oneOf')
84
+ # Build reverse map: $ref -> discriminant value, from discriminator.mapping if present
85
+ discriminant_by_ref = {}
86
+ if @current.dig('discriminator', 'mapping').is_a?(Hash)
87
+ @current['discriminator']['mapping'].each do |value, ref|
88
+ discriminant_by_ref[ref] = value
89
+ end
90
+ end
91
+ discriminant_property = @current.dig('discriminator', 'propertyName')
92
+ @current['oneOf'].each do |variant_node|
93
+ ref = variant_node['$ref']
94
+ variant_reader = ref ? resolve_ref(ref) : Reader.new(@root, variant_node)
95
+ discriminant_value = ref ? discriminant_by_ref[ref] : nil
96
+ on_variant&.call(variant_reader, discriminant_property, discriminant_value)
97
+ variant_reader.each_property(prefix, on_variant: on_variant, &block)
98
+ end
99
+ return
100
+ end
52
101
  properties = dig('properties')
53
102
  properties.current.each_key do |name|
54
103
  property_full_name = "#{prefix}#{name}"
@@ -61,11 +110,11 @@ module Aspera
61
110
  # Recursively process nested structures
62
111
  case node['type']
63
112
  when 'object'
64
- property_schema.each_property("#{property_full_name}.", &block) if node['properties']
113
+ property_schema.each_property("#{property_full_name}.", on_variant: on_variant, &block) if node['properties']
65
114
  when 'array'
66
115
  if node['items']
67
116
  array_item_schema = property_schema.dig('items')
68
- array_item_schema.each_property("#{property_full_name}[].", &block) if array_item_schema.current['properties']
117
+ array_item_schema.each_property("#{property_full_name}[].", on_variant: on_variant, &block) if array_item_schema.current['properties']
69
118
  end
70
119
  end
71
120
  end
@@ -18,11 +18,22 @@ module Aspera
18
18
  LOCATIONS.key?(sym)
19
19
  end
20
20
 
21
- # Get path to request body, no check if it exists
22
- # @return [String] path to request body
21
+ # Get path to request body schema, no check if it exists
22
+ # @param component [String] registry key (e.g. 'faspex', 'aoc')
23
+ # @param endpoint [String] endpoint path without leading slash (e.g. 'packages.post')
24
+ # @return [String] schema path usable in schema: keyword
23
25
  def req_body(component, endpoint)
24
26
  "#{component}:paths./#{endpoint}.requestBody.content.application/json.schema"
25
27
  end
28
+
29
+ # Get path to query parameters for a GET endpoint
30
+ # @param component [String] registry key (e.g. 'faspex', 'aoc')
31
+ # @param endpoint [String] resource path without leading slash (e.g. 'packages')
32
+ # @param method [String] HTTP method (default: 'get')
33
+ # @return [String] schema path usable in query_schema: keyword
34
+ def query_params(component, endpoint, method: 'get')
35
+ "#{component}:paths./#{endpoint}.#{method}#{QUERY_PARAMS_SUFFIX}"
36
+ end
26
37
  end
27
38
 
28
39
  LOCATIONS = {
@@ -42,18 +53,34 @@ module Aspera
42
53
  AOC = 'aoc'
43
54
  FASPEX = 'faspex'
44
55
  ASYNC_TABLES = 'async_tables'
45
- TRANSFER_INFO = "#{OPTIONS}:components.schemas.TransferInfo"
56
+ LOG_OPTIONS = "#{OPTIONS}:components.schemas.LogOptions"
57
+ DIRECT_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.DirectAgentOptions"
58
+ NODE_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.NodeAgentOptions"
59
+ HTTPGW_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.HttpgwAgentOptions"
60
+ TRANSFERD_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.TransferdAgentOptions"
61
+ TRANSFER_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.TransferAgentOptions"
62
+ SMTP_OPTIONS = "#{OPTIONS}:components.schemas.SmtpOptions"
63
+ HTTP_OPTIONS = "#{OPTIONS}:components.schemas.HttpOptions"
64
+ VAULT_OPTIONS = "#{OPTIONS}:components.schemas.VaultOptions"
65
+ IMAGE_OPTIONS = "#{OPTIONS}:components.schemas.ImageOptions"
66
+ PACKAGE_FOLDER_OPTIONS = "#{OPTIONS}:components.schemas.PackageFolderOptions"
46
67
 
47
68
  REQ_BODY = '.requestBody.content.application/json.schema'
69
+ # Suffix appended to a dotted path to signal query-param extraction in reader()
70
+ QUERY_PARAMS_SUFFIX = '.parameters'
48
71
 
49
72
  def initialize
50
73
  @cache = {}
51
74
  @main_folder = File.expand_path('../..', __dir__)
52
75
  end
53
76
 
54
- # Read schema from file or from cache
55
- # @param name_path [String] one of the keys in LOCATIONS, with optional :<path> suffix
56
- # @return [Reader] schema
77
+ # Read schema from file or from cache.
78
+ # When name_path ends with QUERY_PARAMS_SUFFIX, the OAS `parameters` array at that path
79
+ # is synthesised into an object schema via Reader.from_query_params instead of navigating
80
+ # into the tree.
81
+ # @param name_path [String] registry key with optional colon-separated dotted path suffix,
82
+ # e.g. "faspex:paths./packages.get.parameters" or "faspex:paths./packages.post.requestBody..."
83
+ # @return [Reader] schema reader
57
84
  def reader(name_path)
58
85
  name, path = name_path.split(':', 2)
59
86
  sym = name.to_sym
@@ -61,6 +88,12 @@ module Aspera
61
88
  spec_file = File.join(@main_folder, LOCATIONS[sym])
62
89
  @cache[sym] = Yaml.safe_load(File.read(spec_file)) if spec_file.end_with?('.yaml') && !@cache.key?(sym)
63
90
  @cache[sym] = JSON.parse(File.read(spec_file)) if spec_file.end_with?('.json') && !@cache.key?(sym)
91
+ # Query-params path: strip the suffix, navigate to the operation node, extract parameters
92
+ if path&.end_with?(QUERY_PARAMS_SUFFIX)
93
+ parent_path = path.delete_suffix(QUERY_PARAMS_SUFFIX)
94
+ node = @cache[sym].dig(*parent_path.split('.'))
95
+ return Reader.from_query_params(node&.fetch('parameters', []) || [])
96
+ end
64
97
  reader = Reader.new(@cache[sym])
65
98
  return reader unless path
66
99
  reader.dig(*path.split('.'))
@@ -45,7 +45,7 @@ module Aspera
45
45
  private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :KEY_FALSE_POSITIVES, :REGEX_LOG_REPLACES
46
46
  attr_accessor :log_secrets
47
47
 
48
- # @return new log formatter that hides secrets
48
+ # @return [Proc] new log formatter that hides secrets
49
49
  def log_formatter(original_formatter)
50
50
  original_formatter ||= Logger::Formatter.new
51
51
  # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
@@ -59,11 +59,16 @@ module Aspera
59
59
  end
60
60
  end
61
61
 
62
+ # Replace private key content in a string with the hidden password placeholder
63
+ # @param value [String] Input string possibly containing a private key
64
+ # @return [String] String with private key replaced by placeholder
62
65
  def hide_secrets_in_string(value)
63
66
  return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
64
67
  end
65
68
 
66
- # @return true if the key denotes a secret
69
+ # @param keyword [String, Symbol] Key name to check
70
+ # @param value [String] Associated value (must be a String to be a secret)
71
+ # @return [Boolean] true if the key denotes a secret
67
72
  def secret?(keyword, value)
68
73
  keyword = keyword.to_s if keyword.is_a?(Symbol)
69
74
  # only Strings can be secrets, not booleans, or hash, arrays
data/lib/aspera/ssl.rb CHANGED
@@ -12,16 +12,16 @@ module Aspera
12
12
  @extra_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
13
13
  attr_reader :extra_options
14
14
 
15
- def option_list=(v)
16
- Aspera.assert_type(v, Array){'ssl_options'}
17
- v.each do |opt|
15
+ def option_list=(options)
16
+ Aspera.assert_type(options, Array){'ssl_options'}
17
+ options.each do |opt|
18
18
  Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
19
19
  case opt
20
20
  when Integer
21
21
  @extra_options = opt
22
22
  when String
23
23
  name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
24
- raise Cli::BadArgument, "Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}" if !OpenSSL::SSL.const_defined?(name)
24
+ Aspera.assert(OpenSSL::SSL.const_defined?(name), type: Cli::BadArgument){"Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}"}
25
25
  if opt.start_with?('-')
26
26
  @extra_options &= ~OpenSSL::SSL.const_get(name)
27
27
  else
@@ -637,7 +637,7 @@ properties:
637
637
  Partial transfers may exist if communication disruptions caused the underlying ascp processes to terminate early.
638
638
  Note that transfer resumption can only happen if the `reset` option is disabled.
639
639
  If an async session starts with `reset` enabled and resume enabled, transfers interrupted during that
640
- session will be resumeable, but only if async is then restarted with 'reset' disabled.
640
+ session will be resumable, but only if async is then restarted with 'reset' disabled.
641
641
  type: object
642
642
  properties:
643
643
  enabled:
@@ -654,7 +654,7 @@ properties:
654
654
  Sets the age limit in days for temporary files that will be preserved
655
655
  on cleanup (usually at async's start and stop) for potential transfer resume.
656
656
  Temp files older than the given value will be removed
657
- regardless of whether they might be resumeable.
657
+ regardless of whether they might be resumable.
658
658
  type: integer
659
659
  default: "5"
660
660
  x-cli-option: --resume-age-days
@@ -4,6 +4,7 @@
4
4
 
5
5
  require 'aspera/ascp/installation'
6
6
  require 'aspera/agent/direct'
7
+ require 'aspera/exec_spec'
7
8
  require 'aspera/command_line_converter'
8
9
  require 'aspera/command_line_builder'
9
10
  require 'aspera/log'
@@ -90,10 +91,7 @@ module Aspera
90
91
  Log.dump(:sync_params_initial, sync_info)
91
92
  Aspera.assert_type(sync_info, Hash)
92
93
  Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
93
- env_args = {
94
- args: [],
95
- env: {}
96
- }
94
+ exec_spec = ExecSpec.new(exec: :async)
97
95
  if sync_info.key?('local')
98
96
  # `conf` format
99
97
  Aspera.assert_type(sync_info['local'], Hash){'local'}
@@ -115,10 +113,10 @@ module Aspera
115
113
  remote['private_key_paths'].concat(add_certificates)
116
114
  end
117
115
  # '--exclusive-mgmt-port=12345', '--arg-err-path=-',
118
- env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_info))}"]
116
+ exec_spec.args = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_info))}"]
119
117
  Log.dump(:sync_conf, sync_info)
120
118
  agent = Agent::Direct.new
121
- agent.start_and_monitor_process(session: {}, name: :async, **env_args)
119
+ agent.start_and_monitor_process(session: {}, exec_spec: exec_spec)
122
120
  else
123
121
  # `args` format
124
122
  raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
@@ -141,16 +139,16 @@ module Aspera
141
139
  Aspera.assert_type(sync_info['instance'], Hash)
142
140
  instance_builder = CommandLineBuilder.new(sync_info['instance'], ARGS_INSTANCE_SCHEMA, CommandLineConverter)
143
141
  instance_builder.process_params
144
- instance_builder.add_env_args(env_args)
142
+ instance_builder.add_to_exec_spec(exec_spec)
145
143
  end
146
144
  sync_info['sessions'].each do |session_params|
147
145
  Aspera.assert_type(session_params, Hash)
148
146
  Aspera.assert(session_params.key?('name'), 'session must contain at least: name')
149
147
  session_builder = CommandLineBuilder.new(session_params, ARGS_SESSION_SCHEMA, CommandLineConverter)
150
148
  session_builder.process_params
151
- session_builder.add_env_args(env_args)
149
+ session_builder.add_to_exec_spec(exec_spec)
152
150
  end
153
- Environment.secure_execute(Ascp::Installation.instance.path(:async), *env_args[:args], env: env_args[:env])
151
+ Environment.secure_execute(Ascp::Installation.instance.path(:async), *exec_spec.args, env: exec_spec.env)
154
152
  end
155
153
  return
156
154
  end
@@ -216,7 +214,7 @@ module Aspera
216
214
  return sync_info['local_db_dir']
217
215
  elsif (local_path = sync_info.dig('local', 'path'))
218
216
  return local_path
219
- elsif exception
217
+ else
220
218
  raise Error, 'Missing either local_db_dir or local.path'
221
219
  end
222
220
  else
@@ -226,11 +224,10 @@ module Aspera
226
224
  return session['local_db_dir']
227
225
  elsif session.key?('local_dir')
228
226
  return session['local_dir']
229
- elsif exception
227
+ else
230
228
  raise Error, 'Missing either local_db_dir or local_dir'
231
229
  end
232
230
  end
233
- nil
234
231
  end
235
232
 
236
233
  def session_name(sync_info)
@@ -325,7 +322,7 @@ module Aspera
325
322
  end
326
323
  end
327
324
  path, props = find_option(CONF_SCHEMA, [], option)
328
- raise "Option not found: #{option}" if path.nil?
325
+ Aspera.assert(!path.nil?){"Option not found: #{option}"}
329
326
  last_key = path.pop
330
327
  # navigate in the current result to insert the value
331
328
  current = result
@@ -52,7 +52,7 @@ module Aspera
52
52
  end
53
53
 
54
54
  # Ensure that provided folder exists, or create it, generate a unique filename
55
- # @return path to that unique file
55
+ # @return [String] path to that unique file
56
56
  def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
57
57
  FileUtils.mkdir_p(temp_folder)
58
58
  new_file = File.join(temp_folder, [prefix, SecureRandom.uuid, suffix].compact.join('-'))
@@ -3,7 +3,7 @@
3
3
  module Aspera
4
4
  # trigger returns true only if the delay has passed since the last trigger
5
5
  class TimerLimiter
6
- # @param delay in seconds (float)
6
+ # @param delay [Float] delay in seconds
7
7
  def initialize(delay)
8
8
  @delay = delay
9
9
  @last_trigger_time = nil
@@ -11,42 +11,55 @@ module Aspera
11
11
  SIZE_UNITS = %w[k m g t p e].freeze
12
12
  private_constant :SCHEME, :PREFIX, :SIZE_UNITS
13
13
  class << self
14
- # @return nil if not a faux: scheme, else a FauxFile instance
14
+ # Parse a faux URI and return a `FauxFile` instance, or `nil` if the URI does not use the faux scheme.
15
+ # URI format: `faux:///<path>?<size>`, where `<size>` is a decimal integer with an optional
16
+ # case-insensitive unit suffix: `k`, `m`, `g`, `t`, `p`, `e` (powers of 1024).
17
+ # When no suffix is given, the size is interpreted as raw bytes.
18
+ # Examples: `faux:///file.bin?10` (10 bytes), `faux:///file.bin?10m` (10 MiB)
19
+ # @param name [String] source file name, possibly a faux URI
20
+ # @return [FauxFile, nil] `nil` if not a faux scheme, else a `FauxFile` instance
15
21
  def create(name)
16
22
  return unless name.start_with?(PREFIX)
17
23
  name_params = name.delete_prefix(PREFIX).split('?', 2)
18
- raise Error, 'Format: #{PREFIX}<file path>?<size>' unless name_params.length.eql?(2)
19
- raise Error, "Format: <integer>[#{SIZE_UNITS.join(',')}]" unless (m = name_params[1].downcase.match(/^(\d+)([#{SIZE_UNITS.join('')}])$/))
20
- size = m[1].to_i
21
- suffix = m[2]
22
- SIZE_UNITS.each do |s|
23
- size *= 1024
24
- break if s.eql?(suffix)
25
- end
24
+ Aspera.assert(name_params.length.eql?(2), type: Error){"Format: #{PREFIX}<file path>?<size>"}
25
+ m = name_params[1].downcase.match(/^(\d+)([#{SIZE_UNITS.join('')}]?)$/)
26
+ Aspera.assert(m, type: Error){"Format: <integer>[#{SIZE_UNITS.join(',')}]"}
27
+ size = m[2].empty? ? m[1].to_i : m[1].to_i * (1024**(SIZE_UNITS.index(m[2]) + 1))
26
28
  return FauxFile.new(name_params[0], size)
27
29
  end
28
30
  end
31
+ # @return [String] virtual file path
32
+ # @return [Integer] total size in bytes
29
33
  attr_reader :path, :size
30
34
 
35
+ # @param path [String] virtual file path (from the faux URI)
36
+ # @param size [Integer] total number of bytes to produce
31
37
  def initialize(path, size)
32
38
  @path = path
33
39
  @size = size
34
40
  @offset = 0
35
- # we cache large chunks, anyway most of them will be the same size
41
+ # cache chunks by size so repeated reads of the same chunk length reuse the same buffer
36
42
  @chunk_by_size = {}
37
43
  end
38
44
 
45
+ # Read up to `chunk_size` bytes from the stream and advance the internal offset.
46
+ # Returns `nil` when the stream is exhausted.
47
+ # @param chunk_size [Integer] maximum number of bytes to read
48
+ # @return [String, nil] null-byte string of the bytes actually read, or `nil` at EOF
39
49
  def read(chunk_size)
40
50
  return if eof?
41
51
  bytes_to_read = [chunk_size, @size - @offset].min
42
52
  @offset += bytes_to_read
43
- @chunk_by_size[bytes_to_read] = "\x00" * bytes_to_read unless @chunk_by_size.key?(bytes_to_read)
53
+ @chunk_by_size[bytes_to_read] ||= "\x00" * bytes_to_read
44
54
  return @chunk_by_size[bytes_to_read]
45
55
  end
46
56
 
57
+ # No-op: required to satisfy the IO-like interface used by transfer agents.
58
+ # @return [nil]
47
59
  def close
48
60
  end
49
61
 
62
+ # @return [Boolean] true when all bytes have been read
50
63
  def eof?
51
64
  return @offset >= @size
52
65
  end