aspera-cli 4.27.5 → 4.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +160 -4
  4. data/CONTRIBUTING.md +4 -2
  5. data/README.md +1 -1
  6. data/bin/asession +1 -1
  7. data/docs/README.md +1086 -331
  8. data/docs/test-mcp-with-ai.md +5 -3
  9. data/lib/aspera/agent/direct.rb +15 -4
  10. data/lib/aspera/agent/factory.rb +2 -1
  11. data/lib/aspera/agent/node.rb +1 -1
  12. data/lib/aspera/api/cos_node.rb +1 -1
  13. data/lib/aspera/api/faspex.rb +1 -1
  14. data/lib/aspera/api/node.rb +6 -2
  15. data/lib/aspera/ascp/installation.rb +47 -11
  16. data/lib/aspera/ascp/management.rb +89 -43
  17. data/lib/aspera/cli/ascp_actions.rb +5 -19
  18. data/lib/aspera/cli/bootstrapper.rb +1 -2
  19. data/lib/aspera/cli/command_registry.rb +43 -3
  20. data/lib/aspera/cli/command_spec.rb +3 -0
  21. data/lib/aspera/cli/completion/ascli.bash +39 -5
  22. data/lib/aspera/cli/completion/ascli.fish +16 -8
  23. data/lib/aspera/cli/completion/ascli.ps1 +41 -0
  24. data/lib/aspera/cli/completion/ascli.zsh +20 -4
  25. data/lib/aspera/cli/completion_actions.rb +360 -0
  26. data/lib/aspera/cli/error.rb +42 -0
  27. data/lib/aspera/cli/extended_value.rb +30 -5
  28. data/lib/aspera/cli/formatter.rb +118 -56
  29. data/lib/aspera/cli/hints.rb +0 -1
  30. data/lib/aspera/cli/http.rb +6 -5
  31. data/lib/aspera/cli/info.rb +10 -4
  32. data/lib/aspera/cli/mailer.rb +1 -1
  33. data/lib/aspera/cli/mcp/http_servlet.rb +156 -0
  34. data/lib/aspera/cli/mcp/secrets.rb +143 -0
  35. data/lib/aspera/cli/mcp/tool.rb +231 -0
  36. data/lib/aspera/cli/option_value.rb +9 -1
  37. data/lib/aspera/cli/options.schema.yaml +390 -39
  38. data/lib/aspera/cli/parser.rb +14 -5
  39. data/lib/aspera/cli/plugins/aoc.rb +8 -11
  40. data/lib/aspera/cli/plugins/base.rb +36 -15
  41. data/lib/aspera/cli/plugins/config.rb +15 -44
  42. data/lib/aspera/cli/plugins/console.rb +3 -1
  43. data/lib/aspera/cli/plugins/factory.rb +1 -1
  44. data/lib/aspera/cli/plugins/faspex5.rb +9 -19
  45. data/lib/aspera/cli/plugins/faspio.rb +36 -5
  46. data/lib/aspera/cli/plugins/httpgw.rb +1 -0
  47. data/lib/aspera/cli/plugins/mcp.rb +177 -118
  48. data/lib/aspera/cli/plugins/node.rb +14 -8
  49. data/lib/aspera/cli/plugins/orchestrator.rb +351 -38
  50. data/lib/aspera/cli/plugins/server.rb +3 -1
  51. data/lib/aspera/cli/plugins/shares.rb +205 -35
  52. data/lib/aspera/cli/preset_actions.rb +15 -7
  53. data/lib/aspera/cli/preset_manager.rb +24 -2
  54. data/lib/aspera/cli/prompt.rb +3 -1
  55. data/lib/aspera/cli/result.rb +17 -4
  56. data/lib/aspera/cli/runner.rb +133 -43
  57. data/lib/aspera/cli/security.rb +134 -0
  58. data/lib/aspera/cli/table_fitter.rb +116 -0
  59. data/lib/aspera/cli/terminal_formatter.rb +23 -34
  60. data/lib/aspera/cli/transfer_agent.rb +4 -1
  61. data/lib/aspera/cli/vault_manager.rb +6 -5
  62. data/lib/aspera/cli/version.rb +1 -1
  63. data/lib/aspera/cli/web_secret.html.erb +61 -0
  64. data/lib/aspera/cli/web_secret.rb +220 -0
  65. data/lib/aspera/cli/wizard.rb +19 -9
  66. data/lib/aspera/command_line_builder.rb +26 -0
  67. data/lib/aspera/data_repository.rb +38 -24
  68. data/lib/aspera/duration.rb +42 -0
  69. data/lib/aspera/environment.rb +88 -24
  70. data/lib/aspera/exec_spec.rb +6 -5
  71. data/lib/aspera/faspex_gw.rb +98 -32
  72. data/lib/aspera/faspex_postproc.rb +187 -44
  73. data/lib/aspera/formatter_interface.rb +0 -1
  74. data/lib/aspera/id_generator.rb +2 -1
  75. data/lib/aspera/json_rpc/client.rb +51 -32
  76. data/lib/aspera/json_rpc/version.rb +1 -0
  77. data/lib/aspera/keychain/base.rb +5 -0
  78. data/lib/aspera/keychain/encrypted_hash.rb +6 -2
  79. data/lib/aspera/keychain/factory.rb +2 -3
  80. data/lib/aspera/keychain/hashicorp_vault.rb +7 -4
  81. data/lib/aspera/keychain/macos_security.rb +119 -13
  82. data/lib/aspera/keychain/one_password_api.rb +8 -3
  83. data/lib/aspera/keychain/one_password_base.rb +14 -9
  84. data/lib/aspera/keychain/one_password_cli.rb +4 -2
  85. data/lib/aspera/log.rb +3 -6
  86. data/lib/aspera/markdown.rb +152 -115
  87. data/lib/aspera/mime.rb +6 -0
  88. data/lib/aspera/nagios.rb +3 -1
  89. data/lib/aspera/node_emulator.rb +98 -101
  90. data/lib/aspera/oauth/{base.rb → client/base.rb} +5 -4
  91. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +3 -2
  92. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +7 -0
  93. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +1 -1
  94. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +1 -1
  95. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +1 -1
  96. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +1 -1
  97. data/lib/aspera/oauth/{web.rb → client/web.rb} +2 -2
  98. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  99. data/lib/aspera/oauth/server/pkce.rb +426 -0
  100. data/lib/aspera/orchestrator_workflow_checker.rb +561 -0
  101. data/lib/aspera/preview/terminal.rb +3 -2
  102. data/lib/aspera/rainbow.rb +27 -1
  103. data/lib/aspera/rest/aspera_errors.rb +51 -59
  104. data/lib/aspera/rest/client.rb +21 -14
  105. data/lib/aspera/rest/error_analyzer.rb +9 -0
  106. data/lib/aspera/rest/list.rb +6 -6
  107. data/lib/aspera/rest/util.rb +7 -1
  108. data/lib/aspera/rest.rb +0 -1
  109. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -2
  110. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4001 -0
  111. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  112. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +35 -35
  113. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  114. data/lib/aspera/schema/documentation.rb +50 -78
  115. data/lib/aspera/schema/reader.rb +39 -11
  116. data/lib/aspera/schema/registry.rb +8 -2
  117. data/lib/aspera/secret_hider.rb +94 -15
  118. data/lib/aspera/secret_strength.rb +114 -0
  119. data/lib/aspera/servlet_helpers.rb +111 -0
  120. data/lib/aspera/sync/args.schema.yaml +3 -1
  121. data/lib/aspera/sync/conf.schema.yaml +16 -6
  122. data/lib/aspera/sync/operations.rb +21 -1
  123. data/lib/aspera/temp_file_manager.rb +79 -20
  124. data/lib/aspera/transfer/parameters.rb +5 -0
  125. data/lib/aspera/transfer/spec.rb +52 -12
  126. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  127. data/lib/aspera/web_auth/index.html.erb +16 -0
  128. data/lib/aspera/web_auth/server.rb +18 -10
  129. data/lib/aspera/web_page.html.erb +262 -0
  130. data/lib/aspera/web_page.rb +61 -0
  131. data/lib/aspera/web_server_simple.rb +76 -10
  132. data.tar.gz.sig +0 -0
  133. metadata +47 -32
  134. metadata.gz.sig +0 -0
  135. data/lib/aspera/cli/mcp_tool.rb +0 -189
  136. data/lib/aspera/oauth.rb +0 -9
  137. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  138. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  139. data/lib/aspera/web_auth/index.html +0 -147
@@ -6,7 +6,10 @@ require 'aspera/environment'
6
6
  require 'aspera/yaml'
7
7
  require 'aspera/log'
8
8
  require 'aspera/assert'
9
+ require 'aspera/secret_hider'
9
10
  require 'aspera/cli/error'
11
+ require 'aspera/cli/prompt'
12
+ require 'aspera/cli/security'
10
13
  require 'json'
11
14
  require 'base64'
12
15
  require 'zlib'
@@ -73,6 +76,16 @@ module Aspera
73
76
  Aspera.assert(value.empty?, type: BadArgument) { "no value allowed for extended value type: #{ext_type}" }
74
77
  end
75
78
 
79
+ # Ask the user for a secret: on a web page if the user interface is graphical (option `ui`), else on terminal
80
+ # @param prompt [String] Name of the secret, empty for default
81
+ # @return [String] The secret
82
+ def read_secret(prompt)
83
+ prompt = 'secret' if prompt.empty?
84
+ return Prompt.prompt_user_input(prompt, sensitive: true) unless Environment.instance.url_method.eql?(:graphical)
85
+ require 'aspera/cli/web_secret'
86
+ WebSecret.ask(prompt)
87
+ end
88
+
76
89
  def read_stdin(mode)
77
90
  case mode
78
91
  when '' then $stdin.read
@@ -95,7 +108,7 @@ module Aspera
95
108
  base64: lambda { |i| Base64.decode64(i) },
96
109
  csvt: lambda { |i| ExtendedValue.decode_csvt(i) },
97
110
  env: lambda { |i| ENV.fetch(i, nil) },
98
- file: lambda { |i| File.read(File.expand_path(i)) },
111
+ file: lambda { |i| File.read(Security.instance.readable_path(i)) },
99
112
  uri: lambda { |i| UriReader.read(i) },
100
113
  json: lambda { |i| ExtendedValue.JSON_parse(i) },
101
114
  lines: lambda { |i| i.split("\n") },
@@ -105,7 +118,7 @@ module Aspera
105
118
  re: lambda { |i| Regexp.new(i, Regexp::MULTILINE) },
106
119
  ruby: lambda { |i| Environment.secure_eval(i, __FILE__, __LINE__) },
107
120
  s: lambda(&:to_s),
108
- secret: lambda { |i| prompt = i.empty? ? 'secret' : i; $stdin.getpass("#{prompt}> ") }, # rubocop:disable Style/Semicolon
121
+ secret: lambda { |i| ExtendedValue.read_secret(i) },
109
122
  stdin: lambda { |i| ExtendedValue.read_stdin(i) },
110
123
  yaml: lambda { |i| YAML.safe_load(i) },
111
124
  zlib: lambda { |i| Zlib::Inflate.inflate(i) },
@@ -170,8 +183,15 @@ module Aspera
170
183
  break if SPECIAL_HANDLERS.include?(handler)
171
184
  end
172
185
  Log.log.trace1 { "evaluating: #{handlers_reversed}, value: #{value.class}:#{value}" }
173
- handlers_reversed.each do |handler|
186
+ handlers_reversed.each_with_index do |handler, index|
187
+ Security.instance.check_extended_value(handler)
188
+ # The value read by the previous handler may be a secret
189
+ Security.instance.check_secret_processing("@#{handler}") { SecretHider.instance.contains_secret?(value) } if index.positive?
174
190
  value = @handlers[handler].call(value)
191
+ # Values read from a secret source are hidden wherever they appear: logs, results
192
+ if SECRET_HANDLERS.include?(handler)
193
+ value.is_a?(String) ? SecretHider.instance.add_secret_values(value) : SecretHider.instance.add_secrets_in(value)
194
+ end
175
195
  rescue => e
176
196
  raise BadArgument, "Evaluation of #{handler} for #{context}: #{e.message}"
177
197
  end
@@ -184,7 +204,10 @@ module Aspera
184
204
  while (m = value.match(@regex_extend))
185
205
  sub_value = "@#{m[2]}:#{m[3]}"
186
206
  Log.log.debug { "evaluating #{sub_value}" }
187
- value = "#{m[1]}#{evaluate(sub_value, context: 'composite extended value')}#{m[4]}"
207
+ sub_value = evaluate(sub_value, context: 'composite extended value')
208
+ # In a text, a secret is not the whole value
209
+ Security.instance.check_secret_processing('@extend') { SecretHider.instance.contains_secret?(sub_value) }
210
+ value = "#{m[1]}#{sub_value}#{m[4]}"
188
211
  end
189
212
  return value
190
213
  end
@@ -199,10 +222,12 @@ module Aspera
199
222
  # :extend includes processing of other handlers in itself
200
223
  # :val keeps the value intact
201
224
  SPECIAL_HANDLERS = %i[extend val].freeze
225
+ # Handlers that read a secret
226
+ SECRET_HANDLERS = %i[secret vault].freeze
202
227
 
203
228
  # Array and Hash types:
204
229
  DEFAULT_PARSER_TYPES = [Array, Hash].freeze
205
- private_constant :MARKER_START, :MARKER_END, :MARKER_IN_END, :SPECIAL_HANDLERS, :DEFAULT_PARSER_TYPES
230
+ private_constant :MARKER_START, :MARKER_END, :MARKER_IN_END, :SPECIAL_HANDLERS, :SECRET_HANDLERS, :DEFAULT_PARSER_TYPES
206
231
  end
207
232
  end
208
233
  end
@@ -5,6 +5,7 @@ require 'aspera/schema/registry'
5
5
  require 'aspera/cli/option_declarator'
6
6
  require 'aspera/cli/special_values'
7
7
  require 'aspera/cli/terminal_formatter'
8
+ require 'aspera/cli/table_fitter'
8
9
  require 'aspera/preview/terminal'
9
10
  require 'aspera/secret_hider'
10
11
  require 'aspera/environment'
@@ -12,12 +13,11 @@ require 'aspera/log'
12
13
  require 'aspera/assert'
13
14
  require 'aspera/markdown'
14
15
  require 'aspera/dot_container'
15
- require 'terminal-table'
16
16
  require 'tty-spinner'
17
+ require 'json'
17
18
  require 'yaml'
18
19
  require 'pp'
19
20
  require 'csv'
20
- require 'word_wrap'
21
21
  require 'aspera/rainbow'
22
22
  using Rainbow
23
23
 
@@ -37,10 +37,12 @@ module Aspera
37
37
  SINGLE_OBJECT_COLUMN_NAMES = %i[field value].freeze
38
38
  # Terminal: vertical separator for list of strings.
39
39
  STR_LST_SEP_VERT = "\n"
40
+ # Sub-options of option `out`, for help and errors
41
+ OUT_SUB_OPTIONS = 'format, level, file, fields, select, query, table[.pivot], flat, secrets, colors, utf8, img'
40
42
 
41
- private_constant :FIELDS_LESS, :DISPLAY_FORMATS, :DISPLAY_LEVELS, :SINGLE_OBJECT_COLUMN_NAMES, :STR_LST_SEP_VERT
43
+ private_constant :FIELDS_LESS, :DISPLAY_FORMATS, :DISPLAY_LEVELS, :SINGLE_OBJECT_COLUMN_NAMES, :STR_LST_SEP_VERT, :OUT_SUB_OPTIONS
42
44
 
43
- option :out, description: 'Output rendering options (dot-notation: format, level, file, fields, select, table[.pivot], flat, secrets, colors, utf8, img)', schema: Schema::Registry::OUT_OPTIONS
45
+ option :out, description: "Output rendering options (dot-notation: #{OUT_SUB_OPTIONS})", schema: Schema::Registry::OUT_OPTIONS
44
46
  option :display, description: 'Output only some information', allowed: DISPLAY_LEVELS, default: :data, deprecation: {last: '4.27.0', message: 'use --out.level'}
45
47
  option :format, description: 'Output format (also: --out.format)', allowed: DISPLAY_FORMATS, default: :table
46
48
  option :output, description: 'Destination for results', deprecation: {last: '4.27.0', message: 'use --out.file'}
@@ -56,35 +58,23 @@ module Aspera
56
58
 
57
59
  class << self
58
60
  # Replace special values with a readable version on terminal
61
+ # @param input_hash [Hash] Object to display (not modified)
62
+ # @param string_list_separator [String] Separator of the items of a list of strings
63
+ # @return [Hash] Copy of input_hash with special values replaced, also in nested objects
59
64
  def replace_specific_for_terminal(input_hash, string_list_separator)
60
- hash_to_process = [input_hash]
61
- until hash_to_process.empty?
62
- current = hash_to_process.pop
63
- current.each do |key, value|
64
- case value
65
- when NilClass
66
- current[key] = TerminalFormatter.special_format('null')
67
- when String
68
- current[key] = TerminalFormatter.special_format('empty string') if value.empty?
69
- when Proc
70
- current[key] = TerminalFormatter.special_format('lambda')
71
- when Array
72
- if value.empty?
73
- current[key] = TerminalFormatter.special_format('empty list')
74
- elsif value.all?(String)
75
- current[key] = value.join(string_list_separator)
76
- else
77
- value.each do |item|
78
- hash_to_process.push(item) if item.is_a?(Hash)
79
- end
80
- end
81
- when Hash
82
- if value.empty?
83
- current[key] = TerminalFormatter.special_format('empty dict')
84
- else
85
- hash_to_process.push(value)
86
- end
65
+ input_hash.transform_values do |value|
66
+ case value
67
+ when NilClass then TerminalFormatter.special_format('null')
68
+ when String then value.empty? ? TerminalFormatter.special_format('empty string') : value
69
+ when Proc then TerminalFormatter.special_format('lambda')
70
+ when Array
71
+ if value.empty? then TerminalFormatter.special_format('empty list')
72
+ elsif value.all?(String) then value.join(string_list_separator)
73
+ else value.map { |item| item.is_a?(Hash) ? replace_specific_for_terminal(item, string_list_separator) : item }
87
74
  end
75
+ when Hash
76
+ value.empty? ? TerminalFormatter.special_format('empty dict') : replace_specific_for_terminal(value, string_list_separator)
77
+ else value
88
78
  end
89
79
  end
90
80
  end
@@ -102,8 +92,12 @@ module Aspera
102
92
  @options = {}
103
93
  # Last value of option `out`
104
94
  @out_dispatched = {}
105
- # Colors auto-detected by Rainbow at load time, restored when `--out.colors` is reset
95
+ # Colors auto-detected at load time (aspera/rainbow), restored when `--out.colors` is reset
106
96
  @colors_detected = Rainbow.enabled
97
+ # Value of `--out.colors`: `true`, `false`, or `nil` for detection
98
+ @colors = nil
99
+ # `true` if results are written to a file (`--out.file`)
100
+ @output_to_file = false
107
101
  @spinner = nil
108
102
  end
109
103
 
@@ -185,9 +179,12 @@ module Aspera
185
179
  when :flat then @parser.set_option(:flat_hash, v, warn_deprecation: false)
186
180
  when :secrets then @parser.set_option(:show_secrets, v, warn_deprecation: false)
187
181
  when :img then @parser.set_option(:image, v, warn_deprecation: false)
188
- when :colors then Rainbow.enabled = v.nil? ? @colors_detected : v
182
+ when :query then self.query = v
189
183
  when :utf8 then Environment.unicode = v
190
- else Aspera.error_unexpected_value(k) { 'out sub-option (format, level, file, fields, select, table[.pivot], flat, secrets, colors, utf8, img)' }
184
+ when :colors
185
+ @colors = v
186
+ apply_colors
187
+ else Aspera.error_unexpected_value(k) { "out sub-option (#{OUT_SUB_OPTIONS})" }
191
188
  end
192
189
  end
193
190
  return
@@ -199,11 +196,13 @@ module Aspera
199
196
  # The default output level depends on the format, unless the level is set explicitly (e.g. `--out.level`)
200
197
  @options[:display] = value.eql?(:table) ? :info : :data if @parser.nil? || @parser.option_def(:display).source.eql?(:default)
201
198
  when :output
202
- $stdout = if value.eql?('-')
203
- STDOUT # rubocop:disable Style/GlobalStdStream
204
- else
199
+ @output_to_file = !value.eql?('-')
200
+ $stdout = if @output_to_file
205
201
  File.open(value, 'w')
202
+ else
203
+ STDOUT # rubocop:disable Style/GlobalStdStream
206
204
  end
205
+ apply_colors
207
206
  when :image
208
207
  # get list if key arguments of method
209
208
  allowed_options = Preview::Terminal.method(:build).parameters.select { |i| i[0].eql?(:key) }.map { |i| i[1] }
@@ -214,6 +213,28 @@ module Aspera
214
213
  nil
215
214
  end
216
215
 
216
+ # Enable colors as set by `--out.colors`, else as detected, but not when results go to a file
217
+ # @return [nil]
218
+ def apply_colors
219
+ Rainbow.enabled = @colors.nil? ? @colors_detected && !@output_to_file : @colors
220
+ nil
221
+ end
222
+
223
+ # Set the query of `--out.query`, syntax of a JMESPath expression is checked before the command is executed
224
+ # @param expression [String, Proc, nil] JMESPath expression, or Ruby lambda, or `nil` for none
225
+ def query=(expression)
226
+ Aspera.assert_type(expression, String, Proc, NilClass, type: Cli::BadArgument) { 'out.query' }
227
+ if expression.is_a?(String)
228
+ require 'jmespath'
229
+ begin
230
+ JMESPath::Parser.new.parse(expression)
231
+ rescue JMESPath::Errors::SyntaxError => e
232
+ raise Cli::BadArgument, "JMESPath: #{e.message}"
233
+ end
234
+ end
235
+ @options[:query] = expression
236
+ end
237
+
217
238
  # Main output method for displaying messages to the user
218
239
  # This method is used by Result classes to output formatted data
219
240
  # @param message_level [Symbol] The level of the message - must be one of: :data, :info, :error
@@ -238,14 +259,15 @@ module Aspera
238
259
  display_message(:info, status, **kwargs)
239
260
  end
240
261
 
241
- # Check if secrets should be hidden
262
+ # Check if secrets should be hidden: always, unless explicitly requested (`--out.secrets=yes`)
242
263
  def hide_secrets?
243
- !@options[:show_secrets] && !@options[:display].eql?(:data)
264
+ !@options[:show_secrets]
244
265
  end
245
266
 
246
- # Hides secrets in Hash or Array
267
+ # @param data [Object] Data of a result
268
+ # @return [Object] Copy of data with secrets hidden (by key name, and declared values), or data itself if secrets are shown
247
269
  def hide_secrets(data)
248
- SecretHider.instance.deep_remove_secret(data) if hide_secrets?
270
+ hide_secrets? ? SecretHider.instance.hidden_copy(data) : data
249
271
  end
250
272
 
251
273
  # Display results using the Visitor pattern
@@ -260,8 +282,10 @@ module Aspera
260
282
  Log.dump(:data, result.data, level: :trace1)
261
283
  Log.dump(:fields, result.fields, level: :trace1)
262
284
 
263
- # Hide secrets in data
264
- hide_secrets(result.data)
285
+ # Hide secrets in a copy of data: it may be shared (e.g. a preset of the configuration)
286
+ result = result.with_data(hide_secrets(result.data)) if hide_secrets?
287
+ # Secrets are hidden before the expression may rename their keys
288
+ result = query_result(result) unless @options[:query].nil?
265
289
 
266
290
  # Use the Visitor pattern: delegate formatting to the result object
267
291
  result.format(self)
@@ -357,9 +381,9 @@ module Aspera
357
381
  # no filter for single element
358
382
  return data unless data.is_a?(Array)
359
383
  # by default, keep all data intact
360
- return data if @options[:fields].eql?(SpecialValues::DEF) && @options[:select].nil?
384
+ return data if @options[:fields].eql?(SpecialValues::DEF) && [nil, {}].include?(@options[:select])
361
385
  Aspera.assert_array_all(data, Hash) { 'filter or select' }
362
- filter_columns_on_select(data)
386
+ data = filter_columns_on_select(data)
363
387
  return data if @options[:fields].eql?(SpecialValues::DEF)
364
388
  selected_fields = compute_fields(data, @options[:fields])
365
389
  return data.map { |i| i[selected_fields.first] } if selected_fields.length == 1
@@ -367,20 +391,54 @@ module Aspera
367
391
  end
368
392
 
369
393
  # Filter the list of items on the select option
370
- # @param data [Array<Hash>] List of items
371
- # @return [nil]
394
+ # @param data [Array<Hash>] List of items (not modified)
395
+ # @return [Array<Hash>] Selected items
372
396
  def filter_columns_on_select(data)
373
397
  case @options[:select]
374
398
  when Proc
375
399
  begin
376
- data.select! { |i| @options[:select].call(i) }
400
+ data.select { |i| @options[:select].call(i) }
377
401
  rescue StandardError => e
378
402
  raise Cli::BadArgument, "Error in user-provided ruby lambda code during select: #{e.message}"
379
403
  end
380
404
  when Hash
381
- @options[:select].each { |k, v| data.select! { |i| i[k].eql?(v) } }
405
+ @options[:select].reduce(data) { |items, (k, v)| items.select { |i| i[k].eql?(v) } }
406
+ else data
382
407
  end
383
- nil
408
+ end
409
+
410
+ # Apply the query of `--out.query` to the data of an object or list result
411
+ # @param result [Result] Result of command
412
+ # @return [Result] Result with the value of the expression, or `result` if no query, or not an object or list
413
+ def query_result(result)
414
+ return result if @options[:query].nil?
415
+ data =
416
+ case result
417
+ when Result::SingleObject, Result::ObjectList, Result::ValueList then result.data
418
+ when Result::Empty then []
419
+ else return result
420
+ end
421
+ value =
422
+ if @options[:query].is_a?(Proc)
423
+ begin
424
+ # JSON view of the value, as with JMESPath: keys, and objects like `Time`, become strings
425
+ JSON.parse(JSON.generate(@options[:query].call(data)))
426
+ rescue StandardError => e
427
+ raise Cli::BadArgument, "Error in user-provided ruby lambda code during query: #{e.message}"
428
+ end
429
+ else
430
+ begin
431
+ JMESPath.search(@options[:query], data)
432
+ rescue JMESPath::Errors::Error => e
433
+ raise Cli::BadArgument, "JMESPath: #{e.message}"
434
+ end
435
+ end
436
+ # Lists of objects, and lists of scalar values are displayed as tables
437
+ return Result.auto(value) unless value.is_a?(Array) && !value.all?(Hash) && !value.all? { |i| Result::SCALAR_TYPES.include?(i.class) }
438
+ Aspera.assert(Result::STRUCTURED_FORMATS.include?(@options[:format]), type: Cli::BadArgument) do
439
+ "Result of query is a list of lists or of mixed values: use a JMESPath multi-select hash (e.g. `[].{a: x, b: y}`), or a format among: #{Result::STRUCTURED_FORMATS.join(', ')}"
440
+ end
441
+ Result.new(data: value)
384
442
  end
385
443
 
386
444
  # Displays a list of objects
@@ -396,11 +454,11 @@ module Aspera
396
454
  display_message(:info, TerminalFormatter.special_format('empty')) if @options[:format].eql?(:table)
397
455
  return
398
456
  end
399
- filter_columns_on_select(object_array)
457
+ object_array = filter_columns_on_select(object_array)
400
458
  format_style = @options[:table_style].symbolize_keys
401
459
  string_list_separator = format_style.delete(:str_lst_sep) || STR_LST_SEP_VERT
402
460
  # convert data to string, and keep only display fields
403
- object_array.each { |i| self.class.replace_specific_for_terminal(i, string_list_separator) }
461
+ object_array = object_array.map { |i| self.class.replace_specific_for_terminal(i, string_list_separator) }
404
462
  # if table has only one element, and only one field, display the value
405
463
  if object_array.length == 1 && fields.length == 1
406
464
  Log.log.debug("single element, field: #{fields.first}")
@@ -423,22 +481,26 @@ module Aspera
423
481
  case @options[:format]
424
482
  when :table
425
483
  format_style[:border] = :unicode_round if Environment.terminal_supports_unicode?
484
+ # fit in the requested maximum width, or in the terminal
485
+ width = format_style.delete(:width) || Environment.terminal_columns
426
486
  if single || @options[:multi_single].eql?(:yes) ||
427
487
  (@options[:multi_single].eql?(:single) && final_table_rows.length.eql?(1))
428
488
  # display multiple objects as multiple transposed tables
429
489
  final_table_rows.each do |row|
430
- display_message(:data, Terminal::Table.new(
490
+ display_message(:data, TableFitter.table(
431
491
  headings: SINGLE_OBJECT_COLUMN_NAMES,
432
492
  rows: fields.zip(row),
433
- style: format_style
493
+ style: format_style,
494
+ width: width
434
495
  ))
435
496
  end
436
497
  else
437
498
  # display the table as single table
438
- display_message(:data, Terminal::Table.new(
499
+ display_message(:data, TableFitter.table(
439
500
  headings: fields,
440
501
  rows: final_table_rows,
441
- style: format_style
502
+ style: format_style,
503
+ width: width
442
504
  ))
443
505
  end
444
506
  when :csv
@@ -137,7 +137,6 @@ module Aspera
137
137
  matches = hint[:match]
138
138
  matches = [matches] unless matches.is_a?(Array)
139
139
  matches.each do |m|
140
- Aspera.assert_values(m.class, [String, Regexp])
141
140
  case m
142
141
  when String
143
142
  next unless message.eql?(m)
@@ -7,7 +7,7 @@ require 'aspera/line_logger'
7
7
  require 'aspera/schema/registry'
8
8
  require 'aspera/cli/option_declarator'
9
9
  require 'aspera/rest'
10
- require 'aspera/oauth'
10
+ require 'aspera/oauth/client/factory'
11
11
  require 'aspera/ssl'
12
12
  require 'openssl'
13
13
 
@@ -28,7 +28,7 @@ module Aspera
28
28
 
29
29
  # `on_set` callbacks are methods of the instance given as `target:` to `declare_options`
30
30
  option :insecure, description: 'HTTP/S: Do not validate any certificate', allowed: Type::BOOLEAN, default: false, on_set: :insecure=
31
- option :ignore_certificate, description: 'HTTP/S: Do not validate certificate for these URLs', allowed: [Array, NilClass], on_set: :ignore_cert_host_port=
31
+ option :ignore_certificate, description: 'HTTP/S: Do not validate certificate for these URLs', allowed: Type::STRING_ARRAY, on_set: :ignore_cert_host_port=
32
32
  option :warn_insecure, description: 'HTTP/S: Issue a warning if certificate is ignored', allowed: Type::BOOLEAN, default: true, on_set: :warn_insecure=
33
33
  option :cert_stores, description: 'HTTP/S: List of folder with trusted certificates', allowed: Type::STRING_ARRAY, on_set: :trusted_cert_locations=
34
34
  option :http_options, schema: Schema::Registry::HTTP_OPTIONS, on_set: :http_options=
@@ -87,9 +87,10 @@ module Aspera
87
87
  # Per-URL certificate ignore list
88
88
  # ------------------------------------------------------------------
89
89
 
90
- # @param url_list [Array<String>, nil] URLs for which certificate is not validated
90
+ # @param url_list [Array<String>] URLs for which certificate is not validated
91
91
  def ignore_cert_host_port=(url_list)
92
- @ignore_cert_host_port = (url_list || []).map do |url|
92
+ Aspera.assert_type(url_list, Array) { 'ignore certificate URLs' }
93
+ @ignore_cert_host_port = url_list.map do |url|
93
94
  uri = URI.parse(url)
94
95
  Aspera.assert(uri.scheme.eql?('https')) { "Expecting https scheme: #{url}" }
95
96
  [uri.host, uri.port].freeze
@@ -148,7 +149,7 @@ module Aspera
148
149
  paths_to_add.each do |p|
149
150
  pp = [File.realpath(p)]
150
151
  if File.directory?(p)
151
- pp = Dir.entries(p)
152
+ pp = Dir.children(p)
152
153
  .map { |e| File.realpath(File.join(p, e)) }
153
154
  .select { |entry| File.file?(entry) }
154
155
  .select { |entry| CERT_EXT.any? { |ext| entry.end_with?(ext) } }
@@ -12,14 +12,20 @@ module Aspera
12
12
  GEM_URL = "https://rubygems.org/gems/#{GEM_NAME}"
13
13
  SRC_URL = 'https://github.com/IBM/aspera-cli'
14
14
  CONTAINER = 'docker.io/martinlaurent/ascli'
15
+ # Minimum required Ruby version: single reference, used by `required_ruby_version` in gemspec file and by the documentation
16
+ # Policy: a Ruby version is supported until 18 months after its end of official support
17
+ # Consistency with other files: `rake tools:check_ruby`
18
+ RUBY_MINIMUM_VERSION = '3.2'
15
19
  # Set this to warn in advance when minimum required ruby version will increase
16
- # See also required_ruby_version in gemspec file
17
- RUBY_FUTURE_MINIMUM_VERSION = '3.2'
20
+ RUBY_FUTURE_MINIMUM_VERSION = '3.3'
18
21
  # Version with which this version of CLI was tested
19
22
  SDK_VERSION = '1.1.9'
20
- # RubyInstaller version packaged in the Windows installer zip (release tag without `RubyInstaller-` prefix)
23
+ # Ruby version with which this version of CLI was tested, packaged in container image and portable packages
24
+ RUBY_TESTED_VERSION = '4.0.7'
25
+ # Suffix of the RubyInstaller release of `RUBY_TESTED_VERSION`, packaged in the Windows zip
26
+ # (release tag: `RubyInstaller-<Ruby version><suffix>`)
21
27
  # https://github.com/oneclick/rubyinstaller2/releases
22
- WINDOWS_RUBY_INSTALLER_VERSION = '4.0.7-1'
28
+ WINDOWS_RUBY_INSTALLER_EXT = '-2'
23
29
  end
24
30
  end
25
31
  end
@@ -58,7 +58,7 @@ module Aspera
58
58
  %i[server port domain].each do |n|
59
59
  Aspera.assert(smtp.key?(n)) { "Missing mandatory smtp parameter: #{n}" }
60
60
  end
61
- Log.dump(:smtp, smtp.reject { |k, _| k == :password })
61
+ Log.dump(:smtp, smtp)
62
62
  return smtp
63
63
  end
64
64
 
@@ -0,0 +1,156 @@
1
+ # frozen_string_literal: true
2
+
3
+ # cspell:ignore webrick
4
+
5
+ require 'aspera/log'
6
+ require 'aspera/servlet_helpers'
7
+ require 'aspera/cli/web_secret'
8
+ require 'json'
9
+ require 'openssl'
10
+ require 'stringio'
11
+ require 'webrick'
12
+
13
+ module Aspera
14
+ module Cli
15
+ module Mcp
16
+ # Servlet of the HTTP transport of the MCP server, on WEBrick (already a project dependency), mounted on `/` of a WebServerSimple.
17
+ # Rack 3 removed Rack::Handler: the servlet calls the Rack app of the MCP transport directly.
18
+ #
19
+ # Routes:
20
+ # / Discovery: server metadata, no authentication
21
+ # /.well-known/oauth-authorization-server, /oauth/*: OAuth 2.0 PKCE server, if enabled
22
+ # /secret/<id> Secret pages (see WebSecret), protected by their access code
23
+ # other paths MCP endpoints: static Bearer token or OAuth access token, if configured
24
+ # Requests that a web page could send are refused on all paths (see ServletHelpers#check_request),
25
+ # as the MCP transport does for its own endpoints, with the same `allowed_hosts` and `allowed_origins`.
26
+ class HttpServlet < WEBrick::HTTPServlet::AbstractServlet
27
+ include ServletHelpers
28
+
29
+ # Paths with a JSON body, not handled by the MCP transport
30
+ JSON_PATHS = %w[/oauth/register].freeze
31
+ private_constant :JSON_PATHS
32
+
33
+ # @param server [WEBrick::HTTPServer]
34
+ # @param options [Hash] With keys:
35
+ # app: [#call] Rack app of the MCP transport
36
+ # server_info: [Hash] Name, version and description of the server, for discovery
37
+ # web_secret: [WebSecret] Secret pages
38
+ # auth_token: [String, nil] Static Bearer token, if any
39
+ # oauth_server: [OAuth::Server::Pkce, nil] OAuth server, if enabled
40
+ # allowed_hosts: [Array<String>, nil] Other values accepted in header `Host`
41
+ # allowed_origins: [Array<String>, nil] Other values accepted in header `Origin`
42
+ def initialize(server, options)
43
+ super(server)
44
+ @app = options.fetch(:app)
45
+ @server_info = options.fetch(:server_info)
46
+ @web_secret = options.fetch(:web_secret)
47
+ @auth_token = options[:auth_token]
48
+ @oauth_server = options[:oauth_server]
49
+ @allowed = {allowed_hosts: Array(options[:allowed_hosts]), allowed_origins: Array(options[:allowed_origins])}
50
+ end
51
+
52
+ def do_GET(req, res) = dispatch(req, res)
53
+ def do_POST(req, res) = dispatch(req, res)
54
+ def do_DELETE(req, res) = dispatch(req, res)
55
+
56
+ private
57
+
58
+ def dispatch(req, res)
59
+ check_request(req, json: JSON_PATHS.include?(req.path), **@allowed)
60
+ case req.path
61
+ when '/'
62
+ # Discovery endpoint (server metadata, no auth required)
63
+ json_response(res, 200, @server_info)
64
+ when '/.well-known/oauth-authorization-server' then oauth(req, res, :handle_discovery)
65
+ when '/oauth/register' then oauth(req, res, :handle_register, post_only: true)
66
+ when '/oauth/authorize' then oauth(req, res, :handle_authorize)
67
+ when '/oauth/token' then oauth(req, res, :handle_token, post_only: true)
68
+ when ->(path) { path.start_with?(WebSecret::PATH_PREFIX) }
69
+ # Page opened by the user in a browser (no bearer token), protected by the access code
70
+ @web_secret.handle(req, res)
71
+ else
72
+ # All other paths are MCP endpoints: require authorization
73
+ return unauthorized(res) unless authorized?(req)
74
+ call_app(req, res)
75
+ end
76
+ rescue WEBrick::HTTPStatus::Error => e
77
+ Log.log.warn { "Rejected request: #{e.message}" }
78
+ json_error(res, e.code, 'invalid_request', e.message)
79
+ end
80
+
81
+ # Route to the OAuth server, if enabled
82
+ # @param handler [Symbol] Method of the OAuth server
83
+ # @param post_only [Boolean] Accept only method POST
84
+ def oauth(req, res, handler, post_only: false)
85
+ return json_error(res, 405, 'method_not_allowed') if post_only && !req.request_method.eql?('POST')
86
+ return json_error(res, 404, 'not_found') if @oauth_server.nil?
87
+ @oauth_server.public_send(handler, req, res)
88
+ end
89
+
90
+ # @return [Boolean] true if no auth is configured, or the request carries a valid token
91
+ def authorized?(req)
92
+ return true if @auth_token.nil? && @oauth_server.nil?
93
+ token = authorization(req, 'Bearer')
94
+ return false if token.nil?
95
+ # Static token, or OAuth access token
96
+ (!@auth_token.nil? && OpenSSL.secure_compare(token, @auth_token.to_s)) ||
97
+ (!@oauth_server.nil? && @oauth_server.access_token_valid?(token))
98
+ end
99
+
100
+ def unauthorized(res)
101
+ json_error(res, 401, 'Unauthorized', headers: {'WWW-Authenticate' => @oauth_server.nil? ? 'Bearer' : @oauth_server.www_authenticate_header})
102
+ end
103
+
104
+ # Call the Rack app, with streaming of the response body if it is a callable (SSE)
105
+ def call_app(req, res)
106
+ status, headers, rack_body = @app.call(rack_env(req))
107
+ res.status = status
108
+ headers.each { |k, v| res[k] = v }
109
+ if rack_body.respond_to?(:call)
110
+ res.chunked = true
111
+ res.body = proc do |socket|
112
+ stream = Object.new
113
+ stream.define_singleton_method(:write) { |data| socket.write(data) }
114
+ stream.define_singleton_method(:flush) { socket.flush rescue nil }
115
+ stream.define_singleton_method(:close) { socket.close rescue nil }
116
+ rack_body.call(stream)
117
+ end
118
+ else
119
+ buf = +''
120
+ rack_body.each { |chunk| buf << chunk }
121
+ rack_body.close if rack_body.respond_to?(:close)
122
+ res.body = buf
123
+ end
124
+ end
125
+
126
+ # @return [Hash] Rack environment of the WEBrick request
127
+ def rack_env(req)
128
+ {
129
+ 'REQUEST_METHOD' => req.request_method,
130
+ 'SCRIPT_NAME' => '',
131
+ 'PATH_INFO' => req.path,
132
+ 'QUERY_STRING' => req.query_string || '',
133
+ 'SERVER_NAME' => req.host,
134
+ 'SERVER_PORT' => req.port.to_s,
135
+ 'HTTP_VERSION' => req.http_version,
136
+ 'rack.input' => StringIO.new(req.body.to_s),
137
+ 'rack.errors' => $stderr,
138
+ 'rack.url_scheme' => req.ssl? ? 'https' : 'http',
139
+ 'rack.multithread' => true,
140
+ 'rack.multiprocess' => false,
141
+ 'rack.run_once' => false
142
+ }.tap do |env|
143
+ req.header.each do |key, values|
144
+ http_key = "HTTP_#{key.upcase.tr('-', '_')}"
145
+ case key.downcase
146
+ when 'content-type' then env['CONTENT_TYPE'] = values.first
147
+ when 'content-length' then env['CONTENT_LENGTH'] = values.first
148
+ else env[http_key] = values.join(', ')
149
+ end
150
+ end
151
+ end
152
+ end
153
+ end
154
+ end
155
+ end
156
+ end