aspera-cli 4.29.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +2 -1
  3. data/CHANGELOG.md +76 -2
  4. data/README.md +20 -1
  5. data/SECURITY.md +14 -0
  6. data/docs/README.md +277 -38
  7. data/lib/aspera/api/node.rb +9 -2
  8. data/lib/aspera/ascp/installation.rb +26 -4
  9. data/lib/aspera/ascp/log_analyzer/lines.rb +2 -2
  10. data/lib/aspera/ascp/log_analyzer.rb +19 -3
  11. data/lib/aspera/ascp/log_locations.rb +163 -0
  12. data/lib/aspera/cli/ascp_logs_actions.rb +12 -0
  13. data/lib/aspera/cli/bootstrapper.rb +1 -0
  14. data/lib/aspera/cli/command_line.rb +39 -2
  15. data/lib/aspera/cli/command_registry.rb +3 -0
  16. data/lib/aspera/cli/command_spec.rb +13 -0
  17. data/lib/aspera/cli/error.rb +3 -0
  18. data/lib/aspera/cli/extended_value.rb +5 -4
  19. data/lib/aspera/cli/formatter.rb +14 -8
  20. data/lib/aspera/cli/http.rb +26 -1
  21. data/lib/aspera/cli/info.rb +1 -1
  22. data/lib/aspera/cli/mcp/tool.rb +5 -3
  23. data/lib/aspera/cli/option_value.rb +14 -8
  24. data/lib/aspera/cli/options.schema.yaml +156 -19
  25. data/lib/aspera/cli/parser.rb +15 -3
  26. data/lib/aspera/cli/plugins/aoc.rb +4 -2
  27. data/lib/aspera/cli/plugins/ats.rb +1 -1
  28. data/lib/aspera/cli/plugins/base.rb +33 -6
  29. data/lib/aspera/cli/plugins/config.rb +12 -1
  30. data/lib/aspera/cli/plugins/faspex5.rb +3 -1
  31. data/lib/aspera/cli/plugins/mcp.rb +14 -0
  32. data/lib/aspera/cli/plugins/node.rb +8 -8
  33. data/lib/aspera/cli/plugins/oauth.rb +18 -5
  34. data/lib/aspera/cli/plugins/orchestrator.rb +28 -24
  35. data/lib/aspera/cli/plugins/preview.rb +3 -1
  36. data/lib/aspera/cli/plugins/server.rb +3 -3
  37. data/lib/aspera/cli/plugins/shares.rb +1 -2
  38. data/lib/aspera/cli/plugins/transferd.rb +7 -2
  39. data/lib/aspera/cli/poll.rb +236 -0
  40. data/lib/aspera/cli/preset_manager.rb +63 -19
  41. data/lib/aspera/cli/prompt.rb +153 -8
  42. data/lib/aspera/cli/runner.rb +25 -7
  43. data/lib/aspera/cli/security.rb +10 -0
  44. data/lib/aspera/cli/sync_actions.rb +2 -0
  45. data/lib/aspera/cli/transfer_agent.rb +14 -0
  46. data/lib/aspera/cli/transfer_progress.rb +6 -5
  47. data/lib/aspera/cli/version.rb +1 -1
  48. data/lib/aspera/environment.rb +3 -1
  49. data/lib/aspera/id_generator.rb +16 -3
  50. data/lib/aspera/inflate.rb +43 -0
  51. data/lib/aspera/oauth/client/base.rb +38 -8
  52. data/lib/aspera/oauth/client/factory.rb +35 -15
  53. data/lib/aspera/oauth/client/json_credentials.rb +3 -2
  54. data/lib/aspera/oauth/client/shares_web.rb +13 -4
  55. data/lib/aspera/oauth/client/url_json.rb +3 -2
  56. data/lib/aspera/oauth/client/web.rb +2 -2
  57. data/lib/aspera/orchestrator_workflow_checker.rb +10 -1
  58. data/lib/aspera/preview/terminal.rb +8 -2
  59. data/lib/aspera/products/desktop.rb +13 -1
  60. data/lib/aspera/rest/client.rb +23 -5
  61. data/lib/aspera/schema/registry.rb +1 -0
  62. data/lib/aspera/sealed_hash.rb +2 -18
  63. data/lib/aspera/secret/hider.rb +13 -1
  64. data/lib/aspera/secret/store/one_password_cli.rb +13 -11
  65. data/lib/aspera/sync/operations.rb +4 -4
  66. data/lib/aspera/transfer/result.rb +20 -0
  67. data/lib/aspera/transfer/spec.rb +2 -2
  68. data/lib/aspera/{transfer/units.rb → units/decimal.rb} +4 -12
  69. data/lib/aspera/units/duration.rb +109 -0
  70. data/lib/aspera/uri_reader.rb +11 -6
  71. data/lib/aspera/web_auth/reply.html.erb +1 -1
  72. data/llms-install.md +88 -0
  73. data/llms.txt +6 -0
  74. data/skills/ascli/SKILL.md +6 -0
  75. data.tar.gz.sig +0 -0
  76. metadata +7 -3
  77. metadata.gz.sig +0 -0
  78. data/lib/aspera/duration.rb +0 -45
@@ -52,11 +52,14 @@ module Aspera
52
52
  # Loads YAML from cloud with locations of SDK archives for all platforms
53
53
  # Tries each URL of option `locations_url` (default if empty) in order, until one provides a valid list.
54
54
  # All but the last URL are read with a short timeout and no retry, so that fallback is quick.
55
+ # The request tells which application reads the list, and what it downloads (header `Referer`, see `referer_url`):
56
+ # the default URL counts requests per `Referer` (download statistics).
57
+ # @param download [Hash, nil] SDK about to be downloaded (`sdk`: version, `platform`), `nil` if the list is only read
55
58
  # @return [Array<Hash>] location structure
56
- def sdk_locations
59
+ def sdk_locations(download: nil)
57
60
  Aspera.assert_type(@transferd_urls, Array) { 'locations_url' }
58
61
  location_urls = @transferd_urls.empty? ? TRANSFERD_ARCHIVE_LOCATION_URLS : @transferd_urls
59
- return UriReader.read_first(location_urls, what: 'SDK locations') do |content|
62
+ return UriReader.read_first(location_urls, what: 'SDK locations', headers: locations_headers(download)) do |content|
60
63
  locations = Yaml.safe_load(content)
61
64
  Aspera.assert_type(locations, Array) { 'SDK locations' }
62
65
  locations
@@ -234,6 +237,11 @@ module Aspera
234
237
  ascp_data = file_paths
235
238
  ascp_data.merge!(ascp_info_from_log)
236
239
  ascp_data.merge!(ascp_info_from_file)
240
+ # `ascp` is run with `-L-` to read its paths: report where it logs without `-L`
241
+ if ascp_data['log'].eql?('-')
242
+ require_relative 'log_locations'
243
+ ascp_data['log'] = LogLocations.ascp_log_location(path(:ascp), root: ascp_data['root'], conf_file: ascp_data['conf'])
244
+ end
237
245
  return ascp_data
238
246
  end
239
247
 
@@ -241,8 +249,8 @@ module Aspera
241
249
  # @param version [nil, String] Version, if nil: latest
242
250
  # @return [Hash] the entry of SDK locations for the given platform and version
243
251
  def sdk_location_for_platform(platform: nil, version: nil)
244
- all_locations = sdk_locations
245
252
  platform = Environment.instance.architecture if platform.nil?
253
+ all_locations = sdk_locations(download: {sdk: version || LATEST_VERSION, platform: platform})
246
254
  locations = all_locations.select { |l| l['platform'].eql?(platform) }
247
255
  Aspera.assert(!locations.empty?) { "No SDK for platform: #{platform}, available: #{all_locations.map { |i| i['platform'] }.uniq}" }
248
256
  version = locations.max_by { |entry| Gem::Version.new(entry['version']) }['version'] if version.nil?
@@ -435,8 +443,18 @@ module Aspera
435
443
  # @return [Array<String>] URLs of SDK locations, tried in order
436
444
  attr_accessor :transferd_urls
437
445
 
446
+ # @return [String, nil] URL that identifies the application and its version, sent in header `Referer` when SDK locations are read, `nil`: not sent
447
+ attr_accessor :referer_url
448
+
438
449
  private
439
450
 
451
+ # @param download [Hash, nil] See `sdk_locations`
452
+ # @return [Hash] HTTP headers of the request for SDK locations
453
+ def locations_headers(download)
454
+ return {} if @referer_url.nil?
455
+ {'Referer' => download.nil? ? @referer_url : "#{@referer_url}?#{URI.encode_www_form(download)}"}
456
+ end
457
+
440
458
  # Run the block with another SDK folder (files of the SDK are located from it), then set the previous one back
441
459
  # @param folder [String] Temporary SDK folder
442
460
  def with_sdk_directory(folder)
@@ -476,14 +494,18 @@ module Aspera
476
494
  'https://ibm.biz/sdk_location',
477
495
  'https://raw.githubusercontent.com/IBM/aspera-cli/refs/heads/main/docs/sdk_location.yaml'
478
496
  ].freeze
497
+ # Version reported in statistics when the latest version is requested (not known before the list is read)
498
+ LATEST_VERSION = 'latest'
479
499
  # filename for ascp with optional extension (Windows)
480
- private_constant :DEFAULT_ASPERA_CONF, :EXE_FILES, :SDK_FILES, :TRANSFERD_ARCHIVE_LOCATION_URLS
500
+ private_constant :DEFAULT_ASPERA_CONF, :EXE_FILES, :SDK_FILES, :TRANSFERD_ARCHIVE_LOCATION_URLS, :LATEST_VERSION
481
501
 
482
502
  def initialize
483
503
  # cache for installed products found
484
504
  @found_products = nil
485
505
  # URLs of SDK locations, tried in order
486
506
  @transferd_urls = TRANSFERD_ARCHIVE_LOCATION_URLS
507
+ # set by the application, `nil`: no `Referer` sent
508
+ @referer_url = nil
487
509
  # product selected with option `sdk_folder`, or nil to use SDK folder
488
510
  @ascp_product = nil
489
511
  # cache for folder of `ascp` of selected product
@@ -4,11 +4,11 @@
4
4
  # SPDX-License-Identifier: Apache-2.0
5
5
 
6
6
  # cspell:words getlocal MSGID PROCID rsyslog
7
- require_relative '../log_analyzer'
8
-
9
7
  module Aspera
10
8
  module Ascp
11
9
  class LogAnalyzer
10
+ # Path that designates the standard input
11
+ STDIN_PATH = '-'
12
12
  # Formats and timestamps of log lines: shared by LogFile and Records, which define `@path` and `@timezone` (nil: time zone of the analyst)
13
13
  module Lines
14
14
  MONTHS = %w[Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec].freeze
@@ -21,8 +21,6 @@ module Aspera
21
21
  class LogAnalyzer
22
22
  # Oldest ascp version for which the analysis is designed
23
23
  MIN_VERSION = '4.0'
24
- # Path that designates the standard input
25
- STDIN_PATH = '-'
26
24
  # First two bytes of a gzip file
27
25
  GZIP_MAGIC = "\x1F\x8B".b.freeze
28
26
  # Files analyzed when scanning a folder
@@ -132,6 +130,16 @@ module Aspera
132
130
  end
133
131
  end
134
132
 
133
+ # Reads the file up to its first `ascp` line only
134
+ # @param path [String] log file, plain or compressed with gzip
135
+ # @return [Boolean] true if the file contains `ascp` log lines
136
+ def ascp_log?(path)
137
+ read_lines(path, Encoding::BINARY) { |line| return true if line.match?(Lines::ASCP_LINE) }
138
+ false
139
+ rescue Zlib::Error, Errno::EPIPE
140
+ false
141
+ end
142
+
135
143
  private
136
144
 
137
145
  # `ascp` creates `<name>.log` as a hard link to the current rotated file `<name>.<n>.log`: keep only one of them.
@@ -147,7 +155,15 @@ module Aspera
147
155
 
148
156
  # @return [Array<String>] files, with the log files of folder added
149
157
  def scan_folder(root, folder, files)
150
- Dir.children(folder).sort.each do |name|
158
+ children = begin
159
+ Dir.children(folder)
160
+ rescue SystemCallError => e
161
+ # A sub-folder that cannot be read is skipped, the folder given is not
162
+ raise if folder.eql?(root)
163
+ Log.log.warn { "Ignored, cannot read #{folder}: #{e.message}" }
164
+ []
165
+ end
166
+ children.sort.each do |name|
151
167
  next if name.start_with?('.')
152
168
  path = File.join(folder, name)
153
169
  if File.directory?(path)
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../environment'
7
+ require_relative '../log'
8
+ require_relative '../products/desktop'
9
+ require_relative 'log_analyzer'
10
+ require 'json'
11
+ require 'set'
12
+ require 'xmlsimple'
13
+
14
+ module Aspera
15
+ module Ascp
16
+ # Location of the `ascp` log files of the local system.
17
+ # `ascp` logs in the folder of option `-L` (set by Connect, transferd, IBM Aspera for Desktop),
18
+ # else in `<logging><directory>` of `aspera.conf`, else in a default location that depends on the OS:
19
+ # syslog on Linux and Unix, `~/Library/Logs/Aspera` on macOS, `var\log` of the product (or folder of `ascp`) on Windows.
20
+ module LogLocations
21
+ # Linux and Unix: `ascp` logs to syslog (facility `local2`), in one of these files depending on the syslog configuration
22
+ SYSLOG_FILES = %w[/var/log/aspera.log /var/log/messages /var/log/syslog].freeze
23
+ # Log location of `ascp` on Linux and Unix without `-L` nor `<logging><directory>`
24
+ SYSLOG = 'syslog'
25
+ # Source of a location that is not specific to a product
26
+ DEFAULT_SOURCE = 'ascp default'
27
+ # Source of a location given by an `ascp` found in `PATH`, but not in the locations of known products
28
+ PATH_SOURCE = 'ascp in PATH'
29
+ # Names of the log files written by `ascp` in a folder (`-L` or `aspera.conf`): current file, rotations, compressed
30
+ ASCP_FILE = /\A(?:aspera-scp-transfer|ascp4?)(?:\.\d+)?\.log(?:\.gz)?\z/
31
+ class << self
32
+ # @param products [Array<Hash>] installed products, see `Installation#installed_products`
33
+ # @param defaults [Array<Array>] default locations: `[path, source]`
34
+ # @return [Array<Hash>] files that contain `ascp` log lines: `path`, `size`, `modified`, `source`, most recent first
35
+ def find(products, defaults: default_locations)
36
+ seen = Set.new
37
+ # Default locations first: they are shared by products (e.g. `~/Library/Logs/Aspera`)
38
+ locations = defaults + (products + path_products(products)).flat_map { |product| product_locations(product) }
39
+ files = locations.flat_map { |location, source| location_files(location).map { |file| [file, source] } }
40
+ files.filter_map do |file, source|
41
+ stat = File.stat(file)
42
+ # Same file in several locations, or `<name>.log` hard link to `<name>.<n>.log`
43
+ next unless seen.add?(stat.ino.zero? ? File.expand_path(file) : [stat.dev, stat.ino])
44
+ next unless LogAnalyzer.ascp_log?(file)
45
+ {'path' => file, 'size' => stat.size, 'modified' => stat.mtime, 'source' => source}
46
+ end.sort_by { |row| -row['modified'].to_f }
47
+ end
48
+
49
+ # @return [Array<Array>] locations not specific to an installed product: `[path, source]`
50
+ def default_locations
51
+ locations = case Environment.instance.os
52
+ when Environment::OS_WINDOWS then []
53
+ when Environment::OS_MACOS then [File.join(Dir.home, 'Library', 'Logs', 'Aspera'), File.join('', 'Library', 'Logs', 'Aspera')]
54
+ else SYSLOG_FILES
55
+ end.map { |path| [path, DEFAULT_SOURCE] }
56
+ desktop = Products::Desktop.log_folder
57
+ locations.push([desktop, Products::Desktop::APP_NAME]) unless desktop.nil?
58
+ service = transferd_service_log_folder
59
+ locations.push([service, 'transferd service']) unless service.nil?
60
+ locations
61
+ end
62
+
63
+ # Location of the log of `ascp` when it runs without option `-L`
64
+ # @param ascp [String] path of `ascp`
65
+ # @param root [String, nil] root folder of the product, as reported by `ascp` (Windows)
66
+ # @param conf_file [String, nil] `aspera.conf` used by `ascp`
67
+ # @return [String] log folder, or `syslog` (Linux and Unix)
68
+ def ascp_log_location(ascp, root: nil, conf_file: nil)
69
+ folder = conf_directory(conf_file)
70
+ return folder unless folder.nil?
71
+ case Environment.instance.os
72
+ when Environment::OS_MACOS
73
+ File.join(Dir.home.start_with?('/var/root') ? '' : Dir.home, 'Library', 'Logs', 'Aspera')
74
+ when Environment::OS_WINDOWS
75
+ var_log = File.join(root || File.dirname(File.dirname(ascp)), 'var', 'log')
76
+ File.directory?(var_log) ? var_log : File.dirname(ascp)
77
+ else SYSLOG
78
+ end
79
+ end
80
+
81
+ private
82
+
83
+ # @param products [Array<Hash>] installed products
84
+ # @return [Array<Hash>] products for the `ascp` found in `PATH` that are not one of `products`
85
+ def path_products(products)
86
+ known = products.filter_map { |product| real_path(product[:ascp_path]) }
87
+ ENV.fetch('PATH', '').split(File::PATH_SEPARATOR).filter_map do |folder|
88
+ ascp = real_path(File.join(folder, Environment.instance.exe_file('ascp')))
89
+ next if ascp.nil? || known.include?(ascp)
90
+ known.push(ascp)
91
+ bin = File.dirname(ascp)
92
+ {name: PATH_SOURCE, ascp_path: ascp, app_root: File.basename(bin).eql?('bin') ? File.dirname(bin) : bin}
93
+ end
94
+ end
95
+
96
+ # @param product [Hash] installed product
97
+ # @return [Array<Array>] locations of logs of the product: `[path, source]`
98
+ def product_locations(product)
99
+ locations = [product[:log_root], conf_log_folder(product), File.join(product[:app_root], 'var', 'log')]
100
+ # Windows: without `var\log`, `ascp` logs in its own folder
101
+ locations.push(File.dirname(product[:ascp_path])) if Environment.instance.os.eql?(Environment::OS_WINDOWS)
102
+ locations.compact.uniq.map { |path| [path, product[:name]] }
103
+ end
104
+
105
+ # @param product [Hash] installed product
106
+ # @return [String, nil] `<default><logging><directory>` of `aspera.conf` of the product: in `etc`, or in the folder of `ascp` (SDK)
107
+ def conf_log_folder(product)
108
+ conf_directory([File.join(product[:app_root], 'etc', 'aspera.conf'), File.join(File.dirname(product[:ascp_path]), 'aspera.conf')].find { |f| File.file?(f) })
109
+ end
110
+
111
+ # @param conf_file [String, nil] `aspera.conf`
112
+ # @return [String, nil] `<default><logging><directory>` of the file
113
+ def conf_directory(conf_file)
114
+ return if conf_file.nil? || !File.file?(conf_file)
115
+ folder = XmlSimple.xml_in(File.read(conf_file), {'ForceArray' => false}).dig('default', 'logging', 'directory')
116
+ folder if folder.is_a?(String) && !folder.empty?
117
+ rescue StandardError => e
118
+ Log.log.debug { "Cannot read #{conf_file}: #{e.message}" }
119
+ nil
120
+ end
121
+
122
+ # @return [String, nil] `fasp_runtime.log.dir` (folder given to `ascp -L`) of the transferd service installed by `transferd service install`
123
+ def transferd_service_log_folder
124
+ require_relative '../products/transferd_service'
125
+ file = Products::TransferdService::CONFIG_FILE
126
+ return unless File.file?(file) && File.readable?(file)
127
+ folder = JSON.parse(File.read(file)).dig('fasp_runtime', 'log', 'dir')
128
+ folder if folder.is_a?(String) && File.absolute_path?(folder)
129
+ rescue StandardError => e
130
+ Log.log.debug { "Cannot read transferd configuration: #{e.message}" }
131
+ nil
132
+ end
133
+
134
+ # @return [Boolean] true if readable, else warn
135
+ def readable?(location)
136
+ return true if File.readable?(location)
137
+ Log.log.warn { "Cannot read #{location}: permission denied" }
138
+ false
139
+ end
140
+
141
+ # @return [String, nil] real path of the file, nil if it does not exist
142
+ def real_path(file)
143
+ File.realpath(file) if file && File.file?(file)
144
+ end
145
+
146
+ # @param location [String] log folder, or syslog file (with its rotations: `<file>-<date>`, `<file>.<n>`, compressed or not)
147
+ # @return [Array<String>] readable files of the location: `ascp` log files of a folder, or syslog files
148
+ def location_files(location)
149
+ files = if File.directory?(location)
150
+ return [] unless readable?(location)
151
+ LogAnalyzer.log_paths(location).select { |file| File.basename(file).match?(ASCP_FILE) }
152
+ else
153
+ Dir.glob("#{location}*").select { |file| File.file?(file) }.sort
154
+ end
155
+ files.select { |file| readable?(file) }
156
+ rescue SystemCallError => e
157
+ Log.log.warn { "Cannot read #{location}: #{e.message}" }
158
+ []
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end
@@ -62,6 +62,18 @@ module Aspera
62
62
  end
63
63
  end
64
64
 
65
+ # Local log files: of installed products, of `ascp` in `PATH`, and default locations
66
+ def action_ascp_logs_find(**)
67
+ require_relative '../ascp/log_locations'
68
+ products = begin
69
+ Ascp::Installation.instance.installed_products
70
+ rescue StandardError => e
71
+ Log.log.debug { "No installed product: #{e.message}" }
72
+ []
73
+ end
74
+ Result::ObjectList.new(Ascp::LogLocations.find(products), fields: %w[path size modified source])
75
+ end
76
+
65
77
  def action_ascp_logs_summary(paths:, **)
66
78
  Result::SingleObject.new(analyze_ascp_logs(paths).summary)
67
79
  end
@@ -176,6 +176,7 @@ module Aspera
176
176
  Rest::Parameters.instance.session_cb = ->(http_session) { @context.http_config.update_session(http_session) }
177
177
  Rest::Parameters.instance.spinner_cb = ->(title = nil, action: :spin) { @context.formatter.long_operation(title, action: action) }
178
178
  OAuth::Web.additional_info = "#{Info::CMD_NAME} v#{Cli::VERSION}"
179
+ Ascp::Installation.instance.referer_url = "#{Info::GEM_URL}/versions/#{Cli::VERSION}"
179
180
  Transfer::Parameters.file_list_folder = File.join(@context.main_folder, FILE_LIST_FOLDER_NAME)
180
181
  Rest::ErrorAnalyzer.instance.log_file = File.join(@context.main_folder, REST_EXCEPTIONS_LOG_FILENAME)
181
182
  end
@@ -7,6 +7,9 @@ require_relative 'error'
7
7
  require_relative '../dot_container'
8
8
  require_relative '../log'
9
9
  require_relative '../assert'
10
+ require_relative '../secret/hider'
11
+ require 'json'
12
+ require 'yaml'
10
13
 
11
14
  module Aspera
12
15
  module Cli
@@ -68,8 +71,11 @@ module Aspera
68
71
  # @return [String, nil] inline value, or value of next token
69
72
  def value = @inline_value || @value_token&.value
70
73
 
71
- # @return [String] raw token, followed by its separate value if any
72
- def to_s = @value_token.nil? ? @raw : "#{@raw} #{@value_token.value}"
74
+ # @return [String] raw token, with declared secret value hidden (logs, messages)
75
+ def raw_hidden = inline? ? "#{@raw.delete_suffix(@inline_value)}#{Secret::Hider.instance.hidden_copy(@inline_value)}" : @raw
76
+
77
+ # @return [String] raw token, followed by its separate value if any, with declared secret value hidden (logs, messages)
78
+ def to_s = @value_token.nil? ? raw_hidden : "#{@raw} #{Secret::Hider.instance.hidden_copy(@value_token.value)}"
73
79
 
74
80
  class << self
75
81
  # @param token [String] command line token
@@ -111,6 +117,13 @@ module Aspera
111
117
  # unless that token looks like an option.
112
118
  # If the option turns out to be a flag, the claimed token is given back to positional arguments.
113
119
  class CommandLine
120
+ # Decoders of extended values whose content is checked for secrets by key name
121
+ STRUCTURED_DECODERS = {
122
+ '@json:' => ->(v) { JSON.parse(v) },
123
+ '@yaml:' => ->(v) { YAML.safe_load(v) }
124
+ }.freeze
125
+ private_constant :STRUCTURED_DECODERS
126
+
114
127
  # @param argv [Array<String>] command line arguments
115
128
  def initialize(argv)
116
129
  # @type [Array<Option, Argument>]
@@ -139,6 +152,8 @@ module Aspera
139
152
  @tokens.push(argument)
140
153
  end
141
154
  end
155
+ declare_secrets
156
+ Log.dump(:argv, level: :trace2) { @tokens.map { |t| t.is_a?(Option) ? t.raw_hidden : Secret::Hider.instance.hidden_copy(t.value) } }
142
157
  Log.log.trace1 { "arguments=#{pending_arguments},options=#{pending_options}" }
143
158
  end
144
159
 
@@ -246,6 +261,28 @@ module Aspera
246
261
 
247
262
  private
248
263
 
264
+ # Declare secret values of long options, by option name (last key for dot notation), as those of presets and env vars,
265
+ # so that they are hidden wherever they appear (logs, results), before the option is declared.
266
+ # A structured value (`@json:`, `@yaml:`) that contains a secret is declared as a whole, with its content.
267
+ def declare_secrets
268
+ hider = Secret::Hider.instance
269
+ @tokens.each do |tok|
270
+ next unless tok.is_a?(Option) && tok.short_char.nil? && tok.value.is_a?(String)
271
+ hider.add_named_secrets((tok.dot_path || [tok.name]).last, tok.value)
272
+ prefix, decoder = STRUCTURED_DECODERS.find { |p, _| tok.value.start_with?(p) }
273
+ next if prefix.nil?
274
+ content = tok.value.delete_prefix(prefix)
275
+ decoded = begin
276
+ decoder.call(content)
277
+ rescue StandardError
278
+ next
279
+ end
280
+ next unless hider.contains_secret?(decoded)
281
+ hider.add_secrets_in(decoded)
282
+ hider.add_secret_values(tok.value, content)
283
+ end
284
+ end
285
+
249
286
  # Give back claimed token to positional arguments.
250
287
  # @param tok [Option] flag option token
251
288
  # @return [nil]
@@ -262,6 +262,7 @@ module Aspera
262
262
  raise ArgumentError, "#{path.inspect}: mount arguments must be mandatory" unless mount.arguments.all?(&:mandatory)
263
263
  validate_arguments(path, mount.arguments, plugin_class)
264
264
  raise ArgumentError, "#{path.inspect}: mount and action: are exclusive" if spec.action
265
+ raise ArgumentError, "#{path.inspect}: mount and read_only: are exclusive" if spec.read_only.eql?(true)
265
266
  raise ArgumentError, "#{path.inspect}: mount at #{mount.at.inspect} not found in #{mount.plugin}" unless mount.at.empty? || mount.registry[mount.at]
266
267
  target_ids = mount.registry.children_of(mount.at).keys
267
268
  unknown = Array(mount.only) + Array(mount.except) - target_ids
@@ -271,6 +272,8 @@ module Aspera
271
272
  next
272
273
  end
273
274
 
275
+ # Rule: only a leaf is executed, so only a leaf is read-only
276
+ raise ArgumentError, "#{path.inspect}: read_only: only on a leaf" if spec.read_only.eql?(true) && @children_index[path]&.any?
274
277
  next if @children_index[path]&.any? # intermediate node: skip
275
278
  action = spec.action
276
279
  if action.nil?
@@ -191,6 +191,9 @@ module Aspera
191
191
  end
192
192
  end
193
193
 
194
+ # Ids of the commands read-only by default (`read_only: :default`): they change nothing
195
+ READ_ONLY_IDS = %i[list show status info health ls browse find].freeze
196
+
194
197
  # Declares a single command node in the flat registry.
195
198
  #
196
199
  # @!attribute id [Symbol] Unique identifier within its parent's namespace
@@ -207,6 +210,8 @@ module Aspera
207
210
  # @!attribute query_schema [String, nil] Schema path for --query help; when set, the runner hints `--query=help`
208
211
  # @!attribute mount [MountSpec, Hash, nil] Expose children of another plugin's registry under this node
209
212
  # @!attribute security [Symbol, nil] Most restrictive security level at which the command is available (`:low`: executes code), see `Security`
213
+ # @!attribute read_only [Boolean, :default] Leaf only: the command changes nothing (on the server or locally), so it can be repeated (option `poll`).
214
+ # `:default` (default): read-only if the id is one of READ_ONLY_IDS (`list`, `show`, ...)
210
215
  CommandSpec = Struct.new(
211
216
  :id,
212
217
  :parent,
@@ -220,10 +225,13 @@ module Aspera
220
225
  :query_schema,
221
226
  :mount,
222
227
  :security,
228
+ :read_only,
223
229
  keyword_init: true
224
230
  ) do
225
231
  def initialize(**kwargs)
226
232
  Aspera.assert_values(kwargs[:security], Security::LEVELS + [nil]) { "#{kwargs[:id]}: security" }
233
+ kwargs[:read_only] = :default if kwargs[:read_only].nil?
234
+ Aspera.assert_values(kwargs[:read_only], [true, false, :default]) { "#{kwargs[:id]}: read_only" }
227
235
  # Coerce each element of arguments: from Hash to ArgumentSpec if needed
228
236
  if kwargs[:arguments]
229
237
  kwargs[:arguments] = kwargs[:arguments].map do |a|
@@ -244,6 +252,11 @@ module Aspera
244
252
  end
245
253
  end
246
254
 
255
+ # @return [Boolean] `true` if the command changes nothing: explicit `read_only`, or from its id (READ_ONLY_IDS)
256
+ def read_only?
257
+ read_only.eql?(:default) ? READ_ONLY_IDS.include?(id) : read_only
258
+ end
259
+
247
260
  # Compute the full path as Array<Symbol> from parent + id.
248
261
  # @return [Array<Symbol>]
249
262
  def full_path
@@ -70,6 +70,9 @@ module Aspera
70
70
 
71
71
  class NoSuchElement < Error; end
72
72
 
73
+ # Raised when the condition of option `poll` (`until`) is not met after the last run.
74
+ class PollConditionNotMet < Error; end
75
+
73
76
  # Raised when a lookup for a specific entity fails to return exactly one result.
74
77
  #
75
78
  # Provides a formatted message indicating whether the entity was missing
@@ -110,7 +110,7 @@ module Aspera
110
110
  base64: lambda { |i| Base64.decode64(i) },
111
111
  csvt: lambda { |i| ExtendedValue.decode_csvt(i) },
112
112
  env: lambda { |i| ENV.fetch(i, nil) },
113
- file: lambda { |i| File.read(Security.instance.readable_path(i)) },
113
+ file: lambda { |i| Security.instance.read_file(i) },
114
114
  uri: lambda { |i| UriReader.read(i) },
115
115
  json: lambda { |i| ExtendedValue.JSON_parse(i) },
116
116
  lines: lambda { |i| i.split("\n") },
@@ -213,9 +213,10 @@ module Aspera
213
213
  handler = m[1].to_sym
214
214
  handlers_reversed.unshift(handler)
215
215
  value = m[2]
216
- break if SPECIAL_HANDLERS.include?(handler)
216
+ # `@val:` keeps the rest as is; `@extend:` too, unless the rest is a single extended value (e.g. `@extend:@file:x`): decoded first, then extended
217
+ break if SPECIAL_HANDLERS.include?(handler) && (handler.eql?(:val) || @regex_inner.match?(value))
217
218
  end
218
- Log.log.trace1 { "evaluating: #{handlers_reversed}, value: #{value.class}:#{value}" }
219
+ Log.log.trace1 { "evaluating: #{handlers_reversed}, value: #{value.class}:#{Secret::Hider.instance.hidden_copy(value)}" }
219
220
  handlers_reversed.each_with_index do |handler, index|
220
221
  Security.instance.check_extended_value(handler)
221
222
  # The value read by the previous handler may be a secret
@@ -253,7 +254,7 @@ module Aspera
253
254
  MARKER_IN_END = '@'
254
255
 
255
256
  # Special handlers stop processing of handlers on right
256
- # :extend includes processing of other handlers in itself
257
+ # :extend includes processing of other handlers in itself, except if the value on right has no embedded value
257
258
  # :val keeps the value intact
258
259
  SPECIAL_HANDLERS = %i[extend val].freeze
259
260
  # Handlers that read a secret
@@ -81,6 +81,19 @@ module Aspera
81
81
  end
82
82
  end
83
83
 
84
+ # Check the syntax of a JMESPath expression, e.g. of `--out.query`, before the command is executed
85
+ # @param expression [String] JMESPath expression
86
+ # @param label [String, nil] Name of the option, for the error message
87
+ # @raise [Cli::BadArgument] if the syntax is invalid
88
+ # @return [nil]
89
+ def check_jmespath(expression, label = nil)
90
+ require 'jmespath'
91
+ JMESPath::Parser.new.parse(expression)
92
+ nil
93
+ rescue JMESPath::Errors::SyntaxError => e
94
+ raise Cli::BadArgument, "#{"#{label}: " if label}JMESPath: #{e.message}"
95
+ end
96
+
84
97
  # @param fields [Array<String>] List of fields to not display
85
98
  # @return [Array<String>] List of fields to display, including special values (`ALL`) and tagged (`-`) fields
86
99
  def all_but(*fields)
@@ -226,14 +239,7 @@ module Aspera
226
239
  # @param expression [String, Proc, nil] JMESPath expression, or Ruby lambda, or `nil` for none
227
240
  def query=(expression)
228
241
  Aspera.assert_type(expression, String, Proc, NilClass, type: Cli::BadArgument) { 'out.query' }
229
- if expression.is_a?(String)
230
- require 'jmespath'
231
- begin
232
- JMESPath::Parser.new.parse(expression)
233
- rescue JMESPath::Errors::SyntaxError => e
234
- raise Cli::BadArgument, "JMESPath: #{e.message}"
235
- end
236
- end
242
+ self.class.check_jmespath(expression) if expression.is_a?(String)
237
243
  @options[:query] = expression
238
244
  end
239
245
 
@@ -7,6 +7,7 @@ require_relative 'special_values'
7
7
  require_relative '../log'
8
8
  require_relative '../assert'
9
9
  require_relative '../line_logger'
10
+ require_relative '../units/duration'
10
11
  require_relative '../schema/registry'
11
12
  require_relative 'option_declarator'
12
13
  require_relative '../rest'
@@ -27,7 +28,24 @@ module Aspera
27
28
  # OpenSSL constant that disables peer verification (VERIFY_NONE)
28
29
  SELF_SIGNED_CERT = OpenSSL::SSL.const_get(:enon_yfirev.to_s.upcase.reverse) # cspell: disable-line
29
30
 
30
- private_constant :CERT_EXT, :SELF_SIGNED_CERT
31
+ # Keys of `http_options` that are durations in seconds: a number, or a string decoded by {Units::Duration}
32
+ DURATION_KEYS = %i[
33
+ read_timeout write_timeout open_timeout keep_alive_timeout
34
+ retry_sleep
35
+ token_cache_max_age token_refresh_threshold token_refresh_max_age jwt_accepted_offset_sec jwt_expiry_offset_sec
36
+ ].freeze
37
+
38
+ private_constant :CERT_EXT, :SELF_SIGNED_CERT, :DURATION_KEYS
39
+
40
+ class << self
41
+ # @param value [Numeric, String] Duration: number of seconds, or string, e.g. `12h`, `1d`, `1h 30m`, `0.5`
42
+ # @return [Numeric] Number of seconds (integer, unless the string has milliseconds)
43
+ def duration_seconds(value)
44
+ return value if value.is_a?(Numeric)
45
+ milliseconds = Units::Duration.new(value).to_unit('ms')
46
+ (milliseconds % 1000).zero? ? milliseconds / 1000 : milliseconds / 1000.0
47
+ end
48
+ end
31
49
 
32
50
  # `on_set` callbacks are methods of the instance given as `target:` to `declare_options`
33
51
  option :insecure, description: 'HTTP/S: Do not validate any certificate', allowed: Type::BOOLEAN, default: false, on_set: :insecure=
@@ -56,12 +74,19 @@ module Aspera
56
74
  # in @http_options for Net::HTTP session configuration in update_session.
57
75
  # This runs on every assignment (JSON hash, dotted notation, preset merge) so timing
58
76
  # of option parsing never matters.
77
+ # Durations (`DURATION_KEYS`) are converted to seconds first.
59
78
  # @param new_options [Hash] merged http_options hash
60
79
  # @return [nil]
80
+ # @raise [BadArgument] if a duration is invalid
61
81
  def http_options=(new_options)
62
82
  Aspera.assert_type(new_options, Hash)
63
83
  kept = {}
64
84
  new_options.each do |k, v|
85
+ begin
86
+ v = self.class.duration_seconds(v) if DURATION_KEYS.include?(k.to_sym)
87
+ rescue ParameterError => e
88
+ raise BadArgument, "Option http_options: #{k}: #{e.message}"
89
+ end
65
90
  method = "#{k}=".to_sym
66
91
  if Rest::Parameters.instance.respond_to?(method)
67
92
  Rest::Parameters.instance.send(method, v)
@@ -26,7 +26,7 @@ module Aspera
26
26
  # Version with which this version of CLI was tested
27
27
  SDK_VERSION = '1.1.9'
28
28
  # Ruby version with which this version of CLI was tested, packaged in container image and portable packages
29
- RUBY_TESTED_VERSION = '4.0.9'
29
+ RUBY_TESTED_VERSION = '4.0.10'
30
30
  # Suffix of the RubyInstaller release of `RUBY_TESTED_VERSION`, packaged in the Windows zip
31
31
  # (release tag: `RubyInstaller-<Ruby version><suffix>`)
32
32
  # https://github.com/oneclick/rubyinstaller2/releases
@@ -15,8 +15,8 @@ require 'json'
15
15
  unless defined?(MCP::Tool)
16
16
  begin
17
17
  require 'mcp'
18
- rescue LoadError
19
- raise Cli::Error, "The 'mcp' and 'rack' gems are required. Install them with: gem install mcp rack"
18
+ rescue LoadError => e
19
+ raise Aspera::Cli::Error, "The 'mcp' gem is required (and 'rack' for transport http). Install with: gem install mcp rack (#{e.message})"
20
20
  end
21
21
  end
22
22
 
@@ -51,7 +51,7 @@ module Aspera
51
51
  "@val:..." — literal string (no further decoding)
52
52
  "@secret:name" — secret entered by the user on a web page, never visible to you:
53
53
  use it for a secret the user did not give, e.g. "--password=@secret:password"
54
- Note: @stdin: is disabled; @ruby:, @file:, @uri:, @env: depend on the security level of the server.
54
+ Note: @stdin: is disabled; @ruby:, @file:, @uri:, @env:, @zlib: depend on the security level of the server.
55
55
  A secret (from @secret:, @vault:, @preset:) must be the whole value: no other prefix before it, not in @extend:.
56
56
 
57
57
  AUTOMATIC FLAGS
@@ -152,6 +152,8 @@ module Aspera
152
152
  Log.dump(:mcp_execute, effective_args)
153
153
  runner = Runner.new(effective_args)
154
154
  result = runner.run_with_result
155
+ # The command of a poll is executed when the result is displayed, on the command line only
156
+ raise Cli::BadArgument, 'Option poll: not available in MCP server mode, call the tool again instead' if result.is_a?(Result::Poll)
155
157
  # Secrets never reach the AI client, whatever the output format requested in args.
156
158
  # As on the command line, `--out.query` applies to the data of an object or list, after secrets are removed
157
159
  if result.is_a?(Result::SingleObject) || result.is_a?(Result::ObjectList) || result.is_a?(Result::ValueList)