aspera-cli 4.29.0 → 4.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -1
- data/CHANGELOG.md +76 -2
- data/README.md +20 -1
- data/SECURITY.md +14 -0
- data/docs/README.md +277 -38
- data/lib/aspera/api/node.rb +9 -2
- data/lib/aspera/ascp/installation.rb +26 -4
- data/lib/aspera/ascp/log_analyzer/lines.rb +2 -2
- data/lib/aspera/ascp/log_analyzer.rb +19 -3
- data/lib/aspera/ascp/log_locations.rb +163 -0
- data/lib/aspera/cli/ascp_logs_actions.rb +12 -0
- data/lib/aspera/cli/bootstrapper.rb +1 -0
- data/lib/aspera/cli/command_line.rb +39 -2
- data/lib/aspera/cli/command_registry.rb +3 -0
- data/lib/aspera/cli/command_spec.rb +13 -0
- data/lib/aspera/cli/error.rb +3 -0
- data/lib/aspera/cli/extended_value.rb +5 -4
- data/lib/aspera/cli/formatter.rb +14 -8
- data/lib/aspera/cli/http.rb +26 -1
- data/lib/aspera/cli/info.rb +1 -1
- data/lib/aspera/cli/mcp/tool.rb +5 -3
- data/lib/aspera/cli/option_value.rb +14 -8
- data/lib/aspera/cli/options.schema.yaml +156 -19
- data/lib/aspera/cli/parser.rb +15 -3
- data/lib/aspera/cli/plugins/aoc.rb +4 -2
- data/lib/aspera/cli/plugins/ats.rb +1 -1
- data/lib/aspera/cli/plugins/base.rb +33 -6
- data/lib/aspera/cli/plugins/config.rb +12 -1
- data/lib/aspera/cli/plugins/faspex5.rb +3 -1
- data/lib/aspera/cli/plugins/mcp.rb +14 -0
- data/lib/aspera/cli/plugins/node.rb +8 -8
- data/lib/aspera/cli/plugins/oauth.rb +18 -5
- data/lib/aspera/cli/plugins/orchestrator.rb +28 -24
- data/lib/aspera/cli/plugins/preview.rb +3 -1
- data/lib/aspera/cli/plugins/server.rb +3 -3
- data/lib/aspera/cli/plugins/shares.rb +1 -2
- data/lib/aspera/cli/plugins/transferd.rb +7 -2
- data/lib/aspera/cli/poll.rb +236 -0
- data/lib/aspera/cli/preset_manager.rb +63 -19
- data/lib/aspera/cli/prompt.rb +153 -8
- data/lib/aspera/cli/runner.rb +25 -7
- data/lib/aspera/cli/security.rb +10 -0
- data/lib/aspera/cli/sync_actions.rb +2 -0
- data/lib/aspera/cli/transfer_agent.rb +14 -0
- data/lib/aspera/cli/transfer_progress.rb +6 -5
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/environment.rb +3 -1
- data/lib/aspera/id_generator.rb +16 -3
- data/lib/aspera/inflate.rb +43 -0
- data/lib/aspera/oauth/client/base.rb +38 -8
- data/lib/aspera/oauth/client/factory.rb +35 -15
- data/lib/aspera/oauth/client/json_credentials.rb +3 -2
- data/lib/aspera/oauth/client/shares_web.rb +13 -4
- data/lib/aspera/oauth/client/url_json.rb +3 -2
- data/lib/aspera/oauth/client/web.rb +2 -2
- data/lib/aspera/orchestrator_workflow_checker.rb +10 -1
- data/lib/aspera/preview/terminal.rb +8 -2
- data/lib/aspera/products/desktop.rb +13 -1
- data/lib/aspera/rest/client.rb +23 -5
- data/lib/aspera/schema/registry.rb +1 -0
- data/lib/aspera/sealed_hash.rb +2 -18
- data/lib/aspera/secret/hider.rb +13 -1
- data/lib/aspera/secret/store/one_password_cli.rb +13 -11
- data/lib/aspera/sync/operations.rb +4 -4
- data/lib/aspera/transfer/result.rb +20 -0
- data/lib/aspera/transfer/spec.rb +2 -2
- data/lib/aspera/{transfer/units.rb → units/decimal.rb} +4 -12
- data/lib/aspera/units/duration.rb +109 -0
- data/lib/aspera/uri_reader.rb +11 -6
- data/lib/aspera/web_auth/reply.html.erb +1 -1
- data/llms-install.md +88 -0
- data/llms.txt +6 -0
- data/skills/ascli/SKILL.md +6 -0
- data.tar.gz.sig +0 -0
- metadata +7 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/duration.rb +0 -45
|
@@ -52,11 +52,14 @@ module Aspera
|
|
|
52
52
|
# Loads YAML from cloud with locations of SDK archives for all platforms
|
|
53
53
|
# Tries each URL of option `locations_url` (default if empty) in order, until one provides a valid list.
|
|
54
54
|
# All but the last URL are read with a short timeout and no retry, so that fallback is quick.
|
|
55
|
+
# The request tells which application reads the list, and what it downloads (header `Referer`, see `referer_url`):
|
|
56
|
+
# the default URL counts requests per `Referer` (download statistics).
|
|
57
|
+
# @param download [Hash, nil] SDK about to be downloaded (`sdk`: version, `platform`), `nil` if the list is only read
|
|
55
58
|
# @return [Array<Hash>] location structure
|
|
56
|
-
def sdk_locations
|
|
59
|
+
def sdk_locations(download: nil)
|
|
57
60
|
Aspera.assert_type(@transferd_urls, Array) { 'locations_url' }
|
|
58
61
|
location_urls = @transferd_urls.empty? ? TRANSFERD_ARCHIVE_LOCATION_URLS : @transferd_urls
|
|
59
|
-
return UriReader.read_first(location_urls, what: 'SDK locations') do |content|
|
|
62
|
+
return UriReader.read_first(location_urls, what: 'SDK locations', headers: locations_headers(download)) do |content|
|
|
60
63
|
locations = Yaml.safe_load(content)
|
|
61
64
|
Aspera.assert_type(locations, Array) { 'SDK locations' }
|
|
62
65
|
locations
|
|
@@ -234,6 +237,11 @@ module Aspera
|
|
|
234
237
|
ascp_data = file_paths
|
|
235
238
|
ascp_data.merge!(ascp_info_from_log)
|
|
236
239
|
ascp_data.merge!(ascp_info_from_file)
|
|
240
|
+
# `ascp` is run with `-L-` to read its paths: report where it logs without `-L`
|
|
241
|
+
if ascp_data['log'].eql?('-')
|
|
242
|
+
require_relative 'log_locations'
|
|
243
|
+
ascp_data['log'] = LogLocations.ascp_log_location(path(:ascp), root: ascp_data['root'], conf_file: ascp_data['conf'])
|
|
244
|
+
end
|
|
237
245
|
return ascp_data
|
|
238
246
|
end
|
|
239
247
|
|
|
@@ -241,8 +249,8 @@ module Aspera
|
|
|
241
249
|
# @param version [nil, String] Version, if nil: latest
|
|
242
250
|
# @return [Hash] the entry of SDK locations for the given platform and version
|
|
243
251
|
def sdk_location_for_platform(platform: nil, version: nil)
|
|
244
|
-
all_locations = sdk_locations
|
|
245
252
|
platform = Environment.instance.architecture if platform.nil?
|
|
253
|
+
all_locations = sdk_locations(download: {sdk: version || LATEST_VERSION, platform: platform})
|
|
246
254
|
locations = all_locations.select { |l| l['platform'].eql?(platform) }
|
|
247
255
|
Aspera.assert(!locations.empty?) { "No SDK for platform: #{platform}, available: #{all_locations.map { |i| i['platform'] }.uniq}" }
|
|
248
256
|
version = locations.max_by { |entry| Gem::Version.new(entry['version']) }['version'] if version.nil?
|
|
@@ -435,8 +443,18 @@ module Aspera
|
|
|
435
443
|
# @return [Array<String>] URLs of SDK locations, tried in order
|
|
436
444
|
attr_accessor :transferd_urls
|
|
437
445
|
|
|
446
|
+
# @return [String, nil] URL that identifies the application and its version, sent in header `Referer` when SDK locations are read, `nil`: not sent
|
|
447
|
+
attr_accessor :referer_url
|
|
448
|
+
|
|
438
449
|
private
|
|
439
450
|
|
|
451
|
+
# @param download [Hash, nil] See `sdk_locations`
|
|
452
|
+
# @return [Hash] HTTP headers of the request for SDK locations
|
|
453
|
+
def locations_headers(download)
|
|
454
|
+
return {} if @referer_url.nil?
|
|
455
|
+
{'Referer' => download.nil? ? @referer_url : "#{@referer_url}?#{URI.encode_www_form(download)}"}
|
|
456
|
+
end
|
|
457
|
+
|
|
440
458
|
# Run the block with another SDK folder (files of the SDK are located from it), then set the previous one back
|
|
441
459
|
# @param folder [String] Temporary SDK folder
|
|
442
460
|
def with_sdk_directory(folder)
|
|
@@ -476,14 +494,18 @@ module Aspera
|
|
|
476
494
|
'https://ibm.biz/sdk_location',
|
|
477
495
|
'https://raw.githubusercontent.com/IBM/aspera-cli/refs/heads/main/docs/sdk_location.yaml'
|
|
478
496
|
].freeze
|
|
497
|
+
# Version reported in statistics when the latest version is requested (not known before the list is read)
|
|
498
|
+
LATEST_VERSION = 'latest'
|
|
479
499
|
# filename for ascp with optional extension (Windows)
|
|
480
|
-
private_constant :DEFAULT_ASPERA_CONF, :EXE_FILES, :SDK_FILES, :TRANSFERD_ARCHIVE_LOCATION_URLS
|
|
500
|
+
private_constant :DEFAULT_ASPERA_CONF, :EXE_FILES, :SDK_FILES, :TRANSFERD_ARCHIVE_LOCATION_URLS, :LATEST_VERSION
|
|
481
501
|
|
|
482
502
|
def initialize
|
|
483
503
|
# cache for installed products found
|
|
484
504
|
@found_products = nil
|
|
485
505
|
# URLs of SDK locations, tried in order
|
|
486
506
|
@transferd_urls = TRANSFERD_ARCHIVE_LOCATION_URLS
|
|
507
|
+
# set by the application, `nil`: no `Referer` sent
|
|
508
|
+
@referer_url = nil
|
|
487
509
|
# product selected with option `sdk_folder`, or nil to use SDK folder
|
|
488
510
|
@ascp_product = nil
|
|
489
511
|
# cache for folder of `ascp` of selected product
|
|
@@ -4,11 +4,11 @@
|
|
|
4
4
|
# SPDX-License-Identifier: Apache-2.0
|
|
5
5
|
|
|
6
6
|
# cspell:words getlocal MSGID PROCID rsyslog
|
|
7
|
-
require_relative '../log_analyzer'
|
|
8
|
-
|
|
9
7
|
module Aspera
|
|
10
8
|
module Ascp
|
|
11
9
|
class LogAnalyzer
|
|
10
|
+
# Path that designates the standard input
|
|
11
|
+
STDIN_PATH = '-'
|
|
12
12
|
# Formats and timestamps of log lines: shared by LogFile and Records, which define `@path` and `@timezone` (nil: time zone of the analyst)
|
|
13
13
|
module Lines
|
|
14
14
|
MONTHS = %w[Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec].freeze
|
|
@@ -21,8 +21,6 @@ module Aspera
|
|
|
21
21
|
class LogAnalyzer
|
|
22
22
|
# Oldest ascp version for which the analysis is designed
|
|
23
23
|
MIN_VERSION = '4.0'
|
|
24
|
-
# Path that designates the standard input
|
|
25
|
-
STDIN_PATH = '-'
|
|
26
24
|
# First two bytes of a gzip file
|
|
27
25
|
GZIP_MAGIC = "\x1F\x8B".b.freeze
|
|
28
26
|
# Files analyzed when scanning a folder
|
|
@@ -132,6 +130,16 @@ module Aspera
|
|
|
132
130
|
end
|
|
133
131
|
end
|
|
134
132
|
|
|
133
|
+
# Reads the file up to its first `ascp` line only
|
|
134
|
+
# @param path [String] log file, plain or compressed with gzip
|
|
135
|
+
# @return [Boolean] true if the file contains `ascp` log lines
|
|
136
|
+
def ascp_log?(path)
|
|
137
|
+
read_lines(path, Encoding::BINARY) { |line| return true if line.match?(Lines::ASCP_LINE) }
|
|
138
|
+
false
|
|
139
|
+
rescue Zlib::Error, Errno::EPIPE
|
|
140
|
+
false
|
|
141
|
+
end
|
|
142
|
+
|
|
135
143
|
private
|
|
136
144
|
|
|
137
145
|
# `ascp` creates `<name>.log` as a hard link to the current rotated file `<name>.<n>.log`: keep only one of them.
|
|
@@ -147,7 +155,15 @@ module Aspera
|
|
|
147
155
|
|
|
148
156
|
# @return [Array<String>] files, with the log files of folder added
|
|
149
157
|
def scan_folder(root, folder, files)
|
|
150
|
-
|
|
158
|
+
children = begin
|
|
159
|
+
Dir.children(folder)
|
|
160
|
+
rescue SystemCallError => e
|
|
161
|
+
# A sub-folder that cannot be read is skipped, the folder given is not
|
|
162
|
+
raise if folder.eql?(root)
|
|
163
|
+
Log.log.warn { "Ignored, cannot read #{folder}: #{e.message}" }
|
|
164
|
+
[]
|
|
165
|
+
end
|
|
166
|
+
children.sort.each do |name|
|
|
151
167
|
next if name.start_with?('.')
|
|
152
168
|
path = File.join(folder, name)
|
|
153
169
|
if File.directory?(path)
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../environment'
|
|
7
|
+
require_relative '../log'
|
|
8
|
+
require_relative '../products/desktop'
|
|
9
|
+
require_relative 'log_analyzer'
|
|
10
|
+
require 'json'
|
|
11
|
+
require 'set'
|
|
12
|
+
require 'xmlsimple'
|
|
13
|
+
|
|
14
|
+
module Aspera
|
|
15
|
+
module Ascp
|
|
16
|
+
# Location of the `ascp` log files of the local system.
|
|
17
|
+
# `ascp` logs in the folder of option `-L` (set by Connect, transferd, IBM Aspera for Desktop),
|
|
18
|
+
# else in `<logging><directory>` of `aspera.conf`, else in a default location that depends on the OS:
|
|
19
|
+
# syslog on Linux and Unix, `~/Library/Logs/Aspera` on macOS, `var\log` of the product (or folder of `ascp`) on Windows.
|
|
20
|
+
module LogLocations
|
|
21
|
+
# Linux and Unix: `ascp` logs to syslog (facility `local2`), in one of these files depending on the syslog configuration
|
|
22
|
+
SYSLOG_FILES = %w[/var/log/aspera.log /var/log/messages /var/log/syslog].freeze
|
|
23
|
+
# Log location of `ascp` on Linux and Unix without `-L` nor `<logging><directory>`
|
|
24
|
+
SYSLOG = 'syslog'
|
|
25
|
+
# Source of a location that is not specific to a product
|
|
26
|
+
DEFAULT_SOURCE = 'ascp default'
|
|
27
|
+
# Source of a location given by an `ascp` found in `PATH`, but not in the locations of known products
|
|
28
|
+
PATH_SOURCE = 'ascp in PATH'
|
|
29
|
+
# Names of the log files written by `ascp` in a folder (`-L` or `aspera.conf`): current file, rotations, compressed
|
|
30
|
+
ASCP_FILE = /\A(?:aspera-scp-transfer|ascp4?)(?:\.\d+)?\.log(?:\.gz)?\z/
|
|
31
|
+
class << self
|
|
32
|
+
# @param products [Array<Hash>] installed products, see `Installation#installed_products`
|
|
33
|
+
# @param defaults [Array<Array>] default locations: `[path, source]`
|
|
34
|
+
# @return [Array<Hash>] files that contain `ascp` log lines: `path`, `size`, `modified`, `source`, most recent first
|
|
35
|
+
def find(products, defaults: default_locations)
|
|
36
|
+
seen = Set.new
|
|
37
|
+
# Default locations first: they are shared by products (e.g. `~/Library/Logs/Aspera`)
|
|
38
|
+
locations = defaults + (products + path_products(products)).flat_map { |product| product_locations(product) }
|
|
39
|
+
files = locations.flat_map { |location, source| location_files(location).map { |file| [file, source] } }
|
|
40
|
+
files.filter_map do |file, source|
|
|
41
|
+
stat = File.stat(file)
|
|
42
|
+
# Same file in several locations, or `<name>.log` hard link to `<name>.<n>.log`
|
|
43
|
+
next unless seen.add?(stat.ino.zero? ? File.expand_path(file) : [stat.dev, stat.ino])
|
|
44
|
+
next unless LogAnalyzer.ascp_log?(file)
|
|
45
|
+
{'path' => file, 'size' => stat.size, 'modified' => stat.mtime, 'source' => source}
|
|
46
|
+
end.sort_by { |row| -row['modified'].to_f }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# @return [Array<Array>] locations not specific to an installed product: `[path, source]`
|
|
50
|
+
def default_locations
|
|
51
|
+
locations = case Environment.instance.os
|
|
52
|
+
when Environment::OS_WINDOWS then []
|
|
53
|
+
when Environment::OS_MACOS then [File.join(Dir.home, 'Library', 'Logs', 'Aspera'), File.join('', 'Library', 'Logs', 'Aspera')]
|
|
54
|
+
else SYSLOG_FILES
|
|
55
|
+
end.map { |path| [path, DEFAULT_SOURCE] }
|
|
56
|
+
desktop = Products::Desktop.log_folder
|
|
57
|
+
locations.push([desktop, Products::Desktop::APP_NAME]) unless desktop.nil?
|
|
58
|
+
service = transferd_service_log_folder
|
|
59
|
+
locations.push([service, 'transferd service']) unless service.nil?
|
|
60
|
+
locations
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Location of the log of `ascp` when it runs without option `-L`
|
|
64
|
+
# @param ascp [String] path of `ascp`
|
|
65
|
+
# @param root [String, nil] root folder of the product, as reported by `ascp` (Windows)
|
|
66
|
+
# @param conf_file [String, nil] `aspera.conf` used by `ascp`
|
|
67
|
+
# @return [String] log folder, or `syslog` (Linux and Unix)
|
|
68
|
+
def ascp_log_location(ascp, root: nil, conf_file: nil)
|
|
69
|
+
folder = conf_directory(conf_file)
|
|
70
|
+
return folder unless folder.nil?
|
|
71
|
+
case Environment.instance.os
|
|
72
|
+
when Environment::OS_MACOS
|
|
73
|
+
File.join(Dir.home.start_with?('/var/root') ? '' : Dir.home, 'Library', 'Logs', 'Aspera')
|
|
74
|
+
when Environment::OS_WINDOWS
|
|
75
|
+
var_log = File.join(root || File.dirname(File.dirname(ascp)), 'var', 'log')
|
|
76
|
+
File.directory?(var_log) ? var_log : File.dirname(ascp)
|
|
77
|
+
else SYSLOG
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
# @param products [Array<Hash>] installed products
|
|
84
|
+
# @return [Array<Hash>] products for the `ascp` found in `PATH` that are not one of `products`
|
|
85
|
+
def path_products(products)
|
|
86
|
+
known = products.filter_map { |product| real_path(product[:ascp_path]) }
|
|
87
|
+
ENV.fetch('PATH', '').split(File::PATH_SEPARATOR).filter_map do |folder|
|
|
88
|
+
ascp = real_path(File.join(folder, Environment.instance.exe_file('ascp')))
|
|
89
|
+
next if ascp.nil? || known.include?(ascp)
|
|
90
|
+
known.push(ascp)
|
|
91
|
+
bin = File.dirname(ascp)
|
|
92
|
+
{name: PATH_SOURCE, ascp_path: ascp, app_root: File.basename(bin).eql?('bin') ? File.dirname(bin) : bin}
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# @param product [Hash] installed product
|
|
97
|
+
# @return [Array<Array>] locations of logs of the product: `[path, source]`
|
|
98
|
+
def product_locations(product)
|
|
99
|
+
locations = [product[:log_root], conf_log_folder(product), File.join(product[:app_root], 'var', 'log')]
|
|
100
|
+
# Windows: without `var\log`, `ascp` logs in its own folder
|
|
101
|
+
locations.push(File.dirname(product[:ascp_path])) if Environment.instance.os.eql?(Environment::OS_WINDOWS)
|
|
102
|
+
locations.compact.uniq.map { |path| [path, product[:name]] }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# @param product [Hash] installed product
|
|
106
|
+
# @return [String, nil] `<default><logging><directory>` of `aspera.conf` of the product: in `etc`, or in the folder of `ascp` (SDK)
|
|
107
|
+
def conf_log_folder(product)
|
|
108
|
+
conf_directory([File.join(product[:app_root], 'etc', 'aspera.conf'), File.join(File.dirname(product[:ascp_path]), 'aspera.conf')].find { |f| File.file?(f) })
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# @param conf_file [String, nil] `aspera.conf`
|
|
112
|
+
# @return [String, nil] `<default><logging><directory>` of the file
|
|
113
|
+
def conf_directory(conf_file)
|
|
114
|
+
return if conf_file.nil? || !File.file?(conf_file)
|
|
115
|
+
folder = XmlSimple.xml_in(File.read(conf_file), {'ForceArray' => false}).dig('default', 'logging', 'directory')
|
|
116
|
+
folder if folder.is_a?(String) && !folder.empty?
|
|
117
|
+
rescue StandardError => e
|
|
118
|
+
Log.log.debug { "Cannot read #{conf_file}: #{e.message}" }
|
|
119
|
+
nil
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# @return [String, nil] `fasp_runtime.log.dir` (folder given to `ascp -L`) of the transferd service installed by `transferd service install`
|
|
123
|
+
def transferd_service_log_folder
|
|
124
|
+
require_relative '../products/transferd_service'
|
|
125
|
+
file = Products::TransferdService::CONFIG_FILE
|
|
126
|
+
return unless File.file?(file) && File.readable?(file)
|
|
127
|
+
folder = JSON.parse(File.read(file)).dig('fasp_runtime', 'log', 'dir')
|
|
128
|
+
folder if folder.is_a?(String) && File.absolute_path?(folder)
|
|
129
|
+
rescue StandardError => e
|
|
130
|
+
Log.log.debug { "Cannot read transferd configuration: #{e.message}" }
|
|
131
|
+
nil
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# @return [Boolean] true if readable, else warn
|
|
135
|
+
def readable?(location)
|
|
136
|
+
return true if File.readable?(location)
|
|
137
|
+
Log.log.warn { "Cannot read #{location}: permission denied" }
|
|
138
|
+
false
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# @return [String, nil] real path of the file, nil if it does not exist
|
|
142
|
+
def real_path(file)
|
|
143
|
+
File.realpath(file) if file && File.file?(file)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# @param location [String] log folder, or syslog file (with its rotations: `<file>-<date>`, `<file>.<n>`, compressed or not)
|
|
147
|
+
# @return [Array<String>] readable files of the location: `ascp` log files of a folder, or syslog files
|
|
148
|
+
def location_files(location)
|
|
149
|
+
files = if File.directory?(location)
|
|
150
|
+
return [] unless readable?(location)
|
|
151
|
+
LogAnalyzer.log_paths(location).select { |file| File.basename(file).match?(ASCP_FILE) }
|
|
152
|
+
else
|
|
153
|
+
Dir.glob("#{location}*").select { |file| File.file?(file) }.sort
|
|
154
|
+
end
|
|
155
|
+
files.select { |file| readable?(file) }
|
|
156
|
+
rescue SystemCallError => e
|
|
157
|
+
Log.log.warn { "Cannot read #{location}: #{e.message}" }
|
|
158
|
+
[]
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
@@ -62,6 +62,18 @@ module Aspera
|
|
|
62
62
|
end
|
|
63
63
|
end
|
|
64
64
|
|
|
65
|
+
# Local log files: of installed products, of `ascp` in `PATH`, and default locations
|
|
66
|
+
def action_ascp_logs_find(**)
|
|
67
|
+
require_relative '../ascp/log_locations'
|
|
68
|
+
products = begin
|
|
69
|
+
Ascp::Installation.instance.installed_products
|
|
70
|
+
rescue StandardError => e
|
|
71
|
+
Log.log.debug { "No installed product: #{e.message}" }
|
|
72
|
+
[]
|
|
73
|
+
end
|
|
74
|
+
Result::ObjectList.new(Ascp::LogLocations.find(products), fields: %w[path size modified source])
|
|
75
|
+
end
|
|
76
|
+
|
|
65
77
|
def action_ascp_logs_summary(paths:, **)
|
|
66
78
|
Result::SingleObject.new(analyze_ascp_logs(paths).summary)
|
|
67
79
|
end
|
|
@@ -176,6 +176,7 @@ module Aspera
|
|
|
176
176
|
Rest::Parameters.instance.session_cb = ->(http_session) { @context.http_config.update_session(http_session) }
|
|
177
177
|
Rest::Parameters.instance.spinner_cb = ->(title = nil, action: :spin) { @context.formatter.long_operation(title, action: action) }
|
|
178
178
|
OAuth::Web.additional_info = "#{Info::CMD_NAME} v#{Cli::VERSION}"
|
|
179
|
+
Ascp::Installation.instance.referer_url = "#{Info::GEM_URL}/versions/#{Cli::VERSION}"
|
|
179
180
|
Transfer::Parameters.file_list_folder = File.join(@context.main_folder, FILE_LIST_FOLDER_NAME)
|
|
180
181
|
Rest::ErrorAnalyzer.instance.log_file = File.join(@context.main_folder, REST_EXCEPTIONS_LOG_FILENAME)
|
|
181
182
|
end
|
|
@@ -7,6 +7,9 @@ require_relative 'error'
|
|
|
7
7
|
require_relative '../dot_container'
|
|
8
8
|
require_relative '../log'
|
|
9
9
|
require_relative '../assert'
|
|
10
|
+
require_relative '../secret/hider'
|
|
11
|
+
require 'json'
|
|
12
|
+
require 'yaml'
|
|
10
13
|
|
|
11
14
|
module Aspera
|
|
12
15
|
module Cli
|
|
@@ -68,8 +71,11 @@ module Aspera
|
|
|
68
71
|
# @return [String, nil] inline value, or value of next token
|
|
69
72
|
def value = @inline_value || @value_token&.value
|
|
70
73
|
|
|
71
|
-
# @return [String] raw token,
|
|
72
|
-
def
|
|
74
|
+
# @return [String] raw token, with declared secret value hidden (logs, messages)
|
|
75
|
+
def raw_hidden = inline? ? "#{@raw.delete_suffix(@inline_value)}#{Secret::Hider.instance.hidden_copy(@inline_value)}" : @raw
|
|
76
|
+
|
|
77
|
+
# @return [String] raw token, followed by its separate value if any, with declared secret value hidden (logs, messages)
|
|
78
|
+
def to_s = @value_token.nil? ? raw_hidden : "#{@raw} #{Secret::Hider.instance.hidden_copy(@value_token.value)}"
|
|
73
79
|
|
|
74
80
|
class << self
|
|
75
81
|
# @param token [String] command line token
|
|
@@ -111,6 +117,13 @@ module Aspera
|
|
|
111
117
|
# unless that token looks like an option.
|
|
112
118
|
# If the option turns out to be a flag, the claimed token is given back to positional arguments.
|
|
113
119
|
class CommandLine
|
|
120
|
+
# Decoders of extended values whose content is checked for secrets by key name
|
|
121
|
+
STRUCTURED_DECODERS = {
|
|
122
|
+
'@json:' => ->(v) { JSON.parse(v) },
|
|
123
|
+
'@yaml:' => ->(v) { YAML.safe_load(v) }
|
|
124
|
+
}.freeze
|
|
125
|
+
private_constant :STRUCTURED_DECODERS
|
|
126
|
+
|
|
114
127
|
# @param argv [Array<String>] command line arguments
|
|
115
128
|
def initialize(argv)
|
|
116
129
|
# @type [Array<Option, Argument>]
|
|
@@ -139,6 +152,8 @@ module Aspera
|
|
|
139
152
|
@tokens.push(argument)
|
|
140
153
|
end
|
|
141
154
|
end
|
|
155
|
+
declare_secrets
|
|
156
|
+
Log.dump(:argv, level: :trace2) { @tokens.map { |t| t.is_a?(Option) ? t.raw_hidden : Secret::Hider.instance.hidden_copy(t.value) } }
|
|
142
157
|
Log.log.trace1 { "arguments=#{pending_arguments},options=#{pending_options}" }
|
|
143
158
|
end
|
|
144
159
|
|
|
@@ -246,6 +261,28 @@ module Aspera
|
|
|
246
261
|
|
|
247
262
|
private
|
|
248
263
|
|
|
264
|
+
# Declare secret values of long options, by option name (last key for dot notation), as those of presets and env vars,
|
|
265
|
+
# so that they are hidden wherever they appear (logs, results), before the option is declared.
|
|
266
|
+
# A structured value (`@json:`, `@yaml:`) that contains a secret is declared as a whole, with its content.
|
|
267
|
+
def declare_secrets
|
|
268
|
+
hider = Secret::Hider.instance
|
|
269
|
+
@tokens.each do |tok|
|
|
270
|
+
next unless tok.is_a?(Option) && tok.short_char.nil? && tok.value.is_a?(String)
|
|
271
|
+
hider.add_named_secrets((tok.dot_path || [tok.name]).last, tok.value)
|
|
272
|
+
prefix, decoder = STRUCTURED_DECODERS.find { |p, _| tok.value.start_with?(p) }
|
|
273
|
+
next if prefix.nil?
|
|
274
|
+
content = tok.value.delete_prefix(prefix)
|
|
275
|
+
decoded = begin
|
|
276
|
+
decoder.call(content)
|
|
277
|
+
rescue StandardError
|
|
278
|
+
next
|
|
279
|
+
end
|
|
280
|
+
next unless hider.contains_secret?(decoded)
|
|
281
|
+
hider.add_secrets_in(decoded)
|
|
282
|
+
hider.add_secret_values(tok.value, content)
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
|
|
249
286
|
# Give back claimed token to positional arguments.
|
|
250
287
|
# @param tok [Option] flag option token
|
|
251
288
|
# @return [nil]
|
|
@@ -262,6 +262,7 @@ module Aspera
|
|
|
262
262
|
raise ArgumentError, "#{path.inspect}: mount arguments must be mandatory" unless mount.arguments.all?(&:mandatory)
|
|
263
263
|
validate_arguments(path, mount.arguments, plugin_class)
|
|
264
264
|
raise ArgumentError, "#{path.inspect}: mount and action: are exclusive" if spec.action
|
|
265
|
+
raise ArgumentError, "#{path.inspect}: mount and read_only: are exclusive" if spec.read_only.eql?(true)
|
|
265
266
|
raise ArgumentError, "#{path.inspect}: mount at #{mount.at.inspect} not found in #{mount.plugin}" unless mount.at.empty? || mount.registry[mount.at]
|
|
266
267
|
target_ids = mount.registry.children_of(mount.at).keys
|
|
267
268
|
unknown = Array(mount.only) + Array(mount.except) - target_ids
|
|
@@ -271,6 +272,8 @@ module Aspera
|
|
|
271
272
|
next
|
|
272
273
|
end
|
|
273
274
|
|
|
275
|
+
# Rule: only a leaf is executed, so only a leaf is read-only
|
|
276
|
+
raise ArgumentError, "#{path.inspect}: read_only: only on a leaf" if spec.read_only.eql?(true) && @children_index[path]&.any?
|
|
274
277
|
next if @children_index[path]&.any? # intermediate node: skip
|
|
275
278
|
action = spec.action
|
|
276
279
|
if action.nil?
|
|
@@ -191,6 +191,9 @@ module Aspera
|
|
|
191
191
|
end
|
|
192
192
|
end
|
|
193
193
|
|
|
194
|
+
# Ids of the commands read-only by default (`read_only: :default`): they change nothing
|
|
195
|
+
READ_ONLY_IDS = %i[list show status info health ls browse find].freeze
|
|
196
|
+
|
|
194
197
|
# Declares a single command node in the flat registry.
|
|
195
198
|
#
|
|
196
199
|
# @!attribute id [Symbol] Unique identifier within its parent's namespace
|
|
@@ -207,6 +210,8 @@ module Aspera
|
|
|
207
210
|
# @!attribute query_schema [String, nil] Schema path for --query help; when set, the runner hints `--query=help`
|
|
208
211
|
# @!attribute mount [MountSpec, Hash, nil] Expose children of another plugin's registry under this node
|
|
209
212
|
# @!attribute security [Symbol, nil] Most restrictive security level at which the command is available (`:low`: executes code), see `Security`
|
|
213
|
+
# @!attribute read_only [Boolean, :default] Leaf only: the command changes nothing (on the server or locally), so it can be repeated (option `poll`).
|
|
214
|
+
# `:default` (default): read-only if the id is one of READ_ONLY_IDS (`list`, `show`, ...)
|
|
210
215
|
CommandSpec = Struct.new(
|
|
211
216
|
:id,
|
|
212
217
|
:parent,
|
|
@@ -220,10 +225,13 @@ module Aspera
|
|
|
220
225
|
:query_schema,
|
|
221
226
|
:mount,
|
|
222
227
|
:security,
|
|
228
|
+
:read_only,
|
|
223
229
|
keyword_init: true
|
|
224
230
|
) do
|
|
225
231
|
def initialize(**kwargs)
|
|
226
232
|
Aspera.assert_values(kwargs[:security], Security::LEVELS + [nil]) { "#{kwargs[:id]}: security" }
|
|
233
|
+
kwargs[:read_only] = :default if kwargs[:read_only].nil?
|
|
234
|
+
Aspera.assert_values(kwargs[:read_only], [true, false, :default]) { "#{kwargs[:id]}: read_only" }
|
|
227
235
|
# Coerce each element of arguments: from Hash to ArgumentSpec if needed
|
|
228
236
|
if kwargs[:arguments]
|
|
229
237
|
kwargs[:arguments] = kwargs[:arguments].map do |a|
|
|
@@ -244,6 +252,11 @@ module Aspera
|
|
|
244
252
|
end
|
|
245
253
|
end
|
|
246
254
|
|
|
255
|
+
# @return [Boolean] `true` if the command changes nothing: explicit `read_only`, or from its id (READ_ONLY_IDS)
|
|
256
|
+
def read_only?
|
|
257
|
+
read_only.eql?(:default) ? READ_ONLY_IDS.include?(id) : read_only
|
|
258
|
+
end
|
|
259
|
+
|
|
247
260
|
# Compute the full path as Array<Symbol> from parent + id.
|
|
248
261
|
# @return [Array<Symbol>]
|
|
249
262
|
def full_path
|
data/lib/aspera/cli/error.rb
CHANGED
|
@@ -70,6 +70,9 @@ module Aspera
|
|
|
70
70
|
|
|
71
71
|
class NoSuchElement < Error; end
|
|
72
72
|
|
|
73
|
+
# Raised when the condition of option `poll` (`until`) is not met after the last run.
|
|
74
|
+
class PollConditionNotMet < Error; end
|
|
75
|
+
|
|
73
76
|
# Raised when a lookup for a specific entity fails to return exactly one result.
|
|
74
77
|
#
|
|
75
78
|
# Provides a formatted message indicating whether the entity was missing
|
|
@@ -110,7 +110,7 @@ module Aspera
|
|
|
110
110
|
base64: lambda { |i| Base64.decode64(i) },
|
|
111
111
|
csvt: lambda { |i| ExtendedValue.decode_csvt(i) },
|
|
112
112
|
env: lambda { |i| ENV.fetch(i, nil) },
|
|
113
|
-
file: lambda { |i|
|
|
113
|
+
file: lambda { |i| Security.instance.read_file(i) },
|
|
114
114
|
uri: lambda { |i| UriReader.read(i) },
|
|
115
115
|
json: lambda { |i| ExtendedValue.JSON_parse(i) },
|
|
116
116
|
lines: lambda { |i| i.split("\n") },
|
|
@@ -213,9 +213,10 @@ module Aspera
|
|
|
213
213
|
handler = m[1].to_sym
|
|
214
214
|
handlers_reversed.unshift(handler)
|
|
215
215
|
value = m[2]
|
|
216
|
-
|
|
216
|
+
# `@val:` keeps the rest as is; `@extend:` too, unless the rest is a single extended value (e.g. `@extend:@file:x`): decoded first, then extended
|
|
217
|
+
break if SPECIAL_HANDLERS.include?(handler) && (handler.eql?(:val) || @regex_inner.match?(value))
|
|
217
218
|
end
|
|
218
|
-
Log.log.trace1 { "evaluating: #{handlers_reversed}, value: #{value.class}:#{value}" }
|
|
219
|
+
Log.log.trace1 { "evaluating: #{handlers_reversed}, value: #{value.class}:#{Secret::Hider.instance.hidden_copy(value)}" }
|
|
219
220
|
handlers_reversed.each_with_index do |handler, index|
|
|
220
221
|
Security.instance.check_extended_value(handler)
|
|
221
222
|
# The value read by the previous handler may be a secret
|
|
@@ -253,7 +254,7 @@ module Aspera
|
|
|
253
254
|
MARKER_IN_END = '@'
|
|
254
255
|
|
|
255
256
|
# Special handlers stop processing of handlers on right
|
|
256
|
-
# :extend includes processing of other handlers in itself
|
|
257
|
+
# :extend includes processing of other handlers in itself, except if the value on right has no embedded value
|
|
257
258
|
# :val keeps the value intact
|
|
258
259
|
SPECIAL_HANDLERS = %i[extend val].freeze
|
|
259
260
|
# Handlers that read a secret
|
data/lib/aspera/cli/formatter.rb
CHANGED
|
@@ -81,6 +81,19 @@ module Aspera
|
|
|
81
81
|
end
|
|
82
82
|
end
|
|
83
83
|
|
|
84
|
+
# Check the syntax of a JMESPath expression, e.g. of `--out.query`, before the command is executed
|
|
85
|
+
# @param expression [String] JMESPath expression
|
|
86
|
+
# @param label [String, nil] Name of the option, for the error message
|
|
87
|
+
# @raise [Cli::BadArgument] if the syntax is invalid
|
|
88
|
+
# @return [nil]
|
|
89
|
+
def check_jmespath(expression, label = nil)
|
|
90
|
+
require 'jmespath'
|
|
91
|
+
JMESPath::Parser.new.parse(expression)
|
|
92
|
+
nil
|
|
93
|
+
rescue JMESPath::Errors::SyntaxError => e
|
|
94
|
+
raise Cli::BadArgument, "#{"#{label}: " if label}JMESPath: #{e.message}"
|
|
95
|
+
end
|
|
96
|
+
|
|
84
97
|
# @param fields [Array<String>] List of fields to not display
|
|
85
98
|
# @return [Array<String>] List of fields to display, including special values (`ALL`) and tagged (`-`) fields
|
|
86
99
|
def all_but(*fields)
|
|
@@ -226,14 +239,7 @@ module Aspera
|
|
|
226
239
|
# @param expression [String, Proc, nil] JMESPath expression, or Ruby lambda, or `nil` for none
|
|
227
240
|
def query=(expression)
|
|
228
241
|
Aspera.assert_type(expression, String, Proc, NilClass, type: Cli::BadArgument) { 'out.query' }
|
|
229
|
-
if expression.is_a?(String)
|
|
230
|
-
require 'jmespath'
|
|
231
|
-
begin
|
|
232
|
-
JMESPath::Parser.new.parse(expression)
|
|
233
|
-
rescue JMESPath::Errors::SyntaxError => e
|
|
234
|
-
raise Cli::BadArgument, "JMESPath: #{e.message}"
|
|
235
|
-
end
|
|
236
|
-
end
|
|
242
|
+
self.class.check_jmespath(expression) if expression.is_a?(String)
|
|
237
243
|
@options[:query] = expression
|
|
238
244
|
end
|
|
239
245
|
|
data/lib/aspera/cli/http.rb
CHANGED
|
@@ -7,6 +7,7 @@ require_relative 'special_values'
|
|
|
7
7
|
require_relative '../log'
|
|
8
8
|
require_relative '../assert'
|
|
9
9
|
require_relative '../line_logger'
|
|
10
|
+
require_relative '../units/duration'
|
|
10
11
|
require_relative '../schema/registry'
|
|
11
12
|
require_relative 'option_declarator'
|
|
12
13
|
require_relative '../rest'
|
|
@@ -27,7 +28,24 @@ module Aspera
|
|
|
27
28
|
# OpenSSL constant that disables peer verification (VERIFY_NONE)
|
|
28
29
|
SELF_SIGNED_CERT = OpenSSL::SSL.const_get(:enon_yfirev.to_s.upcase.reverse) # cspell: disable-line
|
|
29
30
|
|
|
30
|
-
|
|
31
|
+
# Keys of `http_options` that are durations in seconds: a number, or a string decoded by {Units::Duration}
|
|
32
|
+
DURATION_KEYS = %i[
|
|
33
|
+
read_timeout write_timeout open_timeout keep_alive_timeout
|
|
34
|
+
retry_sleep
|
|
35
|
+
token_cache_max_age token_refresh_threshold token_refresh_max_age jwt_accepted_offset_sec jwt_expiry_offset_sec
|
|
36
|
+
].freeze
|
|
37
|
+
|
|
38
|
+
private_constant :CERT_EXT, :SELF_SIGNED_CERT, :DURATION_KEYS
|
|
39
|
+
|
|
40
|
+
class << self
|
|
41
|
+
# @param value [Numeric, String] Duration: number of seconds, or string, e.g. `12h`, `1d`, `1h 30m`, `0.5`
|
|
42
|
+
# @return [Numeric] Number of seconds (integer, unless the string has milliseconds)
|
|
43
|
+
def duration_seconds(value)
|
|
44
|
+
return value if value.is_a?(Numeric)
|
|
45
|
+
milliseconds = Units::Duration.new(value).to_unit('ms')
|
|
46
|
+
(milliseconds % 1000).zero? ? milliseconds / 1000 : milliseconds / 1000.0
|
|
47
|
+
end
|
|
48
|
+
end
|
|
31
49
|
|
|
32
50
|
# `on_set` callbacks are methods of the instance given as `target:` to `declare_options`
|
|
33
51
|
option :insecure, description: 'HTTP/S: Do not validate any certificate', allowed: Type::BOOLEAN, default: false, on_set: :insecure=
|
|
@@ -56,12 +74,19 @@ module Aspera
|
|
|
56
74
|
# in @http_options for Net::HTTP session configuration in update_session.
|
|
57
75
|
# This runs on every assignment (JSON hash, dotted notation, preset merge) so timing
|
|
58
76
|
# of option parsing never matters.
|
|
77
|
+
# Durations (`DURATION_KEYS`) are converted to seconds first.
|
|
59
78
|
# @param new_options [Hash] merged http_options hash
|
|
60
79
|
# @return [nil]
|
|
80
|
+
# @raise [BadArgument] if a duration is invalid
|
|
61
81
|
def http_options=(new_options)
|
|
62
82
|
Aspera.assert_type(new_options, Hash)
|
|
63
83
|
kept = {}
|
|
64
84
|
new_options.each do |k, v|
|
|
85
|
+
begin
|
|
86
|
+
v = self.class.duration_seconds(v) if DURATION_KEYS.include?(k.to_sym)
|
|
87
|
+
rescue ParameterError => e
|
|
88
|
+
raise BadArgument, "Option http_options: #{k}: #{e.message}"
|
|
89
|
+
end
|
|
65
90
|
method = "#{k}=".to_sym
|
|
66
91
|
if Rest::Parameters.instance.respond_to?(method)
|
|
67
92
|
Rest::Parameters.instance.send(method, v)
|
data/lib/aspera/cli/info.rb
CHANGED
|
@@ -26,7 +26,7 @@ module Aspera
|
|
|
26
26
|
# Version with which this version of CLI was tested
|
|
27
27
|
SDK_VERSION = '1.1.9'
|
|
28
28
|
# Ruby version with which this version of CLI was tested, packaged in container image and portable packages
|
|
29
|
-
RUBY_TESTED_VERSION = '4.0.
|
|
29
|
+
RUBY_TESTED_VERSION = '4.0.10'
|
|
30
30
|
# Suffix of the RubyInstaller release of `RUBY_TESTED_VERSION`, packaged in the Windows zip
|
|
31
31
|
# (release tag: `RubyInstaller-<Ruby version><suffix>`)
|
|
32
32
|
# https://github.com/oneclick/rubyinstaller2/releases
|
data/lib/aspera/cli/mcp/tool.rb
CHANGED
|
@@ -15,8 +15,8 @@ require 'json'
|
|
|
15
15
|
unless defined?(MCP::Tool)
|
|
16
16
|
begin
|
|
17
17
|
require 'mcp'
|
|
18
|
-
rescue LoadError
|
|
19
|
-
raise Cli::Error, "The 'mcp' and 'rack'
|
|
18
|
+
rescue LoadError => e
|
|
19
|
+
raise Aspera::Cli::Error, "The 'mcp' gem is required (and 'rack' for transport http). Install with: gem install mcp rack (#{e.message})"
|
|
20
20
|
end
|
|
21
21
|
end
|
|
22
22
|
|
|
@@ -51,7 +51,7 @@ module Aspera
|
|
|
51
51
|
"@val:..." — literal string (no further decoding)
|
|
52
52
|
"@secret:name" — secret entered by the user on a web page, never visible to you:
|
|
53
53
|
use it for a secret the user did not give, e.g. "--password=@secret:password"
|
|
54
|
-
Note: @stdin: is disabled; @ruby:, @file:, @uri:, @env: depend on the security level of the server.
|
|
54
|
+
Note: @stdin: is disabled; @ruby:, @file:, @uri:, @env:, @zlib: depend on the security level of the server.
|
|
55
55
|
A secret (from @secret:, @vault:, @preset:) must be the whole value: no other prefix before it, not in @extend:.
|
|
56
56
|
|
|
57
57
|
AUTOMATIC FLAGS
|
|
@@ -152,6 +152,8 @@ module Aspera
|
|
|
152
152
|
Log.dump(:mcp_execute, effective_args)
|
|
153
153
|
runner = Runner.new(effective_args)
|
|
154
154
|
result = runner.run_with_result
|
|
155
|
+
# The command of a poll is executed when the result is displayed, on the command line only
|
|
156
|
+
raise Cli::BadArgument, 'Option poll: not available in MCP server mode, call the tool again instead' if result.is_a?(Result::Poll)
|
|
155
157
|
# Secrets never reach the AI client, whatever the output format requested in args.
|
|
156
158
|
# As on the command line, `--out.query` applies to the data of an object or list, after secrets are removed
|
|
157
159
|
if result.is_a?(Result::SingleObject) || result.is_a?(Result::ObjectList) || result.is_a?(Result::ValueList)
|