aspera-cli 4.27.5 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +301 -4
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +139 -58
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +103 -26
- data/SECURITY.md +70 -0
- data/bin/ascli +8 -5
- data/bin/asession +10 -7
- data/docs/README.md +2551 -772
- data/docs/test-mcp-with-ai.md +13 -11
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +34 -22
- data/lib/aspera/agent/factory.rb +8 -4
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +8 -5
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +23 -19
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +84 -42
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +354 -0
- data/lib/aspera/ascp/management.rb +102 -48
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +16 -37
- data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +37 -34
- data/lib/aspera/cli/command_line.rb +24 -6
- data/lib/aspera/cli/command_registry.rb +49 -4
- data/lib/aspera/cli/command_spec.rb +34 -4
- data/lib/aspera/cli/completion/ascli.bash +53 -5
- data/lib/aspera/cli/completion/ascli.fish +27 -8
- data/lib/aspera/cli/completion/ascli.ps1 +63 -0
- data/lib/aspera/cli/completion/ascli.zsh +36 -4
- data/lib/aspera/cli/completion_actions.rb +594 -0
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +45 -0
- data/lib/aspera/cli/extended_value.rb +83 -24
- data/lib/aspera/cli/formatter.rb +142 -73
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -11
- data/lib/aspera/cli/http.rb +17 -13
- data/lib/aspera/cli/info.rb +17 -6
- data/lib/aspera/cli/mailer.rb +7 -4
- data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
- data/lib/aspera/cli/mcp/secrets.rb +146 -0
- data/lib/aspera/cli/mcp/tool.rb +232 -0
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +72 -11
- data/lib/aspera/cli/options.schema.yaml +856 -81
- data/lib/aspera/cli/parser.rb +64 -31
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +250 -44
- data/lib/aspera/cli/plugins/ats.rb +9 -8
- data/lib/aspera/cli/plugins/base.rb +51 -25
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +207 -141
- data/lib/aspera/cli/plugins/console.rb +9 -4
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +22 -8
- data/lib/aspera/cli/plugins/faspex5.rb +37 -44
- data/lib/aspera/cli/plugins/faspio.rb +41 -8
- data/lib/aspera/cli/plugins/httpgw.rb +8 -4
- data/lib/aspera/cli/plugins/mcp.rb +186 -121
- data/lib/aspera/cli/plugins/node.rb +41 -34
- data/lib/aspera/cli/plugins/oauth.rb +5 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
- data/lib/aspera/cli/plugins/preview.rb +69 -44
- data/lib/aspera/cli/plugins/server.rb +66 -12
- data/lib/aspera/cli/plugins/shares.rb +288 -48
- data/lib/aspera/cli/plugins/transferd.rb +223 -0
- data/lib/aspera/cli/preset_actions.rb +155 -33
- data/lib/aspera/cli/preset_manager.rb +72 -16
- data/lib/aspera/cli/prompt.rb +7 -2
- data/lib/aspera/cli/result.rb +28 -11
- data/lib/aspera/cli/runner.rb +193 -80
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +204 -0
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +13 -10
- data/lib/aspera/cli/table_fitter.rb +119 -0
- data/lib/aspera/cli/terminal_formatter.rb +32 -40
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +31 -19
- data/lib/aspera/cli/transfer_progress.rb +13 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +197 -50
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.html.erb +61 -0
- data/lib/aspera/cli/web_secret.rb +222 -0
- data/lib/aspera/cli/wizard.rb +58 -45
- data/lib/aspera/command_line_builder.rb +35 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +42 -25
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/duration.rb +45 -0
- data/lib/aspera/environment.rb +106 -33
- data/lib/aspera/exec_spec.rb +9 -5
- data/lib/aspera/faspex_gw.rb +105 -35
- data/lib/aspera/faspex_postproc.rb +193 -47
- data/lib/aspera/formatter_interface.rb +4 -2
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +6 -3
- data/lib/aspera/json_rpc/client.rb +55 -35
- data/lib/aspera/json_rpc/version.rb +4 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +20 -10
- data/lib/aspera/markdown.rb +155 -115
- data/lib/aspera/mime.rb +9 -0
- data/lib/aspera/nagios.rb +9 -5
- data/lib/aspera/node_emulator.rb +111 -112
- data/lib/aspera/oauth/client/base.rb +163 -0
- data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
- data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
- data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
- data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
- data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +85 -0
- data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
- data/lib/aspera/oauth/client/web.rb +84 -0
- data/lib/aspera/oauth/server/authorize.html.erb +34 -0
- data/lib/aspera/oauth/server/pkce.rb +427 -0
- data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +11 -6
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +4 -1
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +30 -1
- data/lib/aspera/rest/aspera_errors.rb +55 -60
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +42 -42
- data/lib/aspera/rest/error_analyzer.rb +14 -2
- data/lib/aspera/rest/list.rb +14 -10
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +13 -4
- data/lib/aspera/rest.rb +9 -7
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +56 -78
- data/lib/aspera/schema/reader.rb +46 -11
- data/lib/aspera/schema/registry.rb +45 -24
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +166 -0
- data/lib/aspera/secret/hider.rb +201 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +110 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +114 -0
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +6 -1
- data/lib/aspera/sync/conf.schema.yaml +25 -10
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +34 -12
- data/lib/aspera/temp_file_manager.rb +85 -22
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +22 -14
- data/lib/aspera/transfer/result.rb +3 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +45 -18
- data/lib/aspera/transfer/spec.schema.yaml +8 -5
- data/lib/aspera/transfer/units.rb +75 -0
- data/lib/aspera/uri_reader.rb +57 -7
- data/lib/aspera/web_auth/index.html.erb +20 -0
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +59 -14
- data/lib/aspera/web_page.html.erb +262 -0
- data/lib/aspera/web_page.rb +66 -0
- data/lib/aspera/web_server_simple.rb +82 -13
- data/lib/aspera/yaml.rb +5 -1
- data/llms.txt +81 -0
- data/skills/ascli/SKILL.md +88 -0
- data.tar.gz.sig +0 -0
- metadata +116 -48
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/mcp_tool.rb +0 -189
- data/lib/aspera/keychain/base.rb +0 -25
- data/lib/aspera/keychain/encrypted_hash.rb +0 -188
- data/lib/aspera/keychain/factory.rb +0 -59
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
- data/lib/aspera/keychain/macos_security.rb +0 -171
- data/lib/aspera/keychain/one_password_api.rb +0 -86
- data/lib/aspera/keychain/one_password_base.rb +0 -34
- data/lib/aspera/keychain/one_password_cli.rb +0 -98
- data/lib/aspera/oauth/base.rb +0 -141
- data/lib/aspera/oauth/web.rb +0 -57
- data/lib/aspera/oauth.rb +0 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
- data/lib/aspera/secret_hider.rb +0 -117
- data/lib/aspera/web_auth/index.html +0 -147
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
# cspell:ignore asperascp asws getname lastlog lsass pcap tokengen trapd userdata winuser wtmp
|
|
7
|
+
require_relative '../assert'
|
|
8
|
+
require_relative '../log'
|
|
9
|
+
require_relative 'log_analyzer/log_file'
|
|
10
|
+
require_relative 'log_analyzer/records'
|
|
11
|
+
require 'resolv'
|
|
12
|
+
require 'set'
|
|
13
|
+
require 'stringio'
|
|
14
|
+
require 'time'
|
|
15
|
+
require 'zlib'
|
|
16
|
+
|
|
17
|
+
module Aspera
|
|
18
|
+
module Ascp
|
|
19
|
+
# Analysis of ascp transfer logs: sessions, statistics and diagnostics.
|
|
20
|
+
# Results are tables: Hash (one object) or Array of Hash (list of objects).
|
|
21
|
+
class LogAnalyzer
|
|
22
|
+
# Oldest ascp version for which the analysis is designed
|
|
23
|
+
MIN_VERSION = '4.0'
|
|
24
|
+
# Path that designates the standard input
|
|
25
|
+
STDIN_PATH = '-'
|
|
26
|
+
# First two bytes of a gzip file
|
|
27
|
+
GZIP_MAGIC = "\x1F\x8B".b.freeze
|
|
28
|
+
# Files analyzed when scanning a folder
|
|
29
|
+
INCLUDED_FILES = /transfer|messages|log/i
|
|
30
|
+
# Files not analyzed when scanning a folder (path relative to folder)
|
|
31
|
+
EXCLUDED_FILES = %r{async|ssl_|\.dat|\.xml|\.html|graph|winuser|manager|asperascp\.20|background|asws|node|plugin|opt/aspera/var|\.exe|trapd|rtf|lsass|lastlog|wtmp|
|
|
32
|
+
process|stats|production|mongrel|console|ntp\.|prepost|aspx|aspshell|tokengen|userdata|http|asconfigurator|ascmd|\.jpg|\.png|ui\.|connect|ssh|
|
|
33
|
+
stderr|stdout|central|sync|\.zip|\.msi|\.tgz|\.tbz|\.tar|\.rar|\.bz2?|\.pcap|analysis\.txt|\.7z}ix
|
|
34
|
+
# Folders not scanned
|
|
35
|
+
EXCLUDED_FOLDERS = /\$|recycle/i
|
|
36
|
+
# Statistics summed over log files
|
|
37
|
+
SUMMED_STATS = %i[total_sessions success failed cancelled http_fallback smart_reset win_clock retries] + LogFile::FINDINGS.keys
|
|
38
|
+
# Severities of diagnostics, most severe first
|
|
39
|
+
SEVERITIES = %w[error warning info].freeze
|
|
40
|
+
# Maximum values of log files
|
|
41
|
+
MAXIMA = {max_total: 'max_throughput', max_out: 'max_send', max_in: 'max_receive', max_sess: 'max_sessions'}.freeze
|
|
42
|
+
|
|
43
|
+
# Default diagnostic thresholds
|
|
44
|
+
DEFAULT_THRESHOLDS = {
|
|
45
|
+
efficiency: 90.0,
|
|
46
|
+
control_loss_percent: 5.0,
|
|
47
|
+
control_loss_sent: 10,
|
|
48
|
+
retry_window: 3600,
|
|
49
|
+
last_error_window: 60,
|
|
50
|
+
rate_decline_ratio: 0.7,
|
|
51
|
+
rate_decline_min_samples: 10,
|
|
52
|
+
rate_decline_delay: 20,
|
|
53
|
+
rate_decline_end_fraction: 0.1,
|
|
54
|
+
license_expiry_days: 30,
|
|
55
|
+
small_files_min_count: 100,
|
|
56
|
+
small_files_max_mean: 1_000_000,
|
|
57
|
+
small_mtu_max: 1400
|
|
58
|
+
}.freeze
|
|
59
|
+
|
|
60
|
+
# @return [Array<LogFile>] analyzed log files
|
|
61
|
+
attr_reader :log_files, :thresholds
|
|
62
|
+
|
|
63
|
+
# @param period [Integer] Snapshot period in seconds
|
|
64
|
+
# @param zero_bytes [Boolean] Include sessions without transferred bytes
|
|
65
|
+
# @param mtu [Integer, nil] MTU to use when not found in logs
|
|
66
|
+
# @param resolve [Boolean] Resolve peer addresses to host names
|
|
67
|
+
# @param thresholds [Hash] Diagnostic thresholds
|
|
68
|
+
# @param timezone [String, nil] Time zone of the log lines: `UTC` or `+HH:MM`. Time zone of the analyst if nil.
|
|
69
|
+
def initialize(period: 60, zero_bytes: false, mtu: nil, resolve: false, thresholds: {}, timezone: nil)
|
|
70
|
+
self.class.check_timezone(timezone)
|
|
71
|
+
@timezone = timezone
|
|
72
|
+
thresholds = thresholds.transform_keys(&:to_sym)
|
|
73
|
+
unknown = thresholds.keys - DEFAULT_THRESHOLDS.keys
|
|
74
|
+
Aspera.assert(unknown.empty?, type: ArgumentError) { "Unknown threshold: #{unknown.join(', ')}" }
|
|
75
|
+
@thresholds = DEFAULT_THRESHOLDS.merge(thresholds)
|
|
76
|
+
@options = {period: period, zero_bytes: zero_bytes, mtu: mtu, thresholds: @thresholds, timezone: timezone}
|
|
77
|
+
@resolve = resolve
|
|
78
|
+
@log_files = []
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
class << self
|
|
82
|
+
# @param timezone [String, nil] Time zone of the log lines: `UTC` or `+HH:MM`
|
|
83
|
+
# @raise [ParameterError] if the time zone is not valid
|
|
84
|
+
def check_timezone(timezone)
|
|
85
|
+
return if timezone.nil?
|
|
86
|
+
Time.at(0, in: timezone)
|
|
87
|
+
rescue ArgumentError
|
|
88
|
+
raise ParameterError, "Invalid time zone: #{timezone.inspect}, expected `UTC` or `+HH:MM`"
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# @param path [String] log file, `-` (standard input), or folder
|
|
92
|
+
# @return [Array<String>] the log file, or the log files in the folder (recursively)
|
|
93
|
+
def log_paths(path)
|
|
94
|
+
return [path] unless File.directory?(path)
|
|
95
|
+
without_links(scan_folder(path, path, []))
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Read the lines of a log file: plain, or compressed with gzip (detected with its first bytes)
|
|
99
|
+
# @param path [String] Log file, or `-` for the standard input
|
|
100
|
+
# @param encoding [Encoding] Encoding of lines
|
|
101
|
+
# @yieldparam line [String]
|
|
102
|
+
def read_lines(path, encoding, &block)
|
|
103
|
+
io = path == STDIN_PATH ? StringIO.new($stdin.binmode.read) : File.open(path, 'rb')
|
|
104
|
+
gzip = io.read(2) == GZIP_MAGIC
|
|
105
|
+
io.rewind
|
|
106
|
+
io = Zlib::GzipReader.new(io, external_encoding: encoding) if gzip
|
|
107
|
+
io.set_encoding(encoding) unless gzip
|
|
108
|
+
io.each_line(&block)
|
|
109
|
+
ensure
|
|
110
|
+
io&.close
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Load the log files of a path
|
|
114
|
+
# In a folder, a file that is not an ascp log is ignored. A file given explicitly must be an ascp log.
|
|
115
|
+
# @param path [String] log file, `-` (standard input), or folder
|
|
116
|
+
# @yieldparam file [String] log file
|
|
117
|
+
# @yieldreturn [#ascp?] the loaded log
|
|
118
|
+
# @return [Array] the loaded logs that are ascp logs
|
|
119
|
+
def load_logs(path)
|
|
120
|
+
explicit = !File.directory?(path)
|
|
121
|
+
log_paths(path).filter_map do |file|
|
|
122
|
+
log = yield(file)
|
|
123
|
+
next log if log.ascp?
|
|
124
|
+
Aspera.assert(!explicit, type: ParameterError) { "Not an ascp log: #{file}" }
|
|
125
|
+
Log.log.debug { "Ignored, not an ascp log: #{file}" }
|
|
126
|
+
nil
|
|
127
|
+
rescue Zlib::Error, Errno::EPIPE => e
|
|
128
|
+
# JRuby raises Errno::EPIPE (instead of Zlib::Error) on truncated gzip input
|
|
129
|
+
raise ParameterError, "Cannot read #{file}: #{e.message}" if explicit
|
|
130
|
+
Log.log.warn("Ignored, cannot read #{file}: #{e.message}")
|
|
131
|
+
nil
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
private
|
|
136
|
+
|
|
137
|
+
# `ascp` creates `<name>.log` as a hard link to the current rotated file `<name>.<n>.log`: keep only one of them.
|
|
138
|
+
# @return [Array<String>] files, without those that are the same file as a previous one
|
|
139
|
+
def without_links(files)
|
|
140
|
+
seen = Set.new
|
|
141
|
+
files.select do |file|
|
|
142
|
+
stat = File.stat(file)
|
|
143
|
+
# No inode number on some platforms
|
|
144
|
+
stat.ino.zero? || seen.add?([stat.dev, stat.ino])
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# @return [Array<String>] files, with the log files of folder added
|
|
149
|
+
def scan_folder(root, folder, files)
|
|
150
|
+
Dir.children(folder).sort.each do |name|
|
|
151
|
+
next if name.start_with?('.')
|
|
152
|
+
path = File.join(folder, name)
|
|
153
|
+
if File.directory?(path)
|
|
154
|
+
scan_folder(root, path, files) unless name.match?(EXCLUDED_FOLDERS)
|
|
155
|
+
elsif name.match?(INCLUDED_FILES) && !path.delete_prefix(root).match?(EXCLUDED_FILES)
|
|
156
|
+
files.push(path)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
files
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Analyze a log file, or the log files in a folder (recursively)
|
|
164
|
+
# @param path [String] file, `-` (standard input) or folder
|
|
165
|
+
# @return [self]
|
|
166
|
+
def add(path)
|
|
167
|
+
@log_files.concat(self.class.load_logs(path) { |file| LogFile.new(file, **@options).analyze })
|
|
168
|
+
self
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# @return [Hash] statistics over all log files
|
|
172
|
+
def summary
|
|
173
|
+
stats = Hash.new(0)
|
|
174
|
+
errors = Hash.new(0)
|
|
175
|
+
categories = Hash.new(0)
|
|
176
|
+
result = {}
|
|
177
|
+
@log_files.each do |log|
|
|
178
|
+
SUMMED_STATS.each { |key| stats[key] += log.stats[key] }
|
|
179
|
+
stats[:stuck_nack_loss] = [stats[:stuck_nack_loss], log.stats[:stuck_nack_loss]].max
|
|
180
|
+
log.stats[:errors].each { |error, count| errors[error] += count }
|
|
181
|
+
log.stats[:error_categories].each { |category, count| categories[category] += count }
|
|
182
|
+
MAXIMA.each do |key, name|
|
|
183
|
+
next if log.stats[key] <= result.fetch(name, 0)
|
|
184
|
+
result[name] = log.stats[key]
|
|
185
|
+
result["#{name}_time"] = iso(log.stats[:"#{key}_time"])
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
first = @log_files.filter_map(&:first_time).min
|
|
189
|
+
last = @log_files.filter_map(&:last_time).max
|
|
190
|
+
{
|
|
191
|
+
'files' => @log_files.size,
|
|
192
|
+
'first_log' => iso(first),
|
|
193
|
+
'last_log' => iso(last),
|
|
194
|
+
'sessions' => stats[:total_sessions],
|
|
195
|
+
'success' => stats[:success],
|
|
196
|
+
'failed' => stats[:failed],
|
|
197
|
+
'cancelled' => stats[:cancelled],
|
|
198
|
+
'success_rate' => ((100.0 * (stats[:total_sessions] - stats[:failed]) / stats[:total_sessions]).round(2) if stats[:total_sessions].positive?),
|
|
199
|
+
'http_fallback' => stats[:http_fallback],
|
|
200
|
+
'retries' => stats[:retries],
|
|
201
|
+
'multi_session' => @log_files.sum { |log| log.multi_sessions.size },
|
|
202
|
+
'errors' => errors.sort_by { |_, count| -count }.to_h,
|
|
203
|
+
'error_categories' => categories.sort_by { |_, count| -count }.to_h,
|
|
204
|
+
'diagnostics' => SEVERITIES.to_h { |severity| [severity, 0] }.merge(diagnostics.map { |row| row['severity'] }.tally).reject { |_, count| count.zero? }
|
|
205
|
+
}.merge(result).merge(
|
|
206
|
+
'findings' => LogFile.findings(stats),
|
|
207
|
+
'warnings' => warnings
|
|
208
|
+
).compact
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# @return [Array<String>] warnings about log files
|
|
212
|
+
def warnings
|
|
213
|
+
@log_files.flat_map do |log|
|
|
214
|
+
log.old_versions.map { |version| "#{log.path}: ascp #{version} is older than #{MIN_VERSION}, analysis may be inaccurate" }
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# @return [Array<Hash>] one row per session, with the other side if its log is also analyzed
|
|
219
|
+
def sessions
|
|
220
|
+
rows = @log_files.flat_map(&:session_reports)
|
|
221
|
+
join_sides(rows)
|
|
222
|
+
single_file(rows)
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# @return [Array<Hash>] one row per diagnostic of sessions: id, severity, message, recommendation (if the diagnostic has one)
|
|
226
|
+
def diagnostics
|
|
227
|
+
single_file(@log_files.flat_map(&:diagnostics))
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# @return [Array<Hash>] one row per multi-session transfer
|
|
231
|
+
def multi_sessions
|
|
232
|
+
single_file(@log_files.flat_map(&:multi_sessions))
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# @return [Array<Hash>] one row per snapshot period: aggregate rates
|
|
236
|
+
def snapshots
|
|
237
|
+
single_file(@log_files.flat_map(&:snapshots))
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# @return [Array<Hash>] one row per session in each snapshot period
|
|
241
|
+
def snapshot_sessions
|
|
242
|
+
single_file(@log_files.flat_map(&:snapshot_sessions))
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# @return [Array<Hash>] sessions and errors by peer address, most failures first
|
|
246
|
+
def peers
|
|
247
|
+
rows = {}
|
|
248
|
+
each_session do |_, s|
|
|
249
|
+
peer = s[:peer] or next
|
|
250
|
+
row = (rows[peer] ||= {'peer' => peer, 'users' => Set.new, 'sessions' => 0, 'success' => 0, 'failed' => 0, 'cancelled' => 0, 'errors' => Hash.new(0)})
|
|
251
|
+
row['users'].add(s[:user]) if s[:user]
|
|
252
|
+
row['sessions'] += 1
|
|
253
|
+
case s[:status]
|
|
254
|
+
when 'success' then row['success'] += 1
|
|
255
|
+
when 'failed'
|
|
256
|
+
row[s[:error].to_s.match?(/cancelled/i) ? 'cancelled' : 'failed'] += 1
|
|
257
|
+
row['errors'][s[:error]] += 1 if s[:error]
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
rows.values.sort_by { |row| -row['failed'] }.map do |row|
|
|
261
|
+
row.merge('users' => row['users'].to_a, 'errors' => row['errors'].to_h).merge(hostname(row['peer']))
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# @return [Array<Hash>] one row per transferred file
|
|
266
|
+
def transfers
|
|
267
|
+
result = []
|
|
268
|
+
each_session do |log, s|
|
|
269
|
+
s.files.each do |name, file|
|
|
270
|
+
result.push({
|
|
271
|
+
'file' => log.path,
|
|
272
|
+
'session' => s[:uuid],
|
|
273
|
+
'pid' => s.pid,
|
|
274
|
+
'user' => s[:user],
|
|
275
|
+
'peer' => s[:peer],
|
|
276
|
+
'destination' => s[:dest],
|
|
277
|
+
'start' => iso(s[:start]),
|
|
278
|
+
'stop' => iso(s[:stop]),
|
|
279
|
+
'path' => name,
|
|
280
|
+
'direction' => file[:op] == 'recv' ? 'receive' : file[:op],
|
|
281
|
+
'size' => file[:size],
|
|
282
|
+
'status' => file[:status],
|
|
283
|
+
'error' => file[:error]
|
|
284
|
+
}.compact)
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
single_file(result)
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# @return [Array<Hash>] licenses of local and peer systems
|
|
291
|
+
def licenses
|
|
292
|
+
@log_files.each_with_object({}) do |log, all|
|
|
293
|
+
log.licenses.each { |key, license| all[key] = license.merge(file: log.path) }
|
|
294
|
+
end.map do |(number, address), license|
|
|
295
|
+
{'license' => number, 'address' => address, 'product' => license[:product], 'account' => license[:account], 'os' => license[:os], 'file' => license[:file]}.compact
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# @return [Array<Hash>] by peer: number of sessions with loss due to overdriving of network (shallow queues) or storage
|
|
300
|
+
def overdrives
|
|
301
|
+
counts = Hash.new(0)
|
|
302
|
+
each_session do |_, s|
|
|
303
|
+
cause = s.overdrive
|
|
304
|
+
counts[[s[:peer], cause]] += 1 if cause
|
|
305
|
+
end
|
|
306
|
+
counts.sort_by { |_, count| -count }.map do |(peer, cause), count|
|
|
307
|
+
{'peer' => peer, 'cause' => cause.to_s, 'occurrences' => count}.merge(hostname(peer))
|
|
308
|
+
end
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
private
|
|
312
|
+
|
|
313
|
+
def iso(time)
|
|
314
|
+
Time.at(time).getlocal(@timezone).iso8601 if time
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
def each_session
|
|
318
|
+
@log_files.each { |log| log.sessions.each { |s| yield(log, s) } }
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
# Same session in the logs of sender and receiver: add the other side to each row
|
|
322
|
+
# @param rows [Array<Hash>] sessions
|
|
323
|
+
def join_sides(rows)
|
|
324
|
+
rows.group_by { |row| row['session'] }.each do |uuid, same|
|
|
325
|
+
next if uuid.nil? || same.size != 2 || same.map { |row| row['direction'] }.uniq.size != 2
|
|
326
|
+
same.each_with_index do |row, index|
|
|
327
|
+
other = same[1 - index]
|
|
328
|
+
row['peer_side'] = other.slice('file', 'pid', 'role', 'status', 'error').compact
|
|
329
|
+
row['peer_diagnostics'] = other['diagnostics'] if other['diagnostics']&.any?
|
|
330
|
+
end
|
|
331
|
+
end
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
# Remove column `file` when only one file is analyzed
|
|
335
|
+
def single_file(rows)
|
|
336
|
+
return rows if @log_files.size > 1
|
|
337
|
+
rows.each { |row| row.delete('file') }
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# @return [Hash] host name of address, if resolution is enabled and successful
|
|
341
|
+
def hostname(address)
|
|
342
|
+
return {} unless @resolve && address
|
|
343
|
+
@hostnames ||= {}
|
|
344
|
+
@hostnames[address] ||= Resolv::DNS.open do |dns|
|
|
345
|
+
dns.timeouts = 5
|
|
346
|
+
dns.getname(address).to_s
|
|
347
|
+
rescue Resolv::ResolvError, ArgumentError
|
|
348
|
+
''
|
|
349
|
+
end
|
|
350
|
+
@hostnames[address].empty? ? {} : {'hostname' => @hostnames[address]}
|
|
351
|
+
end
|
|
352
|
+
end
|
|
353
|
+
end
|
|
354
|
+
end
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../assert'
|
|
7
|
+
require_relative '../log'
|
|
8
|
+
require_relative '../string_ext'
|
|
5
9
|
|
|
6
10
|
module Aspera
|
|
7
11
|
module Ascp
|
|
@@ -9,15 +13,15 @@ module Aspera
|
|
|
9
13
|
# Reference: `mgmtmess.c`
|
|
10
14
|
class Management
|
|
11
15
|
# References:
|
|
12
|
-
# https://www.google.com/search?q=FASP+error+codes
|
|
13
16
|
# https://www.ibm.com/support/pages/error-code-reference-tables
|
|
14
|
-
#
|
|
17
|
+
# `asmgmt_err.h` : codes, mnemonics and messages
|
|
18
|
+
# `mgmtmess.c` : `faspmgmt_err_name_str` (additional info) and `as_mgmt_err_is_retryable`
|
|
15
19
|
# Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
|
|
16
20
|
# rubocop:disable-next Layout/FirstHashElementLineBreak
|
|
17
21
|
ERRORS = {
|
|
18
22
|
# id retry-able mnemo/code message additional info
|
|
19
23
|
0 => {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'},
|
|
20
|
-
1 => {r: false, c: 'FASP_PROTO', m: 'Generic fasp(tm) protocol error', a: '
|
|
24
|
+
1 => {r: false, c: 'FASP_PROTO', m: 'Generic fasp(tm) protocol error', a: 'FASP error'},
|
|
21
25
|
2 => {r: false, c: 'ASCP', m: 'Generic SCP error', a: 'ASCP error'},
|
|
22
26
|
3 => {r: false, c: 'AMBIGUOUS_TARGET', m: 'Target incorrectly specified', a: 'Ambiguous target'},
|
|
23
27
|
4 => {r: false, c: 'NO_SUCH_FILE', m: 'No such file or directory', a: 'No such file or directory'},
|
|
@@ -29,13 +33,13 @@ module Aspera
|
|
|
29
33
|
10 => {r: false, c: 'LIC_RATE_EXCEEDED', m: 'Rate exceeds the cap imposed by license', a: 'Rate exceeds cap imposed by license'},
|
|
30
34
|
11 => {r: false, c: 'INTERNAL_ERROR', m: 'Internal error (unexpected error)', a: 'Internal error'},
|
|
31
35
|
12 => {r: true, c: 'TRANSFER_ERROR', m: 'Error establishing control connection',
|
|
32
|
-
|
|
36
|
+
a: 'Error establishing SSH connection (check SSH port and firewall)'},
|
|
33
37
|
13 => {r: true, c: 'TRANSFER_TIMEOUT', m: 'Timeout establishing control connection',
|
|
34
|
-
|
|
38
|
+
a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
|
|
35
39
|
14 => {r: true, c: 'CONNECTION_ERROR', m: 'Error establishing data connection',
|
|
36
|
-
|
|
40
|
+
a: 'Error establishing UDP connection (check UDP port and firewall)'},
|
|
37
41
|
15 => {r: true, c: 'CONNECTION_TIMEOUT', m: 'Timeout establishing data connection',
|
|
38
|
-
|
|
42
|
+
a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
|
|
39
43
|
16 => {r: true, c: 'CONNECTION_LOST', m: 'Connection lost', a: 'Connection lost'},
|
|
40
44
|
17 => {r: true, c: 'RCVR_SEND_ERROR', m: 'Receiver fails to send feedback', a: 'Network failure (receiver can\'t send feedback)'},
|
|
41
45
|
18 => {r: true, c: 'RCVR_RECV_ERROR', m: 'Receiver fails to receive data packets', a: 'Network failure (receiver can\'t receive UDP data)'},
|
|
@@ -67,34 +71,38 @@ module Aspera
|
|
|
67
71
|
44 => {r: true, c: 'PMTU_BRTT_ERROR', m: 'UDP session initiation fatal error', a: 'UDP session initiation fatal error'},
|
|
68
72
|
45 => {r: true, c: 'BWMEAS_ERROR', m: 'Bandwidth measurement fatal error', a: 'Bandwidth measurement fatal error'},
|
|
69
73
|
46 => {r: false, c: 'VLINK_ERROR', m: 'Virtual link error', a: 'Virtual link error'},
|
|
70
|
-
47 => {r:
|
|
71
|
-
|
|
72
|
-
48 => {r: false, c: 'FILE_ENCRYPTION_ERROR', m: 'File encryption error, e.g. corrupt file',
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
50 => {r: false, c: 'BAD_CONFIGURATION', m: 'Aspera.conf contains invalid data and was rejected', a: 'Invalid configuration'},
|
|
74
|
+
47 => {r: true, c: 'CONNECTION_ERROR_HTTP', m: 'Error establishing HTTP connection',
|
|
75
|
+
a: 'Error establishing HTTP connection (check HTTP port and firewall)'},
|
|
76
|
+
48 => {r: false, c: 'FILE_ENCRYPTION_ERROR', m: 'File encryption error, e.g. corrupt file', a: 'File encryption/decryption error, e.g. corrupt file'},
|
|
77
|
+
49 => {r: false, c: 'FILE_DECRYPTION_PASS', m: 'File decryption bad passphrase', a: 'File decryption error, bad passphrase'},
|
|
78
|
+
50 => {r: false, c: 'BAD_CONFIGURATION', m: 'Aspera.conf contains invalid data and was rejected', a: 'Invalid configuration'},
|
|
76
79
|
51 => {r: false, c: 'INSECURE_CONNECTION', m: 'Remote-host key check failure', a: 'Remote host is not who we expected'},
|
|
77
80
|
52 => {r: false, c: 'START_VALIDATION_FAILED', m: 'File start validation failed', a: 'File start validation failed'},
|
|
78
81
|
53 => {r: false, c: 'STOP_VALIDATION_FAILED', m: 'File stop validation failed', a: 'File stop validation failed'},
|
|
79
82
|
54 => {r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed', a: 'File threshold validation failed'},
|
|
80
83
|
55 => {r: false, c: 'FILEPATH_TOO_LONG', m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
|
|
81
84
|
56 => {r: false, c: 'ILLEGAL_CHARS_IN_PATH', m: 'Windows path contains illegal characters',
|
|
82
|
-
|
|
83
|
-
57 => {r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT', m: 'Chunk size/start must be aligned with storage',
|
|
84
|
-
|
|
85
|
+
a: 'Path being written to Windows file system contains illegal characters'},
|
|
86
|
+
57 => {r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT', m: 'Chunk size/start must be aligned with storage',
|
|
87
|
+
a: 'Data size must match storage chunk size and alignment requirements'},
|
|
88
|
+
58 => {r: false, c: 'VALIDATION_SESSION_ABORT', m: 'Session aborted due to validation error(s)', a: 'Session aborted due to validation error(s)'},
|
|
85
89
|
59 => {r: false, c: 'REMOTE_STORAGE_ERROR', m: 'Remote storage errored', a: 'Remote storage errored'},
|
|
86
|
-
60 => {r: false, c: 'LUA_SCRIPT_ABORTED_SESSION', m: 'Session aborted due to Lua script abort', a: 'Session aborted
|
|
90
|
+
60 => {r: false, c: 'LUA_SCRIPT_ABORTED_SESSION', m: 'Session aborted due to Lua script abort', a: 'Session aborted'},
|
|
87
91
|
61 => {r: true, c: 'SSEAR_RETRYABLE', m: 'Transfer failed because of a retryable Encryption at Rest error',
|
|
88
|
-
|
|
89
|
-
62 => {r: false, c: 'SSEAR_FATAL', m: 'Transfer failed because of a fatal Encryption at Rest error',
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
92
|
+
a: 'Server-side Encryption At Rest unavailable'},
|
|
93
|
+
62 => {r: false, c: 'SSEAR_FATAL', m: 'Transfer failed because of a fatal Encryption at Rest error', a: 'Server-side Encryption At Rest failed'},
|
|
94
|
+
63 => {r: false, c: 'LINK_LOOP', m: 'Path refers to a symbolic link loop', a: 'Symbolic link loop'},
|
|
95
|
+
64 => {r: false, c: 'CANNOT_RENAME_PARTIAL', m: 'Can\'t rename a partial file', a: 'Cannot rename partial file'},
|
|
96
|
+
65 => {r: false, c: 'CIPHER_NON_COMPAT_FIPS', m: 'Can\'t use this cipher with FIPS mode enabled', a: 'Cypher not supported in FIPS mode'},
|
|
97
|
+
66 => {r: false, c: 'PEER_REQUIRES_FIPS', m: 'Peer rejects cipher due to FIPS mode enabled on peer', a: 'Peer requires FIPS be enabled'},
|
|
98
|
+
67 => {r: false, c: 'UPLOAD_IN_PROGRESS', m: 'Conflict between two transfers', a: 'File upload already in progress'},
|
|
99
|
+
68 => {r: false, c: 'MIN_VERSION_CHECK', m: 'Minimum version check failed', a: 'Failed to start due to minimum version check'},
|
|
100
|
+
69 => {r: true, c: 'MAX_ASCP', m: 'Maximum ascp processes on a node exceeded',
|
|
101
|
+
a: 'Failed to start session maximum transfer sessions exceeded'}
|
|
96
102
|
}.freeze
|
|
97
|
-
#
|
|
103
|
+
# Mnemonic => error, see ERRORS
|
|
104
|
+
ERRORS_BY_MNEMONIC = ERRORS.values.to_h { |error| [error[:c], error] }.freeze
|
|
105
|
+
# Same as `as_mgmt_cmd_name_str` (`NOTIFICATION` is twice there too)
|
|
98
106
|
OPERATIONS = %w[
|
|
99
107
|
NOP
|
|
100
108
|
START
|
|
@@ -117,8 +125,9 @@ module Aspera
|
|
|
117
125
|
CLOSE
|
|
118
126
|
SKIP
|
|
119
127
|
ARGSTOP
|
|
120
|
-
]
|
|
128
|
+
].freeze
|
|
121
129
|
|
|
130
|
+
# Same as `as_mgmt_arg_name_str`
|
|
122
131
|
PARAMETERS = %w[
|
|
123
132
|
Type
|
|
124
133
|
File
|
|
@@ -241,7 +250,7 @@ module Aspera
|
|
|
241
250
|
FilesEncrypt
|
|
242
251
|
FilesDecrypt
|
|
243
252
|
DatagramSize
|
|
244
|
-
|
|
253
|
+
Stalled
|
|
245
254
|
XoptFlags
|
|
246
255
|
VLinkVersion
|
|
247
256
|
PeerVLinkVersion
|
|
@@ -273,26 +282,44 @@ module Aspera
|
|
|
273
282
|
PostTransferValidation
|
|
274
283
|
OverwritePolicyCap
|
|
275
284
|
ExtraCreatePolicy
|
|
276
|
-
|
|
285
|
+
FipsModeEnabled
|
|
286
|
+
].freeze
|
|
277
287
|
# Management port start message
|
|
278
288
|
PROT_VERSION = '2'
|
|
279
|
-
MGT_HEADER = "FASPMGR #{PROT_VERSION}"
|
|
289
|
+
MGT_HEADER = "FASPMGR #{PROT_VERSION}".freeze
|
|
280
290
|
# empty line is separator to end event information
|
|
281
291
|
MGT_FRAME_SEPARATOR = ''
|
|
282
292
|
# fields description for JSON generation
|
|
283
|
-
#
|
|
284
|
-
INTEGER_FIELDS = %w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy
|
|
293
|
+
# Integer fields may also be a percentage (e.g. `Rate: 50%`): kept as String
|
|
294
|
+
INTEGER_FIELDS = %w[Bytescont FaspFileArgIndex StartByte EndByte Rate MinRate Port Priority RateCap MinRateCap PriorityCap TCPPort CreatePolicy TimePolicy
|
|
285
295
|
DatagramSize XoptFlags VLinkVersion PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize
|
|
286
|
-
ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec
|
|
287
|
-
|
|
296
|
+
ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes PartialPreTransferBytes TransferBytes PMTU Elapsedusec Delay
|
|
297
|
+
Code RetryTimeout XferRetry MoveRangeLow MoveRangeHigh
|
|
298
|
+
PreTransferFiles PreTransferDirs PreTransferSpecial PreTransferFailed PreTransferExcluded
|
|
299
|
+
ArgScansAttempted ArgScansCompleted PathScansAttempted PathScansFailed PathScansIrregular PathScansExcluded
|
|
300
|
+
DirScansCompleted FileScansCompleted DirCreatesAttempted DirCreatesFailed DirCreatesPassed
|
|
301
|
+
TransfersAttempted TransfersFailed TransfersPassed TransfersSkipped
|
|
302
|
+
ArgTransfersAttempted ArgTransfersFailed ArgTransfersPassed ArgTransfersSkipped].freeze
|
|
288
303
|
BOOLEAN_FIELDS = %w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled
|
|
289
|
-
MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect].freeze
|
|
304
|
+
MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect Stalled PostTransferValidation FipsModeEnabled].freeze
|
|
290
305
|
BOOLEAN_TRUE = 'Yes'
|
|
306
|
+
INTEGER_VALUE = /\A-?\d+\z/
|
|
307
|
+
# Fields with path values, escaped on management port (`as_mgmt_escape_path`)
|
|
308
|
+
PATH_FIELDS = %w[Source Destination File].freeze
|
|
309
|
+
# Characters escaped in path values: `ESC` followed by the replacement
|
|
310
|
+
PATH_ESCAPE = "\e"
|
|
311
|
+
PATH_ESCAPES = {"\n" => 'n', "\r" => 'r', PATH_ESCAPE => PATH_ESCAPE}.freeze
|
|
291
312
|
|
|
292
|
-
private_constant :OPERATIONS, :PARAMETERS, :MGT_HEADER, :MGT_FRAME_SEPARATOR, :INTEGER_FIELDS, :BOOLEAN_FIELDS, :BOOLEAN_TRUE
|
|
293
|
-
|
|
313
|
+
private_constant :ERRORS_BY_MNEMONIC, :OPERATIONS, :PARAMETERS, :MGT_HEADER, :MGT_FRAME_SEPARATOR, :INTEGER_FIELDS, :BOOLEAN_FIELDS, :BOOLEAN_TRUE, :INTEGER_VALUE,
|
|
314
|
+
:PATH_FIELDS, :PATH_ESCAPE, :PATH_ESCAPES
|
|
294
315
|
|
|
295
316
|
class << self
|
|
317
|
+
# @param mnemonic [String] mnemonic of error, e.g. `CONNECTION_LOST`
|
|
318
|
+
# @return [Hash, nil] error of management port (see ERRORS), nil if unknown
|
|
319
|
+
def error_by_mnemonic(mnemonic)
|
|
320
|
+
ERRORS_BY_MNEMONIC[mnemonic]
|
|
321
|
+
end
|
|
322
|
+
|
|
296
323
|
# Translate native event name to snake case
|
|
297
324
|
def field_native_to_snake(name)
|
|
298
325
|
case name
|
|
@@ -305,7 +332,7 @@ module Aspera
|
|
|
305
332
|
# Translate snake case event name to native
|
|
306
333
|
# @param name [String] Field name
|
|
307
334
|
def field_snake_to_native(name)
|
|
308
|
-
field = name.delete('_')
|
|
335
|
+
field = name.to_s.delete('_')
|
|
309
336
|
result = PARAMETERS.find { |w| w.casecmp?(field) }
|
|
310
337
|
Aspera.assert(!result.nil?) { "No such field: #{name}" }
|
|
311
338
|
result
|
|
@@ -315,7 +342,7 @@ module Aspera
|
|
|
315
342
|
def event_native_to_snake(event)
|
|
316
343
|
event.each_with_object({}) do |(key, value), h|
|
|
317
344
|
h[field_native_to_snake(key)] =
|
|
318
|
-
if INTEGER_FIELDS.include?(key) then value.to_i
|
|
345
|
+
if INTEGER_FIELDS.include?(key) && INTEGER_VALUE.match?(value) then value.to_i
|
|
319
346
|
elsif BOOLEAN_FIELDS.include?(key) then value.eql?(BOOLEAN_TRUE)
|
|
320
347
|
else
|
|
321
348
|
value
|
|
@@ -323,12 +350,36 @@ module Aspera
|
|
|
323
350
|
end
|
|
324
351
|
end
|
|
325
352
|
|
|
353
|
+
# Escape path value for management port: line breaks and escape char
|
|
354
|
+
# @param value [String] Path
|
|
355
|
+
# @return [String] Escaped path
|
|
356
|
+
def escape_path(value)
|
|
357
|
+
value.gsub(/[\n\r\e]/) { |char| "#{PATH_ESCAPE}#{PATH_ESCAPES[char]}" }
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
# Reverse of `escape_path`
|
|
361
|
+
# @param value [String] Escaped path
|
|
362
|
+
# @return [String] Path
|
|
363
|
+
def unescape_path(value)
|
|
364
|
+
value.gsub(/\e([nr\e])/) { PATH_ESCAPES.key(Regexp.last_match(1)) }
|
|
365
|
+
end
|
|
366
|
+
|
|
326
367
|
# Build command to send on management port
|
|
368
|
+
# Frame: header, one line per field, empty line (last empty element adds the final line break)
|
|
327
369
|
# @param data [Hash] keys are snake case: e.g. {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
|
328
370
|
# @return [String] frame to send on management port
|
|
329
371
|
def command_to_stream(data)
|
|
330
372
|
data
|
|
331
|
-
.map
|
|
373
|
+
.map do |key, value|
|
|
374
|
+
name = field_snake_to_native(key)
|
|
375
|
+
value = value.to_s
|
|
376
|
+
if PATH_FIELDS.include?(name)
|
|
377
|
+
value = escape_path(value)
|
|
378
|
+
else
|
|
379
|
+
Aspera.assert(!value.match?(/[\r\n]/)) { "mgt port: line break in value of #{name}" }
|
|
380
|
+
end
|
|
381
|
+
"#{name}: #{value}"
|
|
382
|
+
end
|
|
332
383
|
.unshift(MGT_HEADER)
|
|
333
384
|
.push(MGT_FRAME_SEPARATOR, '')
|
|
334
385
|
.join("\n")
|
|
@@ -347,21 +398,24 @@ module Aspera
|
|
|
347
398
|
# @param line [String] line of mgt port event
|
|
348
399
|
# @return [Hash] event hash or nil if event is not yet complete
|
|
349
400
|
def process_line(line)
|
|
350
|
-
# Log.log.debug{"line=[#{line}]"}
|
|
351
401
|
case line
|
|
352
402
|
when MGT_HEADER
|
|
353
403
|
# begin event
|
|
404
|
+
Log.log.warn { "mgt port: incomplete event discarded: #{@event_build}" } unless @event_build.nil?
|
|
354
405
|
@event_build = {}
|
|
355
|
-
when /^([^:]+): (.*)$/
|
|
356
|
-
Aspera.assert_type(@event_build, Hash) { 'mgt port: unexpected line: data without header' }
|
|
357
|
-
# event field
|
|
358
|
-
@event_build[Regexp.last_match(1)] = Regexp.last_match(2)
|
|
359
406
|
when MGT_FRAME_SEPARATOR
|
|
360
407
|
Aspera.assert_type(@event_build, Hash) { 'mgt port: unexpected line: end frame without header' }
|
|
361
408
|
@last_event = @event_build
|
|
362
409
|
@event_build = nil
|
|
363
410
|
return @last_event
|
|
364
|
-
else
|
|
411
|
+
else
|
|
412
|
+
# event field: name up to first colon, optional single space (as `as_mgmt_buffered_read_msg`)
|
|
413
|
+
name, value = line.split(':', 2)
|
|
414
|
+
Aspera.error_unexpected_value(line) { 'mgt port' } if value.nil?
|
|
415
|
+
Aspera.assert_type(@event_build, Hash) { 'mgt port: unexpected line: data without header' }
|
|
416
|
+
value = value.delete_prefix(' ')
|
|
417
|
+
value = self.class.unescape_path(value) if PATH_FIELDS.include?(name)
|
|
418
|
+
@event_build[name] = value
|
|
365
419
|
end
|
|
366
420
|
return
|
|
367
421
|
end
|