aspera-cli 4.28.0 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +143 -2
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +135 -56
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +103 -26
- data/SECURITY.md +70 -0
- data/bin/ascli +8 -5
- data/bin/asession +9 -6
- data/docs/README.md +1633 -609
- data/docs/test-mcp-with-ai.md +8 -8
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +20 -19
- data/lib/aspera/agent/factory.rb +6 -3
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +7 -4
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +20 -20
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +71 -65
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +354 -0
- data/lib/aspera/ascp/management.rb +16 -8
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +12 -19
- data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +37 -33
- data/lib/aspera/cli/command_line.rb +24 -6
- data/lib/aspera/cli/command_registry.rb +6 -1
- data/lib/aspera/cli/command_spec.rb +34 -7
- data/lib/aspera/cli/completion/ascli.bash +19 -5
- data/lib/aspera/cli/completion/ascli.fish +13 -2
- data/lib/aspera/cli/completion/ascli.ps1 +31 -9
- data/lib/aspera/cli/completion/ascli.zsh +22 -6
- data/lib/aspera/cli/completion_actions.rb +303 -69
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +3 -0
- data/lib/aspera/cli/extended_value.rb +61 -27
- data/lib/aspera/cli/formatter.rb +29 -22
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -10
- data/lib/aspera/cli/http.rb +12 -9
- data/lib/aspera/cli/info.rb +12 -7
- data/lib/aspera/cli/mailer.rb +6 -3
- data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
- data/lib/aspera/cli/mcp/secrets.rb +8 -5
- data/lib/aspera/cli/mcp/tool.rb +15 -14
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +65 -12
- data/lib/aspera/cli/options.schema.yaml +487 -63
- data/lib/aspera/cli/parser.rb +56 -32
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +244 -35
- data/lib/aspera/cli/plugins/ats.rb +9 -8
- data/lib/aspera/cli/plugins/base.rb +17 -12
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +198 -103
- data/lib/aspera/cli/plugins/console.rb +6 -3
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +21 -7
- data/lib/aspera/cli/plugins/faspex5.rb +28 -25
- data/lib/aspera/cli/plugins/faspio.rb +7 -5
- data/lib/aspera/cli/plugins/httpgw.rb +7 -4
- data/lib/aspera/cli/plugins/mcp.rb +28 -22
- data/lib/aspera/cli/plugins/node.rb +31 -30
- data/lib/aspera/cli/plugins/oauth.rb +5 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
- data/lib/aspera/cli/plugins/preview.rb +69 -44
- data/lib/aspera/cli/plugins/server.rb +63 -11
- data/lib/aspera/cli/plugins/shares.rb +86 -16
- data/lib/aspera/cli/plugins/transferd.rb +223 -0
- data/lib/aspera/cli/preset_actions.rb +144 -30
- data/lib/aspera/cli/preset_manager.rb +52 -18
- data/lib/aspera/cli/prompt.rb +4 -1
- data/lib/aspera/cli/result.rb +12 -8
- data/lib/aspera/cli/runner.rb +69 -46
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +79 -9
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +13 -10
- data/lib/aspera/cli/table_fitter.rb +3 -0
- data/lib/aspera/cli/terminal_formatter.rb +9 -6
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +29 -20
- data/lib/aspera/cli/transfer_progress.rb +13 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +188 -42
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.rb +12 -10
- data/lib/aspera/cli/wizard.rb +42 -39
- data/lib/aspera/command_line_builder.rb +9 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +4 -1
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/duration.rb +5 -2
- data/lib/aspera/environment.rb +21 -12
- data/lib/aspera/exec_spec.rb +3 -0
- data/lib/aspera/faspex_gw.rb +13 -9
- data/lib/aspera/faspex_postproc.rb +15 -12
- data/lib/aspera/formatter_interface.rb +4 -1
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +4 -2
- data/lib/aspera/json_rpc/client.rb +6 -5
- data/lib/aspera/json_rpc/version.rb +3 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +18 -5
- data/lib/aspera/markdown.rb +5 -2
- data/lib/aspera/mime.rb +3 -0
- data/lib/aspera/nagios.rb +6 -4
- data/lib/aspera/node_emulator.rb +15 -13
- data/lib/aspera/oauth/client/base.rb +91 -70
- data/lib/aspera/oauth/client/boot.rb +7 -4
- data/lib/aspera/oauth/client/factory.rb +6 -3
- data/lib/aspera/oauth/client/generic.rb +5 -1
- data/lib/aspera/oauth/client/json_credentials.rb +4 -1
- data/lib/aspera/oauth/client/jwt.rb +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +85 -0
- data/lib/aspera/oauth/client/url_json.rb +4 -1
- data/lib/aspera/oauth/client/web.rb +39 -12
- data/lib/aspera/oauth/server/pkce.rb +6 -5
- data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +8 -4
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +4 -1
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +3 -0
- data/lib/aspera/rest/aspera_errors.rb +4 -1
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +22 -29
- data/lib/aspera/rest/error_analyzer.rb +5 -2
- data/lib/aspera/rest/list.rb +8 -4
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +8 -5
- data/lib/aspera/rest.rb +9 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +8 -2
- data/lib/aspera/schema/reader.rb +7 -0
- data/lib/aspera/schema/registry.rb +43 -28
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +166 -0
- data/lib/aspera/secret/hider.rb +201 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +110 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +5 -2
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +3 -0
- data/lib/aspera/sync/conf.schema.yaml +9 -4
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +16 -14
- data/lib/aspera/temp_file_manager.rb +8 -4
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +17 -14
- data/lib/aspera/transfer/result.rb +3 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +9 -22
- data/lib/aspera/transfer/units.rb +75 -0
- data/lib/aspera/uri_reader.rb +57 -7
- data/lib/aspera/web_auth/index.html.erb +6 -2
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +47 -10
- data/lib/aspera/web_page.rb +7 -2
- data/lib/aspera/web_server_simple.rb +6 -3
- data/lib/aspera/yaml.rb +5 -1
- data/llms.txt +81 -0
- data/skills/ascli/SKILL.md +88 -0
- data.tar.gz.sig +0 -0
- metadata +72 -19
- metadata.gz.sig +0 -0
- data/lib/aspera/keychain/base.rb +0 -30
- data/lib/aspera/keychain/encrypted_hash.rb +0 -192
- data/lib/aspera/keychain/factory.rb +0 -58
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
- data/lib/aspera/keychain/macos_security.rb +0 -277
- data/lib/aspera/keychain/one_password_api.rb +0 -91
- data/lib/aspera/keychain/one_password_base.rb +0 -39
- data/lib/aspera/keychain/one_password_cli.rb +0 -100
- data/lib/aspera/secret_hider.rb +0 -196
- data/lib/aspera/secret_strength.rb +0 -114
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
# cspell:ignore abrtxfer
|
|
7
|
+
require_relative 'rules'
|
|
8
|
+
require_relative 'lines'
|
|
9
|
+
require_relative '../../schema/registry'
|
|
10
|
+
require_relative '../../assert'
|
|
11
|
+
require 'strscan'
|
|
12
|
+
require 'time'
|
|
13
|
+
|
|
14
|
+
module Aspera
|
|
15
|
+
module Ascp
|
|
16
|
+
class LogAnalyzer
|
|
17
|
+
# Lines of an ascp log file parsed into records, with the field names of the log lines:
|
|
18
|
+
# - samples: `Receiver bl` or `Sender bl` line, merged with the `DS Qs` line that follows it
|
|
19
|
+
# - sessions: `Session Start`, `Session Params`, `Session Stop` and `Session Statistics` lines of a session,
|
|
20
|
+
# and the lines of the process about the session: license (`Initializing FASP version`), MTU and rate parameters of the peer
|
|
21
|
+
# - transfers: `Transfer Start` and `Transfer Stop` lines of a file
|
|
22
|
+
# Fields are described in schema `ascp_log`.
|
|
23
|
+
class Records
|
|
24
|
+
include Lines
|
|
25
|
+
|
|
26
|
+
# Kind of record => schema
|
|
27
|
+
SCHEMAS = {
|
|
28
|
+
samples: "#{Schema::Registry::ASCP_LOG}:components.schemas.Sample",
|
|
29
|
+
sessions: "#{Schema::Registry::ASCP_LOG}:components.schemas.Session",
|
|
30
|
+
transfers: "#{Schema::Registry::ASCP_LOG}:components.schemas.Transfer"
|
|
31
|
+
}.freeze
|
|
32
|
+
# Message of a line of ascp: `ascp[1234]: LOG <message>` or `[4c58-140234] LOG <message>`
|
|
33
|
+
MESSAGE = /\]:?\s+LOG\s+(.*?)\s*\z/
|
|
34
|
+
# Line message => method called with (pid, time, MatchData)
|
|
35
|
+
MESSAGES = {
|
|
36
|
+
/\A(Receiver|Sender) (bl .*)/ => :sample_line,
|
|
37
|
+
/\A(Receiver|Sender) DS (.*)/ => :queues_line,
|
|
38
|
+
/\AFASP Session (Start|Stop) (.*)/ => :session_line,
|
|
39
|
+
/\AFASP Session Params (.*)/ => :params_line,
|
|
40
|
+
/\AFASP Session Statistics \[\w+\] (.*)/ => :statistics_line,
|
|
41
|
+
/\AInitializing FASP version (.+?), license\b/ => :initializing_line,
|
|
42
|
+
/\AMeasured pMTU: (\d+) Bytes, start_brtt: (\d+) ms/ => :measured_mtu_line,
|
|
43
|
+
/\Adatagram size (\d+)B, block size (\d+)B(?:, path MTU (\d+)B)?/ => :datagram_line,
|
|
44
|
+
/\APeer requested rate params adopted,\s+target_rate (\d+) min_rate (\d+) transfer_policy (\w+)/ => :peer_rate_params_line,
|
|
45
|
+
/\AFASP Transfer (Start|Stop) (.*)/ => :transfer_line
|
|
46
|
+
}.freeze
|
|
47
|
+
# Group of a sample line: `name k1/k2=v1/v2`, or `key=value` alone
|
|
48
|
+
GROUP_FIELDS = %r{(?<![A-Za-z_/])(?:([A-Za-z_]++)\s++)?([A-Za-z_]++(?:/[A-Za-z_]++)*+)=(-?\d++(?:/-?\d++)*+)}
|
|
49
|
+
# `mgmt backlog i/s/n = 0/3/0 1/4/0` of `DS Qs` lines: one triplet per management socket
|
|
50
|
+
MGMT_BACKLOG = %r{mgmt backlog ([a-z/]++) =((?:\s++-?\d++(?:/-?\d++)*+)*+)}
|
|
51
|
+
# `[peer errcode=27 errstr="..."]`: fields of the peer
|
|
52
|
+
PREFIXED = /\[(\w+)\s+([^\]]*)\]/
|
|
53
|
+
KEY = %r{([A-Za-z_][\w/]*)=}
|
|
54
|
+
# Value of `key=value`: quoted, in brackets, or up to the next key (it may contain spaces, e.g. `ver=4.4.8.2596 b12cd13`)
|
|
55
|
+
QUOTED = /"(.*?)"(?=\s|\z)/
|
|
56
|
+
BRACKETS = /\[([^\]]*)\](?=\s|\z)/
|
|
57
|
+
UNQUOTED = %r{.*?(?=\s+(?:\[\w+\s|[A-Za-z_][\w/]*=)|\s*\z)}
|
|
58
|
+
# Group of `Session Statistics`: `(name: key value key value)`, or `(name: value unit)`
|
|
59
|
+
STATISTICS_GROUP = /\(([a-z ]+):\s*([^()]*)\)/
|
|
60
|
+
# Never output: contains user names and emails
|
|
61
|
+
HIDDEN_FIELDS = %w[cookie].freeze
|
|
62
|
+
# Keys with a slash: name of each value
|
|
63
|
+
SPLIT_KEYS = {'exclude_older/newer_than' => %w[exclude_older_than exclude_newer_than]}.freeze
|
|
64
|
+
# Number, with optional unit
|
|
65
|
+
NUMBER = /\A(-?\d+(?:\.\d+)?)\s*([KMG]?bps|ms|s|%)?\z/
|
|
66
|
+
RATE_UNITS = {'Kbps' => 1e3, 'Mbps' => 1e6, 'Gbps' => 1e9}.freeze
|
|
67
|
+
|
|
68
|
+
class << self
|
|
69
|
+
# @param kind [Symbol] One of the keys of SCHEMAS
|
|
70
|
+
# @param paths [Array<String>] Log files, `-` (standard input) or folders
|
|
71
|
+
# @param timezone [String, nil] Time zone of the log lines: `UTC` or `+HH:MM`. Time zone of the analyst if nil.
|
|
72
|
+
# @return [Array<Hash>] records of all log files, with field `log_file` when there are several log files
|
|
73
|
+
def list(kind, paths, timezone: nil)
|
|
74
|
+
Aspera.assert_values(kind, SCHEMAS.keys)
|
|
75
|
+
LogAnalyzer.check_timezone(timezone)
|
|
76
|
+
logs = paths.flat_map { |path| LogAnalyzer.load_logs(path) { |file| new(file, timezone: timezone).parse } }
|
|
77
|
+
logs.flat_map do |log|
|
|
78
|
+
rows = log.public_send(kind)
|
|
79
|
+
logs.size > 1 ? rows.map { |row| {'log_file' => log.path}.merge(row) } : rows
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# @param kind [Symbol] One of the keys of SCHEMAS
|
|
84
|
+
# @return [Hash{String => String}] dot path of field => JSON type
|
|
85
|
+
def field_types(kind)
|
|
86
|
+
(@field_types ||= {})[kind] ||= {}.tap do |types|
|
|
87
|
+
Schema::Registry.instance.reader(SCHEMAS[kind]).each_property do |schema, _name, full_name, _required|
|
|
88
|
+
types[full_name] = schema.current['type']
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# @param text [String] Value in log line
|
|
94
|
+
# @param type [String, nil] JSON type of field, `nil` if not in schema
|
|
95
|
+
# @return [String, Integer, Float] value: numbers in bps, ms, s or %, without unit
|
|
96
|
+
def value(text, type)
|
|
97
|
+
return text if type.eql?('string')
|
|
98
|
+
if type.nil?
|
|
99
|
+
return text.to_i if text.match?(/\A-?\d+\z/)
|
|
100
|
+
return text.to_f if text.match?(/\A-?\d+\.\d+\z/)
|
|
101
|
+
return text
|
|
102
|
+
end
|
|
103
|
+
m = text.match(NUMBER) or return text
|
|
104
|
+
number = m[1].to_f * RATE_UNITS.fetch(m[2], 1)
|
|
105
|
+
type.eql?('integer') ? number.round : number
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# @param text [String] groups of a sample line: `name k1/k2=v1/v2`, or `key=value`
|
|
109
|
+
# @return [Hash] name => {key => value}, and key => value. A repeated group gets a number: `bl`, `bl2`
|
|
110
|
+
def group_fields(text)
|
|
111
|
+
seen = Hash.new(0)
|
|
112
|
+
text.scan(GROUP_FIELDS).each_with_object({}) do |(group, keys, numbers), result|
|
|
113
|
+
values = keys.split('/').zip(numbers.split('/').map(&:to_i)).to_h
|
|
114
|
+
if group.nil?
|
|
115
|
+
result.merge!(values)
|
|
116
|
+
else
|
|
117
|
+
seen[group] += 1
|
|
118
|
+
result[seen[group] > 1 ? "#{group}#{seen[group]}" : group] = values
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# @param text [String] `Qs ... mgmt backlog ...` of a `DS Qs` line
|
|
124
|
+
# @return [Hash] queue groups and `mgmt_backlog` array
|
|
125
|
+
def queues_fields(text)
|
|
126
|
+
backlog = text.match(MGMT_BACKLOG)
|
|
127
|
+
text = backlog.pre_match + backlog.post_match if backlog
|
|
128
|
+
queues = group_fields(text)
|
|
129
|
+
if backlog
|
|
130
|
+
keys = backlog[1].split('/')
|
|
131
|
+
queues['mgmt_backlog'] = backlog[2].split.map { |values| keys.zip(values.split('/').map(&:to_i)).to_h }
|
|
132
|
+
end
|
|
133
|
+
queues
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# @param text [String] `key=value key="value" key=[k=v,k=v] [prefix key=value]`
|
|
137
|
+
# @return [Hash] key => value (String), or Hash for brackets
|
|
138
|
+
def key_values(text)
|
|
139
|
+
fields = {}
|
|
140
|
+
scanner = StringScanner.new(text)
|
|
141
|
+
loop do
|
|
142
|
+
scanner.skip(/\s+/)
|
|
143
|
+
break if scanner.eos?
|
|
144
|
+
if scanner.scan(PREFIXED)
|
|
145
|
+
prefix = scanner[1]
|
|
146
|
+
key_values(scanner[2]).each { |key, value| fields["#{prefix}_#{key}"] = value }
|
|
147
|
+
elsif scanner.scan(KEY)
|
|
148
|
+
key = scanner[1]
|
|
149
|
+
value =
|
|
150
|
+
if scanner.scan(QUOTED)
|
|
151
|
+
scanner[1]
|
|
152
|
+
elsif scanner.scan(BRACKETS)
|
|
153
|
+
scanner[1].include?('=') ? scanner[1].split(',').to_h { |pair| pair.split('=', 2) } : scanner[1]
|
|
154
|
+
else
|
|
155
|
+
scanner.scan(UNQUOTED)
|
|
156
|
+
end
|
|
157
|
+
add_field(fields, key, value)
|
|
158
|
+
else
|
|
159
|
+
scanner.skip(/\S+/)
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
fields.except(*HIDDEN_FIELDS)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# A key with a slash has one value per name: `exclude_older/newer_than=0/0`
|
|
166
|
+
def add_field(fields, key, value)
|
|
167
|
+
if key.include?('/') && value.is_a?(String)
|
|
168
|
+
SPLIT_KEYS.fetch(key) { key.split('/') }.zip(value.split('/')) { |name, part| fields[name] = part }
|
|
169
|
+
else
|
|
170
|
+
fields[key] = value
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# @param text [String] `key=value`, `key value` and `(group: key value key value)` of `Session Statistics`
|
|
175
|
+
# @return [Hash] key => value, group => {key => value}. Key of `a->b.key`: `key`
|
|
176
|
+
def statistics_fields(text)
|
|
177
|
+
fields = {}
|
|
178
|
+
scanner = StringScanner.new(text)
|
|
179
|
+
loop do
|
|
180
|
+
scanner.skip(/\s+/)
|
|
181
|
+
break if scanner.eos?
|
|
182
|
+
if scanner.scan(STATISTICS_GROUP)
|
|
183
|
+
tokens = scanner[2].split
|
|
184
|
+
fields[scanner[1].split.join('_')] = tokens.first&.match?(/\A-?\d/) ? tokens.first : pairs(tokens)
|
|
185
|
+
elsif scanner.scan(/([^\s=]+)=(\S*)/) || scanner.scan(/(\S+)\s+(-?\d\S*)/)
|
|
186
|
+
fields[scanner[1].split(/->|\./).last] = scanner[2]
|
|
187
|
+
else
|
|
188
|
+
scanner.skip(/\S+/)
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
fields
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# @param tokens [Array<String>] `key value key value`
|
|
195
|
+
# @return [Hash] key => value. A repeated key with a percentage: `<key>_pct`, e.g. `lost 1 lost 1.47%`
|
|
196
|
+
def pairs(tokens)
|
|
197
|
+
tokens.each_slice(2).with_object({}) do |(key, value), result|
|
|
198
|
+
key = "#{key}_pct" if result.key?(key) && value.to_s.end_with?('%')
|
|
199
|
+
result[key] = value
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# @param fields [Hash] Values from log line (String)
|
|
204
|
+
# @param kind [Symbol] Kind of record
|
|
205
|
+
# @return [Hash] values with the type of the field in the schema
|
|
206
|
+
def typed(fields, kind, prefix = '')
|
|
207
|
+
types = field_types(kind)
|
|
208
|
+
fields.to_h do |key, value|
|
|
209
|
+
path = "#{prefix}#{key}"
|
|
210
|
+
value =
|
|
211
|
+
case value
|
|
212
|
+
when Hash then typed(value, kind, "#{path}.")
|
|
213
|
+
when String then self.value(value, types[path])
|
|
214
|
+
else value
|
|
215
|
+
end
|
|
216
|
+
[key, value]
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# @return [String] Log file
|
|
222
|
+
attr_reader :path
|
|
223
|
+
# @return [Array<Hash>] one record per sample of a session
|
|
224
|
+
attr_reader :samples
|
|
225
|
+
|
|
226
|
+
# @param path [String] Log file
|
|
227
|
+
# @param timezone [String, nil] Time zone of the log lines: `UTC` or `+HH:MM`. Time zone of the analyst if nil.
|
|
228
|
+
def initialize(path, timezone: nil)
|
|
229
|
+
@path = path
|
|
230
|
+
@timezone = timezone
|
|
231
|
+
@samples = []
|
|
232
|
+
# uuid => session record
|
|
233
|
+
@sessions = {}
|
|
234
|
+
@transfers = []
|
|
235
|
+
# PID => uuid of current session of process
|
|
236
|
+
@uuids = {}
|
|
237
|
+
# PID => last sample, until its `DS Qs` line
|
|
238
|
+
@last_sample = {}
|
|
239
|
+
# [uuid, file] => transfer record, until its `Transfer Stop` line
|
|
240
|
+
@open_transfers = {}
|
|
241
|
+
# PID => fields of lines logged before the `Session Start` line of the process
|
|
242
|
+
@pending = {}
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Read and parse the log file
|
|
246
|
+
# @return [self]
|
|
247
|
+
def parse
|
|
248
|
+
LogAnalyzer.read_lines(@path, Encoding::UTF_8) do |line|
|
|
249
|
+
line = line.scrub unless line.valid_encoding?
|
|
250
|
+
parse_line(normalize(line))
|
|
251
|
+
end
|
|
252
|
+
self
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# @return [Array<Hash>] one record per session
|
|
256
|
+
def sessions
|
|
257
|
+
@sessions.values.map(&:compact)
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# @return [Array<Hash>] one record per transferred file
|
|
261
|
+
def transfers
|
|
262
|
+
@transfers.map(&:compact)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
private
|
|
266
|
+
|
|
267
|
+
def parse_line(line)
|
|
268
|
+
track_ascp(line)
|
|
269
|
+
return unless line.include?(' LOG ')
|
|
270
|
+
pid = line[PID, 1] or return
|
|
271
|
+
process = line[PROCESS, 1]
|
|
272
|
+
return unless process.nil? || process.match?(ASCP_PROCESS)
|
|
273
|
+
message = line[MESSAGE, 1] or return
|
|
274
|
+
MESSAGES.each do |regex, method|
|
|
275
|
+
m = message.match(regex) or next
|
|
276
|
+
stamp = parse_time(line[LINE_TIME, 1])
|
|
277
|
+
send(method, pid, iso(stamp), m)
|
|
278
|
+
break
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# @return [Hash] first fields of a sample
|
|
283
|
+
def sample_record(pid, time, side)
|
|
284
|
+
{'time' => time, 'pid' => pid, 'uuid' => @uuids[pid], 'side' => side}.compact
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
# `Receiver bl ...` or `Sender bl ...`
|
|
288
|
+
def sample_line(pid, time, m)
|
|
289
|
+
record = sample_record(pid, time, m[1]).merge(self.class.group_fields(m[2]))
|
|
290
|
+
@samples.push(record)
|
|
291
|
+
@last_sample[pid] = record
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
# `Receiver DS Qs ...` or `Sender DS Qs ...`: in the sample logged before
|
|
295
|
+
def queues_line(pid, time, m)
|
|
296
|
+
queues = self.class.queues_fields(m[2])
|
|
297
|
+
record = @last_sample.delete(pid)
|
|
298
|
+
if record.nil? || record['side'] != m[1]
|
|
299
|
+
record = sample_record(pid, time, m[1])
|
|
300
|
+
@samples.push(record)
|
|
301
|
+
end
|
|
302
|
+
record['DS'] = queues
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# @return [Hash] session record of uuid: created on first line
|
|
306
|
+
def session_record(uuid, pid)
|
|
307
|
+
@uuids[pid] = uuid
|
|
308
|
+
@sessions[uuid] ||= {'uuid' => uuid, 'pid' => pid, 'start' => nil, 'stop' => nil}.merge(@pending.delete(pid) || {})
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
# @return [Hash] fields kept until the `Session Start` line of the process, else the session record of the process
|
|
312
|
+
def process_session(pid)
|
|
313
|
+
@pending[pid] || (@uuids.key?(pid) ? @sessions[@uuids[pid]] : (@pending[pid] = {}))
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# `Initializing FASP version`: starts a process. License fields are only the version, the maximum rate and the expiry date:
|
|
317
|
+
# `account no.` and `license no.` are not output.
|
|
318
|
+
def initializing_line(pid, _time, m)
|
|
319
|
+
rate = m.string[LICENSE_RATE, 1]
|
|
320
|
+
@pending[pid] = {
|
|
321
|
+
'fasp_version' => m[1], 'license_max_rate' => rate && (rate.to_i * 1_000_000), 'license_expires' => m.string[LICENSE_EXPIRES, 1]
|
|
322
|
+
}.compact
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
# `Measured pMTU: 1492 Bytes, start_brtt: 56 ms`
|
|
326
|
+
def measured_mtu_line(pid, _time, m)
|
|
327
|
+
process_session(pid).merge!('pMTU' => m[1].to_i, 'start_brtt' => m[2].to_i)
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
# `datagram size 1472B, block size 1408B, path MTU 1492B` (no path MTU in old versions)
|
|
331
|
+
def datagram_line(pid, _time, m)
|
|
332
|
+
process_session(pid).merge!({'datagram_size' => m[1].to_i, 'block_size' => m[2].to_i, 'path_mtu' => m[3]&.to_i}.compact)
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
# `Peer requested rate params adopted, target_rate 100000000 min_rate 0 transfer_policy fair`: one item per change
|
|
336
|
+
def peer_rate_params_line(pid, time, m)
|
|
337
|
+
(process_session(pid)['peer_rate_params'] ||= []).push({'time' => time, 'target_rate' => m[1].to_i, 'min_rate' => m[2].to_i, 'transfer_policy' => m[3]})
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# `Session Start` or `Session Stop`: same fields, the ones of `Session Stop` replace the ones of `Session Start`
|
|
341
|
+
def session_line(pid, time, m)
|
|
342
|
+
fields = self.class.key_values(m[2])
|
|
343
|
+
uuid = fields['uuid'] or return
|
|
344
|
+
record = session_record(uuid, pid)
|
|
345
|
+
record[m[1].downcase] = time
|
|
346
|
+
record.merge!(self.class.typed(fields, :sessions))
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def params_line(pid, _time, m)
|
|
350
|
+
fields = self.class.key_values(m[1])
|
|
351
|
+
uuid = fields['uuid'] or return
|
|
352
|
+
session_record(uuid, pid).merge!(self.class.typed(fields, :sessions))
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
def statistics_line(pid, _time, m)
|
|
356
|
+
fields = self.class.statistics_fields(m[1])
|
|
357
|
+
uuid = fields.delete('id') or return
|
|
358
|
+
session_record(uuid, pid).merge!(self.class.typed(fields, :sessions))
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
# `Transfer Start` or `Transfer Stop`: same fields, the ones of `Transfer Stop` replace the ones of `Transfer Start`
|
|
362
|
+
def transfer_line(pid, time, m)
|
|
363
|
+
fields = self.class.key_values(m[2])
|
|
364
|
+
uuid = fields['uuid'] or return
|
|
365
|
+
@uuids[pid] = uuid
|
|
366
|
+
key = [uuid, fields['file']]
|
|
367
|
+
record = m[1].eql?('Stop') ? @open_transfers.delete(key) : nil
|
|
368
|
+
if record.nil?
|
|
369
|
+
record = {'uuid' => uuid, 'pid' => pid, 'start' => nil, 'stop' => nil}
|
|
370
|
+
@transfers.push(record)
|
|
371
|
+
@open_transfers[key] = record if m[1].eql?('Start')
|
|
372
|
+
end
|
|
373
|
+
record[m[1].downcase] = time
|
|
374
|
+
record.merge!(self.class.typed(fields, :transfers))
|
|
375
|
+
end
|
|
376
|
+
end
|
|
377
|
+
end
|
|
378
|
+
end
|
|
379
|
+
end
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
# cspell:ignore alloc ossl aspmover bdrate brcvd dups rcvd sndr aspsimplesync astokengen fdlimit filewrite pulseaudio sesserr varation
|
|
7
|
+
module Aspera
|
|
8
|
+
module Ascp
|
|
9
|
+
class LogAnalyzer
|
|
10
|
+
# Timestamp at beginning of log line: `May 29 06:53:47` (syslog) or `2017-05-24 02:33:56` (log file)
|
|
11
|
+
LINE_TIME = /^((?:\w+\s+\d+|\d+-\d+-\d+)\s+\d\d:\d\d:\d\d)/
|
|
12
|
+
# Timestamp of sessions and samples
|
|
13
|
+
STAMP = /^(.*?(?:\s+\d+[^:])?\s+\d\d:\d\d:\d\d)/
|
|
14
|
+
# Process identifier: `ascp[1234]` (syslog) or `[4c58-140234]` (log file: process-thread)
|
|
15
|
+
PID = /^.*?\[(\w+)(?:-\w+)?\]/
|
|
16
|
+
# Error message: `ascp[1234]: ERR <message>` or `[4c58-140234] ERR <message>`
|
|
17
|
+
LOG_ERROR = /\]:?\s+ERR\s+(.+)/
|
|
18
|
+
# Process name in syslog line: `host ascp[1234]:` (none in `ascp` log files)
|
|
19
|
+
PROCESS = /\s([\w.-]+)\[\d+\]:/
|
|
20
|
+
# Processes whose lines are analyzed (other processes: only time is used)
|
|
21
|
+
ASCP_PROCESS = /\Aascp\d*\z/
|
|
22
|
+
# Lines that are never relevant: containing one of these texts, or matching
|
|
23
|
+
IGNORED_TEXTS = [
|
|
24
|
+
'async[', 'LOG Queue:', 'snmpd', 'asconfigurator', 'asuserdata', 'message repeated', 'asperacentral', 'ERR Unrecognized FASP TLV',
|
|
25
|
+
'LOG fdlimit set', 'astokengen', 'varation_index', 'LOG brtt', 'Action command:', 'meta_cmp_attrs:'
|
|
26
|
+
].freeze
|
|
27
|
+
IGNORED_LINES = Regexp.union(/pulseaudio\[\d+\]/, /\s+DBG\s+/, /10061$/, /kernel:(?!\s*ascp)/, /sshd.*?Failed password/)
|
|
28
|
+
# Lines of other Aspera processes, when they contain the PID
|
|
29
|
+
OTHER_PROCESSES = /prepost|aspmover|aspsimplesync/
|
|
30
|
+
SESSION_START_STOP = /#{STAMP.source}.*?Session\s+(Start|Stop)/
|
|
31
|
+
ERRSTR = /errstr="([^"]*)"(?:.*?errstr="([^"]*)")?/
|
|
32
|
+
# Error code and message of `Session Stop`: local, then peer
|
|
33
|
+
ERRCODE = /errcode=(\d+) errstr="([^"]*)"/
|
|
34
|
+
# Transport ciphers (`FASP_TRANSPORT_CIPHER_*` of ascp), by identifier
|
|
35
|
+
CIPHERS = %w[none aes-128 aes-192 aes-256 aes-128-cfb aes-192-cfb aes-256-cfb aes-128-gcm aes-192-gcm aes-256-gcm].freeze
|
|
36
|
+
# `Session Params` field => session data
|
|
37
|
+
SESSION_PARAMS = {
|
|
38
|
+
'user' => :user, 'targetrate' => :target_rate, 'minrate' => :min_rate, 'rate_policy' => :policy, 'cipher' => :secure,
|
|
39
|
+
'resume' => :resume, 'ovr' => :overwrite, 'rd_sz' => :read_block_size, 'wr_sz' => :write_block_size,
|
|
40
|
+
'xfer_id' => :xfer_id, 'cluster_num_nodes' => :cluster_nodes, 'cluster_node_id' => :cluster_node
|
|
41
|
+
}.freeze
|
|
42
|
+
# License in `Initializing FASP version` line: maximum rate in Mbps (none if unlimited), expiry date (none if no expiry)
|
|
43
|
+
LICENSE_RATE = /\bmax rate=(\d+) Mbps/
|
|
44
|
+
LICENSE_EXPIRES = /\blicense expires on (\w{3} \w{3} +\d+ [\d:]+ \d{4})/
|
|
45
|
+
AS_BUF_IO = /ERR as_buf_io:\s+IO/
|
|
46
|
+
SERVER_MGMT_RESET = {set: %i[server_management_connection_reset], clear: %i[peer_abort]}.freeze
|
|
47
|
+
DATA_TIMEOUT = %i[data_transfer_timeout].freeze
|
|
48
|
+
|
|
49
|
+
# Log line conditions, and their effect on the session. Only the first matching rule applies.
|
|
50
|
+
# match: Regexp, or Array of Regexp that must all match
|
|
51
|
+
# set: flags to set, clear: flags to clear, error: session error, status: session status
|
|
52
|
+
# unless: flags that cancel the effect, call: method of LogFile called with (session, MatchData)
|
|
53
|
+
# No effect: line is recognized, so that a later rule does not match it
|
|
54
|
+
LINE_RULES = [
|
|
55
|
+
[/\sLOG (?:IBM )?(Aspera [\w -]+?) version \d/, {call: :product_name}],
|
|
56
|
+
[/\sLOG ASPERA_AUTH_TOKEN is set to TOKEN/, {call: :token_auth}],
|
|
57
|
+
[/\sLOG Node storage access_key=/, {call: :node_auth}],
|
|
58
|
+
[/end File Transfer statistics/, {call: :end_of_statistics}],
|
|
59
|
+
[/(?:Receiver|Sender)\s+bl\b/, {call: :add_sample}],
|
|
60
|
+
[/LOG Falling back to HTTP/, {set: %i[http_fallback]}],
|
|
61
|
+
[/Transfer\s+Stop\s+(.*)/, {call: :transfer_stop}],
|
|
62
|
+
[/base\s+RTT\s+reset/, {call: :brtt_reset}],
|
|
63
|
+
[/(Peer requested|Negotiated) rate params.*?target_rate\s+(\d+)\s+min_rate\s+(\d+)\s+(?:transfer_)?policy\s+(\w+)/, {call: :rate_params}],
|
|
64
|
+
[/pMTU:\s+(\d+).*?start_brtt:\s(\d+)/, {call: :path_mtu}],
|
|
65
|
+
[/server configuration specifies transit encryption cipher (\d+), user requests (\d+)/, {call: :cipher_override}],
|
|
66
|
+
[/Downgrading resume policy to (\S+)/, {call: :resume_downgrade}],
|
|
67
|
+
[/changing session job size from (\d+) to (\d+) to match server configuration/, {call: :job_size_change}],
|
|
68
|
+
[/vlink init: id=(\d+), type=(\w+), capacity=(\d+)/, {call: :vlink_init}],
|
|
69
|
+
[/datagram size\s+(\d+)B, block size (\d+)B(?:, path MTU (\d+)B)?/, {call: :datagram_size}],
|
|
70
|
+
[/User aborted session/, {set: %i[user_abort]}],
|
|
71
|
+
[/FASP session cancelled/, {set: %i[user_abort]}],
|
|
72
|
+
[/ERR Server unable to hear from client/i, {set: %i[server_nohear_client]}],
|
|
73
|
+
[/ERR Client unable to connect to server/i, {set: %i[client_nohear_server]}],
|
|
74
|
+
[/ERR.*?Token expired\s+\d+\s+seconds ago/i, {set: %i[token_expired]}],
|
|
75
|
+
[/ERR\s+Invalid argument/i, {set: %i[invalid_argument]}],
|
|
76
|
+
[/ERR\sInvalid configuration file/i, {set: %i[invalid_conf_file]}],
|
|
77
|
+
[/ERR Disk read failed/i, {set: %i[read]}],
|
|
78
|
+
[/ERR Disk write failed/i, {set: %i[write]}],
|
|
79
|
+
[/Timeout receiving data \(60 sec\)/, {set: %i[receiver_data_transfer_timeout] + DATA_TIMEOUT, error: 'Receiver data transfer timeout'}],
|
|
80
|
+
[/Timeout establishing UDP connection/, {set: %i[timeout_establishing_udp_connection]}],
|
|
81
|
+
[/ERR failed to find a valid pair of \(pMTU, brtt\)/, {set: %i[timeout_establishing_udp_connection]}],
|
|
82
|
+
[/bind udp socket failed/, {set: %i[bind_udp_socket_failed]}],
|
|
83
|
+
[/ERR Sender network failure \(UDP recv got "No route to host"\)/, {set: %i[send_no_route]}],
|
|
84
|
+
[/Transmitter inactivity timer fired/, {set: %i[inactivity_timer_fired] + DATA_TIMEOUT, error: 'Data transfer timeout - inactivity timer fired'}],
|
|
85
|
+
[/fc_filewrite_sz:.*errno=11672/, {set: %i[device_not_connected]}],
|
|
86
|
+
[/fc_filewrite_sz:.*errno=(?:28|39|112)/, {set: %i[out_of_disk]}],
|
|
87
|
+
[/fc_filewrite_sz:.*errno=59/, {set: %i[net_error]}],
|
|
88
|
+
[/ReadFile failed.*error=53/, {set: %i[net_path_not_found]}],
|
|
89
|
+
[/fc_filewrite_sz:.*errno=64/, {set: %i[net_name_gone]}],
|
|
90
|
+
[/errno=1450/, {set: %i[out_of_resources]}],
|
|
91
|
+
[/errcode\s+-2009/, {set: %i[decryption_error]}],
|
|
92
|
+
[/Insufficient system resources/, {set: %i[out_of_resources]}],
|
|
93
|
+
[/ERR Could not open.*?errno 32/, {set: %i[sharing_violation], error: 'Sharing violation'}],
|
|
94
|
+
[/Insufficient permissions/, {set: %i[insufficient_permissions], unless: %i[sharing_violation]}],
|
|
95
|
+
[/ERR fc_create: failed to alloc cache/, {set: %i[out_of_memory]}],
|
|
96
|
+
[/ERR fc_filewrite_sz: Could not write cache bytes/, {set: %i[disk_write_failed]}],
|
|
97
|
+
[/ERR Server requires content protection/, {set: %i[content_protection]}],
|
|
98
|
+
[/ERR required management connection to localhost:\d+/, {set: %i[cannot_connect_to_management_socket]}],
|
|
99
|
+
[/ERR Failed to connect to management socket, exiting/, {set: %i[cannot_connect_to_management_socket], status: 'failed', error: 'Cannot connect to management socket'}],
|
|
100
|
+
[/ERR aspera.conf parsing error\)?(.*)/, {call: :conf_parse_error}],
|
|
101
|
+
[/ERR\s+Mandatory management socket localhost:\d+, Connection reset by peer\s+\(server\)/, SERVER_MGMT_RESET],
|
|
102
|
+
[/ERR.*?Mandatory management socket localhost:\d+, An existing connection was forcibly closed by the remote host.\s+\(server\)/, SERVER_MGMT_RESET],
|
|
103
|
+
[/ERR Cannot read from management socket localhost.*?server/, SERVER_MGMT_RESET],
|
|
104
|
+
[/ERR\s+Server aborted session: Failed to connect to management socket/, SERVER_MGMT_RESET],
|
|
105
|
+
[/ERR\s+Cannot write to management socket\s+localhost:\d+.*?Broken pipe.*?server/, SERVER_MGMT_RESET],
|
|
106
|
+
[/ERR\s+Cannot write to management socket\s+localhost:\d+.*?Broken pipe/, {}],
|
|
107
|
+
[/ERR cannot write to management socket localhost:\d+, Unknown error \(server\)/, {set: %i[server_management_connection_reset]}],
|
|
108
|
+
[/ERR\s+Cannot write to management socket localhost:\d+.*\(client\)/, {set: %i[client_management_connection_reset]}],
|
|
109
|
+
[/ERR PEER ABORT: Received sesserr notification from peer, reason FASP_ABORT_LOCAL/, {set: %i[peer_abort]}],
|
|
110
|
+
[/ERR Peer aborted session/, {set: %i[peer_abort]}],
|
|
111
|
+
[/ERR Mandatory management socket localhost:\d+, Connection reset by peer/, {}],
|
|
112
|
+
[/ERR Cannot read from management socket localhost:\d+, Connection reset by peer/, {}],
|
|
113
|
+
[/ERR Cannot read from management socket localhost.*?client/, {set: %i[client_management_connection_lost]}],
|
|
114
|
+
[/ERR Server aborted session: Cannot connect to management port localhost/, {set: %i[server_management_connection_reset]}],
|
|
115
|
+
[[AS_BUF_IO, /errno=10054/], {set: %i[remote_reset]}],
|
|
116
|
+
[[AS_BUF_IO, /errno=10053/], {set: %i[local_reset]}],
|
|
117
|
+
[[AS_BUF_IO, /errno=10055/], {set: %i[no_buffer_mem], error: 'Out of buffer memory'}],
|
|
118
|
+
[AS_BUF_IO, {set: %i[as_buf_io]}],
|
|
119
|
+
[/ERR Connection lost in midst of data session/, {set: %i[ssh_timeout]}],
|
|
120
|
+
[/\[asssh\] SSH connection startup failed/, {set: %i[ssh_startup_failed]}],
|
|
121
|
+
[/\[asssh\] failure during public-key authentication/, {set: %i[ssh_key_authentication]}],
|
|
122
|
+
[/\[asssh\] SSH session open failed/, {set: %i[ssh_session_failed]}],
|
|
123
|
+
[/Failed to start remote command, libssh2 result/, {set: %i[ssh_command_failed]}],
|
|
124
|
+
[%r{\[asssh\] Remote failed to forward TCP/IP}, {set: %i[ssh_forward_failed]}],
|
|
125
|
+
[/\[asssh\] (?:channel|session) [a-z ]+ rc=-?\d+/, {call: :ssh_disconnect_error}],
|
|
126
|
+
[/speed exceeded/, {set: %i[exceeded]}],
|
|
127
|
+
[/segfault/, {call: :segfault}],
|
|
128
|
+
[/(Receiver|Sender) has not received a packet for (\d+) seconds/, {call: :no_data}],
|
|
129
|
+
[/ERR \S*ossl-modules not found/, {set: %i[ossl_modules_missing]}],
|
|
130
|
+
[/ERR Warning: unable to submit change event/, {set: %i[change_event_failed]}],
|
|
131
|
+
[/Warning: rex timer overflow auto-adjusted/, {call: :rex_timer_overflow}],
|
|
132
|
+
[/\bFASP Session Statistics \[\w+\] (.*)/, {call: :session_statistics}]
|
|
133
|
+
].freeze
|
|
134
|
+
|
|
135
|
+
# When the session stop error is `Undefined`: first matching flag sets the error
|
|
136
|
+
UNDEFINED_ERROR_RULES = [
|
|
137
|
+
[%i[invalid_conf_file], {}],
|
|
138
|
+
[%i[send_no_route], {error: 'No route to host'}],
|
|
139
|
+
[%i[peer_abort], {error: 'Peer aborted session'}]
|
|
140
|
+
].freeze
|
|
141
|
+
|
|
142
|
+
# Session stop errors of failed sessions, refined with flags: first matching rule applies
|
|
143
|
+
ERROR_REFINEMENTS = {
|
|
144
|
+
'No error' => [
|
|
145
|
+
[%i[write], {error: 'Disk write failed'}],
|
|
146
|
+
[%i[client_management_connection_reset], {error: 'Client connection to management socket reset'}],
|
|
147
|
+
[%i[client_management_connection_lost], {error: 'Client management connection lost'}],
|
|
148
|
+
[%i[user_abort], {error: 'User aborted session'}],
|
|
149
|
+
[%i[peer_abort], {error: 'Peer aborted session'}]
|
|
150
|
+
]
|
|
151
|
+
}.freeze
|
|
152
|
+
|
|
153
|
+
# Sample lines: `Receiver bl t/o/r/d/ts/c/h=1/2/3/4/5/6/7 rex_rtt l/h/s/o=...`: group, field names, values
|
|
154
|
+
SAMPLE = /#{STAMP.source}.*?\[(\w+)(?:-\w+)?\].*?(Receiver|Sender)\s+(bl\b.*)/
|
|
155
|
+
# Sample value => path in sample records
|
|
156
|
+
SAMPLE_COMMON = {
|
|
157
|
+
blocks: %w[bl t], orig: %w[bl o], rex: %w[bl r], target_rate: %w[rate t], min_rate: %w[rate m], calc_rate: %w[rate c],
|
|
158
|
+
tx_bytes: %w[prog t], written: %w[prog f], elapsed_us: %w[prog e]
|
|
159
|
+
}.freeze
|
|
160
|
+
RECEIVER_SAMPLE = SAMPLE_COMMON.merge(
|
|
161
|
+
rex_srtt: %w[rex_rtt s], brtt: %w[rate_rtt b], rtt_high: %w[rate_rtt h], srtt: %w[rate_rtt s], brtt_resets: %w[rate_rtt r],
|
|
162
|
+
rex_count: %w[rex n], lost: %w[bl2 l], ooo: %w[bl2 o], ring_high: %w[disk h], ring_full: %w[disk b],
|
|
163
|
+
net_rate: %w[rate n], vlink_local: %w[rate vl], vlink_remote: %w[rate vr], disk_rate: %w[rate r]
|
|
164
|
+
).freeze
|
|
165
|
+
# Sender: rates of the receiver (ascp 4): network, virtual links, disk
|
|
166
|
+
SENDER_SAMPLE = SAMPLE_COMMON.merge(
|
|
167
|
+
rex_received: %w[rex b], rex_count: %w[rex n],
|
|
168
|
+
peer_net_rate: %w[rate n], peer_vlink_local: %w[rate vl], peer_vlink_remote: %w[rate vr], peer_disk_rate: %w[rate r]
|
|
169
|
+
).freeze
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|