aspera-cli 4.27.5 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +301 -4
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +139 -58
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +103 -26
- data/SECURITY.md +70 -0
- data/bin/ascli +8 -5
- data/bin/asession +10 -7
- data/docs/README.md +2551 -772
- data/docs/test-mcp-with-ai.md +13 -11
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +34 -22
- data/lib/aspera/agent/factory.rb +8 -4
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +8 -5
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +23 -19
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +84 -42
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +354 -0
- data/lib/aspera/ascp/management.rb +102 -48
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +16 -37
- data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +37 -34
- data/lib/aspera/cli/command_line.rb +24 -6
- data/lib/aspera/cli/command_registry.rb +49 -4
- data/lib/aspera/cli/command_spec.rb +34 -4
- data/lib/aspera/cli/completion/ascli.bash +53 -5
- data/lib/aspera/cli/completion/ascli.fish +27 -8
- data/lib/aspera/cli/completion/ascli.ps1 +63 -0
- data/lib/aspera/cli/completion/ascli.zsh +36 -4
- data/lib/aspera/cli/completion_actions.rb +594 -0
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +45 -0
- data/lib/aspera/cli/extended_value.rb +83 -24
- data/lib/aspera/cli/formatter.rb +142 -73
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -11
- data/lib/aspera/cli/http.rb +17 -13
- data/lib/aspera/cli/info.rb +17 -6
- data/lib/aspera/cli/mailer.rb +7 -4
- data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
- data/lib/aspera/cli/mcp/secrets.rb +146 -0
- data/lib/aspera/cli/mcp/tool.rb +232 -0
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +72 -11
- data/lib/aspera/cli/options.schema.yaml +856 -81
- data/lib/aspera/cli/parser.rb +64 -31
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +250 -44
- data/lib/aspera/cli/plugins/ats.rb +9 -8
- data/lib/aspera/cli/plugins/base.rb +51 -25
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +207 -141
- data/lib/aspera/cli/plugins/console.rb +9 -4
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +22 -8
- data/lib/aspera/cli/plugins/faspex5.rb +37 -44
- data/lib/aspera/cli/plugins/faspio.rb +41 -8
- data/lib/aspera/cli/plugins/httpgw.rb +8 -4
- data/lib/aspera/cli/plugins/mcp.rb +186 -121
- data/lib/aspera/cli/plugins/node.rb +41 -34
- data/lib/aspera/cli/plugins/oauth.rb +5 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
- data/lib/aspera/cli/plugins/preview.rb +69 -44
- data/lib/aspera/cli/plugins/server.rb +66 -12
- data/lib/aspera/cli/plugins/shares.rb +288 -48
- data/lib/aspera/cli/plugins/transferd.rb +223 -0
- data/lib/aspera/cli/preset_actions.rb +155 -33
- data/lib/aspera/cli/preset_manager.rb +72 -16
- data/lib/aspera/cli/prompt.rb +7 -2
- data/lib/aspera/cli/result.rb +28 -11
- data/lib/aspera/cli/runner.rb +193 -80
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +204 -0
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +13 -10
- data/lib/aspera/cli/table_fitter.rb +119 -0
- data/lib/aspera/cli/terminal_formatter.rb +32 -40
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +31 -19
- data/lib/aspera/cli/transfer_progress.rb +13 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +197 -50
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.html.erb +61 -0
- data/lib/aspera/cli/web_secret.rb +222 -0
- data/lib/aspera/cli/wizard.rb +58 -45
- data/lib/aspera/command_line_builder.rb +35 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +42 -25
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/duration.rb +45 -0
- data/lib/aspera/environment.rb +106 -33
- data/lib/aspera/exec_spec.rb +9 -5
- data/lib/aspera/faspex_gw.rb +105 -35
- data/lib/aspera/faspex_postproc.rb +193 -47
- data/lib/aspera/formatter_interface.rb +4 -2
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +6 -3
- data/lib/aspera/json_rpc/client.rb +55 -35
- data/lib/aspera/json_rpc/version.rb +4 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +20 -10
- data/lib/aspera/markdown.rb +155 -115
- data/lib/aspera/mime.rb +9 -0
- data/lib/aspera/nagios.rb +9 -5
- data/lib/aspera/node_emulator.rb +111 -112
- data/lib/aspera/oauth/client/base.rb +163 -0
- data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
- data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
- data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
- data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
- data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +85 -0
- data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
- data/lib/aspera/oauth/client/web.rb +84 -0
- data/lib/aspera/oauth/server/authorize.html.erb +34 -0
- data/lib/aspera/oauth/server/pkce.rb +427 -0
- data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +11 -6
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +4 -1
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +30 -1
- data/lib/aspera/rest/aspera_errors.rb +55 -60
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +42 -42
- data/lib/aspera/rest/error_analyzer.rb +14 -2
- data/lib/aspera/rest/list.rb +14 -10
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +13 -4
- data/lib/aspera/rest.rb +9 -7
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +56 -78
- data/lib/aspera/schema/reader.rb +46 -11
- data/lib/aspera/schema/registry.rb +45 -24
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +166 -0
- data/lib/aspera/secret/hider.rb +201 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +110 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +114 -0
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +6 -1
- data/lib/aspera/sync/conf.schema.yaml +25 -10
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +34 -12
- data/lib/aspera/temp_file_manager.rb +85 -22
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +22 -14
- data/lib/aspera/transfer/result.rb +3 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +45 -18
- data/lib/aspera/transfer/spec.schema.yaml +8 -5
- data/lib/aspera/transfer/units.rb +75 -0
- data/lib/aspera/uri_reader.rb +57 -7
- data/lib/aspera/web_auth/index.html.erb +20 -0
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +59 -14
- data/lib/aspera/web_page.html.erb +262 -0
- data/lib/aspera/web_page.rb +66 -0
- data/lib/aspera/web_server_simple.rb +82 -13
- data/lib/aspera/yaml.rb +5 -1
- data/llms.txt +81 -0
- data/skills/ascli/SKILL.md +88 -0
- data.tar.gz.sig +0 -0
- metadata +116 -48
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/mcp_tool.rb +0 -189
- data/lib/aspera/keychain/base.rb +0 -25
- data/lib/aspera/keychain/encrypted_hash.rb +0 -188
- data/lib/aspera/keychain/factory.rb +0 -59
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
- data/lib/aspera/keychain/macos_security.rb +0 -171
- data/lib/aspera/keychain/one_password_api.rb +0 -86
- data/lib/aspera/keychain/one_password_base.rb +0 -34
- data/lib/aspera/keychain/one_password_cli.rb +0 -98
- data/lib/aspera/oauth/base.rb +0 -141
- data/lib/aspera/oauth/web.rb +0 -57
- data/lib/aspera/oauth.rb +0 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
- data/lib/aspera/secret_hider.rb +0 -117
- data/lib/aspera/web_auth/index.html +0 -147
|
@@ -0,0 +1,593 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
# cspell:words nstore pids throughputs
|
|
7
|
+
require_relative '../../assert'
|
|
8
|
+
require_relative 'rules'
|
|
9
|
+
require_relative 'advice'
|
|
10
|
+
require_relative 'lines'
|
|
11
|
+
require_relative 'session'
|
|
12
|
+
require 'set'
|
|
13
|
+
require 'time'
|
|
14
|
+
|
|
15
|
+
module Aspera
|
|
16
|
+
module Ascp
|
|
17
|
+
class LogAnalyzer
|
|
18
|
+
# Analysis of one ascp log file: sessions, snapshot periods and statistics
|
|
19
|
+
class LogFile
|
|
20
|
+
include Lines
|
|
21
|
+
|
|
22
|
+
# Statistic => finding, when not zero
|
|
23
|
+
FINDINGS = {
|
|
24
|
+
stuck_nack: 'Retransmission requests (NACKs) were stuck in %d transfers: check the logs.',
|
|
25
|
+
filename_too_long: '%d files failed because their Windows path is longer than 260 characters.',
|
|
26
|
+
gap: 'ascp was blocked (IO wait) %d times.',
|
|
27
|
+
gap_ssh: 'ascp was blocked (IO wait) %d times, with an SSH timeout.',
|
|
28
|
+
buffer_died: 'The system ran out of buffer memory, and ascp exited, %d times.',
|
|
29
|
+
timeout_died: 'A data or SSH timeout made ascp exit %d times.',
|
|
30
|
+
decryption_died: 'A file decryption error made ascp exit %d times.',
|
|
31
|
+
table_died: 'ascp exited at the end of %d sessions, without logging the statistics table.',
|
|
32
|
+
died: 'ascp exited, was blocked (IO wait), or stopped logging, %d times: check the logs.',
|
|
33
|
+
win_clock: 'The clock of the system moves backwards: update the firmware of the motherboard.',
|
|
34
|
+
vm: 'This system may run on a VM, or have another clock problem, which affected %d transfers.',
|
|
35
|
+
disable_reverse: 'Reverse path congestion inference skew affected %d transfers: check the logs, and disable it if appropriate.',
|
|
36
|
+
vm_peer: 'Peers may run on a VM, or have another clock problem, which affected %d transfers.',
|
|
37
|
+
ossl_modules_missing: 'The OpenSSL modules folder of ascp was not found in %d sessions (configuration, e.g. installation of the HTTP gateway): no effect unless an OpenSSL provider (e.g. FIPS) is needed.',
|
|
38
|
+
change_event_failed: 'A change of the schedule of the virtual link capacity could not be registered in %d sessions: the schedule is not applied.'
|
|
39
|
+
}.freeze
|
|
40
|
+
# Totals of a snapshot period
|
|
41
|
+
PERIOD_TOTALS = %i[rate rate_in rate_out calc calc_in calc_out target target_in target_out loss_in loss_out].freeze
|
|
42
|
+
# A retry has the same values for these
|
|
43
|
+
RETRY_KEYS = %i[peer op source dest].freeze
|
|
44
|
+
|
|
45
|
+
attr_reader :path, :stats, :licenses, :first_time, :last_time, :thresholds
|
|
46
|
+
|
|
47
|
+
# @param path [String] Log file
|
|
48
|
+
# @param period [Integer] Snapshot period in seconds
|
|
49
|
+
# @param zero_bytes [Boolean] Include sessions without transferred bytes
|
|
50
|
+
# @param mtu [Integer, nil] MTU to use when not found in logs
|
|
51
|
+
# @param thresholds [Hash] Diagnostic thresholds
|
|
52
|
+
# @param timezone [String, nil] Time zone of the log lines: `UTC` or `+HH:MM`. Time zone of the analyst if nil.
|
|
53
|
+
def initialize(path, period: 60, zero_bytes: false, mtu: nil, thresholds: {}, timezone: nil)
|
|
54
|
+
@path = path
|
|
55
|
+
@timezone = timezone
|
|
56
|
+
@period = period
|
|
57
|
+
@zero_bytes = zero_bytes
|
|
58
|
+
@mtu = mtu
|
|
59
|
+
thresholds = thresholds.transform_keys(&:to_sym)
|
|
60
|
+
unknown = thresholds.keys - DEFAULT_THRESHOLDS.keys
|
|
61
|
+
Aspera.assert(unknown.empty?, type: ArgumentError) { "Unknown threshold: #{unknown.join(', ')}" }
|
|
62
|
+
@thresholds = DEFAULT_THRESHOLDS.merge(thresholds)
|
|
63
|
+
# PID => current Session of process
|
|
64
|
+
@current = {}
|
|
65
|
+
# All sessions, in order of creation
|
|
66
|
+
@sessions = []
|
|
67
|
+
# PID => user and version from configuration lines
|
|
68
|
+
@config = Hash.new { |h, k| h[k] = {} }
|
|
69
|
+
@stats = Hash.new(0)
|
|
70
|
+
@stats[:errors] = Hash.new(0)
|
|
71
|
+
@stats[:error_categories] = Hash.new(0)
|
|
72
|
+
# Start time => {start:, stop:, slots: {Session => Hash}, totals: Hash}
|
|
73
|
+
@periods = {}
|
|
74
|
+
# Start time of the latest period with samples
|
|
75
|
+
@bucket = nil
|
|
76
|
+
# [license number, IP] => {product:, account:, os:}
|
|
77
|
+
@licenses = {}
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Read and analyze the log file
|
|
81
|
+
# @return [self]
|
|
82
|
+
def analyze
|
|
83
|
+
LogAnalyzer.read_lines(@path, Encoding::BINARY) { |line| parse_line(normalize(line)) }
|
|
84
|
+
@sessions.each(&:evaluate_atm)
|
|
85
|
+
@periods.sort.each { |time, period| end_of_period(time, period) }
|
|
86
|
+
compute_statistics
|
|
87
|
+
self
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# @return [Array<Session>] transfer sessions to report, sorted by date
|
|
91
|
+
def sessions
|
|
92
|
+
transfer_sessions.select { |s| counted?(s) }.sort_by { |s| [s.num(:date), s[:user].to_s] }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# @return [Array<Hash>] one row per session
|
|
96
|
+
def session_reports
|
|
97
|
+
windows_log = @path.match?(/\w:/)
|
|
98
|
+
sessions.map { |s| {'file' => @path}.merge(s.report(windows_log: windows_log)) }
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
class << self
|
|
102
|
+
# @param stats [Hash] statistics of one or several log files
|
|
103
|
+
# @return [Array<String>] findings from statistics
|
|
104
|
+
def findings(stats)
|
|
105
|
+
result = FINDINGS.filter_map do |key, message|
|
|
106
|
+
if stats[key].positive?
|
|
107
|
+
message.include?('%d') ? format(message, stats[key]) : message
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
result.insert(1, 'The stuck retransmission requests may have been caused by high packet loss.') if stats[:stuck_nack_loss] > 0.4
|
|
111
|
+
if stats[:smart_reset].positive?
|
|
112
|
+
result.push("ascp reset the base RTT #{stats[:smart_reset]} times.#{' Consider changing the rtt_autocorrect parameters.' if stats[:smart_reset] > 100}")
|
|
113
|
+
end
|
|
114
|
+
result
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# @return [Array<Hash>] one row per multi-session transfer: sessions with the same transfer identifier
|
|
119
|
+
def multi_sessions
|
|
120
|
+
parallel = transfer_sessions.select { |s| s.num(:cluster_nodes) > 1 && s[:xfer_id] }
|
|
121
|
+
parallel.group_by { |s| s[:xfer_id] }.map do |xfer_id, group|
|
|
122
|
+
group = group.sort_by { |s| s.num(:cluster_node) }
|
|
123
|
+
start = group.filter_map { |s| s[:start] }.min
|
|
124
|
+
stop = group.filter_map { |s| s[:stop] || s[:last] }.max
|
|
125
|
+
bytes = group.sum { |s| s.num(:tx_bytes) }
|
|
126
|
+
{
|
|
127
|
+
'file' => @path,
|
|
128
|
+
'transfer_id' => xfer_id,
|
|
129
|
+
'nodes' => group.first[:cluster_nodes],
|
|
130
|
+
'pids' => group.map(&:pid),
|
|
131
|
+
'peer' => group.first[:peer],
|
|
132
|
+
'direction' => group.first.direction,
|
|
133
|
+
'status' => group.all? { |s| s[:status] == 'success' } ? 'success' : 'failed',
|
|
134
|
+
'start' => iso(start),
|
|
135
|
+
'stop' => iso(stop),
|
|
136
|
+
'transmitted_bytes' => bytes,
|
|
137
|
+
'average_rate' => (bytes * 8 / (stop - start) if start && stop && stop > start)
|
|
138
|
+
}.compact
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# @return [Array<Hash>] one row per diagnostic of sessions: id, severity, message, recommendation (if any)
|
|
143
|
+
def diagnostics
|
|
144
|
+
windows_log = @path.match?(/\w:/)
|
|
145
|
+
sessions.flat_map do |s|
|
|
146
|
+
s.report(windows_log: windows_log)
|
|
147
|
+
s.diagnostics.map do |diagnostic|
|
|
148
|
+
{'file' => @path, 'pid' => s.pid, 'session' => s[:uuid]}.compact.merge(diagnostic, 'recommendation' => RECOMMENDATIONS[diagnostic['id'].to_sym]).compact
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# @return [Array<String>] ascp versions found in log, older than supported
|
|
154
|
+
def old_versions
|
|
155
|
+
transfer_sessions.filter_map { |s| s[:ver] if Session.version_older?(s[:ver], MIN_VERSION) }.uniq
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# @return [Array<Hash>] one row per snapshot period
|
|
159
|
+
def snapshots
|
|
160
|
+
reported_periods.map do |time, period|
|
|
161
|
+
t = period[:totals]
|
|
162
|
+
row = {'file' => @path, 'time' => iso(time), 'sessions' => active_sessions(period[:slots])}
|
|
163
|
+
if @period > 600 && period[:stop].to_i > period[:start].to_i
|
|
164
|
+
duration = period[:stop] - period[:start]
|
|
165
|
+
row.merge!('rate' => t[:data] / duration, 'rate_in' => t[:data_in] / duration, 'rate_out' => t[:data_out] / duration)
|
|
166
|
+
else
|
|
167
|
+
row.merge!(PERIOD_TOTALS.to_h { |k| [k.to_s, t[k]] })
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# @return [Array<Hash>] one row per session in each snapshot period
|
|
173
|
+
def snapshot_sessions
|
|
174
|
+
reported_periods.flat_map do |time, period|
|
|
175
|
+
period[:slots].sort_by { |session, _| session.pid.to_i }.filter_map do |session, slot|
|
|
176
|
+
next if !@zero_bytes && slot[:net_rate].to_i.zero?
|
|
177
|
+
progress = (100.0 * slot[:tx_bytes].to_i / session[:tx_bytes]).round(2) if session.flag?(:statistics) && session.num(:tx_bytes).nonzero?
|
|
178
|
+
{
|
|
179
|
+
'file' => @path,
|
|
180
|
+
'time' => iso(time),
|
|
181
|
+
'pid' => session.pid,
|
|
182
|
+
'session' => session[:uuid],
|
|
183
|
+
'rate' => slot[:net_rate].to_i,
|
|
184
|
+
'calculated' => slot[:avg_calc],
|
|
185
|
+
'target' => slot[:target_rate],
|
|
186
|
+
'loss' => slot[:loss_rate]&.round(2),
|
|
187
|
+
'progress' => progress,
|
|
188
|
+
'direction' => slot[:receiver] ? 'receive' : 'send',
|
|
189
|
+
'constraint' => slot[:calc_1]&.join(',')
|
|
190
|
+
}.compact
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
private
|
|
196
|
+
|
|
197
|
+
# @return [Boolean] true if session is counted in statistics and reported
|
|
198
|
+
def counted?(session)
|
|
199
|
+
@zero_bytes || session.num(:tx_bytes).nonzero? || session[:status] == 'failed'
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# @return [Session] current session of process
|
|
203
|
+
def session(pid)
|
|
204
|
+
@current[pid] || new_session(pid)
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# A process starts a new session
|
|
208
|
+
# @return [Session] new current session of process
|
|
209
|
+
def new_session(pid)
|
|
210
|
+
s = Session.new(pid, thresholds: @thresholds, timezone: @timezone)
|
|
211
|
+
@sessions.push(s)
|
|
212
|
+
@current[pid] = s
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# A new process (configuration line) reuses the PID of a process whose session ended
|
|
216
|
+
def check_pid_reuse(pid)
|
|
217
|
+
current = @current[pid]
|
|
218
|
+
new_session(pid) if current && (current.flag?(:statistics) || current[:stop])
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# @return [Boolean] true if line is not relevant
|
|
222
|
+
def ignored?(line, pid)
|
|
223
|
+
return true unless line.match?(/\w/)
|
|
224
|
+
return true if IGNORED_TEXTS.any? { |text| line.include?(text) } || line.match?(IGNORED_LINES)
|
|
225
|
+
line.match?(OTHER_PROCESSES) && line.match?(/(?:#{OTHER_PROCESSES.source}).*?#{pid}/)
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def parse_line(line)
|
|
229
|
+
track_ascp(line)
|
|
230
|
+
pid = line[PID, 1] or return
|
|
231
|
+
time = parse_time(line[LINE_TIME, 1])
|
|
232
|
+
if time
|
|
233
|
+
@first_time = time if @first_time.nil? || time < @first_time
|
|
234
|
+
@last_time = time if @last_time.nil? || time > @last_time
|
|
235
|
+
end
|
|
236
|
+
if line.include?('rex buffer full')
|
|
237
|
+
session(pid).set(:rex_buffer_full)
|
|
238
|
+
return
|
|
239
|
+
end
|
|
240
|
+
return if ignored?(line, pid)
|
|
241
|
+
@stats[:win_clock] = 1 if line.include?('_usec_since') && line.match?(/get(?:nstore)?_usec_since/)
|
|
242
|
+
process = line[PROCESS, 1]
|
|
243
|
+
ascp = process.nil? || process.match?(ASCP_PROCESS)
|
|
244
|
+
if ascp && line.include?('Configuration:') && (user = line[/Configuration:\s+using.*?user\s+(.*)/, 1])
|
|
245
|
+
check_pid_reuse(pid)
|
|
246
|
+
@config[pid][:user] = user
|
|
247
|
+
session(pid)[:role] = user == '-' ? 'client' : 'server'
|
|
248
|
+
end
|
|
249
|
+
if ascp && line.include?('Initializing') && (version = line[/Initializing\s+FASP\s+version\s+((?:\d\.?)+)/, 1])
|
|
250
|
+
check_pid_reuse(pid)
|
|
251
|
+
session(pid).set(:transfer)
|
|
252
|
+
@config[pid][:ver] = version
|
|
253
|
+
# Authentication is logged after this line (the product name before or after it, depending on the version)
|
|
254
|
+
@config[pid].delete(:auth)
|
|
255
|
+
license(@config[pid], line)
|
|
256
|
+
end
|
|
257
|
+
return if line.include?('ascmd') || line.include?('aspshell')
|
|
258
|
+
return unless ascp
|
|
259
|
+
s = session(pid)
|
|
260
|
+
if line.include?('ERR') && (error = line[LOG_ERROR, 1])
|
|
261
|
+
s[:last_error] = error.strip
|
|
262
|
+
s[:last_error_time] = time
|
|
263
|
+
end
|
|
264
|
+
s.set(:no_reverse_infer) if line.include?('Disabling reverse path congestion inference')
|
|
265
|
+
if line.include?('Session') && (m = line.match(SESSION_START_STOP))
|
|
266
|
+
s = session_line(s, pid, m[2].downcase.to_sym, m[1], line)
|
|
267
|
+
end
|
|
268
|
+
if time && s[:last] && time - s[:last] > 23
|
|
269
|
+
gap = time - s[:last] - 21
|
|
270
|
+
s[:gap] = gap if gap > s.num(:gap)
|
|
271
|
+
s[:gap_last] = time
|
|
272
|
+
end
|
|
273
|
+
s[:last] = time
|
|
274
|
+
s.session_params(Records.key_values(line[/Session\s+Params\s+(.*)/, 1] || line)) if line.include?('FASP Session Params')
|
|
275
|
+
if line.include?(' DS Qs ')
|
|
276
|
+
parse_ds_qs(pid, line)
|
|
277
|
+
else
|
|
278
|
+
apply_rules(s, line)
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# Apply the first matching rule of LINE_RULES
|
|
283
|
+
def apply_rules(s, line)
|
|
284
|
+
match, effects = LINE_RULES.find { |regexes, _| Array(regexes).all? { |re| line.match?(re) } }
|
|
285
|
+
return unless effects
|
|
286
|
+
effects[:call] ? send(effects[:call], s, line.match(Array(match).first)) : s.apply(effects)
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# Maximum rate (bps, nil if unlimited) and expiry date (nil if none) of the license, from the `Initializing FASP version` line
|
|
290
|
+
def license(config, line)
|
|
291
|
+
mbps = line[LICENSE_RATE, 1]
|
|
292
|
+
config[:license_rate] = mbps && (mbps.to_i * 1_000_000)
|
|
293
|
+
config[:license_expires] = line[LICENSE_EXPIRES, 1]
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# `Session Start` or `Session Stop` line
|
|
297
|
+
# @return [Session] session of line: a new one if the process starts another session
|
|
298
|
+
def session_line(s, pid, kind, stamp, line)
|
|
299
|
+
fields = Records.key_values(line[/Session\s+(?:Start|Stop)\s+(.*)/, 1] || line)
|
|
300
|
+
f = {
|
|
301
|
+
uuid: fields['uuid'] || fields['id'], op: fields['op'], user: fields['user'],
|
|
302
|
+
source: (fields['source']&.match?(/\s/) ? fields['source'].split.first : fields['source']),
|
|
303
|
+
dest: (fields['dest']&.match?(/\s/) ? fields['dest'].split.first : fields['dest']),
|
|
304
|
+
transport: (fields.key?('wss_port') ? 'wss' : ('ssh' if fields.key?('tcp_port'))),
|
|
305
|
+
os: fields['os'], ver: fields['ver']&.split&.first, peeros: fields['peeros'], peerver: fields['peerver']&.split&.first, lic: fields['lic'], peerlic: fields['peerlic']
|
|
306
|
+
}.compact
|
|
307
|
+
if f[:uuid] && s[:uuid] && s[:uuid] != f[:uuid]
|
|
308
|
+
role = s[:role]
|
|
309
|
+
s = new_session(pid)
|
|
310
|
+
s[:role] = role
|
|
311
|
+
end
|
|
312
|
+
s[:gap] = 0 if kind.eql?(:start)
|
|
313
|
+
f[:local], f[:l_port] = address(fields['local'])
|
|
314
|
+
f[:peer], f[:p_port] = address(fields['peer'])
|
|
315
|
+
s.set(:transfer) if f[:peer]
|
|
316
|
+
record_license(f[:lic], f[:local], f[:os])
|
|
317
|
+
record_license(f[:peerlic], f[:peer], f[:peeros])
|
|
318
|
+
s.session_line(kind, f, parse_time(stamp), @config[pid])
|
|
319
|
+
s.stop_error(line, fields['status']) if kind.eql?(:stop)
|
|
320
|
+
s
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# @return [Array] IP and port from `IP:port`
|
|
324
|
+
def address(value)
|
|
325
|
+
m = value.to_s.match(/\A(.*):(\d+)\z/) or return [nil, nil]
|
|
326
|
+
[m[1], m[2].to_i]
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
# @param license [String] product:account:license
|
|
330
|
+
def record_license(license, ip, os)
|
|
331
|
+
return if license.nil?
|
|
332
|
+
product, account, number = license.split(':').map(&:to_i)
|
|
333
|
+
@licenses[[number, ip]] = {product: product, account: account, os: os} unless product == 6
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
# --- LINE_RULES handlers ---
|
|
337
|
+
|
|
338
|
+
def end_of_statistics(s, _m)
|
|
339
|
+
s.set(:stats_end)
|
|
340
|
+
s.delete(:last)
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
def transfer_stop(s, m)
|
|
344
|
+
fields = Records.key_values(m[1])
|
|
345
|
+
s[:uuid] ||= fields['uuid']
|
|
346
|
+
@stats[:filename_too_long] += 1 if s.transfer_stop(fields['op'], fields['status'], fields['file'], fields['size'].to_i, fields['start_byte'].to_i, fields['errstr'])
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def brtt_reset(s, _m)
|
|
350
|
+
s.increment(:brtt_reset)
|
|
351
|
+
@stats[:smart_reset] += 1
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
def rate_params(s, m)
|
|
355
|
+
s.rate_params(m[2].to_i, m[3].to_i, m[4], by_peer: m[1].start_with?('Peer'))
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
def path_mtu(s, m)
|
|
359
|
+
s[:mtu] = m[1].to_i
|
|
360
|
+
s[:start_brtt] = m[2].to_i
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
def datagram_size(s, m)
|
|
364
|
+
s[:mtu] = m[1].to_i
|
|
365
|
+
s[:payload] = m[2].to_i
|
|
366
|
+
s[:path_mtu] = m[3].to_i if m[3]
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
def conf_parse_error(s, m)
|
|
370
|
+
s[:conf_parse] = m[1]
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
# Error when closing the SSH connection: keep the first one
|
|
374
|
+
def ssh_disconnect_error(s, m)
|
|
375
|
+
s[:ssh_disconnect] ||= m[0]
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
# Kernel line: error on the PID, and on PID - 11
|
|
379
|
+
def segfault(s, _m)
|
|
380
|
+
s[:error] = 'segfault'
|
|
381
|
+
session((s.pid.to_i - 11).to_s)[:error] = 'segfault'
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
# No packet received: the receiver logs it from 2 seconds, the sender (no feedback of the receiver) from 3 seconds
|
|
385
|
+
def no_data(s, m)
|
|
386
|
+
sender = m[1] == 'Sender'
|
|
387
|
+
key = sender ? :no_feedback : :no_data
|
|
388
|
+
first = sender ? 3 : 2
|
|
389
|
+
seconds = m[2].to_i
|
|
390
|
+
s[key] = seconds if s.num(key) <= seconds
|
|
391
|
+
s.increment(:"#{key}_total", seconds == first ? first : 1)
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
# Product name: `IBM Aspera High-Speed Transfer Server version 4.4.8.2596`, logged once by the process
|
|
395
|
+
def product_name(s, m)
|
|
396
|
+
@config[s.pid][:product_name] = m[1]
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
# Authentication with a token (`ASPERA_AUTH_TOKEN`), also when the line of the access key of the node storage follows
|
|
400
|
+
def token_auth(s, _m)
|
|
401
|
+
@config[s.pid][:auth] = 'token'
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
# Storage of a node access key, without token: authentication of the node user
|
|
405
|
+
def node_auth(s, _m)
|
|
406
|
+
@config[s.pid][:auth] ||= 'node'
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
def rex_timer_overflow(s, _m)
|
|
410
|
+
s.increment(:rex_overflow)
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
def session_statistics(s, m)
|
|
414
|
+
fields = Records.statistics_fields(m[1])
|
|
415
|
+
s.set(:statistics)
|
|
416
|
+
s[:effective_rate] = fields['effective_rate']
|
|
417
|
+
progress = fields['progress'] || {}
|
|
418
|
+
s[:tx_bytes] = progress['tx_bytes'].to_i
|
|
419
|
+
s[:tx_time] = progress['tx_time'].to_i
|
|
420
|
+
s[:retransmitted] = fields['solicited_rex']&.to_f
|
|
421
|
+
s[:duplicates] = fields['rcvd_dups']&.to_f
|
|
422
|
+
s[:efficiency] = fields['effective']&.to_f
|
|
423
|
+
s[:control_loss] = %w[sndr rcvr rex].filter_map { |dir| (ctl = fields["#{dir}_ctl"]) && [dir, [ctl['sent'].to_i, ctl['lost_pct'].to_f]] }.to_h
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
# Cipher of server configuration replaces the requested one
|
|
427
|
+
def cipher_override(s, m)
|
|
428
|
+
s[:cipher_override] = m.captures.map { |id| CIPHERS.fetch(id.to_i, id) }
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
def resume_downgrade(s, m)
|
|
432
|
+
s[:resume_downgrade] = m[1]
|
|
433
|
+
end
|
|
434
|
+
|
|
435
|
+
def job_size_change(s, m)
|
|
436
|
+
s[:job_size] = m.captures.map(&:to_i)
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
# Virtual link: bandwidth limit shared by sessions
|
|
440
|
+
def vlink_init(s, m)
|
|
441
|
+
(s[:vlinks] ||= {})[m[1].to_i] = {'type' => m[2], 'capacity' => m[3].to_i}
|
|
442
|
+
end
|
|
443
|
+
|
|
444
|
+
# --- Snapshot periods ---
|
|
445
|
+
|
|
446
|
+
def add_sample(_s, m)
|
|
447
|
+
sample_match = m.string.match(SAMPLE) or return
|
|
448
|
+
receiver = sample_match[3].eql?('Receiver')
|
|
449
|
+
values = Records.group_fields(sample_match[4])
|
|
450
|
+
data = (receiver ? RECEIVER_SAMPLE : SENDER_SAMPLE).transform_values { |path| values.dig(*path) }
|
|
451
|
+
return if data.values_at(:tx_bytes, :written, :elapsed_us).any?(&:nil?)
|
|
452
|
+
data[:disk_read_rate] = values['drate'] unless receiver
|
|
453
|
+
session = session(sample_match[2])
|
|
454
|
+
session[:dir] = receiver ? 'receive' : 'send'
|
|
455
|
+
time = parse_time(sample_match[1]) or return
|
|
456
|
+
data.merge!(time: time, receiver: receiver, elapsed: format('%.2f', data[:elapsed_us] / 1_000_000.0).to_f)
|
|
457
|
+
bucket = (time / @period) * @period
|
|
458
|
+
# The previous period is complete: ATM detection works on consecutive periods, in log order
|
|
459
|
+
@sessions.each(&:evaluate_atm) if @bucket && bucket > @bucket
|
|
460
|
+
@bucket = bucket if @bucket.nil? || bucket > @bucket
|
|
461
|
+
period = (@periods[bucket] ||= {slots: {}})
|
|
462
|
+
period[:start] ||= time
|
|
463
|
+
period[:stop] = time
|
|
464
|
+
@stats[:smart_reset] += data[:brtt_resets] if data[:brtt_resets].to_i.positive?
|
|
465
|
+
first = !period[:slots].key?(session)
|
|
466
|
+
slot = (period[:slots][session] ||= {})
|
|
467
|
+
session.add_sample(data, slot, first: first, smart_reset: @stats[:smart_reset].positive?, mtu: @mtu)
|
|
468
|
+
@stats[:last] = time
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
# Parse and accumulate a `DS Qs` line into the current session
|
|
472
|
+
def parse_ds_qs(pid, line)
|
|
473
|
+
text = line[/(?:Receiver|Sender)\s+DS\s+(.*)/, 1] or return
|
|
474
|
+
queues = Records.queues_fields(text)
|
|
475
|
+
qs = queues['Qs'] || {}
|
|
476
|
+
qs['mgmt_backlog'] = queues['mgmt_backlog'].sum { |h| h['n'].to_i } if queues['mgmt_backlog']
|
|
477
|
+
session(pid).add_ds_qs_sample(qs) unless qs.empty?
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
# @return [Integer] number of sessions in period
|
|
481
|
+
def active_sessions(slots)
|
|
482
|
+
@zero_bytes ? slots.size : slots.count { |_, slot| slot[:total1].to_i.positive? || slot[:total2].to_i.positive? }
|
|
483
|
+
end
|
|
484
|
+
|
|
485
|
+
# Totals of period, concurrency of sessions, maximum throughputs
|
|
486
|
+
def end_of_period(date, period)
|
|
487
|
+
slots = period[:slots]
|
|
488
|
+
active = active_sessions(slots)
|
|
489
|
+
totals = period[:totals] = Hash.new(0)
|
|
490
|
+
slots.each_key do |s|
|
|
491
|
+
slot = slots[s]
|
|
492
|
+
s.set(:transfer)
|
|
493
|
+
if slot[:elapsed2].to_f == slot[:elapsed1].to_f
|
|
494
|
+
slots.delete(s)
|
|
495
|
+
next
|
|
496
|
+
end
|
|
497
|
+
calc = slot[:calc_rate]
|
|
498
|
+
slot[:avg_calc] = Session.average(calc).to_i
|
|
499
|
+
slot[:avg_calc] = [slot[:avg_calc], calc.max].max if slot[:avg_calc] < slot[:net_rate].to_i
|
|
500
|
+
way = slot[:receiver] ? :in : :out
|
|
501
|
+
data = slot[:total2].to_i - slot[:total1].to_i
|
|
502
|
+
if data.positive?
|
|
503
|
+
totals[:"calc_#{way}"] += slot[:avg_calc]
|
|
504
|
+
totals[:data] += data
|
|
505
|
+
totals[:"data_#{way}"] += data
|
|
506
|
+
totals[:"target_#{way}"] += slot[:target_rate].to_i
|
|
507
|
+
end
|
|
508
|
+
totals[:rate] += slot[:net_rate].to_i
|
|
509
|
+
totals[:"rate_#{way}"] += slot[:net_rate].to_i
|
|
510
|
+
totals[:"loss_#{way}"] += slot[:loss_speed].to_i
|
|
511
|
+
s[:sess_min] = active if s.num(:sess_min).zero? || s[:sess_min] >= active
|
|
512
|
+
if s.num(:sess_max) <= active
|
|
513
|
+
s[:sess_max] = active
|
|
514
|
+
s[:sess_max_time] = date
|
|
515
|
+
end
|
|
516
|
+
end
|
|
517
|
+
totals[:calc] = totals[:calc_in] + totals[:calc_out]
|
|
518
|
+
totals[:target] = totals[:target_in] + totals[:target_out]
|
|
519
|
+
{max_sess: active, max_total: totals[:rate], max_out: totals[:rate_out], max_in: totals[:rate_in]}.each do |key, value|
|
|
520
|
+
next if value <= @stats[key]
|
|
521
|
+
@stats[key] = value
|
|
522
|
+
@stats[:"#{key}_time"] = date
|
|
523
|
+
end
|
|
524
|
+
end
|
|
525
|
+
|
|
526
|
+
# @return [Array] periods with traffic
|
|
527
|
+
def reported_periods
|
|
528
|
+
@periods.sort.select { |_, period| period[:totals] && (@zero_bytes || period[:totals][:rate_in].nonzero? || period[:totals][:rate_out].nonzero?) }
|
|
529
|
+
end
|
|
530
|
+
|
|
531
|
+
# --- Statistics ---
|
|
532
|
+
|
|
533
|
+
def compute_statistics
|
|
534
|
+
starts = transfer_sessions.filter_map { |s| s[:start] }
|
|
535
|
+
transfer_sessions.each do |s|
|
|
536
|
+
s.shared_link(starts)
|
|
537
|
+
s.clear(:vm) if @stats[:smart_reset].positive?
|
|
538
|
+
exit_category = s.check_exit(@stats[:last])
|
|
539
|
+
@stats[exit_category] += 1 if exit_category
|
|
540
|
+
cancelled = s[:error].to_s.match?(/cancelled/i)
|
|
541
|
+
if counted?(s)
|
|
542
|
+
@stats[:total_sessions] += 1
|
|
543
|
+
case s[:status]
|
|
544
|
+
when 'success' then @stats[:success] += 1
|
|
545
|
+
when 'failed' then @stats[:failed] += 1 unless cancelled
|
|
546
|
+
end
|
|
547
|
+
end
|
|
548
|
+
@stats[:http_fallback] += 1 if s.flag?(:http_fallback)
|
|
549
|
+
if s.num(:gap).positive?
|
|
550
|
+
@stats[:gap] += 1 unless s.flag?(:ssh_timeout)
|
|
551
|
+
@stats[:gap_ssh] += 1 if s.flag?(:ssh_timeout)
|
|
552
|
+
end
|
|
553
|
+
if s[:error]
|
|
554
|
+
@stats[:cancelled] += 1 if cancelled
|
|
555
|
+
@stats[:errors][s[:error]] += 1 unless cancelled
|
|
556
|
+
end
|
|
557
|
+
@stats[:ossl_modules_missing] += 1 if s.flag?(:ossl_modules_missing)
|
|
558
|
+
@stats[:change_event_failed] += 1 if s.flag?(:change_event_failed)
|
|
559
|
+
@stats[:vm] += 1 if s.flag?(:vm)
|
|
560
|
+
@stats[:vm_peer] += 1 if s.flag?(:vm_peer)
|
|
561
|
+
@stats[:disable_reverse] += 1 if s.flag?(:disable_reverse) && !s.flag?(:no_reverse_infer)
|
|
562
|
+
if s.stuck_nack?
|
|
563
|
+
@stats[:stuck_nack] += 1
|
|
564
|
+
@stats[:stuck_nack_loss] = [@stats[:stuck_nack_loss], s.num(:loss_rate)].max
|
|
565
|
+
end
|
|
566
|
+
@stats[:error_categories][s.error_category] += 1 if counted?(s) && s[:status] == 'failed' && s.error_category
|
|
567
|
+
end
|
|
568
|
+
link_retries
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
# @return [Array<Session>] all transfer sessions, including those not reported
|
|
572
|
+
def transfer_sessions
|
|
573
|
+
@sessions.select { |s| s.flag?(:transfer) }
|
|
574
|
+
end
|
|
575
|
+
|
|
576
|
+
# Link a failed session to the next session with same peer, direction, source and destination
|
|
577
|
+
def link_retries
|
|
578
|
+
groups = transfer_sessions.select { |s| s[:source] && s[:start] }.group_by { |s| RETRY_KEYS.map { |key| s[key] } }
|
|
579
|
+
groups.each_value do |group|
|
|
580
|
+
group.sort_by { |s| s[:start] }.each_cons(2) do |previous, current|
|
|
581
|
+
next unless previous[:status] == 'failed' && current[:start] > previous[:start]
|
|
582
|
+
next if previous[:xfer_id] && previous[:xfer_id] == current[:xfer_id]
|
|
583
|
+
next if current[:start] - (previous[:stop] || previous[:last] || previous[:start]) > @thresholds[:retry_window]
|
|
584
|
+
previous[:retried_by] = current.pid
|
|
585
|
+
current[:retry_of] = previous.pid
|
|
586
|
+
@stats[:retries] += 1
|
|
587
|
+
end
|
|
588
|
+
end
|
|
589
|
+
end
|
|
590
|
+
end
|
|
591
|
+
end
|
|
592
|
+
end
|
|
593
|
+
end
|