aspera-cli 4.28.0 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +143 -2
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +135 -56
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +103 -26
- data/SECURITY.md +70 -0
- data/bin/ascli +8 -5
- data/bin/asession +9 -6
- data/docs/README.md +1633 -609
- data/docs/test-mcp-with-ai.md +8 -8
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +20 -19
- data/lib/aspera/agent/factory.rb +6 -3
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +7 -4
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +20 -20
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +71 -65
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +354 -0
- data/lib/aspera/ascp/management.rb +16 -8
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +12 -19
- data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +37 -33
- data/lib/aspera/cli/command_line.rb +24 -6
- data/lib/aspera/cli/command_registry.rb +6 -1
- data/lib/aspera/cli/command_spec.rb +34 -7
- data/lib/aspera/cli/completion/ascli.bash +19 -5
- data/lib/aspera/cli/completion/ascli.fish +13 -2
- data/lib/aspera/cli/completion/ascli.ps1 +31 -9
- data/lib/aspera/cli/completion/ascli.zsh +22 -6
- data/lib/aspera/cli/completion_actions.rb +303 -69
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +3 -0
- data/lib/aspera/cli/extended_value.rb +61 -27
- data/lib/aspera/cli/formatter.rb +29 -22
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -10
- data/lib/aspera/cli/http.rb +12 -9
- data/lib/aspera/cli/info.rb +12 -7
- data/lib/aspera/cli/mailer.rb +6 -3
- data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
- data/lib/aspera/cli/mcp/secrets.rb +8 -5
- data/lib/aspera/cli/mcp/tool.rb +15 -14
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +65 -12
- data/lib/aspera/cli/options.schema.yaml +487 -63
- data/lib/aspera/cli/parser.rb +56 -32
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +244 -35
- data/lib/aspera/cli/plugins/ats.rb +9 -8
- data/lib/aspera/cli/plugins/base.rb +17 -12
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +198 -103
- data/lib/aspera/cli/plugins/console.rb +6 -3
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +21 -7
- data/lib/aspera/cli/plugins/faspex5.rb +28 -25
- data/lib/aspera/cli/plugins/faspio.rb +7 -5
- data/lib/aspera/cli/plugins/httpgw.rb +7 -4
- data/lib/aspera/cli/plugins/mcp.rb +28 -22
- data/lib/aspera/cli/plugins/node.rb +31 -30
- data/lib/aspera/cli/plugins/oauth.rb +5 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
- data/lib/aspera/cli/plugins/preview.rb +69 -44
- data/lib/aspera/cli/plugins/server.rb +63 -11
- data/lib/aspera/cli/plugins/shares.rb +86 -16
- data/lib/aspera/cli/plugins/transferd.rb +223 -0
- data/lib/aspera/cli/preset_actions.rb +144 -30
- data/lib/aspera/cli/preset_manager.rb +52 -18
- data/lib/aspera/cli/prompt.rb +4 -1
- data/lib/aspera/cli/result.rb +12 -8
- data/lib/aspera/cli/runner.rb +69 -46
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +79 -9
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +13 -10
- data/lib/aspera/cli/table_fitter.rb +3 -0
- data/lib/aspera/cli/terminal_formatter.rb +9 -6
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +29 -20
- data/lib/aspera/cli/transfer_progress.rb +13 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +188 -42
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.rb +12 -10
- data/lib/aspera/cli/wizard.rb +42 -39
- data/lib/aspera/command_line_builder.rb +9 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +4 -1
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/duration.rb +5 -2
- data/lib/aspera/environment.rb +21 -12
- data/lib/aspera/exec_spec.rb +3 -0
- data/lib/aspera/faspex_gw.rb +13 -9
- data/lib/aspera/faspex_postproc.rb +15 -12
- data/lib/aspera/formatter_interface.rb +4 -1
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +4 -2
- data/lib/aspera/json_rpc/client.rb +6 -5
- data/lib/aspera/json_rpc/version.rb +3 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +18 -5
- data/lib/aspera/markdown.rb +5 -2
- data/lib/aspera/mime.rb +3 -0
- data/lib/aspera/nagios.rb +6 -4
- data/lib/aspera/node_emulator.rb +15 -13
- data/lib/aspera/oauth/client/base.rb +91 -70
- data/lib/aspera/oauth/client/boot.rb +7 -4
- data/lib/aspera/oauth/client/factory.rb +6 -3
- data/lib/aspera/oauth/client/generic.rb +5 -1
- data/lib/aspera/oauth/client/json_credentials.rb +4 -1
- data/lib/aspera/oauth/client/jwt.rb +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +85 -0
- data/lib/aspera/oauth/client/url_json.rb +4 -1
- data/lib/aspera/oauth/client/web.rb +39 -12
- data/lib/aspera/oauth/server/pkce.rb +6 -5
- data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +8 -4
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +4 -1
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +3 -0
- data/lib/aspera/rest/aspera_errors.rb +4 -1
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +22 -29
- data/lib/aspera/rest/error_analyzer.rb +5 -2
- data/lib/aspera/rest/list.rb +8 -4
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +8 -5
- data/lib/aspera/rest.rb +9 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +8 -2
- data/lib/aspera/schema/reader.rb +7 -0
- data/lib/aspera/schema/registry.rb +43 -28
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +166 -0
- data/lib/aspera/secret/hider.rb +201 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +110 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +5 -2
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +3 -0
- data/lib/aspera/sync/conf.schema.yaml +9 -4
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +16 -14
- data/lib/aspera/temp_file_manager.rb +8 -4
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +17 -14
- data/lib/aspera/transfer/result.rb +3 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +9 -22
- data/lib/aspera/transfer/units.rb +75 -0
- data/lib/aspera/uri_reader.rb +57 -7
- data/lib/aspera/web_auth/index.html.erb +6 -2
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +47 -10
- data/lib/aspera/web_page.rb +7 -2
- data/lib/aspera/web_server_simple.rb +6 -3
- data/lib/aspera/yaml.rb +5 -1
- data/llms.txt +81 -0
- data/skills/ascli/SKILL.md +88 -0
- data.tar.gz.sig +0 -0
- metadata +72 -19
- metadata.gz.sig +0 -0
- data/lib/aspera/keychain/base.rb +0 -30
- data/lib/aspera/keychain/encrypted_hash.rb +0 -192
- data/lib/aspera/keychain/factory.rb +0 -58
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
- data/lib/aspera/keychain/macos_security.rb +0 -277
- data/lib/aspera/keychain/one_password_api.rb +0 -91
- data/lib/aspera/keychain/one_password_base.rb +0 -39
- data/lib/aspera/keychain/one_password_cli.rb +0 -100
- data/lib/aspera/secret_hider.rb +0 -196
- data/lib/aspera/secret_strength.rb +0 -114
|
@@ -0,0 +1,782 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
# cspell:words commify Isilon nacks nohear onefs overdriven
|
|
7
|
+
require_relative '../../assert'
|
|
8
|
+
require_relative '../session_errors'
|
|
9
|
+
require_relative 'diagnostics'
|
|
10
|
+
require 'set'
|
|
11
|
+
require 'time'
|
|
12
|
+
|
|
13
|
+
module Aspera
|
|
14
|
+
module Ascp
|
|
15
|
+
class LogAnalyzer
|
|
16
|
+
# Data of one ascp session (process), collected from its log lines and its samples
|
|
17
|
+
class Session
|
|
18
|
+
# Calculated rates (bps) of 1 packet per second, for various payload sizes: maximum disk throttling
|
|
19
|
+
DISK_THROTTLE_RATES = [11_584, 11_712, 11_936, 7_824, 8_048, 3_840, 4_064, 2_144, 2_368].freeze
|
|
20
|
+
DEFAULT_MTU = 1492
|
|
21
|
+
# Sender (ascp 4): limit of the receiver => sample values
|
|
22
|
+
RECEIVER_CAUSES = {'network' => %i[peer_net_rate], 'disk' => %i[peer_disk_rate], 'vlink' => %i[peer_vlink_local peer_vlink_remote]}.freeze
|
|
23
|
+
# Control messages in `Session Statistics`: direction => description
|
|
24
|
+
CONTROL_DIRECTIONS = Diagnostics::CONTROL_DIRECTIONS
|
|
25
|
+
# Constraint on the rate => series of its sample values
|
|
26
|
+
CONSTRAINT_SERIES = {'network' => :network, 'disk' => :disk, 'vlink_local' => :vlink_local, 'vlink_remote' => :vlink_remote, 'Receiver' => :calc}.freeze
|
|
27
|
+
# Receiver: constraint => sample value
|
|
28
|
+
RECEIVER_CONSTRAINTS = {'network' => :net_rate, 'disk' => :disk_rate, 'vlink_local' => :vlink_local, 'vlink_remote' => :vlink_remote}.freeze
|
|
29
|
+
# Receiver: sample value => series, kept after 20 seconds of transfer
|
|
30
|
+
RECEIVER_SERIES = {srtt: :srtt, rex_srtt: :rex_srtt, rtt_high: :rtt, net_rate: :network, vlink_local: :vlink_local, vlink_remote: :vlink_remote, disk_rate: :disk}.freeze
|
|
31
|
+
RESUME = {0 => 'Do not resume', 1 => 'With file attribute check', 2 => 'With sparse checksum', 3 => 'With full checksum'}.freeze
|
|
32
|
+
OVERWRITE = {255 => 'Always overwrite', 1 => 'If source different', 3 => 'If source different and newer', 63 => 'Never overwrite'}.freeze
|
|
33
|
+
# Rate policy in log lines => policy name
|
|
34
|
+
POLICIES = {'0' => 'fixed', '1' => 'fair', '2' => 'low', 'adapt' => 'fair', 'trickle' => 'low'}.freeze
|
|
35
|
+
# Flag => severity and message of diagnostic (identifier: name of flag)
|
|
36
|
+
FLAG_DIAGNOSTICS = Diagnostics::FLAG_DIAGNOSTICS
|
|
37
|
+
|
|
38
|
+
class << self
|
|
39
|
+
# @return [Float] mean of values, 0 if none
|
|
40
|
+
def average(values)
|
|
41
|
+
values.empty? ? 0 : values.sum.fdiv(values.size)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# @return [Numeric] median of values, 0 if none
|
|
45
|
+
def median(values)
|
|
46
|
+
return 0 if values.empty?
|
|
47
|
+
sorted = values.sort
|
|
48
|
+
mid = sorted.size / 2
|
|
49
|
+
sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# @return [Float] population standard deviation of values, 0 if none
|
|
53
|
+
def standard_deviation(values)
|
|
54
|
+
return 0 if values.empty?
|
|
55
|
+
mean = average(values)
|
|
56
|
+
Math.sqrt(values.sum { |v| (v - mean)**2 } / values.size)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# @return [String] integer with thousands separators, for messages
|
|
60
|
+
def commify(value)
|
|
61
|
+
value.to_i.to_s.gsub(/(\d)(?=(\d{3})+\z)/, '\1,')
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @return [Boolean] true if version is known and older than reference
|
|
65
|
+
def version_older?(version, reference)
|
|
66
|
+
number = version.to_s[/\A\d+(?:\.\d+)*/] or return false
|
|
67
|
+
Gem::Version.new(number) < Gem::Version.new(reference)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# @return [Integer, nil] rate in bps, from text with unit, e.g. `3.33Mbps`
|
|
71
|
+
def rate_bps(text)
|
|
72
|
+
m = text.to_s.match(/\A(\d+\.\d+)([A-Za-z]+)/) or return
|
|
73
|
+
(m[1].to_f * {'Kbps' => 1e3, 'Mbps' => 1e6, 'Gbps' => 1e9}.fetch(m[2], 1)).round
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
attr_accessor :pid
|
|
78
|
+
# Files transferred: name => {op:, size:, status:, error:}
|
|
79
|
+
attr_reader :files
|
|
80
|
+
# Diagnostics of last report: {id, severity, message}
|
|
81
|
+
attr_reader :diagnostics, :thresholds, :data, :flags, :series, :constraint, :constraints, :params, :sizes, :receiver_causes, :rate_samples, :ds_qs_samples, :timezone
|
|
82
|
+
|
|
83
|
+
# @param pid [String] Process identifier
|
|
84
|
+
# @param thresholds [Hash] Diagnostic thresholds
|
|
85
|
+
# @param timezone [String, nil] Time zone of the reported times: `UTC` or `+HH:MM`. Time zone of the analyst if nil.
|
|
86
|
+
def initialize(pid, thresholds: {}, timezone: nil)
|
|
87
|
+
@pid = pid
|
|
88
|
+
@timezone = timezone
|
|
89
|
+
thresholds = thresholds.transform_keys(&:to_sym)
|
|
90
|
+
unknown = thresholds.keys - DEFAULT_THRESHOLDS.keys
|
|
91
|
+
Aspera.assert(unknown.empty?, type: ArgumentError) { "Unknown threshold: #{unknown.join(', ')}" }
|
|
92
|
+
@thresholds = DEFAULT_THRESHOLDS.merge(thresholds)
|
|
93
|
+
@data = {}
|
|
94
|
+
@flags = Set.new
|
|
95
|
+
@series = Hash.new { |h, k| h[k] = [] }
|
|
96
|
+
# Constraint name => {min_rate:, base_rtt:, smooth_rtt:}
|
|
97
|
+
@constraint = {}
|
|
98
|
+
# Names of constraints that limited the rate
|
|
99
|
+
@constraints = Set.new
|
|
100
|
+
# Changes of rate parameters during session
|
|
101
|
+
@params = nil
|
|
102
|
+
@files = {}
|
|
103
|
+
# Sizes of transferred files
|
|
104
|
+
@sizes = []
|
|
105
|
+
# Detection of ATM / tail-drop rate limiting
|
|
106
|
+
@atm = nil
|
|
107
|
+
# Detection of stuck NACKs on sender
|
|
108
|
+
@send_nack = nil
|
|
109
|
+
# Detection of clock problems on peer
|
|
110
|
+
@vm = nil
|
|
111
|
+
# Sender: limit of the receiver => number of samples
|
|
112
|
+
@receiver_causes = Hash.new(0)
|
|
113
|
+
# Rate samples after the delay: [elapsed time (us), time, bps], receiver network rate or sender calculated rate
|
|
114
|
+
@rate_samples = []
|
|
115
|
+
# File-queue stats: one hash per DS Qs sample, keys 'ao', 'ap', 'mgmt_backlog'
|
|
116
|
+
@ds_qs_samples = []
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def [](key)
|
|
120
|
+
@data[key]
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def []=(key, value)
|
|
124
|
+
@data[key] = value
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def delete(key)
|
|
128
|
+
@data.delete(key)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# @return [Numeric] value, 0 if not set
|
|
132
|
+
def num(key)
|
|
133
|
+
@data[key] || 0
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def increment(key, value = 1)
|
|
137
|
+
@data[key] = num(key) + value
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# @return [Boolean] true if any of the flags is set
|
|
141
|
+
def flag?(*names)
|
|
142
|
+
names.any? { |n| @flags.include?(n) }
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def set(*names)
|
|
146
|
+
@flags.merge(names)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def clear(*names)
|
|
150
|
+
@flags.subtract(names)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Apply effects of a rule (see LINE_RULES), unless cancelled by a flag
|
|
154
|
+
def apply(effects)
|
|
155
|
+
return if effects[:unless] && flag?(*effects[:unless])
|
|
156
|
+
set(*effects[:set]) if effects[:set]
|
|
157
|
+
clear(*effects[:clear]) if effects[:clear]
|
|
158
|
+
@data[:error] = effects[:error] if effects[:error]
|
|
159
|
+
@data[:status] = effects[:status] if effects[:status]
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Apply the first rule whose flags are all set
|
|
163
|
+
# @param rules [Array] list of [flags, effects]
|
|
164
|
+
def apply_first(rules)
|
|
165
|
+
rule = rules.find { |flags, _| flags.all? { |f| flag?(f) } }
|
|
166
|
+
apply(rule.last) if rule
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Information from a `Session Start` or `Session Stop` line
|
|
170
|
+
# @param kind [Symbol] :start or :stop
|
|
171
|
+
# @param f [Hash] values from line
|
|
172
|
+
# @param time [Integer, nil] time of line
|
|
173
|
+
# @param config [Hash] user and version from configuration lines
|
|
174
|
+
def session_line(kind, f, time, config)
|
|
175
|
+
%i[uuid op user source dest local os ver peeros peerver transport].each { |k| @data[k] = f[k] if f[k] }
|
|
176
|
+
if f[:lic]
|
|
177
|
+
@data[:product] = f[:lic].split(':').first.to_i
|
|
178
|
+
@data[:role] = 'client' if [6, 7].include?(@data[:product])
|
|
179
|
+
end
|
|
180
|
+
if f[:peerlic]
|
|
181
|
+
@data[:peer_product] = f[:peerlic].split(':').first.to_i
|
|
182
|
+
@data[:role] = 'server' if @data[:peer_product] == 6
|
|
183
|
+
end
|
|
184
|
+
@data[:user] ||= config[:user]
|
|
185
|
+
@data[:role] = 'client' if config[:user] && config[:user] != '-' && @data[:user] && !config[:user].match?(/#{Regexp.escape(@data[:user])}/i)
|
|
186
|
+
if f[:l_port]
|
|
187
|
+
@data[:start_l_port] = @data[:l_port] if kind.eql?(:stop) && @data[:l_port] && @data[:l_port] != f[:l_port]
|
|
188
|
+
@data[:l_port] = f[:l_port]
|
|
189
|
+
end
|
|
190
|
+
@data[:peer] ||= f[:peer]
|
|
191
|
+
@data[:p_port] = f[:p_port] if f[:p_port]
|
|
192
|
+
@data[:ver] ||= config[:ver]
|
|
193
|
+
@data[:product_name] ||= config[:product_name]
|
|
194
|
+
@data[:auth] ||= config[:auth]
|
|
195
|
+
@data[:license_rate] ||= config[:license_rate]
|
|
196
|
+
@data[:license_expires] ||= config[:license_expires]
|
|
197
|
+
@data[kind] = time
|
|
198
|
+
@data[:date] = @data[:stop] || @data[:start]
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# Values from `Session Params` line
|
|
202
|
+
# @param fields [Hash] key=value from line
|
|
203
|
+
def session_params(fields)
|
|
204
|
+
SESSION_PARAMS.each do |name, key|
|
|
205
|
+
next unless fields.key?(name)
|
|
206
|
+
value = fields[name]
|
|
207
|
+
@data[key] = value.match?(/\A\d+\z/) && key != :user ? value.to_i : value
|
|
208
|
+
end
|
|
209
|
+
@data[:policy] = POLICIES.fetch(@data[:policy], @data[:policy]) if @data[:policy]
|
|
210
|
+
# `proxy_ip=-`: no proxy
|
|
211
|
+
@data[:proxy] = fields['proxy_ip'] if fields['proxy_ip'] && fields['proxy_ip'] != '-'
|
|
212
|
+
@data[:fips] = fields['fips_enabled'] == 'yes' if fields.key?('fips_enabled')
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Error and status of `Session Stop` line
|
|
216
|
+
def stop_error(line, status)
|
|
217
|
+
e1, e2 = line.match(ERRSTR)&.captures
|
|
218
|
+
error =
|
|
219
|
+
if !e2.to_s.empty? then e2 == 'Undefined' ? e1 : e2
|
|
220
|
+
elsif e1.to_s.empty? then nil
|
|
221
|
+
elsif e1 == 'Undefined' && flag?(:invalid_conf_file) then 'Invalid Configuration File'
|
|
222
|
+
elsif e1 == 'Undefined' && flag?(:invalid_argument) then 'Invalid argument'
|
|
223
|
+
else e1
|
|
224
|
+
end
|
|
225
|
+
@data[:error] = error if error
|
|
226
|
+
# Code of the error retained (local or peer)
|
|
227
|
+
code = line.scan(ERRCODE).to_h { |number, text| [text, number.to_i] }[error]
|
|
228
|
+
@data[:error_code] = code if code&.nonzero?
|
|
229
|
+
@data[:status] = 'success' if status == 'success'
|
|
230
|
+
return unless status == 'failed'
|
|
231
|
+
if e2.to_s.empty? && e1 == 'Undefined'
|
|
232
|
+
apply_first(UNDEFINED_ERROR_RULES)
|
|
233
|
+
@data[:error] = 'Unable to bind to UDP socket' if flag?(:bind_udp_socket_failed)
|
|
234
|
+
end
|
|
235
|
+
apply_first(ERROR_REFINEMENTS[error]) if ERROR_REFINEMENTS.key?(error)
|
|
236
|
+
@data[:status] = 'failed' unless @data[:status] == 'success'
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# Rate parameters changed
|
|
240
|
+
# @param by_peer [Boolean] rate parameters requested by peer
|
|
241
|
+
def rate_params(t_rate, m_rate, policy, by_peer:)
|
|
242
|
+
@params[:multiple] = true if @params
|
|
243
|
+
@params&.delete(:peer) unless by_peer
|
|
244
|
+
policy = POLICIES.fetch(policy, policy)
|
|
245
|
+
change_param(:policy, policy) unless @data[:policy] == policy
|
|
246
|
+
change_param(:target_rate, t_rate) unless num(:target_rate) == t_rate
|
|
247
|
+
change_param(:min_rate, m_rate) unless num(:min_rate) == m_rate
|
|
248
|
+
@params[:peer] = true if by_peer && @params
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# `Transfer Stop` line
|
|
252
|
+
# @return [Boolean] true if file failed because its name is too long for Windows
|
|
253
|
+
def transfer_stop(op, status, file, size, start_byte, errstr = nil)
|
|
254
|
+
too_long = false
|
|
255
|
+
case status
|
|
256
|
+
when 'success'
|
|
257
|
+
if start_byte == size
|
|
258
|
+
increment(:files_skipped)
|
|
259
|
+
else
|
|
260
|
+
size -= start_byte
|
|
261
|
+
increment(:files_success)
|
|
262
|
+
@sizes.push(size)
|
|
263
|
+
end
|
|
264
|
+
when 'failed'
|
|
265
|
+
increment(:files_failed)
|
|
266
|
+
too_long = file.length >= 240 && (file.match?(/\w:/) || windows?)
|
|
267
|
+
increment(:filename_too_long) if too_long
|
|
268
|
+
else
|
|
269
|
+
return false
|
|
270
|
+
end
|
|
271
|
+
increment(:transfer_stop_count)
|
|
272
|
+
entry = (@files[file] ||= {})
|
|
273
|
+
error = errstr&.match?(/errstr="./) ? errstr[/errstr="(.*?)"/, 1] : errstr
|
|
274
|
+
entry[:error] = error if status == 'failed' && error && !error.empty?
|
|
275
|
+
entry.merge!(op: op, size: size, status: status)
|
|
276
|
+
too_long
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# Process a `bl` sample (Receiver or Sender)
|
|
280
|
+
# @param s [Hash] values of sample, see RECEIVER_SAMPLE, SENDER_SAMPLE
|
|
281
|
+
# @param slot [Hash] data of this session in the current snapshot period
|
|
282
|
+
# @param first [Boolean] first sample of this session in the period
|
|
283
|
+
# @param smart_reset [Boolean] base RTT is reset by ascp
|
|
284
|
+
# @param mtu [Integer, nil] MTU to use when not found in logs
|
|
285
|
+
def add_sample(s, slot, first:, smart_reset:, mtu:)
|
|
286
|
+
receiver = s[:receiver]
|
|
287
|
+
elapsed = s[:elapsed]
|
|
288
|
+
@data[:elapsed_start] = s[:elapsed_us] if num(:elapsed_start).zero?
|
|
289
|
+
@data[:elapsed_us] = s[:elapsed_us]
|
|
290
|
+
if receiver
|
|
291
|
+
increment(:brtt_reset, s[:brtt_resets]) if s[:brtt_resets].to_i.positive?
|
|
292
|
+
RECEIVER_SERIES.each { |key, series| @series[series].push(s[key].to_i) } if elapsed > 20
|
|
293
|
+
if s[:net_rate].to_i == s[:calc_rate].to_i
|
|
294
|
+
queue = s[:srtt].to_i - s[:brtt].to_i
|
|
295
|
+
@data[:max_queue] = queue if @data[:max_queue].nil? || queue > @data[:max_queue]
|
|
296
|
+
end
|
|
297
|
+
else
|
|
298
|
+
check_sender_nacks(s)
|
|
299
|
+
@data[:disk_read_rate] = s[:disk_read_rate] if s[:disk_read_rate].to_i.positive?
|
|
300
|
+
end
|
|
301
|
+
@data[:start_block] = s[:blocks] if num(:start_block).zero?
|
|
302
|
+
@data[:mtu] = format('%.0f', s[:tx_bytes].fdiv(s[:blocks]) + 40).to_i if num(:mtu) <= 8000 && s[:blocks].to_i.positive? && s[:tx_bytes].fdiv(s[:blocks]) > 8000
|
|
303
|
+
@data[:mtu] = mtu if mtu && !@data[:payload]
|
|
304
|
+
@data[:mtu] ||= DEFAULT_MTU
|
|
305
|
+
t_bits = s[:blocks].to_i * @data[:mtu] * 8
|
|
306
|
+
@data[:target_rate] = s[:target_rate] if num(:target_rate).zero?
|
|
307
|
+
@data[:min_rate] = s[:min_rate]
|
|
308
|
+
@series[:calc].push(s[:calc_rate].to_i)
|
|
309
|
+
@rate_samples.push([s[:elapsed_us], s[:time], (receiver ? s[:net_rate] : s[:calc_rate]).to_i]) if elapsed > @thresholds[:rate_decline_delay]
|
|
310
|
+
increment(:disk_max) if DISK_THROTTLE_RATES.include?(s[:calc_rate])
|
|
311
|
+
check_stuck_nack(s)
|
|
312
|
+
check_stuck_rex(s) if receiver
|
|
313
|
+
@data[:tx_bytes] = s[:tx_bytes] if s[:tx_bytes] > num(:tx_bytes)
|
|
314
|
+
log_brtt(s, elapsed, first: first, smart_reset: smart_reset)
|
|
315
|
+
if first
|
|
316
|
+
slot.merge!(
|
|
317
|
+
total1: t_bits, blocks1: s[:blocks], loss1: s[:lost], orig1: s[:orig], rex1: s[:rex], elapsed1: elapsed, date1: s[:time],
|
|
318
|
+
target_rate: s[:target_rate], calc_rate: [s[:calc_rate].to_i], receiver: receiver
|
|
319
|
+
)
|
|
320
|
+
else
|
|
321
|
+
(@atm ||= {})[:samples] = @atm[:samples].to_i + 1
|
|
322
|
+
slot[:calc_rate].push(s[:calc_rate].to_i)
|
|
323
|
+
update_slot(s, slot, t_bits, elapsed, receiver)
|
|
324
|
+
end
|
|
325
|
+
@data[:elapsed] = elapsed
|
|
326
|
+
check_high_rtt(s) if receiver
|
|
327
|
+
if s[:written].nonzero?
|
|
328
|
+
rate_constraint(s, slot, elapsed, receiver, first)
|
|
329
|
+
update_calc_points(s[:calc_rate].to_i, t_bits, elapsed)
|
|
330
|
+
if receiver
|
|
331
|
+
@data[:recv_max] = [num(:recv_max), calc_speed].max
|
|
332
|
+
check_atm(s, t_bits, elapsed) unless first
|
|
333
|
+
else
|
|
334
|
+
detect_peer_vm(s) unless first
|
|
335
|
+
@data[:send_max] = [num(:send_max), calc_speed].max
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
@atm = (@atm || {}).merge(total0: t_bits, elapsed0: elapsed, loss0: s[:lost].to_i, orig0: s[:orig].to_i) if first
|
|
339
|
+
if receiver
|
|
340
|
+
buffers = [s[:ring_high], s[:ring_full]].find { |b| b.to_i >= 5 }
|
|
341
|
+
@data[:buffers] = buffers if buffers && buffers > num(:buffers)
|
|
342
|
+
@data[:ooo] = s[:ooo].to_i.fdiv(s[:blocks]) if s[:blocks].to_i.nonzero?
|
|
343
|
+
@data[:loss_rate] = s[:lost].to_i.fdiv(s[:lost].to_i + s[:orig].to_i) if (s[:lost].to_i + s[:orig].to_i).nonzero?
|
|
344
|
+
elsif (!first || s[:written].nonzero?) && s[:blocks].to_i.nonzero?
|
|
345
|
+
@data[:loss_rate] = s[:rex].to_i.fdiv(s[:blocks])
|
|
346
|
+
end
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
# Accumulate file-queue statistics from one `DS Qs` sample
|
|
350
|
+
# @param qs [Hash] keys: `'ao'` (open-async), `'ap'` (attrprep), `'mgmt_backlog'` (bytes buffered)
|
|
351
|
+
def add_ds_qs_sample(qs)
|
|
352
|
+
@ds_qs_samples.push(qs)
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# Tell if another session starts while the rate samples are recorded: it may share the link
|
|
356
|
+
# @param starts [Array<Integer>] start times of the transfer sessions of the log
|
|
357
|
+
def shared_link(starts)
|
|
358
|
+
return if @rate_samples.empty?
|
|
359
|
+
first, last = @rate_samples.map { |sample| sample[1] }.minmax
|
|
360
|
+
@data[:shared_link] = starts.any? { |time| time > first && time <= last }
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# Evaluate ATM / tail-drop rate limiting detection, at end of a snapshot period
|
|
364
|
+
def evaluate_atm
|
|
365
|
+
return if @atm.nil? || @atm[:samples].nil?
|
|
366
|
+
if @atm[:periods].to_i < 2
|
|
367
|
+
@atm[:periods] = @atm[:periods].to_i + 1
|
|
368
|
+
return
|
|
369
|
+
end
|
|
370
|
+
return unless @atm[:samples] == 6
|
|
371
|
+
increment(:atm) if %i[atm_loss atm_ratio atm_queue].all? { |k| @atm[k].to_i / 6.0 >= 0.83 }
|
|
372
|
+
@atm = nil
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
# Final status of session, if ascp stopped logging before end of session
|
|
376
|
+
# @param last_time [Integer] time of last sample in log file
|
|
377
|
+
# @return [Symbol, nil] category of exit
|
|
378
|
+
def check_exit(last_time)
|
|
379
|
+
return if last_time.to_i - num(:last) <= 30 || flag?(:stats_end)
|
|
380
|
+
category =
|
|
381
|
+
if @data[:error] == 'Out of buffer memory'
|
|
382
|
+
@data[:error] = 'Out of buffer memory, and ascp exited.'
|
|
383
|
+
:buffer_died
|
|
384
|
+
elsif flag?(:decryption_error)
|
|
385
|
+
@data[:error] = 'File decryption error, and ascp exited.'
|
|
386
|
+
:decryption_died
|
|
387
|
+
elsif flag?(:data_transfer_timeout)
|
|
388
|
+
@data[:error] = "#{@data[:error]}, and ascp exited."
|
|
389
|
+
:timeout_died
|
|
390
|
+
elsif @data[:error] == 'segfault'
|
|
391
|
+
@data[:error] = 'ascp exited: segfault.'
|
|
392
|
+
:died
|
|
393
|
+
elsif @data[:status]
|
|
394
|
+
return if flag?(:ssh_timeout, :cannot_connect_to_management_socket)
|
|
395
|
+
set(:table_died)
|
|
396
|
+
:table_died
|
|
397
|
+
else
|
|
398
|
+
@data[:error] = 'ascp exited, or was in IOwait or stopped logging'
|
|
399
|
+
:died
|
|
400
|
+
end
|
|
401
|
+
@data[:status] = 'failed' unless @data[:status] == 'success' || category == :table_died
|
|
402
|
+
@data[:exit] = category
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
# @return [Boolean] true if session has the stuck NACK problem
|
|
406
|
+
def stuck_nack?
|
|
407
|
+
return false if @data[:status] == 'success'
|
|
408
|
+
num(:stuck_nack_count) >= 5 || ((flag?(:send_nack_stuck) || num(:sender_stuck_nack_count) >= 5) && num(:loss_rate) < 0.9)
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
def windows?
|
|
412
|
+
[@data[:os], @data[:peeros]].any? { |os| os.to_s.match?(/windows/i) }
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
# @return [String, nil] `send` or `receive`
|
|
416
|
+
def direction
|
|
417
|
+
(@data[:op] == 'recv' ? 'receive' : @data[:op]) || @data[:dir]
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
# @return [Integer, nil] error code of session, if it is in the numbering of supported versions
|
|
421
|
+
def error_code
|
|
422
|
+
@data[:error_code] unless self.class.version_older?(@data[:ver], MIN_VERSION)
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
# @return [String, nil] category of error code, see SessionErrors::CATEGORIES
|
|
426
|
+
def error_category
|
|
427
|
+
code = error_code or return
|
|
428
|
+
SessionErrors.category(code)
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
# @return [Symbol, nil] probable cause of loss on sender: :storage or :network (shallow queues)
|
|
432
|
+
def overdrive
|
|
433
|
+
return unless @data[:dir] == 'send' && num(:loss_rate) > 0.10 && @constraints.size == 1
|
|
434
|
+
return :storage if DISK_THROTTLE_RATES.include?(@constraint.dig('Receiver', :min_rate))
|
|
435
|
+
return if num(:stuck_nack_count) >= 10 || @params
|
|
436
|
+
:network if @constraint.key?('Receiver') && constraint_average('Receiver') >= 0.8 * num(:target_rate)
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
# @param windows_log [Boolean] log file is on Windows
|
|
440
|
+
# @return [Hash] report of session: one table row
|
|
441
|
+
def report(windows_log: false)
|
|
442
|
+
d = @data
|
|
443
|
+
sample_dir = d[:dir]
|
|
444
|
+
avg = d[:effective_rate] ? self.class.rate_bps(d[:effective_rate]).to_i : num(:avg)
|
|
445
|
+
avg = d[:target_rate] if num(:target_rate).positive? && avg > d[:target_rate]
|
|
446
|
+
loss = num(:loss_rate)
|
|
447
|
+
min_rate_high = d[:min_rate] if d[:policy] == 'fair' && (num(:min_rate) >= 5_000_000 || num(:min_rate) > num(:target_rate) / 2.0)
|
|
448
|
+
rtt = series_stats(:rtt)
|
|
449
|
+
srtt = series_stats(:srtt)
|
|
450
|
+
constraints = @constraint.reject { |_, c| c[:min_rate].nil? }.to_h do |name, c|
|
|
451
|
+
series = @series[CONSTRAINT_SERIES[name]]
|
|
452
|
+
entry = {'max' => c[:min_rate], 'average' => constraint_average(name), 'std_dev' => self.class.standard_deviation(series).to_i, 'samples' => series.size}
|
|
453
|
+
if name == 'network'
|
|
454
|
+
entry['base_rtt'] = c[:base_rtt]
|
|
455
|
+
entry['smoothed_rtt'] = c[:smooth_rtt]
|
|
456
|
+
end
|
|
457
|
+
entry['causes'] = @receiver_causes.sort_by { |_, count| -count }.to_h if name == 'Receiver' && @receiver_causes.any?
|
|
458
|
+
[name, entry]
|
|
459
|
+
end
|
|
460
|
+
slowdown = constraints.values.any? { |c| c['max'] < 8 * (num(:mtu) - 40) }
|
|
461
|
+
tx_time = d[:tx_time].to_i.nonzero? ? d[:tx_time] / 1_000_000.0 : d[:elapsed].to_f
|
|
462
|
+
@diagnostics = Diagnostics.new(self, windows_log: windows_log).evaluate(
|
|
463
|
+
loss: loss,
|
|
464
|
+
slowdown: slowdown,
|
|
465
|
+
sample_dir: sample_dir,
|
|
466
|
+
avg: avg,
|
|
467
|
+
min_rate_high: min_rate_high,
|
|
468
|
+
rtt: rtt,
|
|
469
|
+
srtt: srtt
|
|
470
|
+
)
|
|
471
|
+
{
|
|
472
|
+
'pid' => @pid,
|
|
473
|
+
'session' => d[:uuid],
|
|
474
|
+
'user' => d[:user],
|
|
475
|
+
'direction' => direction,
|
|
476
|
+
'role' => d[:role] || role_from_ports,
|
|
477
|
+
'status' => d[:status],
|
|
478
|
+
'error' => d[:error],
|
|
479
|
+
'error_code' => d[:error_code],
|
|
480
|
+
'error_category' => error_category,
|
|
481
|
+
'error_retry' => (SessionErrors.retryable(error_code) if error_code),
|
|
482
|
+
'transfer_id' => d[:xfer_id],
|
|
483
|
+
'multi_session' => ("#{d[:cluster_node].to_i + 1}/#{d[:cluster_nodes]}" if num(:cluster_nodes) > 1),
|
|
484
|
+
'retry_of' => d[:retry_of],
|
|
485
|
+
'retried_by' => d[:retried_by],
|
|
486
|
+
'start' => iso(d[:start]),
|
|
487
|
+
'stop' => iso(d[:stop]),
|
|
488
|
+
'local' => d[:local],
|
|
489
|
+
'local_port' => d[:l_port],
|
|
490
|
+
'version' => d[:ver],
|
|
491
|
+
'os' => d[:os],
|
|
492
|
+
**context_columns,
|
|
493
|
+
'peer' => d[:peer],
|
|
494
|
+
'peer_port' => d[:p_port],
|
|
495
|
+
'peer_version' => d[:peerver],
|
|
496
|
+
'peer_os' => d[:peeros],
|
|
497
|
+
'peer_product' => d[:peer_product],
|
|
498
|
+
'destination' => d[:dest],
|
|
499
|
+
'policy' => d[:policy],
|
|
500
|
+
'encryption' => d[:secure],
|
|
501
|
+
'resume' => RESUME.fetch(d[:resume], d[:resume]),
|
|
502
|
+
'overwrite' => OVERWRITE.fetch(d[:overwrite], d[:overwrite]),
|
|
503
|
+
'block_size' => d[{'send' => :read_block_size, 'receive' => :write_block_size}[sample_dir]],
|
|
504
|
+
'mtu' => d[:mtu],
|
|
505
|
+
'payload' => d[:payload],
|
|
506
|
+
'target_rate' => d[:target_rate],
|
|
507
|
+
'min_rate' => d[:min_rate],
|
|
508
|
+
'average_rate' => avg,
|
|
509
|
+
'average_calculated_rate' => self.class.average(@series[:calc]).to_i,
|
|
510
|
+
'max_rate' => {'send' => d[:send_max], 'receive' => d[:recv_max].to_i.nonzero? || avg}[sample_dir],
|
|
511
|
+
'transmitted_bytes' => num(:tx_bytes),
|
|
512
|
+
'transfer_time' => tx_time.round(2),
|
|
513
|
+
'files' => d[:transfer_stop_count],
|
|
514
|
+
'files_transferred' => d[:files_success],
|
|
515
|
+
'files_skipped' => d[:files_skipped],
|
|
516
|
+
'files_failed' => d[:files_failed],
|
|
517
|
+
'file_size_mean' => (self.class.average(@sizes).to_i if d[:transfer_stop_count]),
|
|
518
|
+
'file_size_median' => (self.class.median(@sizes).to_i if d[:transfer_stop_count]),
|
|
519
|
+
'concurrent_min' => (d[:sess_min].to_i.nonzero? || 1 if d[:sess_max]),
|
|
520
|
+
'concurrent_max' => d[:sess_max],
|
|
521
|
+
'concurrent_max_time' => iso(d[:sess_max_time]),
|
|
522
|
+
'start_rtt' => d[:start_brtt],
|
|
523
|
+
'base_rtt' => d[:brtt],
|
|
524
|
+
'rtt_average' => rtt&.first,
|
|
525
|
+
'rtt_median' => rtt&.last,
|
|
526
|
+
'srtt_average' => srtt&.first,
|
|
527
|
+
'srtt_median' => srtt&.last,
|
|
528
|
+
'loss_rate' => (loss.positive? ? (loss * 100).round(2) : (0.0 if num(:tx_bytes).nonzero?)),
|
|
529
|
+
'out_of_order_rate' => (num(:ooo) * 100).round(2).nonzero?,
|
|
530
|
+
'efficiency' => d[:efficiency],
|
|
531
|
+
'retransmitted_rate' => d[:retransmitted],
|
|
532
|
+
'duplicate_rate' => d[:duplicates],
|
|
533
|
+
'disk_read_rate' => d[:disk_read_rate],
|
|
534
|
+
'constraints' => @constraints.to_a,
|
|
535
|
+
'constraint' => constraints,
|
|
536
|
+
'diagnostics' => @diagnostics.map { |diagnostic| diagnostic['message'] }
|
|
537
|
+
}.compact
|
|
538
|
+
end
|
|
539
|
+
|
|
540
|
+
private
|
|
541
|
+
|
|
542
|
+
FASP_PORTS = (33_000..33_099)
|
|
543
|
+
private_constant :FASP_PORTS
|
|
544
|
+
|
|
545
|
+
# Columns that tell in which context the session ran, to filter sessions
|
|
546
|
+
# @return [Hash] product, transport (`ssh` or `wss`), authentication type (`token` or `node`), proxy address, FIPS mode
|
|
547
|
+
def context_columns
|
|
548
|
+
{
|
|
549
|
+
'product' => @data[:product],
|
|
550
|
+
'product_name' => @data[:product_name],
|
|
551
|
+
'transport' => @data[:transport],
|
|
552
|
+
'authentication' => @data[:auth],
|
|
553
|
+
'proxy' => @data[:proxy],
|
|
554
|
+
'fips' => @data[:fips]
|
|
555
|
+
}
|
|
556
|
+
end
|
|
557
|
+
|
|
558
|
+
def iso(time)
|
|
559
|
+
Time.at(time).getlocal(@timezone).iso8601 if time
|
|
560
|
+
end
|
|
561
|
+
|
|
562
|
+
# @return [Array, nil] average and median of series, as integers
|
|
563
|
+
def series_stats(name)
|
|
564
|
+
series = @series[name]
|
|
565
|
+
[self.class.average(series).to_i, self.class.median(series).to_i] unless series.empty?
|
|
566
|
+
end
|
|
567
|
+
|
|
568
|
+
# @return [Integer] average of values of a constraint
|
|
569
|
+
def constraint_average(name)
|
|
570
|
+
self.class.average(@series[CONSTRAINT_SERIES[name]]).to_i
|
|
571
|
+
end
|
|
572
|
+
|
|
573
|
+
# @return [String, nil] client or server, deduced from UDP ports
|
|
574
|
+
def role_from_ports
|
|
575
|
+
local, peer = @data.values_at(:l_port, :p_port)
|
|
576
|
+
return unless local && peer
|
|
577
|
+
return 'server' if local == 33_001 || (FASP_PORTS.cover?(local) && !FASP_PORTS.cover?(peer))
|
|
578
|
+
'client'
|
|
579
|
+
end
|
|
580
|
+
|
|
581
|
+
def change_param(key, value)
|
|
582
|
+
@params ||= {}
|
|
583
|
+
@params[:"#{key}_old"] = @data[key]
|
|
584
|
+
@params[key] = value
|
|
585
|
+
@data[key] = value
|
|
586
|
+
end
|
|
587
|
+
|
|
588
|
+
# Sender side: NACKs stuck while no progress
|
|
589
|
+
def check_sender_nacks(s)
|
|
590
|
+
if @send_nack.nil?
|
|
591
|
+
@send_nack = {}
|
|
592
|
+
return
|
|
593
|
+
end
|
|
594
|
+
sn = @send_nack
|
|
595
|
+
no_progress = s[:blocks].to_i >= sn[:blocks].to_i && s[:tx_bytes] == sn[:tx_bytes].to_i && s[:written] == sn[:written].to_i &&
|
|
596
|
+
s[:tx_bytes].nonzero? && s[:written].nonzero?
|
|
597
|
+
if no_progress && s[:rex_count].to_i == sn[:rex_count].to_i && s[:rex_received].to_i == sn[:rex_received].to_i
|
|
598
|
+
sn[:count] = sn[:count].to_i + 1
|
|
599
|
+
set(:send_nack_stuck) if sn[:count] > 10
|
|
600
|
+
elsif no_progress && sn[:rex_received].to_i < s[:rex_received].to_i
|
|
601
|
+
increment(:sender_stuck_nack_count)
|
|
602
|
+
else
|
|
603
|
+
@send_nack = s.slice(:rex_count, :blocks, :rex_received, :tx_bytes, :written)
|
|
604
|
+
@data[:sender_stuck_nack_count] = 0
|
|
605
|
+
end
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
# NACKs stuck at same value while no progress
|
|
609
|
+
def check_stuck_nack(s)
|
|
610
|
+
nack = s[:rex_count]
|
|
611
|
+
return if nack.nil? || num(:tx_bytes).zero?
|
|
612
|
+
if num(:stuck_nack).zero?
|
|
613
|
+
@data.merge!(stuck_nack: nack, stuck_nack_count: 1) if num(:tx_bytes) == s[:tx_bytes]
|
|
614
|
+
elsif nack.zero?
|
|
615
|
+
@data.merge!(stuck_nack: 0, stuck_nack_count: 0)
|
|
616
|
+
elsif nack == @data[:stuck_nack] && num(:tx_bytes) == s[:tx_bytes] && s[:calc_rate].to_i.nonzero?
|
|
617
|
+
increment(:stuck_nack_count)
|
|
618
|
+
else
|
|
619
|
+
@data.merge!(stuck_nack: nack, stuck_nack_count: 0)
|
|
620
|
+
end
|
|
621
|
+
end
|
|
622
|
+
|
|
623
|
+
# Retransmissions not progressing while NACKs pending: NACK requests are being lost
|
|
624
|
+
def check_stuck_rex(s)
|
|
625
|
+
rex = s[:rex]
|
|
626
|
+
nack = s[:rex_count].to_i
|
|
627
|
+
return if rex.nil?
|
|
628
|
+
if @data[:stuck_rex].nil? || (nack.nonzero? && rex > @data[:stuck_rex]) then @data[:stuck_rex] = rex
|
|
629
|
+
elsif nack.zero? then @data[:stuck_rex_count] = 0
|
|
630
|
+
elsif rex == @data[:stuck_rex] then increment(:stuck_rex_count)
|
|
631
|
+
end
|
|
632
|
+
end
|
|
633
|
+
|
|
634
|
+
# Base RTT: detect VM (clock problem) and reverse path congestion skew
|
|
635
|
+
def log_brtt(s, elapsed, first:, smart_reset:)
|
|
636
|
+
b_rtt = s[:brtt].to_i
|
|
637
|
+
brtt = num(:brtt)
|
|
638
|
+
if brtt.nonzero? && elapsed > 22
|
|
639
|
+
if brtt - b_rtt >= 20 && b_rtt.fdiv(brtt) <= 0.5
|
|
640
|
+
set(:vm) unless smart_reset
|
|
641
|
+
@data[:c_rate] = s[:calc_rate]
|
|
642
|
+
elsif b_rtt < brtt
|
|
643
|
+
increment(:reverse_path)
|
|
644
|
+
if (num(:start_brtt).zero? || b_rtt.fdiv(@data[:start_brtt]) <= 0.7) && @data[:reverse_path] >= 5
|
|
645
|
+
set(:disable_reverse)
|
|
646
|
+
@data[:n_rate] = s[:net_rate].to_i if num(:n_rate).zero? || num(:n_rate) > s[:net_rate].to_i
|
|
647
|
+
end
|
|
648
|
+
end
|
|
649
|
+
@data[:brtt] = b_rtt if b_rtt < brtt
|
|
650
|
+
elsif first && s[:rtt_high].to_i.positive? && elapsed > 21
|
|
651
|
+
@data[:brtt] = b_rtt
|
|
652
|
+
end
|
|
653
|
+
end
|
|
654
|
+
|
|
655
|
+
def check_high_rtt(s)
|
|
656
|
+
rtt = s[:rtt_high].to_i
|
|
657
|
+
srtt = s[:srtt].to_i
|
|
658
|
+
@data[:big_rtt] = [rtt, srtt].max if (rtt > 3000 || srtt > 3000) && rtt != srtt && [rtt, srtt].max > num(:big_rtt)
|
|
659
|
+
@data[:big_rex_srtt] = s[:rex_srtt] if s[:rex_srtt].to_i > 3000 && s[:rex_srtt] > num(:big_rex_srtt)
|
|
660
|
+
end
|
|
661
|
+
|
|
662
|
+
# Values at end of snapshot period
|
|
663
|
+
def update_slot(s, slot, t_bits, elapsed, receiver)
|
|
664
|
+
slot[:diff] = s[:time] - slot[:date1] if slot[:elapsed2] && elapsed - slot[:elapsed2] < 20 && s[:written] == slot[:written2]
|
|
665
|
+
slot.merge!(written2: s[:written], tx_bytes: s[:tx_bytes], elapsed2: elapsed, total2: t_bits, blocks2: s[:blocks], loss2: s[:lost], orig2: s[:orig], rex2: s[:rex])
|
|
666
|
+
duration = slot[:diff].to_i.nonzero? || (elapsed - slot[:elapsed1].to_f)
|
|
667
|
+
slot[:net_rate] = (t_bits - slot[:total1].to_i).fdiv(duration).to_i if duration.nonzero?
|
|
668
|
+
session_time = (num(:elapsed_us) - num(:elapsed_start)) / 1_000_000.0
|
|
669
|
+
@data[:avg] = (s[:blocks].to_i - num(:start_block)).fdiv(session_time).to_i * @data[:mtu] * 8 if session_time.nonzero?
|
|
670
|
+
lost, total =
|
|
671
|
+
if receiver
|
|
672
|
+
lost = slot[:loss2].to_i - slot[:loss1].to_i
|
|
673
|
+
[lost, lost + slot[:orig2].to_i - slot[:orig1].to_i]
|
|
674
|
+
else
|
|
675
|
+
[slot[:rex2].to_i - slot[:rex1].to_i, slot[:blocks2].to_i - slot[:blocks1].to_i]
|
|
676
|
+
end
|
|
677
|
+
return if total.zero?
|
|
678
|
+
slot[:loss_rate] = 100.0 * lost / total
|
|
679
|
+
slot[:loss_speed] = (slot[:net_rate].to_i * lost.fdiv(total)).to_i
|
|
680
|
+
end
|
|
681
|
+
|
|
682
|
+
# Determine which constraint limits the rate: Receiver (for sender), or network, disk, vlink (for receiver)
|
|
683
|
+
def rate_constraint(s, slot, elapsed, receiver, first)
|
|
684
|
+
t_rate = s[:target_rate].to_i
|
|
685
|
+
c_rate = s[:calc_rate].to_i
|
|
686
|
+
return if t_rate == c_rate
|
|
687
|
+
rates =
|
|
688
|
+
if receiver
|
|
689
|
+
RECEIVER_CONSTRAINTS.transform_values { |key| s[key].to_i }.select { |_, rate| rate < t_rate }
|
|
690
|
+
else
|
|
691
|
+
{'Receiver' => c_rate}
|
|
692
|
+
end
|
|
693
|
+
limit = rates.min_by(&:last)&.first
|
|
694
|
+
@constraints.add(limit) if receiver && limit && !first
|
|
695
|
+
return if elapsed <= 20
|
|
696
|
+
@data[:policy] = 'fair'
|
|
697
|
+
count_receiver_cause(s, t_rate) unless receiver
|
|
698
|
+
(slot[:calc_1] ||= []).push(limit) unless limit.nil? || slot[:calc_1]&.include?(limit)
|
|
699
|
+
@constraints.add('disk') if receiver && s[:disk_rate].to_i < t_rate
|
|
700
|
+
@constraints.add(limit) if limit && (first || !receiver)
|
|
701
|
+
rates.each do |name, rate|
|
|
702
|
+
current = (@constraint[name] ||= {})
|
|
703
|
+
next if current[:min_rate].to_i.nonzero? && current[:min_rate] <= rate
|
|
704
|
+
current[:min_rate] = rate
|
|
705
|
+
next unless name == 'network'
|
|
706
|
+
current[:base_rtt] = s[:brtt]
|
|
707
|
+
current[:smooth_rtt] = s[:srtt] if s[:net_rate].to_i == c_rate
|
|
708
|
+
end
|
|
709
|
+
end
|
|
710
|
+
|
|
711
|
+
# Sender (ascp 4): count which rate of the receiver limits the calculated rate
|
|
712
|
+
def count_receiver_cause(s, t_rate)
|
|
713
|
+
rates = RECEIVER_CAUSES.transform_values { |keys| keys.map { |key| s[key].to_i }.select(&:positive?).min }
|
|
714
|
+
cause = rates.compact.select { |_, rate| rate < t_rate }.min_by(&:last)&.first
|
|
715
|
+
@receiver_causes[cause] += 1 if cause
|
|
716
|
+
end
|
|
717
|
+
|
|
718
|
+
# First and latest points, to compute average rate
|
|
719
|
+
def update_calc_points(c_rate, t_bits, elapsed)
|
|
720
|
+
point = num(:calc0).zero? ? 0 : 1
|
|
721
|
+
@data.merge!("calc#{point}": c_rate, "total#{point}": t_bits, "elapsed#{point}": elapsed)
|
|
722
|
+
end
|
|
723
|
+
|
|
724
|
+
def calc_speed
|
|
725
|
+
duration = @data[:elapsed1].to_f - @data[:elapsed0].to_f
|
|
726
|
+
duration.zero? ? 0 : (@data[:total1].to_i - @data[:total0].to_i).fdiv(duration).to_i
|
|
727
|
+
end
|
|
728
|
+
|
|
729
|
+
# Detect peer clock problems/VM from sender side
|
|
730
|
+
def detect_peer_vm(s)
|
|
731
|
+
c_rate = s[:calc_rate].to_i
|
|
732
|
+
t_rate = s[:target_rate].to_i
|
|
733
|
+
vm = (@vm ||= {})
|
|
734
|
+
if vm[:elapsed1].nil?
|
|
735
|
+
vm[vm[:elapsed0].nil? ? :elapsed0 : :elapsed1] = s[:elapsed]
|
|
736
|
+
else
|
|
737
|
+
vm[:elapsed0] = vm[:elapsed1]
|
|
738
|
+
vm[:elapsed1] = s[:elapsed]
|
|
739
|
+
end
|
|
740
|
+
vm[:calc0] = num(:calc0)
|
|
741
|
+
return if vm[:elapsed0].to_f == vm[:elapsed1].to_f
|
|
742
|
+
if t_rate < vm[:calc0]
|
|
743
|
+
vm.merge!(calc0: t_rate, low: 0, high: 0)
|
|
744
|
+
clear(:vm_peer)
|
|
745
|
+
@data[:vm_c_rate] = 0
|
|
746
|
+
end
|
|
747
|
+
vm[:high] = vm[:high].to_i + 1 if c_rate >= 0.8 * vm[:calc0]
|
|
748
|
+
if c_rate < 0.5 * vm[:calc0] && vm[:high].to_i > 2
|
|
749
|
+
if vm[:low].to_i > 2
|
|
750
|
+
set(:vm_peer)
|
|
751
|
+
@data[:vm_c_rate] = c_rate
|
|
752
|
+
elsif vm[:calc0] - c_rate > 20_000_000 && !DISK_THROTTLE_RATES.include?(c_rate)
|
|
753
|
+
vm[:low] = vm[:low].to_i + 1
|
|
754
|
+
end
|
|
755
|
+
end
|
|
756
|
+
return if c_rate <= 0.6 * vm[:calc0]
|
|
757
|
+
clear(:vm_peer)
|
|
758
|
+
vm[:low] = 0
|
|
759
|
+
end
|
|
760
|
+
|
|
761
|
+
# Detect ATM/tail-drop rate limiting
|
|
762
|
+
def check_atm(s, t_bits, elapsed)
|
|
763
|
+
atm = (@atm ||= {})
|
|
764
|
+
c_rate = s[:calc_rate].to_i
|
|
765
|
+
t_rate = s[:target_rate].to_i
|
|
766
|
+
lost = s[:lost].to_i - atm[:loss0].to_i
|
|
767
|
+
sent = lost + s[:orig].to_i - atm[:orig0].to_i
|
|
768
|
+
atm[:atm_loss] = atm[:atm_loss].to_i + 1 if sent.nonzero? && lost.fdiv(sent) > 0.2
|
|
769
|
+
duration = elapsed - atm[:elapsed0].to_f
|
|
770
|
+
rate = duration.zero? ? 0 : (t_bits - atm[:total0].to_i) / duration
|
|
771
|
+
return if rate.zero?
|
|
772
|
+
atm[:atm_ratio] = atm[:atm_ratio].to_i + 1 if c_rate / rate > 1.3
|
|
773
|
+
queue = if t_rate <= 5_000_000 then 5
|
|
774
|
+
elsif t_rate <= 200_000_000 then 10
|
|
775
|
+
else 40
|
|
776
|
+
end
|
|
777
|
+
atm[:atm_queue] = atm[:atm_queue].to_i + 1 if num(:max_queue) < 1000 && t_rate * queue / rate > 0.75 * num(:max_queue)
|
|
778
|
+
end
|
|
779
|
+
end
|
|
780
|
+
end
|
|
781
|
+
end
|
|
782
|
+
end
|