aspera-cli 4.28.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +217 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +122 -26
  10. data/SECURITY.md +84 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1895 -632
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +29 -22
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +95 -67
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +370 -0
  41. data/lib/aspera/ascp/log_locations.rb +163 -0
  42. data/lib/aspera/ascp/management.rb +16 -8
  43. data/lib/aspera/ascp/session_errors.rb +217 -0
  44. data/lib/aspera/assert.rb +4 -0
  45. data/lib/aspera/cli/ascp_actions.rb +12 -19
  46. data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
  47. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  48. data/lib/aspera/cli/bootstrapper.rb +38 -33
  49. data/lib/aspera/cli/command_line.rb +63 -8
  50. data/lib/aspera/cli/command_registry.rb +9 -1
  51. data/lib/aspera/cli/command_spec.rb +47 -7
  52. data/lib/aspera/cli/completion/ascli.bash +19 -5
  53. data/lib/aspera/cli/completion/ascli.fish +13 -2
  54. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  55. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  56. data/lib/aspera/cli/completion_actions.rb +303 -69
  57. data/lib/aspera/cli/context.rb +9 -4
  58. data/lib/aspera/cli/deprecation.rb +5 -2
  59. data/lib/aspera/cli/error.rb +6 -0
  60. data/lib/aspera/cli/extended_value.rb +66 -31
  61. data/lib/aspera/cli/formatter.rb +43 -30
  62. data/lib/aspera/cli/gem_checker.rb +9 -6
  63. data/lib/aspera/cli/hints.rb +22 -10
  64. data/lib/aspera/cli/http.rb +38 -10
  65. data/lib/aspera/cli/info.rb +12 -7
  66. data/lib/aspera/cli/mailer.rb +6 -3
  67. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  68. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  69. data/lib/aspera/cli/mcp/tool.rb +20 -17
  70. data/lib/aspera/cli/option_declarator.rb +10 -4
  71. data/lib/aspera/cli/option_registry.rb +6 -3
  72. data/lib/aspera/cli/option_types.rb +17 -2
  73. data/lib/aspera/cli/option_value.rb +77 -18
  74. data/lib/aspera/cli/options.schema.yaml +641 -80
  75. data/lib/aspera/cli/parser.rb +71 -35
  76. data/lib/aspera/cli/plugins/alee.rb +7 -4
  77. data/lib/aspera/cli/plugins/aoc.rb +248 -37
  78. data/lib/aspera/cli/plugins/ats.rb +10 -9
  79. data/lib/aspera/cli/plugins/base.rb +50 -18
  80. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  81. data/lib/aspera/cli/plugins/config.rb +210 -104
  82. data/lib/aspera/cli/plugins/console.rb +6 -3
  83. data/lib/aspera/cli/plugins/cos.rb +8 -4
  84. data/lib/aspera/cli/plugins/factory.rb +21 -7
  85. data/lib/aspera/cli/plugins/faspex5.rb +31 -26
  86. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  87. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  88. data/lib/aspera/cli/plugins/mcp.rb +42 -22
  89. data/lib/aspera/cli/plugins/node.rb +39 -38
  90. data/lib/aspera/cli/plugins/oauth.rb +23 -6
  91. data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
  92. data/lib/aspera/cli/plugins/preview.rb +72 -45
  93. data/lib/aspera/cli/plugins/server.rb +66 -14
  94. data/lib/aspera/cli/plugins/shares.rb +87 -18
  95. data/lib/aspera/cli/plugins/transferd.rb +228 -0
  96. data/lib/aspera/cli/poll.rb +236 -0
  97. data/lib/aspera/cli/preset_actions.rb +144 -30
  98. data/lib/aspera/cli/preset_manager.rb +113 -35
  99. data/lib/aspera/cli/prompt.rb +157 -9
  100. data/lib/aspera/cli/result.rb +12 -8
  101. data/lib/aspera/cli/runner.rb +89 -48
  102. data/lib/aspera/cli/secret_finder.rb +4 -1
  103. data/lib/aspera/cli/security.rb +89 -9
  104. data/lib/aspera/cli/special_values.rb +4 -1
  105. data/lib/aspera/cli/sync_actions.rb +15 -10
  106. data/lib/aspera/cli/table_fitter.rb +3 -0
  107. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  108. data/lib/aspera/cli/transfer_actions.rb +11 -8
  109. data/lib/aspera/cli/transfer_agent.rb +43 -20
  110. data/lib/aspera/cli/transfer_progress.rb +14 -45
  111. data/lib/aspera/cli/vault_actions.rb +45 -0
  112. data/lib/aspera/cli/vault_manager.rb +188 -42
  113. data/lib/aspera/cli/version.rb +4 -1
  114. data/lib/aspera/cli/web_secret.rb +12 -10
  115. data/lib/aspera/cli/wizard.rb +42 -39
  116. data/lib/aspera/command_line_builder.rb +9 -5
  117. data/lib/aspera/command_line_converter.rb +4 -1
  118. data/lib/aspera/coverage.rb +103 -33
  119. data/lib/aspera/data_repository.rb +4 -1
  120. data/lib/aspera/dot_container.rb +4 -1
  121. data/lib/aspera/environment.rb +23 -12
  122. data/lib/aspera/exec_spec.rb +3 -0
  123. data/lib/aspera/faspex_gw.rb +13 -9
  124. data/lib/aspera/faspex_postproc.rb +15 -12
  125. data/lib/aspera/formatter_interface.rb +4 -1
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +20 -5
  129. data/lib/aspera/inflate.rb +43 -0
  130. data/lib/aspera/json_rpc/client.rb +6 -5
  131. data/lib/aspera/json_rpc/version.rb +3 -0
  132. data/lib/aspera/line_logger.rb +4 -1
  133. data/lib/aspera/link_header.rb +4 -1
  134. data/lib/aspera/log.rb +18 -5
  135. data/lib/aspera/markdown.rb +5 -2
  136. data/lib/aspera/mime.rb +3 -0
  137. data/lib/aspera/nagios.rb +6 -4
  138. data/lib/aspera/node_emulator.rb +15 -13
  139. data/lib/aspera/oauth/client/base.rb +124 -73
  140. data/lib/aspera/oauth/client/boot.rb +7 -4
  141. data/lib/aspera/oauth/client/factory.rb +41 -18
  142. data/lib/aspera/oauth/client/generic.rb +5 -1
  143. data/lib/aspera/oauth/client/json_credentials.rb +7 -3
  144. data/lib/aspera/oauth/client/jwt.rb +6 -3
  145. data/lib/aspera/oauth/client/shares_web.rb +94 -0
  146. data/lib/aspera/oauth/client/url_json.rb +7 -3
  147. data/lib/aspera/oauth/client/web.rb +40 -13
  148. data/lib/aspera/oauth/server/pkce.rb +6 -5
  149. data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +16 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +17 -2
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +3 -0
  165. data/lib/aspera/rest/aspera_errors.rb +4 -1
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +45 -34
  168. data/lib/aspera/rest/error_analyzer.rb +5 -2
  169. data/lib/aspera/rest/list.rb +8 -4
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +8 -5
  172. data/lib/aspera/rest.rb +9 -6
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  175. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  176. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  177. data/lib/aspera/schema/async_tables.yaml +11 -10
  178. data/lib/aspera/schema/documentation.rb +8 -2
  179. data/lib/aspera/schema/reader.rb +7 -0
  180. data/lib/aspera/schema/registry.rb +44 -28
  181. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  182. data/lib/aspera/schema/validator.rb +4 -1
  183. data/lib/aspera/sealed_hash.rb +150 -0
  184. data/lib/aspera/secret/hider.rb +213 -0
  185. data/lib/aspera/secret/reference.rb +150 -0
  186. data/lib/aspera/secret/resolver.rb +212 -0
  187. data/lib/aspera/secret/store/base.rb +153 -0
  188. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  189. data/lib/aspera/secret/store/factory.rb +103 -0
  190. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  191. data/lib/aspera/secret/store/keeper.rb +141 -0
  192. data/lib/aspera/secret/store/macos_security.rb +284 -0
  193. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  194. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  195. data/lib/aspera/secret/store/one_password_cli.rb +112 -0
  196. data/lib/aspera/secret/store/secret_service.rb +125 -0
  197. data/lib/aspera/secret/strength.rb +119 -0
  198. data/lib/aspera/servlet_helpers.rb +5 -2
  199. data/lib/aspera/ssh.rb +10 -7
  200. data/lib/aspera/ssl.rb +5 -2
  201. data/lib/aspera/string_ext.rb +3 -0
  202. data/lib/aspera/sync/args.schema.yaml +3 -0
  203. data/lib/aspera/sync/conf.schema.yaml +9 -4
  204. data/lib/aspera/sync/database.rb +3 -0
  205. data/lib/aspera/sync/operations.rb +17 -15
  206. data/lib/aspera/temp_file_manager.rb +8 -4
  207. data/lib/aspera/timer_limiter.rb +3 -0
  208. data/lib/aspera/transfer/error.rb +4 -1
  209. data/lib/aspera/transfer/faux_file.rb +4 -1
  210. data/lib/aspera/transfer/parameters.rb +17 -14
  211. data/lib/aspera/transfer/result.rb +23 -0
  212. data/lib/aspera/transfer/resumer.rb +6 -4
  213. data/lib/aspera/transfer/spec.rb +9 -22
  214. data/lib/aspera/units/decimal.rb +67 -0
  215. data/lib/aspera/units/duration.rb +109 -0
  216. data/lib/aspera/uri_reader.rb +64 -9
  217. data/lib/aspera/web_auth/index.html.erb +6 -2
  218. data/lib/aspera/web_auth/paste.rb +109 -0
  219. data/lib/aspera/web_auth/receiver.rb +50 -0
  220. data/lib/aspera/web_auth/relay.rb +76 -0
  221. data/lib/aspera/web_auth/reply.html.erb +150 -0
  222. data/lib/aspera/web_auth/server.rb +47 -10
  223. data/lib/aspera/web_page.rb +7 -2
  224. data/lib/aspera/web_server_simple.rb +6 -3
  225. data/lib/aspera/yaml.rb +5 -1
  226. data/llms-install.md +88 -0
  227. data/llms.txt +87 -0
  228. data/skills/ascli/SKILL.md +94 -0
  229. data.tar.gz.sig +0 -0
  230. metadata +77 -20
  231. metadata.gz.sig +0 -0
  232. data/lib/aspera/duration.rb +0 -42
  233. data/lib/aspera/keychain/base.rb +0 -30
  234. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  235. data/lib/aspera/keychain/factory.rb +0 -58
  236. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  237. data/lib/aspera/keychain/macos_security.rb +0 -277
  238. data/lib/aspera/keychain/one_password_api.rb +0 -91
  239. data/lib/aspera/keychain/one_password_base.rb +0 -39
  240. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  241. data/lib/aspera/secret_hider.rb +0 -196
  242. data/lib/aspera/secret_strength.rb +0 -114
@@ -0,0 +1,782 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:words commify Isilon nacks nohear onefs overdriven
7
+ require_relative '../../assert'
8
+ require_relative '../session_errors'
9
+ require_relative 'diagnostics'
10
+ require 'set'
11
+ require 'time'
12
+
13
+ module Aspera
14
+ module Ascp
15
+ class LogAnalyzer
16
+ # Data of one ascp session (process), collected from its log lines and its samples
17
+ class Session
18
+ # Calculated rates (bps) of 1 packet per second, for various payload sizes: maximum disk throttling
19
+ DISK_THROTTLE_RATES = [11_584, 11_712, 11_936, 7_824, 8_048, 3_840, 4_064, 2_144, 2_368].freeze
20
+ DEFAULT_MTU = 1492
21
+ # Sender (ascp 4): limit of the receiver => sample values
22
+ RECEIVER_CAUSES = {'network' => %i[peer_net_rate], 'disk' => %i[peer_disk_rate], 'vlink' => %i[peer_vlink_local peer_vlink_remote]}.freeze
23
+ # Control messages in `Session Statistics`: direction => description
24
+ CONTROL_DIRECTIONS = Diagnostics::CONTROL_DIRECTIONS
25
+ # Constraint on the rate => series of its sample values
26
+ CONSTRAINT_SERIES = {'network' => :network, 'disk' => :disk, 'vlink_local' => :vlink_local, 'vlink_remote' => :vlink_remote, 'Receiver' => :calc}.freeze
27
+ # Receiver: constraint => sample value
28
+ RECEIVER_CONSTRAINTS = {'network' => :net_rate, 'disk' => :disk_rate, 'vlink_local' => :vlink_local, 'vlink_remote' => :vlink_remote}.freeze
29
+ # Receiver: sample value => series, kept after 20 seconds of transfer
30
+ RECEIVER_SERIES = {srtt: :srtt, rex_srtt: :rex_srtt, rtt_high: :rtt, net_rate: :network, vlink_local: :vlink_local, vlink_remote: :vlink_remote, disk_rate: :disk}.freeze
31
+ RESUME = {0 => 'Do not resume', 1 => 'With file attribute check', 2 => 'With sparse checksum', 3 => 'With full checksum'}.freeze
32
+ OVERWRITE = {255 => 'Always overwrite', 1 => 'If source different', 3 => 'If source different and newer', 63 => 'Never overwrite'}.freeze
33
+ # Rate policy in log lines => policy name
34
+ POLICIES = {'0' => 'fixed', '1' => 'fair', '2' => 'low', 'adapt' => 'fair', 'trickle' => 'low'}.freeze
35
+ # Flag => severity and message of diagnostic (identifier: name of flag)
36
+ FLAG_DIAGNOSTICS = Diagnostics::FLAG_DIAGNOSTICS
37
+
38
+ class << self
39
+ # @return [Float] mean of values, 0 if none
40
+ def average(values)
41
+ values.empty? ? 0 : values.sum.fdiv(values.size)
42
+ end
43
+
44
+ # @return [Numeric] median of values, 0 if none
45
+ def median(values)
46
+ return 0 if values.empty?
47
+ sorted = values.sort
48
+ mid = sorted.size / 2
49
+ sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0
50
+ end
51
+
52
+ # @return [Float] population standard deviation of values, 0 if none
53
+ def standard_deviation(values)
54
+ return 0 if values.empty?
55
+ mean = average(values)
56
+ Math.sqrt(values.sum { |v| (v - mean)**2 } / values.size)
57
+ end
58
+
59
+ # @return [String] integer with thousands separators, for messages
60
+ def commify(value)
61
+ value.to_i.to_s.gsub(/(\d)(?=(\d{3})+\z)/, '\1,')
62
+ end
63
+
64
+ # @return [Boolean] true if version is known and older than reference
65
+ def version_older?(version, reference)
66
+ number = version.to_s[/\A\d+(?:\.\d+)*/] or return false
67
+ Gem::Version.new(number) < Gem::Version.new(reference)
68
+ end
69
+
70
+ # @return [Integer, nil] rate in bps, from text with unit, e.g. `3.33Mbps`
71
+ def rate_bps(text)
72
+ m = text.to_s.match(/\A(\d+\.\d+)([A-Za-z]+)/) or return
73
+ (m[1].to_f * {'Kbps' => 1e3, 'Mbps' => 1e6, 'Gbps' => 1e9}.fetch(m[2], 1)).round
74
+ end
75
+ end
76
+
77
+ attr_accessor :pid
78
+ # Files transferred: name => {op:, size:, status:, error:}
79
+ attr_reader :files
80
+ # Diagnostics of last report: {id, severity, message}
81
+ attr_reader :diagnostics, :thresholds, :data, :flags, :series, :constraint, :constraints, :params, :sizes, :receiver_causes, :rate_samples, :ds_qs_samples, :timezone
82
+
83
+ # @param pid [String] Process identifier
84
+ # @param thresholds [Hash] Diagnostic thresholds
85
+ # @param timezone [String, nil] Time zone of the reported times: `UTC` or `+HH:MM`. Time zone of the analyst if nil.
86
+ def initialize(pid, thresholds: {}, timezone: nil)
87
+ @pid = pid
88
+ @timezone = timezone
89
+ thresholds = thresholds.transform_keys(&:to_sym)
90
+ unknown = thresholds.keys - DEFAULT_THRESHOLDS.keys
91
+ Aspera.assert(unknown.empty?, type: ArgumentError) { "Unknown threshold: #{unknown.join(', ')}" }
92
+ @thresholds = DEFAULT_THRESHOLDS.merge(thresholds)
93
+ @data = {}
94
+ @flags = Set.new
95
+ @series = Hash.new { |h, k| h[k] = [] }
96
+ # Constraint name => {min_rate:, base_rtt:, smooth_rtt:}
97
+ @constraint = {}
98
+ # Names of constraints that limited the rate
99
+ @constraints = Set.new
100
+ # Changes of rate parameters during session
101
+ @params = nil
102
+ @files = {}
103
+ # Sizes of transferred files
104
+ @sizes = []
105
+ # Detection of ATM / tail-drop rate limiting
106
+ @atm = nil
107
+ # Detection of stuck NACKs on sender
108
+ @send_nack = nil
109
+ # Detection of clock problems on peer
110
+ @vm = nil
111
+ # Sender: limit of the receiver => number of samples
112
+ @receiver_causes = Hash.new(0)
113
+ # Rate samples after the delay: [elapsed time (us), time, bps], receiver network rate or sender calculated rate
114
+ @rate_samples = []
115
+ # File-queue stats: one hash per DS Qs sample, keys 'ao', 'ap', 'mgmt_backlog'
116
+ @ds_qs_samples = []
117
+ end
118
+
119
+ def [](key)
120
+ @data[key]
121
+ end
122
+
123
+ def []=(key, value)
124
+ @data[key] = value
125
+ end
126
+
127
+ def delete(key)
128
+ @data.delete(key)
129
+ end
130
+
131
+ # @return [Numeric] value, 0 if not set
132
+ def num(key)
133
+ @data[key] || 0
134
+ end
135
+
136
+ def increment(key, value = 1)
137
+ @data[key] = num(key) + value
138
+ end
139
+
140
+ # @return [Boolean] true if any of the flags is set
141
+ def flag?(*names)
142
+ names.any? { |n| @flags.include?(n) }
143
+ end
144
+
145
+ def set(*names)
146
+ @flags.merge(names)
147
+ end
148
+
149
+ def clear(*names)
150
+ @flags.subtract(names)
151
+ end
152
+
153
+ # Apply effects of a rule (see LINE_RULES), unless cancelled by a flag
154
+ def apply(effects)
155
+ return if effects[:unless] && flag?(*effects[:unless])
156
+ set(*effects[:set]) if effects[:set]
157
+ clear(*effects[:clear]) if effects[:clear]
158
+ @data[:error] = effects[:error] if effects[:error]
159
+ @data[:status] = effects[:status] if effects[:status]
160
+ end
161
+
162
+ # Apply the first rule whose flags are all set
163
+ # @param rules [Array] list of [flags, effects]
164
+ def apply_first(rules)
165
+ rule = rules.find { |flags, _| flags.all? { |f| flag?(f) } }
166
+ apply(rule.last) if rule
167
+ end
168
+
169
+ # Information from a `Session Start` or `Session Stop` line
170
+ # @param kind [Symbol] :start or :stop
171
+ # @param f [Hash] values from line
172
+ # @param time [Integer, nil] time of line
173
+ # @param config [Hash] user and version from configuration lines
174
+ def session_line(kind, f, time, config)
175
+ %i[uuid op user source dest local os ver peeros peerver transport].each { |k| @data[k] = f[k] if f[k] }
176
+ if f[:lic]
177
+ @data[:product] = f[:lic].split(':').first.to_i
178
+ @data[:role] = 'client' if [6, 7].include?(@data[:product])
179
+ end
180
+ if f[:peerlic]
181
+ @data[:peer_product] = f[:peerlic].split(':').first.to_i
182
+ @data[:role] = 'server' if @data[:peer_product] == 6
183
+ end
184
+ @data[:user] ||= config[:user]
185
+ @data[:role] = 'client' if config[:user] && config[:user] != '-' && @data[:user] && !config[:user].match?(/#{Regexp.escape(@data[:user])}/i)
186
+ if f[:l_port]
187
+ @data[:start_l_port] = @data[:l_port] if kind.eql?(:stop) && @data[:l_port] && @data[:l_port] != f[:l_port]
188
+ @data[:l_port] = f[:l_port]
189
+ end
190
+ @data[:peer] ||= f[:peer]
191
+ @data[:p_port] = f[:p_port] if f[:p_port]
192
+ @data[:ver] ||= config[:ver]
193
+ @data[:product_name] ||= config[:product_name]
194
+ @data[:auth] ||= config[:auth]
195
+ @data[:license_rate] ||= config[:license_rate]
196
+ @data[:license_expires] ||= config[:license_expires]
197
+ @data[kind] = time
198
+ @data[:date] = @data[:stop] || @data[:start]
199
+ end
200
+
201
+ # Values from `Session Params` line
202
+ # @param fields [Hash] key=value from line
203
+ def session_params(fields)
204
+ SESSION_PARAMS.each do |name, key|
205
+ next unless fields.key?(name)
206
+ value = fields[name]
207
+ @data[key] = value.match?(/\A\d+\z/) && key != :user ? value.to_i : value
208
+ end
209
+ @data[:policy] = POLICIES.fetch(@data[:policy], @data[:policy]) if @data[:policy]
210
+ # `proxy_ip=-`: no proxy
211
+ @data[:proxy] = fields['proxy_ip'] if fields['proxy_ip'] && fields['proxy_ip'] != '-'
212
+ @data[:fips] = fields['fips_enabled'] == 'yes' if fields.key?('fips_enabled')
213
+ end
214
+
215
+ # Error and status of `Session Stop` line
216
+ def stop_error(line, status)
217
+ e1, e2 = line.match(ERRSTR)&.captures
218
+ error =
219
+ if !e2.to_s.empty? then e2 == 'Undefined' ? e1 : e2
220
+ elsif e1.to_s.empty? then nil
221
+ elsif e1 == 'Undefined' && flag?(:invalid_conf_file) then 'Invalid Configuration File'
222
+ elsif e1 == 'Undefined' && flag?(:invalid_argument) then 'Invalid argument'
223
+ else e1
224
+ end
225
+ @data[:error] = error if error
226
+ # Code of the error retained (local or peer)
227
+ code = line.scan(ERRCODE).to_h { |number, text| [text, number.to_i] }[error]
228
+ @data[:error_code] = code if code&.nonzero?
229
+ @data[:status] = 'success' if status == 'success'
230
+ return unless status == 'failed'
231
+ if e2.to_s.empty? && e1 == 'Undefined'
232
+ apply_first(UNDEFINED_ERROR_RULES)
233
+ @data[:error] = 'Unable to bind to UDP socket' if flag?(:bind_udp_socket_failed)
234
+ end
235
+ apply_first(ERROR_REFINEMENTS[error]) if ERROR_REFINEMENTS.key?(error)
236
+ @data[:status] = 'failed' unless @data[:status] == 'success'
237
+ end
238
+
239
+ # Rate parameters changed
240
+ # @param by_peer [Boolean] rate parameters requested by peer
241
+ def rate_params(t_rate, m_rate, policy, by_peer:)
242
+ @params[:multiple] = true if @params
243
+ @params&.delete(:peer) unless by_peer
244
+ policy = POLICIES.fetch(policy, policy)
245
+ change_param(:policy, policy) unless @data[:policy] == policy
246
+ change_param(:target_rate, t_rate) unless num(:target_rate) == t_rate
247
+ change_param(:min_rate, m_rate) unless num(:min_rate) == m_rate
248
+ @params[:peer] = true if by_peer && @params
249
+ end
250
+
251
+ # `Transfer Stop` line
252
+ # @return [Boolean] true if file failed because its name is too long for Windows
253
+ def transfer_stop(op, status, file, size, start_byte, errstr = nil)
254
+ too_long = false
255
+ case status
256
+ when 'success'
257
+ if start_byte == size
258
+ increment(:files_skipped)
259
+ else
260
+ size -= start_byte
261
+ increment(:files_success)
262
+ @sizes.push(size)
263
+ end
264
+ when 'failed'
265
+ increment(:files_failed)
266
+ too_long = file.length >= 240 && (file.match?(/\w:/) || windows?)
267
+ increment(:filename_too_long) if too_long
268
+ else
269
+ return false
270
+ end
271
+ increment(:transfer_stop_count)
272
+ entry = (@files[file] ||= {})
273
+ error = errstr&.match?(/errstr="./) ? errstr[/errstr="(.*?)"/, 1] : errstr
274
+ entry[:error] = error if status == 'failed' && error && !error.empty?
275
+ entry.merge!(op: op, size: size, status: status)
276
+ too_long
277
+ end
278
+
279
+ # Process a `bl` sample (Receiver or Sender)
280
+ # @param s [Hash] values of sample, see RECEIVER_SAMPLE, SENDER_SAMPLE
281
+ # @param slot [Hash] data of this session in the current snapshot period
282
+ # @param first [Boolean] first sample of this session in the period
283
+ # @param smart_reset [Boolean] base RTT is reset by ascp
284
+ # @param mtu [Integer, nil] MTU to use when not found in logs
285
+ def add_sample(s, slot, first:, smart_reset:, mtu:)
286
+ receiver = s[:receiver]
287
+ elapsed = s[:elapsed]
288
+ @data[:elapsed_start] = s[:elapsed_us] if num(:elapsed_start).zero?
289
+ @data[:elapsed_us] = s[:elapsed_us]
290
+ if receiver
291
+ increment(:brtt_reset, s[:brtt_resets]) if s[:brtt_resets].to_i.positive?
292
+ RECEIVER_SERIES.each { |key, series| @series[series].push(s[key].to_i) } if elapsed > 20
293
+ if s[:net_rate].to_i == s[:calc_rate].to_i
294
+ queue = s[:srtt].to_i - s[:brtt].to_i
295
+ @data[:max_queue] = queue if @data[:max_queue].nil? || queue > @data[:max_queue]
296
+ end
297
+ else
298
+ check_sender_nacks(s)
299
+ @data[:disk_read_rate] = s[:disk_read_rate] if s[:disk_read_rate].to_i.positive?
300
+ end
301
+ @data[:start_block] = s[:blocks] if num(:start_block).zero?
302
+ @data[:mtu] = format('%.0f', s[:tx_bytes].fdiv(s[:blocks]) + 40).to_i if num(:mtu) <= 8000 && s[:blocks].to_i.positive? && s[:tx_bytes].fdiv(s[:blocks]) > 8000
303
+ @data[:mtu] = mtu if mtu && !@data[:payload]
304
+ @data[:mtu] ||= DEFAULT_MTU
305
+ t_bits = s[:blocks].to_i * @data[:mtu] * 8
306
+ @data[:target_rate] = s[:target_rate] if num(:target_rate).zero?
307
+ @data[:min_rate] = s[:min_rate]
308
+ @series[:calc].push(s[:calc_rate].to_i)
309
+ @rate_samples.push([s[:elapsed_us], s[:time], (receiver ? s[:net_rate] : s[:calc_rate]).to_i]) if elapsed > @thresholds[:rate_decline_delay]
310
+ increment(:disk_max) if DISK_THROTTLE_RATES.include?(s[:calc_rate])
311
+ check_stuck_nack(s)
312
+ check_stuck_rex(s) if receiver
313
+ @data[:tx_bytes] = s[:tx_bytes] if s[:tx_bytes] > num(:tx_bytes)
314
+ log_brtt(s, elapsed, first: first, smart_reset: smart_reset)
315
+ if first
316
+ slot.merge!(
317
+ total1: t_bits, blocks1: s[:blocks], loss1: s[:lost], orig1: s[:orig], rex1: s[:rex], elapsed1: elapsed, date1: s[:time],
318
+ target_rate: s[:target_rate], calc_rate: [s[:calc_rate].to_i], receiver: receiver
319
+ )
320
+ else
321
+ (@atm ||= {})[:samples] = @atm[:samples].to_i + 1
322
+ slot[:calc_rate].push(s[:calc_rate].to_i)
323
+ update_slot(s, slot, t_bits, elapsed, receiver)
324
+ end
325
+ @data[:elapsed] = elapsed
326
+ check_high_rtt(s) if receiver
327
+ if s[:written].nonzero?
328
+ rate_constraint(s, slot, elapsed, receiver, first)
329
+ update_calc_points(s[:calc_rate].to_i, t_bits, elapsed)
330
+ if receiver
331
+ @data[:recv_max] = [num(:recv_max), calc_speed].max
332
+ check_atm(s, t_bits, elapsed) unless first
333
+ else
334
+ detect_peer_vm(s) unless first
335
+ @data[:send_max] = [num(:send_max), calc_speed].max
336
+ end
337
+ end
338
+ @atm = (@atm || {}).merge(total0: t_bits, elapsed0: elapsed, loss0: s[:lost].to_i, orig0: s[:orig].to_i) if first
339
+ if receiver
340
+ buffers = [s[:ring_high], s[:ring_full]].find { |b| b.to_i >= 5 }
341
+ @data[:buffers] = buffers if buffers && buffers > num(:buffers)
342
+ @data[:ooo] = s[:ooo].to_i.fdiv(s[:blocks]) if s[:blocks].to_i.nonzero?
343
+ @data[:loss_rate] = s[:lost].to_i.fdiv(s[:lost].to_i + s[:orig].to_i) if (s[:lost].to_i + s[:orig].to_i).nonzero?
344
+ elsif (!first || s[:written].nonzero?) && s[:blocks].to_i.nonzero?
345
+ @data[:loss_rate] = s[:rex].to_i.fdiv(s[:blocks])
346
+ end
347
+ end
348
+
349
+ # Accumulate file-queue statistics from one `DS Qs` sample
350
+ # @param qs [Hash] keys: `'ao'` (open-async), `'ap'` (attrprep), `'mgmt_backlog'` (bytes buffered)
351
+ def add_ds_qs_sample(qs)
352
+ @ds_qs_samples.push(qs)
353
+ end
354
+
355
+ # Tell if another session starts while the rate samples are recorded: it may share the link
356
+ # @param starts [Array<Integer>] start times of the transfer sessions of the log
357
+ def shared_link(starts)
358
+ return if @rate_samples.empty?
359
+ first, last = @rate_samples.map { |sample| sample[1] }.minmax
360
+ @data[:shared_link] = starts.any? { |time| time > first && time <= last }
361
+ end
362
+
363
+ # Evaluate ATM / tail-drop rate limiting detection, at end of a snapshot period
364
+ def evaluate_atm
365
+ return if @atm.nil? || @atm[:samples].nil?
366
+ if @atm[:periods].to_i < 2
367
+ @atm[:periods] = @atm[:periods].to_i + 1
368
+ return
369
+ end
370
+ return unless @atm[:samples] == 6
371
+ increment(:atm) if %i[atm_loss atm_ratio atm_queue].all? { |k| @atm[k].to_i / 6.0 >= 0.83 }
372
+ @atm = nil
373
+ end
374
+
375
+ # Final status of session, if ascp stopped logging before end of session
376
+ # @param last_time [Integer] time of last sample in log file
377
+ # @return [Symbol, nil] category of exit
378
+ def check_exit(last_time)
379
+ return if last_time.to_i - num(:last) <= 30 || flag?(:stats_end)
380
+ category =
381
+ if @data[:error] == 'Out of buffer memory'
382
+ @data[:error] = 'Out of buffer memory, and ascp exited.'
383
+ :buffer_died
384
+ elsif flag?(:decryption_error)
385
+ @data[:error] = 'File decryption error, and ascp exited.'
386
+ :decryption_died
387
+ elsif flag?(:data_transfer_timeout)
388
+ @data[:error] = "#{@data[:error]}, and ascp exited."
389
+ :timeout_died
390
+ elsif @data[:error] == 'segfault'
391
+ @data[:error] = 'ascp exited: segfault.'
392
+ :died
393
+ elsif @data[:status]
394
+ return if flag?(:ssh_timeout, :cannot_connect_to_management_socket)
395
+ set(:table_died)
396
+ :table_died
397
+ else
398
+ @data[:error] = 'ascp exited, or was in IOwait or stopped logging'
399
+ :died
400
+ end
401
+ @data[:status] = 'failed' unless @data[:status] == 'success' || category == :table_died
402
+ @data[:exit] = category
403
+ end
404
+
405
+ # @return [Boolean] true if session has the stuck NACK problem
406
+ def stuck_nack?
407
+ return false if @data[:status] == 'success'
408
+ num(:stuck_nack_count) >= 5 || ((flag?(:send_nack_stuck) || num(:sender_stuck_nack_count) >= 5) && num(:loss_rate) < 0.9)
409
+ end
410
+
411
+ def windows?
412
+ [@data[:os], @data[:peeros]].any? { |os| os.to_s.match?(/windows/i) }
413
+ end
414
+
415
+ # @return [String, nil] `send` or `receive`
416
+ def direction
417
+ (@data[:op] == 'recv' ? 'receive' : @data[:op]) || @data[:dir]
418
+ end
419
+
420
+ # @return [Integer, nil] error code of session, if it is in the numbering of supported versions
421
+ def error_code
422
+ @data[:error_code] unless self.class.version_older?(@data[:ver], MIN_VERSION)
423
+ end
424
+
425
+ # @return [String, nil] category of error code, see SessionErrors::CATEGORIES
426
+ def error_category
427
+ code = error_code or return
428
+ SessionErrors.category(code)
429
+ end
430
+
431
+ # @return [Symbol, nil] probable cause of loss on sender: :storage or :network (shallow queues)
432
+ def overdrive
433
+ return unless @data[:dir] == 'send' && num(:loss_rate) > 0.10 && @constraints.size == 1
434
+ return :storage if DISK_THROTTLE_RATES.include?(@constraint.dig('Receiver', :min_rate))
435
+ return if num(:stuck_nack_count) >= 10 || @params
436
+ :network if @constraint.key?('Receiver') && constraint_average('Receiver') >= 0.8 * num(:target_rate)
437
+ end
438
+
439
+ # @param windows_log [Boolean] log file is on Windows
440
+ # @return [Hash] report of session: one table row
441
+ def report(windows_log: false)
442
+ d = @data
443
+ sample_dir = d[:dir]
444
+ avg = d[:effective_rate] ? self.class.rate_bps(d[:effective_rate]).to_i : num(:avg)
445
+ avg = d[:target_rate] if num(:target_rate).positive? && avg > d[:target_rate]
446
+ loss = num(:loss_rate)
447
+ min_rate_high = d[:min_rate] if d[:policy] == 'fair' && (num(:min_rate) >= 5_000_000 || num(:min_rate) > num(:target_rate) / 2.0)
448
+ rtt = series_stats(:rtt)
449
+ srtt = series_stats(:srtt)
450
+ constraints = @constraint.reject { |_, c| c[:min_rate].nil? }.to_h do |name, c|
451
+ series = @series[CONSTRAINT_SERIES[name]]
452
+ entry = {'max' => c[:min_rate], 'average' => constraint_average(name), 'std_dev' => self.class.standard_deviation(series).to_i, 'samples' => series.size}
453
+ if name == 'network'
454
+ entry['base_rtt'] = c[:base_rtt]
455
+ entry['smoothed_rtt'] = c[:smooth_rtt]
456
+ end
457
+ entry['causes'] = @receiver_causes.sort_by { |_, count| -count }.to_h if name == 'Receiver' && @receiver_causes.any?
458
+ [name, entry]
459
+ end
460
+ slowdown = constraints.values.any? { |c| c['max'] < 8 * (num(:mtu) - 40) }
461
+ tx_time = d[:tx_time].to_i.nonzero? ? d[:tx_time] / 1_000_000.0 : d[:elapsed].to_f
462
+ @diagnostics = Diagnostics.new(self, windows_log: windows_log).evaluate(
463
+ loss: loss,
464
+ slowdown: slowdown,
465
+ sample_dir: sample_dir,
466
+ avg: avg,
467
+ min_rate_high: min_rate_high,
468
+ rtt: rtt,
469
+ srtt: srtt
470
+ )
471
+ {
472
+ 'pid' => @pid,
473
+ 'session' => d[:uuid],
474
+ 'user' => d[:user],
475
+ 'direction' => direction,
476
+ 'role' => d[:role] || role_from_ports,
477
+ 'status' => d[:status],
478
+ 'error' => d[:error],
479
+ 'error_code' => d[:error_code],
480
+ 'error_category' => error_category,
481
+ 'error_retry' => (SessionErrors.retryable(error_code) if error_code),
482
+ 'transfer_id' => d[:xfer_id],
483
+ 'multi_session' => ("#{d[:cluster_node].to_i + 1}/#{d[:cluster_nodes]}" if num(:cluster_nodes) > 1),
484
+ 'retry_of' => d[:retry_of],
485
+ 'retried_by' => d[:retried_by],
486
+ 'start' => iso(d[:start]),
487
+ 'stop' => iso(d[:stop]),
488
+ 'local' => d[:local],
489
+ 'local_port' => d[:l_port],
490
+ 'version' => d[:ver],
491
+ 'os' => d[:os],
492
+ **context_columns,
493
+ 'peer' => d[:peer],
494
+ 'peer_port' => d[:p_port],
495
+ 'peer_version' => d[:peerver],
496
+ 'peer_os' => d[:peeros],
497
+ 'peer_product' => d[:peer_product],
498
+ 'destination' => d[:dest],
499
+ 'policy' => d[:policy],
500
+ 'encryption' => d[:secure],
501
+ 'resume' => RESUME.fetch(d[:resume], d[:resume]),
502
+ 'overwrite' => OVERWRITE.fetch(d[:overwrite], d[:overwrite]),
503
+ 'block_size' => d[{'send' => :read_block_size, 'receive' => :write_block_size}[sample_dir]],
504
+ 'mtu' => d[:mtu],
505
+ 'payload' => d[:payload],
506
+ 'target_rate' => d[:target_rate],
507
+ 'min_rate' => d[:min_rate],
508
+ 'average_rate' => avg,
509
+ 'average_calculated_rate' => self.class.average(@series[:calc]).to_i,
510
+ 'max_rate' => {'send' => d[:send_max], 'receive' => d[:recv_max].to_i.nonzero? || avg}[sample_dir],
511
+ 'transmitted_bytes' => num(:tx_bytes),
512
+ 'transfer_time' => tx_time.round(2),
513
+ 'files' => d[:transfer_stop_count],
514
+ 'files_transferred' => d[:files_success],
515
+ 'files_skipped' => d[:files_skipped],
516
+ 'files_failed' => d[:files_failed],
517
+ 'file_size_mean' => (self.class.average(@sizes).to_i if d[:transfer_stop_count]),
518
+ 'file_size_median' => (self.class.median(@sizes).to_i if d[:transfer_stop_count]),
519
+ 'concurrent_min' => (d[:sess_min].to_i.nonzero? || 1 if d[:sess_max]),
520
+ 'concurrent_max' => d[:sess_max],
521
+ 'concurrent_max_time' => iso(d[:sess_max_time]),
522
+ 'start_rtt' => d[:start_brtt],
523
+ 'base_rtt' => d[:brtt],
524
+ 'rtt_average' => rtt&.first,
525
+ 'rtt_median' => rtt&.last,
526
+ 'srtt_average' => srtt&.first,
527
+ 'srtt_median' => srtt&.last,
528
+ 'loss_rate' => (loss.positive? ? (loss * 100).round(2) : (0.0 if num(:tx_bytes).nonzero?)),
529
+ 'out_of_order_rate' => (num(:ooo) * 100).round(2).nonzero?,
530
+ 'efficiency' => d[:efficiency],
531
+ 'retransmitted_rate' => d[:retransmitted],
532
+ 'duplicate_rate' => d[:duplicates],
533
+ 'disk_read_rate' => d[:disk_read_rate],
534
+ 'constraints' => @constraints.to_a,
535
+ 'constraint' => constraints,
536
+ 'diagnostics' => @diagnostics.map { |diagnostic| diagnostic['message'] }
537
+ }.compact
538
+ end
539
+
540
+ private
541
+
542
+ FASP_PORTS = (33_000..33_099)
543
+ private_constant :FASP_PORTS
544
+
545
+ # Columns that tell in which context the session ran, to filter sessions
546
+ # @return [Hash] product, transport (`ssh` or `wss`), authentication type (`token` or `node`), proxy address, FIPS mode
547
+ def context_columns
548
+ {
549
+ 'product' => @data[:product],
550
+ 'product_name' => @data[:product_name],
551
+ 'transport' => @data[:transport],
552
+ 'authentication' => @data[:auth],
553
+ 'proxy' => @data[:proxy],
554
+ 'fips' => @data[:fips]
555
+ }
556
+ end
557
+
558
+ def iso(time)
559
+ Time.at(time).getlocal(@timezone).iso8601 if time
560
+ end
561
+
562
+ # @return [Array, nil] average and median of series, as integers
563
+ def series_stats(name)
564
+ series = @series[name]
565
+ [self.class.average(series).to_i, self.class.median(series).to_i] unless series.empty?
566
+ end
567
+
568
+ # @return [Integer] average of values of a constraint
569
+ def constraint_average(name)
570
+ self.class.average(@series[CONSTRAINT_SERIES[name]]).to_i
571
+ end
572
+
573
+ # @return [String, nil] client or server, deduced from UDP ports
574
+ def role_from_ports
575
+ local, peer = @data.values_at(:l_port, :p_port)
576
+ return unless local && peer
577
+ return 'server' if local == 33_001 || (FASP_PORTS.cover?(local) && !FASP_PORTS.cover?(peer))
578
+ 'client'
579
+ end
580
+
581
+ def change_param(key, value)
582
+ @params ||= {}
583
+ @params[:"#{key}_old"] = @data[key]
584
+ @params[key] = value
585
+ @data[key] = value
586
+ end
587
+
588
+ # Sender side: NACKs stuck while no progress
589
+ def check_sender_nacks(s)
590
+ if @send_nack.nil?
591
+ @send_nack = {}
592
+ return
593
+ end
594
+ sn = @send_nack
595
+ no_progress = s[:blocks].to_i >= sn[:blocks].to_i && s[:tx_bytes] == sn[:tx_bytes].to_i && s[:written] == sn[:written].to_i &&
596
+ s[:tx_bytes].nonzero? && s[:written].nonzero?
597
+ if no_progress && s[:rex_count].to_i == sn[:rex_count].to_i && s[:rex_received].to_i == sn[:rex_received].to_i
598
+ sn[:count] = sn[:count].to_i + 1
599
+ set(:send_nack_stuck) if sn[:count] > 10
600
+ elsif no_progress && sn[:rex_received].to_i < s[:rex_received].to_i
601
+ increment(:sender_stuck_nack_count)
602
+ else
603
+ @send_nack = s.slice(:rex_count, :blocks, :rex_received, :tx_bytes, :written)
604
+ @data[:sender_stuck_nack_count] = 0
605
+ end
606
+ end
607
+
608
+ # NACKs stuck at same value while no progress
609
+ def check_stuck_nack(s)
610
+ nack = s[:rex_count]
611
+ return if nack.nil? || num(:tx_bytes).zero?
612
+ if num(:stuck_nack).zero?
613
+ @data.merge!(stuck_nack: nack, stuck_nack_count: 1) if num(:tx_bytes) == s[:tx_bytes]
614
+ elsif nack.zero?
615
+ @data.merge!(stuck_nack: 0, stuck_nack_count: 0)
616
+ elsif nack == @data[:stuck_nack] && num(:tx_bytes) == s[:tx_bytes] && s[:calc_rate].to_i.nonzero?
617
+ increment(:stuck_nack_count)
618
+ else
619
+ @data.merge!(stuck_nack: nack, stuck_nack_count: 0)
620
+ end
621
+ end
622
+
623
+ # Retransmissions not progressing while NACKs pending: NACK requests are being lost
624
+ def check_stuck_rex(s)
625
+ rex = s[:rex]
626
+ nack = s[:rex_count].to_i
627
+ return if rex.nil?
628
+ if @data[:stuck_rex].nil? || (nack.nonzero? && rex > @data[:stuck_rex]) then @data[:stuck_rex] = rex
629
+ elsif nack.zero? then @data[:stuck_rex_count] = 0
630
+ elsif rex == @data[:stuck_rex] then increment(:stuck_rex_count)
631
+ end
632
+ end
633
+
634
+ # Base RTT: detect VM (clock problem) and reverse path congestion skew
635
+ def log_brtt(s, elapsed, first:, smart_reset:)
636
+ b_rtt = s[:brtt].to_i
637
+ brtt = num(:brtt)
638
+ if brtt.nonzero? && elapsed > 22
639
+ if brtt - b_rtt >= 20 && b_rtt.fdiv(brtt) <= 0.5
640
+ set(:vm) unless smart_reset
641
+ @data[:c_rate] = s[:calc_rate]
642
+ elsif b_rtt < brtt
643
+ increment(:reverse_path)
644
+ if (num(:start_brtt).zero? || b_rtt.fdiv(@data[:start_brtt]) <= 0.7) && @data[:reverse_path] >= 5
645
+ set(:disable_reverse)
646
+ @data[:n_rate] = s[:net_rate].to_i if num(:n_rate).zero? || num(:n_rate) > s[:net_rate].to_i
647
+ end
648
+ end
649
+ @data[:brtt] = b_rtt if b_rtt < brtt
650
+ elsif first && s[:rtt_high].to_i.positive? && elapsed > 21
651
+ @data[:brtt] = b_rtt
652
+ end
653
+ end
654
+
655
+ def check_high_rtt(s)
656
+ rtt = s[:rtt_high].to_i
657
+ srtt = s[:srtt].to_i
658
+ @data[:big_rtt] = [rtt, srtt].max if (rtt > 3000 || srtt > 3000) && rtt != srtt && [rtt, srtt].max > num(:big_rtt)
659
+ @data[:big_rex_srtt] = s[:rex_srtt] if s[:rex_srtt].to_i > 3000 && s[:rex_srtt] > num(:big_rex_srtt)
660
+ end
661
+
662
+ # Values at end of snapshot period
663
+ def update_slot(s, slot, t_bits, elapsed, receiver)
664
+ slot[:diff] = s[:time] - slot[:date1] if slot[:elapsed2] && elapsed - slot[:elapsed2] < 20 && s[:written] == slot[:written2]
665
+ slot.merge!(written2: s[:written], tx_bytes: s[:tx_bytes], elapsed2: elapsed, total2: t_bits, blocks2: s[:blocks], loss2: s[:lost], orig2: s[:orig], rex2: s[:rex])
666
+ duration = slot[:diff].to_i.nonzero? || (elapsed - slot[:elapsed1].to_f)
667
+ slot[:net_rate] = (t_bits - slot[:total1].to_i).fdiv(duration).to_i if duration.nonzero?
668
+ session_time = (num(:elapsed_us) - num(:elapsed_start)) / 1_000_000.0
669
+ @data[:avg] = (s[:blocks].to_i - num(:start_block)).fdiv(session_time).to_i * @data[:mtu] * 8 if session_time.nonzero?
670
+ lost, total =
671
+ if receiver
672
+ lost = slot[:loss2].to_i - slot[:loss1].to_i
673
+ [lost, lost + slot[:orig2].to_i - slot[:orig1].to_i]
674
+ else
675
+ [slot[:rex2].to_i - slot[:rex1].to_i, slot[:blocks2].to_i - slot[:blocks1].to_i]
676
+ end
677
+ return if total.zero?
678
+ slot[:loss_rate] = 100.0 * lost / total
679
+ slot[:loss_speed] = (slot[:net_rate].to_i * lost.fdiv(total)).to_i
680
+ end
681
+
682
+ # Determine which constraint limits the rate: Receiver (for sender), or network, disk, vlink (for receiver)
683
+ def rate_constraint(s, slot, elapsed, receiver, first)
684
+ t_rate = s[:target_rate].to_i
685
+ c_rate = s[:calc_rate].to_i
686
+ return if t_rate == c_rate
687
+ rates =
688
+ if receiver
689
+ RECEIVER_CONSTRAINTS.transform_values { |key| s[key].to_i }.select { |_, rate| rate < t_rate }
690
+ else
691
+ {'Receiver' => c_rate}
692
+ end
693
+ limit = rates.min_by(&:last)&.first
694
+ @constraints.add(limit) if receiver && limit && !first
695
+ return if elapsed <= 20
696
+ @data[:policy] = 'fair'
697
+ count_receiver_cause(s, t_rate) unless receiver
698
+ (slot[:calc_1] ||= []).push(limit) unless limit.nil? || slot[:calc_1]&.include?(limit)
699
+ @constraints.add('disk') if receiver && s[:disk_rate].to_i < t_rate
700
+ @constraints.add(limit) if limit && (first || !receiver)
701
+ rates.each do |name, rate|
702
+ current = (@constraint[name] ||= {})
703
+ next if current[:min_rate].to_i.nonzero? && current[:min_rate] <= rate
704
+ current[:min_rate] = rate
705
+ next unless name == 'network'
706
+ current[:base_rtt] = s[:brtt]
707
+ current[:smooth_rtt] = s[:srtt] if s[:net_rate].to_i == c_rate
708
+ end
709
+ end
710
+
711
+ # Sender (ascp 4): count which rate of the receiver limits the calculated rate
712
+ def count_receiver_cause(s, t_rate)
713
+ rates = RECEIVER_CAUSES.transform_values { |keys| keys.map { |key| s[key].to_i }.select(&:positive?).min }
714
+ cause = rates.compact.select { |_, rate| rate < t_rate }.min_by(&:last)&.first
715
+ @receiver_causes[cause] += 1 if cause
716
+ end
717
+
718
+ # First and latest points, to compute average rate
719
+ def update_calc_points(c_rate, t_bits, elapsed)
720
+ point = num(:calc0).zero? ? 0 : 1
721
+ @data.merge!("calc#{point}": c_rate, "total#{point}": t_bits, "elapsed#{point}": elapsed)
722
+ end
723
+
724
+ def calc_speed
725
+ duration = @data[:elapsed1].to_f - @data[:elapsed0].to_f
726
+ duration.zero? ? 0 : (@data[:total1].to_i - @data[:total0].to_i).fdiv(duration).to_i
727
+ end
728
+
729
+ # Detect peer clock problems/VM from sender side
730
+ def detect_peer_vm(s)
731
+ c_rate = s[:calc_rate].to_i
732
+ t_rate = s[:target_rate].to_i
733
+ vm = (@vm ||= {})
734
+ if vm[:elapsed1].nil?
735
+ vm[vm[:elapsed0].nil? ? :elapsed0 : :elapsed1] = s[:elapsed]
736
+ else
737
+ vm[:elapsed0] = vm[:elapsed1]
738
+ vm[:elapsed1] = s[:elapsed]
739
+ end
740
+ vm[:calc0] = num(:calc0)
741
+ return if vm[:elapsed0].to_f == vm[:elapsed1].to_f
742
+ if t_rate < vm[:calc0]
743
+ vm.merge!(calc0: t_rate, low: 0, high: 0)
744
+ clear(:vm_peer)
745
+ @data[:vm_c_rate] = 0
746
+ end
747
+ vm[:high] = vm[:high].to_i + 1 if c_rate >= 0.8 * vm[:calc0]
748
+ if c_rate < 0.5 * vm[:calc0] && vm[:high].to_i > 2
749
+ if vm[:low].to_i > 2
750
+ set(:vm_peer)
751
+ @data[:vm_c_rate] = c_rate
752
+ elsif vm[:calc0] - c_rate > 20_000_000 && !DISK_THROTTLE_RATES.include?(c_rate)
753
+ vm[:low] = vm[:low].to_i + 1
754
+ end
755
+ end
756
+ return if c_rate <= 0.6 * vm[:calc0]
757
+ clear(:vm_peer)
758
+ vm[:low] = 0
759
+ end
760
+
761
+ # Detect ATM/tail-drop rate limiting
762
+ def check_atm(s, t_bits, elapsed)
763
+ atm = (@atm ||= {})
764
+ c_rate = s[:calc_rate].to_i
765
+ t_rate = s[:target_rate].to_i
766
+ lost = s[:lost].to_i - atm[:loss0].to_i
767
+ sent = lost + s[:orig].to_i - atm[:orig0].to_i
768
+ atm[:atm_loss] = atm[:atm_loss].to_i + 1 if sent.nonzero? && lost.fdiv(sent) > 0.2
769
+ duration = elapsed - atm[:elapsed0].to_f
770
+ rate = duration.zero? ? 0 : (t_bits - atm[:total0].to_i) / duration
771
+ return if rate.zero?
772
+ atm[:atm_ratio] = atm[:atm_ratio].to_i + 1 if c_rate / rate > 1.3
773
+ queue = if t_rate <= 5_000_000 then 5
774
+ elsif t_rate <= 200_000_000 then 10
775
+ else 40
776
+ end
777
+ atm[:atm_queue] = atm[:atm_queue].to_i + 1 if num(:max_queue) < 1000 && t_rate * queue / rate > 0.75 * num(:max_queue)
778
+ end
779
+ end
780
+ end
781
+ end
782
+ end