aspera-cli 4.28.0 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +143 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1633 -609
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +20 -20
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +71 -65
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +16 -8
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +12 -19
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -33
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +6 -1
  50. data/lib/aspera/cli/command_spec.rb +34 -7
  51. data/lib/aspera/cli/completion/ascli.bash +19 -5
  52. data/lib/aspera/cli/completion/ascli.fish +13 -2
  53. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  54. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  55. data/lib/aspera/cli/completion_actions.rb +303 -69
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +3 -0
  59. data/lib/aspera/cli/extended_value.rb +61 -27
  60. data/lib/aspera/cli/formatter.rb +29 -22
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -10
  63. data/lib/aspera/cli/http.rb +12 -9
  64. data/lib/aspera/cli/info.rb +12 -7
  65. data/lib/aspera/cli/mailer.rb +6 -3
  66. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  67. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  68. data/lib/aspera/cli/mcp/tool.rb +15 -14
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +65 -12
  73. data/lib/aspera/cli/options.schema.yaml +487 -63
  74. data/lib/aspera/cli/parser.rb +56 -32
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +244 -35
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +17 -12
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +198 -103
  81. data/lib/aspera/cli/plugins/console.rb +6 -3
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +21 -7
  84. data/lib/aspera/cli/plugins/faspex5.rb +28 -25
  85. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  86. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +28 -22
  88. data/lib/aspera/cli/plugins/node.rb +31 -30
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +63 -11
  93. data/lib/aspera/cli/plugins/shares.rb +86 -16
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +144 -30
  96. data/lib/aspera/cli/preset_manager.rb +52 -18
  97. data/lib/aspera/cli/prompt.rb +4 -1
  98. data/lib/aspera/cli/result.rb +12 -8
  99. data/lib/aspera/cli/runner.rb +69 -46
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +79 -9
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +3 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +29 -20
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +188 -42
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.rb +12 -10
  113. data/lib/aspera/cli/wizard.rb +42 -39
  114. data/lib/aspera/command_line_builder.rb +9 -5
  115. data/lib/aspera/command_line_converter.rb +4 -1
  116. data/lib/aspera/coverage.rb +103 -33
  117. data/lib/aspera/data_repository.rb +4 -1
  118. data/lib/aspera/dot_container.rb +4 -1
  119. data/lib/aspera/duration.rb +5 -2
  120. data/lib/aspera/environment.rb +21 -12
  121. data/lib/aspera/exec_spec.rb +3 -0
  122. data/lib/aspera/faspex_gw.rb +13 -9
  123. data/lib/aspera/faspex_postproc.rb +15 -12
  124. data/lib/aspera/formatter_interface.rb +4 -1
  125. data/lib/aspera/graphql.rb +5 -1
  126. data/lib/aspera/hash_ext.rb +3 -0
  127. data/lib/aspera/id_generator.rb +4 -2
  128. data/lib/aspera/json_rpc/client.rb +6 -5
  129. data/lib/aspera/json_rpc/version.rb +3 -0
  130. data/lib/aspera/line_logger.rb +4 -1
  131. data/lib/aspera/link_header.rb +4 -1
  132. data/lib/aspera/log.rb +18 -5
  133. data/lib/aspera/markdown.rb +5 -2
  134. data/lib/aspera/mime.rb +3 -0
  135. data/lib/aspera/nagios.rb +6 -4
  136. data/lib/aspera/node_emulator.rb +15 -13
  137. data/lib/aspera/oauth/client/base.rb +91 -70
  138. data/lib/aspera/oauth/client/boot.rb +7 -4
  139. data/lib/aspera/oauth/client/factory.rb +6 -3
  140. data/lib/aspera/oauth/client/generic.rb +5 -1
  141. data/lib/aspera/oauth/client/json_credentials.rb +4 -1
  142. data/lib/aspera/oauth/client/jwt.rb +6 -3
  143. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  144. data/lib/aspera/oauth/client/url_json.rb +4 -1
  145. data/lib/aspera/oauth/client/web.rb +39 -12
  146. data/lib/aspera/oauth/server/pkce.rb +6 -5
  147. data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
  148. data/lib/aspera/persistency_action_once.rb +5 -3
  149. data/lib/aspera/persistency_folder.rb +6 -3
  150. data/lib/aspera/preview/file_types.rb +6 -3
  151. data/lib/aspera/preview/generator.rb +9 -5
  152. data/lib/aspera/preview/options.schema.yaml +3 -3
  153. data/lib/aspera/preview/terminal.rb +8 -4
  154. data/lib/aspera/preview/utils.rb +6 -4
  155. data/lib/aspera/products/connect.rb +5 -2
  156. data/lib/aspera/products/desktop.rb +4 -1
  157. data/lib/aspera/products/other.rb +4 -2
  158. data/lib/aspera/products/transferd.rb +6 -2
  159. data/lib/aspera/products/transferd_service.rb +610 -0
  160. data/lib/aspera/proxy_auto_config.js +2 -2
  161. data/lib/aspera/proxy_auto_config.rb +5 -2
  162. data/lib/aspera/rainbow.rb +3 -0
  163. data/lib/aspera/rest/aspera_errors.rb +4 -1
  164. data/lib/aspera/rest/call_error.rb +3 -0
  165. data/lib/aspera/rest/client.rb +22 -29
  166. data/lib/aspera/rest/error_analyzer.rb +5 -2
  167. data/lib/aspera/rest/list.rb +8 -4
  168. data/lib/aspera/rest/parameters.rb +3 -0
  169. data/lib/aspera/rest/util.rb +8 -5
  170. data/lib/aspera/rest.rb +9 -6
  171. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  172. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  173. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  174. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  175. data/lib/aspera/schema/async_tables.yaml +11 -10
  176. data/lib/aspera/schema/documentation.rb +8 -2
  177. data/lib/aspera/schema/reader.rb +7 -0
  178. data/lib/aspera/schema/registry.rb +43 -28
  179. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  180. data/lib/aspera/schema/validator.rb +4 -1
  181. data/lib/aspera/sealed_hash.rb +166 -0
  182. data/lib/aspera/secret/hider.rb +201 -0
  183. data/lib/aspera/secret/reference.rb +150 -0
  184. data/lib/aspera/secret/resolver.rb +212 -0
  185. data/lib/aspera/secret/store/base.rb +153 -0
  186. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  187. data/lib/aspera/secret/store/factory.rb +103 -0
  188. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  189. data/lib/aspera/secret/store/keeper.rb +141 -0
  190. data/lib/aspera/secret/store/macos_security.rb +284 -0
  191. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  192. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  193. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  194. data/lib/aspera/secret/store/secret_service.rb +125 -0
  195. data/lib/aspera/secret/strength.rb +119 -0
  196. data/lib/aspera/servlet_helpers.rb +5 -2
  197. data/lib/aspera/ssh.rb +10 -7
  198. data/lib/aspera/ssl.rb +5 -2
  199. data/lib/aspera/string_ext.rb +3 -0
  200. data/lib/aspera/sync/args.schema.yaml +3 -0
  201. data/lib/aspera/sync/conf.schema.yaml +9 -4
  202. data/lib/aspera/sync/database.rb +3 -0
  203. data/lib/aspera/sync/operations.rb +16 -14
  204. data/lib/aspera/temp_file_manager.rb +8 -4
  205. data/lib/aspera/timer_limiter.rb +3 -0
  206. data/lib/aspera/transfer/error.rb +4 -1
  207. data/lib/aspera/transfer/faux_file.rb +4 -1
  208. data/lib/aspera/transfer/parameters.rb +17 -14
  209. data/lib/aspera/transfer/result.rb +3 -0
  210. data/lib/aspera/transfer/resumer.rb +6 -4
  211. data/lib/aspera/transfer/spec.rb +9 -22
  212. data/lib/aspera/transfer/units.rb +75 -0
  213. data/lib/aspera/uri_reader.rb +57 -7
  214. data/lib/aspera/web_auth/index.html.erb +6 -2
  215. data/lib/aspera/web_auth/paste.rb +109 -0
  216. data/lib/aspera/web_auth/receiver.rb +50 -0
  217. data/lib/aspera/web_auth/relay.rb +76 -0
  218. data/lib/aspera/web_auth/reply.html.erb +150 -0
  219. data/lib/aspera/web_auth/server.rb +47 -10
  220. data/lib/aspera/web_page.rb +7 -2
  221. data/lib/aspera/web_server_simple.rb +6 -3
  222. data/lib/aspera/yaml.rb +5 -1
  223. data/llms.txt +81 -0
  224. data/skills/ascli/SKILL.md +88 -0
  225. data.tar.gz.sig +0 -0
  226. metadata +72 -19
  227. metadata.gz.sig +0 -0
  228. data/lib/aspera/keychain/base.rb +0 -30
  229. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  230. data/lib/aspera/keychain/factory.rb +0 -58
  231. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  232. data/lib/aspera/keychain/macos_security.rb +0 -277
  233. data/lib/aspera/keychain/one_password_api.rb +0 -91
  234. data/lib/aspera/keychain/one_password_base.rb +0 -39
  235. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  236. data/lib/aspera/secret_hider.rb +0 -196
  237. data/lib/aspera/secret_strength.rb +0 -114
@@ -0,0 +1,593 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:words nstore pids throughputs
7
+ require_relative '../../assert'
8
+ require_relative 'rules'
9
+ require_relative 'advice'
10
+ require_relative 'lines'
11
+ require_relative 'session'
12
+ require 'set'
13
+ require 'time'
14
+
15
+ module Aspera
16
+ module Ascp
17
+ class LogAnalyzer
18
+ # Analysis of one ascp log file: sessions, snapshot periods and statistics
19
+ class LogFile
20
+ include Lines
21
+
22
+ # Statistic => finding, when not zero
23
+ FINDINGS = {
24
+ stuck_nack: 'Retransmission requests (NACKs) were stuck in %d transfers: check the logs.',
25
+ filename_too_long: '%d files failed because their Windows path is longer than 260 characters.',
26
+ gap: 'ascp was blocked (IO wait) %d times.',
27
+ gap_ssh: 'ascp was blocked (IO wait) %d times, with an SSH timeout.',
28
+ buffer_died: 'The system ran out of buffer memory, and ascp exited, %d times.',
29
+ timeout_died: 'A data or SSH timeout made ascp exit %d times.',
30
+ decryption_died: 'A file decryption error made ascp exit %d times.',
31
+ table_died: 'ascp exited at the end of %d sessions, without logging the statistics table.',
32
+ died: 'ascp exited, was blocked (IO wait), or stopped logging, %d times: check the logs.',
33
+ win_clock: 'The clock of the system moves backwards: update the firmware of the motherboard.',
34
+ vm: 'This system may run on a VM, or have another clock problem, which affected %d transfers.',
35
+ disable_reverse: 'Reverse path congestion inference skew affected %d transfers: check the logs, and disable it if appropriate.',
36
+ vm_peer: 'Peers may run on a VM, or have another clock problem, which affected %d transfers.',
37
+ ossl_modules_missing: 'The OpenSSL modules folder of ascp was not found in %d sessions (configuration, e.g. installation of the HTTP gateway): no effect unless an OpenSSL provider (e.g. FIPS) is needed.',
38
+ change_event_failed: 'A change of the schedule of the virtual link capacity could not be registered in %d sessions: the schedule is not applied.'
39
+ }.freeze
40
+ # Totals of a snapshot period
41
+ PERIOD_TOTALS = %i[rate rate_in rate_out calc calc_in calc_out target target_in target_out loss_in loss_out].freeze
42
+ # A retry has the same values for these
43
+ RETRY_KEYS = %i[peer op source dest].freeze
44
+
45
+ attr_reader :path, :stats, :licenses, :first_time, :last_time, :thresholds
46
+
47
+ # @param path [String] Log file
48
+ # @param period [Integer] Snapshot period in seconds
49
+ # @param zero_bytes [Boolean] Include sessions without transferred bytes
50
+ # @param mtu [Integer, nil] MTU to use when not found in logs
51
+ # @param thresholds [Hash] Diagnostic thresholds
52
+ # @param timezone [String, nil] Time zone of the log lines: `UTC` or `+HH:MM`. Time zone of the analyst if nil.
53
+ def initialize(path, period: 60, zero_bytes: false, mtu: nil, thresholds: {}, timezone: nil)
54
+ @path = path
55
+ @timezone = timezone
56
+ @period = period
57
+ @zero_bytes = zero_bytes
58
+ @mtu = mtu
59
+ thresholds = thresholds.transform_keys(&:to_sym)
60
+ unknown = thresholds.keys - DEFAULT_THRESHOLDS.keys
61
+ Aspera.assert(unknown.empty?, type: ArgumentError) { "Unknown threshold: #{unknown.join(', ')}" }
62
+ @thresholds = DEFAULT_THRESHOLDS.merge(thresholds)
63
+ # PID => current Session of process
64
+ @current = {}
65
+ # All sessions, in order of creation
66
+ @sessions = []
67
+ # PID => user and version from configuration lines
68
+ @config = Hash.new { |h, k| h[k] = {} }
69
+ @stats = Hash.new(0)
70
+ @stats[:errors] = Hash.new(0)
71
+ @stats[:error_categories] = Hash.new(0)
72
+ # Start time => {start:, stop:, slots: {Session => Hash}, totals: Hash}
73
+ @periods = {}
74
+ # Start time of the latest period with samples
75
+ @bucket = nil
76
+ # [license number, IP] => {product:, account:, os:}
77
+ @licenses = {}
78
+ end
79
+
80
+ # Read and analyze the log file
81
+ # @return [self]
82
+ def analyze
83
+ LogAnalyzer.read_lines(@path, Encoding::BINARY) { |line| parse_line(normalize(line)) }
84
+ @sessions.each(&:evaluate_atm)
85
+ @periods.sort.each { |time, period| end_of_period(time, period) }
86
+ compute_statistics
87
+ self
88
+ end
89
+
90
+ # @return [Array<Session>] transfer sessions to report, sorted by date
91
+ def sessions
92
+ transfer_sessions.select { |s| counted?(s) }.sort_by { |s| [s.num(:date), s[:user].to_s] }
93
+ end
94
+
95
+ # @return [Array<Hash>] one row per session
96
+ def session_reports
97
+ windows_log = @path.match?(/\w:/)
98
+ sessions.map { |s| {'file' => @path}.merge(s.report(windows_log: windows_log)) }
99
+ end
100
+
101
+ class << self
102
+ # @param stats [Hash] statistics of one or several log files
103
+ # @return [Array<String>] findings from statistics
104
+ def findings(stats)
105
+ result = FINDINGS.filter_map do |key, message|
106
+ if stats[key].positive?
107
+ message.include?('%d') ? format(message, stats[key]) : message
108
+ end
109
+ end
110
+ result.insert(1, 'The stuck retransmission requests may have been caused by high packet loss.') if stats[:stuck_nack_loss] > 0.4
111
+ if stats[:smart_reset].positive?
112
+ result.push("ascp reset the base RTT #{stats[:smart_reset]} times.#{' Consider changing the rtt_autocorrect parameters.' if stats[:smart_reset] > 100}")
113
+ end
114
+ result
115
+ end
116
+ end
117
+
118
+ # @return [Array<Hash>] one row per multi-session transfer: sessions with the same transfer identifier
119
+ def multi_sessions
120
+ parallel = transfer_sessions.select { |s| s.num(:cluster_nodes) > 1 && s[:xfer_id] }
121
+ parallel.group_by { |s| s[:xfer_id] }.map do |xfer_id, group|
122
+ group = group.sort_by { |s| s.num(:cluster_node) }
123
+ start = group.filter_map { |s| s[:start] }.min
124
+ stop = group.filter_map { |s| s[:stop] || s[:last] }.max
125
+ bytes = group.sum { |s| s.num(:tx_bytes) }
126
+ {
127
+ 'file' => @path,
128
+ 'transfer_id' => xfer_id,
129
+ 'nodes' => group.first[:cluster_nodes],
130
+ 'pids' => group.map(&:pid),
131
+ 'peer' => group.first[:peer],
132
+ 'direction' => group.first.direction,
133
+ 'status' => group.all? { |s| s[:status] == 'success' } ? 'success' : 'failed',
134
+ 'start' => iso(start),
135
+ 'stop' => iso(stop),
136
+ 'transmitted_bytes' => bytes,
137
+ 'average_rate' => (bytes * 8 / (stop - start) if start && stop && stop > start)
138
+ }.compact
139
+ end
140
+ end
141
+
142
+ # @return [Array<Hash>] one row per diagnostic of sessions: id, severity, message, recommendation (if any)
143
+ def diagnostics
144
+ windows_log = @path.match?(/\w:/)
145
+ sessions.flat_map do |s|
146
+ s.report(windows_log: windows_log)
147
+ s.diagnostics.map do |diagnostic|
148
+ {'file' => @path, 'pid' => s.pid, 'session' => s[:uuid]}.compact.merge(diagnostic, 'recommendation' => RECOMMENDATIONS[diagnostic['id'].to_sym]).compact
149
+ end
150
+ end
151
+ end
152
+
153
+ # @return [Array<String>] ascp versions found in log, older than supported
154
+ def old_versions
155
+ transfer_sessions.filter_map { |s| s[:ver] if Session.version_older?(s[:ver], MIN_VERSION) }.uniq
156
+ end
157
+
158
+ # @return [Array<Hash>] one row per snapshot period
159
+ def snapshots
160
+ reported_periods.map do |time, period|
161
+ t = period[:totals]
162
+ row = {'file' => @path, 'time' => iso(time), 'sessions' => active_sessions(period[:slots])}
163
+ if @period > 600 && period[:stop].to_i > period[:start].to_i
164
+ duration = period[:stop] - period[:start]
165
+ row.merge!('rate' => t[:data] / duration, 'rate_in' => t[:data_in] / duration, 'rate_out' => t[:data_out] / duration)
166
+ else
167
+ row.merge!(PERIOD_TOTALS.to_h { |k| [k.to_s, t[k]] })
168
+ end
169
+ end
170
+ end
171
+
172
+ # @return [Array<Hash>] one row per session in each snapshot period
173
+ def snapshot_sessions
174
+ reported_periods.flat_map do |time, period|
175
+ period[:slots].sort_by { |session, _| session.pid.to_i }.filter_map do |session, slot|
176
+ next if !@zero_bytes && slot[:net_rate].to_i.zero?
177
+ progress = (100.0 * slot[:tx_bytes].to_i / session[:tx_bytes]).round(2) if session.flag?(:statistics) && session.num(:tx_bytes).nonzero?
178
+ {
179
+ 'file' => @path,
180
+ 'time' => iso(time),
181
+ 'pid' => session.pid,
182
+ 'session' => session[:uuid],
183
+ 'rate' => slot[:net_rate].to_i,
184
+ 'calculated' => slot[:avg_calc],
185
+ 'target' => slot[:target_rate],
186
+ 'loss' => slot[:loss_rate]&.round(2),
187
+ 'progress' => progress,
188
+ 'direction' => slot[:receiver] ? 'receive' : 'send',
189
+ 'constraint' => slot[:calc_1]&.join(',')
190
+ }.compact
191
+ end
192
+ end
193
+ end
194
+
195
+ private
196
+
197
+ # @return [Boolean] true if session is counted in statistics and reported
198
+ def counted?(session)
199
+ @zero_bytes || session.num(:tx_bytes).nonzero? || session[:status] == 'failed'
200
+ end
201
+
202
+ # @return [Session] current session of process
203
+ def session(pid)
204
+ @current[pid] || new_session(pid)
205
+ end
206
+
207
+ # A process starts a new session
208
+ # @return [Session] new current session of process
209
+ def new_session(pid)
210
+ s = Session.new(pid, thresholds: @thresholds, timezone: @timezone)
211
+ @sessions.push(s)
212
+ @current[pid] = s
213
+ end
214
+
215
+ # A new process (configuration line) reuses the PID of a process whose session ended
216
+ def check_pid_reuse(pid)
217
+ current = @current[pid]
218
+ new_session(pid) if current && (current.flag?(:statistics) || current[:stop])
219
+ end
220
+
221
+ # @return [Boolean] true if line is not relevant
222
+ def ignored?(line, pid)
223
+ return true unless line.match?(/\w/)
224
+ return true if IGNORED_TEXTS.any? { |text| line.include?(text) } || line.match?(IGNORED_LINES)
225
+ line.match?(OTHER_PROCESSES) && line.match?(/(?:#{OTHER_PROCESSES.source}).*?#{pid}/)
226
+ end
227
+
228
+ def parse_line(line)
229
+ track_ascp(line)
230
+ pid = line[PID, 1] or return
231
+ time = parse_time(line[LINE_TIME, 1])
232
+ if time
233
+ @first_time = time if @first_time.nil? || time < @first_time
234
+ @last_time = time if @last_time.nil? || time > @last_time
235
+ end
236
+ if line.include?('rex buffer full')
237
+ session(pid).set(:rex_buffer_full)
238
+ return
239
+ end
240
+ return if ignored?(line, pid)
241
+ @stats[:win_clock] = 1 if line.include?('_usec_since') && line.match?(/get(?:nstore)?_usec_since/)
242
+ process = line[PROCESS, 1]
243
+ ascp = process.nil? || process.match?(ASCP_PROCESS)
244
+ if ascp && line.include?('Configuration:') && (user = line[/Configuration:\s+using.*?user\s+(.*)/, 1])
245
+ check_pid_reuse(pid)
246
+ @config[pid][:user] = user
247
+ session(pid)[:role] = user == '-' ? 'client' : 'server'
248
+ end
249
+ if ascp && line.include?('Initializing') && (version = line[/Initializing\s+FASP\s+version\s+((?:\d\.?)+)/, 1])
250
+ check_pid_reuse(pid)
251
+ session(pid).set(:transfer)
252
+ @config[pid][:ver] = version
253
+ # Authentication is logged after this line (the product name before or after it, depending on the version)
254
+ @config[pid].delete(:auth)
255
+ license(@config[pid], line)
256
+ end
257
+ return if line.include?('ascmd') || line.include?('aspshell')
258
+ return unless ascp
259
+ s = session(pid)
260
+ if line.include?('ERR') && (error = line[LOG_ERROR, 1])
261
+ s[:last_error] = error.strip
262
+ s[:last_error_time] = time
263
+ end
264
+ s.set(:no_reverse_infer) if line.include?('Disabling reverse path congestion inference')
265
+ if line.include?('Session') && (m = line.match(SESSION_START_STOP))
266
+ s = session_line(s, pid, m[2].downcase.to_sym, m[1], line)
267
+ end
268
+ if time && s[:last] && time - s[:last] > 23
269
+ gap = time - s[:last] - 21
270
+ s[:gap] = gap if gap > s.num(:gap)
271
+ s[:gap_last] = time
272
+ end
273
+ s[:last] = time
274
+ s.session_params(Records.key_values(line[/Session\s+Params\s+(.*)/, 1] || line)) if line.include?('FASP Session Params')
275
+ if line.include?(' DS Qs ')
276
+ parse_ds_qs(pid, line)
277
+ else
278
+ apply_rules(s, line)
279
+ end
280
+ end
281
+
282
+ # Apply the first matching rule of LINE_RULES
283
+ def apply_rules(s, line)
284
+ match, effects = LINE_RULES.find { |regexes, _| Array(regexes).all? { |re| line.match?(re) } }
285
+ return unless effects
286
+ effects[:call] ? send(effects[:call], s, line.match(Array(match).first)) : s.apply(effects)
287
+ end
288
+
289
+ # Maximum rate (bps, nil if unlimited) and expiry date (nil if none) of the license, from the `Initializing FASP version` line
290
+ def license(config, line)
291
+ mbps = line[LICENSE_RATE, 1]
292
+ config[:license_rate] = mbps && (mbps.to_i * 1_000_000)
293
+ config[:license_expires] = line[LICENSE_EXPIRES, 1]
294
+ end
295
+
296
+ # `Session Start` or `Session Stop` line
297
+ # @return [Session] session of line: a new one if the process starts another session
298
+ def session_line(s, pid, kind, stamp, line)
299
+ fields = Records.key_values(line[/Session\s+(?:Start|Stop)\s+(.*)/, 1] || line)
300
+ f = {
301
+ uuid: fields['uuid'] || fields['id'], op: fields['op'], user: fields['user'],
302
+ source: (fields['source']&.match?(/\s/) ? fields['source'].split.first : fields['source']),
303
+ dest: (fields['dest']&.match?(/\s/) ? fields['dest'].split.first : fields['dest']),
304
+ transport: (fields.key?('wss_port') ? 'wss' : ('ssh' if fields.key?('tcp_port'))),
305
+ os: fields['os'], ver: fields['ver']&.split&.first, peeros: fields['peeros'], peerver: fields['peerver']&.split&.first, lic: fields['lic'], peerlic: fields['peerlic']
306
+ }.compact
307
+ if f[:uuid] && s[:uuid] && s[:uuid] != f[:uuid]
308
+ role = s[:role]
309
+ s = new_session(pid)
310
+ s[:role] = role
311
+ end
312
+ s[:gap] = 0 if kind.eql?(:start)
313
+ f[:local], f[:l_port] = address(fields['local'])
314
+ f[:peer], f[:p_port] = address(fields['peer'])
315
+ s.set(:transfer) if f[:peer]
316
+ record_license(f[:lic], f[:local], f[:os])
317
+ record_license(f[:peerlic], f[:peer], f[:peeros])
318
+ s.session_line(kind, f, parse_time(stamp), @config[pid])
319
+ s.stop_error(line, fields['status']) if kind.eql?(:stop)
320
+ s
321
+ end
322
+
323
+ # @return [Array] IP and port from `IP:port`
324
+ def address(value)
325
+ m = value.to_s.match(/\A(.*):(\d+)\z/) or return [nil, nil]
326
+ [m[1], m[2].to_i]
327
+ end
328
+
329
+ # @param license [String] product:account:license
330
+ def record_license(license, ip, os)
331
+ return if license.nil?
332
+ product, account, number = license.split(':').map(&:to_i)
333
+ @licenses[[number, ip]] = {product: product, account: account, os: os} unless product == 6
334
+ end
335
+
336
+ # --- LINE_RULES handlers ---
337
+
338
+ def end_of_statistics(s, _m)
339
+ s.set(:stats_end)
340
+ s.delete(:last)
341
+ end
342
+
343
+ def transfer_stop(s, m)
344
+ fields = Records.key_values(m[1])
345
+ s[:uuid] ||= fields['uuid']
346
+ @stats[:filename_too_long] += 1 if s.transfer_stop(fields['op'], fields['status'], fields['file'], fields['size'].to_i, fields['start_byte'].to_i, fields['errstr'])
347
+ end
348
+
349
+ def brtt_reset(s, _m)
350
+ s.increment(:brtt_reset)
351
+ @stats[:smart_reset] += 1
352
+ end
353
+
354
+ def rate_params(s, m)
355
+ s.rate_params(m[2].to_i, m[3].to_i, m[4], by_peer: m[1].start_with?('Peer'))
356
+ end
357
+
358
+ def path_mtu(s, m)
359
+ s[:mtu] = m[1].to_i
360
+ s[:start_brtt] = m[2].to_i
361
+ end
362
+
363
+ def datagram_size(s, m)
364
+ s[:mtu] = m[1].to_i
365
+ s[:payload] = m[2].to_i
366
+ s[:path_mtu] = m[3].to_i if m[3]
367
+ end
368
+
369
+ def conf_parse_error(s, m)
370
+ s[:conf_parse] = m[1]
371
+ end
372
+
373
+ # Error when closing the SSH connection: keep the first one
374
+ def ssh_disconnect_error(s, m)
375
+ s[:ssh_disconnect] ||= m[0]
376
+ end
377
+
378
+ # Kernel line: error on the PID, and on PID - 11
379
+ def segfault(s, _m)
380
+ s[:error] = 'segfault'
381
+ session((s.pid.to_i - 11).to_s)[:error] = 'segfault'
382
+ end
383
+
384
+ # No packet received: the receiver logs it from 2 seconds, the sender (no feedback of the receiver) from 3 seconds
385
+ def no_data(s, m)
386
+ sender = m[1] == 'Sender'
387
+ key = sender ? :no_feedback : :no_data
388
+ first = sender ? 3 : 2
389
+ seconds = m[2].to_i
390
+ s[key] = seconds if s.num(key) <= seconds
391
+ s.increment(:"#{key}_total", seconds == first ? first : 1)
392
+ end
393
+
394
+ # Product name: `IBM Aspera High-Speed Transfer Server version 4.4.8.2596`, logged once by the process
395
+ def product_name(s, m)
396
+ @config[s.pid][:product_name] = m[1]
397
+ end
398
+
399
+ # Authentication with a token (`ASPERA_AUTH_TOKEN`), also when the line of the access key of the node storage follows
400
+ def token_auth(s, _m)
401
+ @config[s.pid][:auth] = 'token'
402
+ end
403
+
404
+ # Storage of a node access key, without token: authentication of the node user
405
+ def node_auth(s, _m)
406
+ @config[s.pid][:auth] ||= 'node'
407
+ end
408
+
409
+ def rex_timer_overflow(s, _m)
410
+ s.increment(:rex_overflow)
411
+ end
412
+
413
+ def session_statistics(s, m)
414
+ fields = Records.statistics_fields(m[1])
415
+ s.set(:statistics)
416
+ s[:effective_rate] = fields['effective_rate']
417
+ progress = fields['progress'] || {}
418
+ s[:tx_bytes] = progress['tx_bytes'].to_i
419
+ s[:tx_time] = progress['tx_time'].to_i
420
+ s[:retransmitted] = fields['solicited_rex']&.to_f
421
+ s[:duplicates] = fields['rcvd_dups']&.to_f
422
+ s[:efficiency] = fields['effective']&.to_f
423
+ s[:control_loss] = %w[sndr rcvr rex].filter_map { |dir| (ctl = fields["#{dir}_ctl"]) && [dir, [ctl['sent'].to_i, ctl['lost_pct'].to_f]] }.to_h
424
+ end
425
+
426
+ # Cipher of server configuration replaces the requested one
427
+ def cipher_override(s, m)
428
+ s[:cipher_override] = m.captures.map { |id| CIPHERS.fetch(id.to_i, id) }
429
+ end
430
+
431
+ def resume_downgrade(s, m)
432
+ s[:resume_downgrade] = m[1]
433
+ end
434
+
435
+ def job_size_change(s, m)
436
+ s[:job_size] = m.captures.map(&:to_i)
437
+ end
438
+
439
+ # Virtual link: bandwidth limit shared by sessions
440
+ def vlink_init(s, m)
441
+ (s[:vlinks] ||= {})[m[1].to_i] = {'type' => m[2], 'capacity' => m[3].to_i}
442
+ end
443
+
444
+ # --- Snapshot periods ---
445
+
446
+ def add_sample(_s, m)
447
+ sample_match = m.string.match(SAMPLE) or return
448
+ receiver = sample_match[3].eql?('Receiver')
449
+ values = Records.group_fields(sample_match[4])
450
+ data = (receiver ? RECEIVER_SAMPLE : SENDER_SAMPLE).transform_values { |path| values.dig(*path) }
451
+ return if data.values_at(:tx_bytes, :written, :elapsed_us).any?(&:nil?)
452
+ data[:disk_read_rate] = values['drate'] unless receiver
453
+ session = session(sample_match[2])
454
+ session[:dir] = receiver ? 'receive' : 'send'
455
+ time = parse_time(sample_match[1]) or return
456
+ data.merge!(time: time, receiver: receiver, elapsed: format('%.2f', data[:elapsed_us] / 1_000_000.0).to_f)
457
+ bucket = (time / @period) * @period
458
+ # The previous period is complete: ATM detection works on consecutive periods, in log order
459
+ @sessions.each(&:evaluate_atm) if @bucket && bucket > @bucket
460
+ @bucket = bucket if @bucket.nil? || bucket > @bucket
461
+ period = (@periods[bucket] ||= {slots: {}})
462
+ period[:start] ||= time
463
+ period[:stop] = time
464
+ @stats[:smart_reset] += data[:brtt_resets] if data[:brtt_resets].to_i.positive?
465
+ first = !period[:slots].key?(session)
466
+ slot = (period[:slots][session] ||= {})
467
+ session.add_sample(data, slot, first: first, smart_reset: @stats[:smart_reset].positive?, mtu: @mtu)
468
+ @stats[:last] = time
469
+ end
470
+
471
+ # Parse and accumulate a `DS Qs` line into the current session
472
+ def parse_ds_qs(pid, line)
473
+ text = line[/(?:Receiver|Sender)\s+DS\s+(.*)/, 1] or return
474
+ queues = Records.queues_fields(text)
475
+ qs = queues['Qs'] || {}
476
+ qs['mgmt_backlog'] = queues['mgmt_backlog'].sum { |h| h['n'].to_i } if queues['mgmt_backlog']
477
+ session(pid).add_ds_qs_sample(qs) unless qs.empty?
478
+ end
479
+
480
+ # @return [Integer] number of sessions in period
481
+ def active_sessions(slots)
482
+ @zero_bytes ? slots.size : slots.count { |_, slot| slot[:total1].to_i.positive? || slot[:total2].to_i.positive? }
483
+ end
484
+
485
+ # Totals of period, concurrency of sessions, maximum throughputs
486
+ def end_of_period(date, period)
487
+ slots = period[:slots]
488
+ active = active_sessions(slots)
489
+ totals = period[:totals] = Hash.new(0)
490
+ slots.each_key do |s|
491
+ slot = slots[s]
492
+ s.set(:transfer)
493
+ if slot[:elapsed2].to_f == slot[:elapsed1].to_f
494
+ slots.delete(s)
495
+ next
496
+ end
497
+ calc = slot[:calc_rate]
498
+ slot[:avg_calc] = Session.average(calc).to_i
499
+ slot[:avg_calc] = [slot[:avg_calc], calc.max].max if slot[:avg_calc] < slot[:net_rate].to_i
500
+ way = slot[:receiver] ? :in : :out
501
+ data = slot[:total2].to_i - slot[:total1].to_i
502
+ if data.positive?
503
+ totals[:"calc_#{way}"] += slot[:avg_calc]
504
+ totals[:data] += data
505
+ totals[:"data_#{way}"] += data
506
+ totals[:"target_#{way}"] += slot[:target_rate].to_i
507
+ end
508
+ totals[:rate] += slot[:net_rate].to_i
509
+ totals[:"rate_#{way}"] += slot[:net_rate].to_i
510
+ totals[:"loss_#{way}"] += slot[:loss_speed].to_i
511
+ s[:sess_min] = active if s.num(:sess_min).zero? || s[:sess_min] >= active
512
+ if s.num(:sess_max) <= active
513
+ s[:sess_max] = active
514
+ s[:sess_max_time] = date
515
+ end
516
+ end
517
+ totals[:calc] = totals[:calc_in] + totals[:calc_out]
518
+ totals[:target] = totals[:target_in] + totals[:target_out]
519
+ {max_sess: active, max_total: totals[:rate], max_out: totals[:rate_out], max_in: totals[:rate_in]}.each do |key, value|
520
+ next if value <= @stats[key]
521
+ @stats[key] = value
522
+ @stats[:"#{key}_time"] = date
523
+ end
524
+ end
525
+
526
+ # @return [Array] periods with traffic
527
+ def reported_periods
528
+ @periods.sort.select { |_, period| period[:totals] && (@zero_bytes || period[:totals][:rate_in].nonzero? || period[:totals][:rate_out].nonzero?) }
529
+ end
530
+
531
+ # --- Statistics ---
532
+
533
+ def compute_statistics
534
+ starts = transfer_sessions.filter_map { |s| s[:start] }
535
+ transfer_sessions.each do |s|
536
+ s.shared_link(starts)
537
+ s.clear(:vm) if @stats[:smart_reset].positive?
538
+ exit_category = s.check_exit(@stats[:last])
539
+ @stats[exit_category] += 1 if exit_category
540
+ cancelled = s[:error].to_s.match?(/cancelled/i)
541
+ if counted?(s)
542
+ @stats[:total_sessions] += 1
543
+ case s[:status]
544
+ when 'success' then @stats[:success] += 1
545
+ when 'failed' then @stats[:failed] += 1 unless cancelled
546
+ end
547
+ end
548
+ @stats[:http_fallback] += 1 if s.flag?(:http_fallback)
549
+ if s.num(:gap).positive?
550
+ @stats[:gap] += 1 unless s.flag?(:ssh_timeout)
551
+ @stats[:gap_ssh] += 1 if s.flag?(:ssh_timeout)
552
+ end
553
+ if s[:error]
554
+ @stats[:cancelled] += 1 if cancelled
555
+ @stats[:errors][s[:error]] += 1 unless cancelled
556
+ end
557
+ @stats[:ossl_modules_missing] += 1 if s.flag?(:ossl_modules_missing)
558
+ @stats[:change_event_failed] += 1 if s.flag?(:change_event_failed)
559
+ @stats[:vm] += 1 if s.flag?(:vm)
560
+ @stats[:vm_peer] += 1 if s.flag?(:vm_peer)
561
+ @stats[:disable_reverse] += 1 if s.flag?(:disable_reverse) && !s.flag?(:no_reverse_infer)
562
+ if s.stuck_nack?
563
+ @stats[:stuck_nack] += 1
564
+ @stats[:stuck_nack_loss] = [@stats[:stuck_nack_loss], s.num(:loss_rate)].max
565
+ end
566
+ @stats[:error_categories][s.error_category] += 1 if counted?(s) && s[:status] == 'failed' && s.error_category
567
+ end
568
+ link_retries
569
+ end
570
+
571
+ # @return [Array<Session>] all transfer sessions, including those not reported
572
+ def transfer_sessions
573
+ @sessions.select { |s| s.flag?(:transfer) }
574
+ end
575
+
576
+ # Link a failed session to the next session with same peer, direction, source and destination
577
+ def link_retries
578
+ groups = transfer_sessions.select { |s| s[:source] && s[:start] }.group_by { |s| RETRY_KEYS.map { |key| s[key] } }
579
+ groups.each_value do |group|
580
+ group.sort_by { |s| s[:start] }.each_cons(2) do |previous, current|
581
+ next unless previous[:status] == 'failed' && current[:start] > previous[:start]
582
+ next if previous[:xfer_id] && previous[:xfer_id] == current[:xfer_id]
583
+ next if current[:start] - (previous[:stop] || previous[:last] || previous[:start]) > @thresholds[:retry_window]
584
+ previous[:retried_by] = current.pid
585
+ current[:retry_of] = previous.pid
586
+ @stats[:retries] += 1
587
+ end
588
+ end
589
+ end
590
+ end
591
+ end
592
+ end
593
+ end