@calltelemetry/cli 0.12.2 → 0.12.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,7 +10,9 @@
10
10
  * keepalive; this bundle surfaces the DB state, NATS KV view, and the log lines
11
11
  * that explain why that keepalive never lands.
12
12
  */
13
- import { writeFile, chmod } from 'node:fs/promises';
13
+ import { writeFile, chmod, mkdir, symlink, unlink } from 'node:fs/promises';
14
+ import { existsSync, statSync } from 'node:fs';
15
+ import { dirname, join } from 'node:path';
14
16
  import { compose, composeExec, composePs } from './compose.js';
15
17
  import { exec } from './exec.js';
16
18
  import { rpc } from './rpc.js';
@@ -31,6 +33,16 @@ const LONG_TIMEOUT = 25_000;
31
33
  // timeout alone (API-1791 gap fix).
32
34
  const PG_OPTIONS_DEFAULT = "-c statement_timeout=5000 -c lock_timeout=1000";
33
35
  const PG_OPTIONS_LONG = "-c statement_timeout=15000 -c lock_timeout=2000";
36
+ // Bridge leg BPF (docker backend leg — plaintext XCC/CURRI after Caddy
37
+ // terminates TLS). Parity with the bash _cd_xcc_pcap_bridge_filter / Go
38
+ // cubeDiagBridgePcapFilter.
39
+ const PCAP_BRIDGE_FILTER = '(tcp port 4080 or tcp port 4000)';
40
+ // Cube-router TLS port clause for the host NIC ingress leg. Parity with the
41
+ // bash/Go port-only fallback when no xcc_hostname/source IP resolves.
42
+ const PCAP_PORT_CLAUSE = '(tcp port 443 or tcp port 80)';
43
+ // Only safe-shaped hostnames/IPs are allowed into a tcpdump BPF filter string
44
+ // (parity with bash _cd_xcc_pcap_host_valid / Go cubeDiagValidPcapHost).
45
+ const PCAP_HOST_RE = /^[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?$/;
34
46
  /** Sleep helper for the heartbeat-trace loop. */
35
47
  function sleep(ms) {
36
48
  return new Promise(resolve => setTimeout(resolve, ms));
@@ -96,6 +108,115 @@ Enum.each(cubes, fn c ->
96
108
  IO.puts(out)
97
109
  end)
98
110
  :ok`;
111
+ /**
112
+ * Snapshot of the ProbeFastDiag fast-path counters (API-1838). Backend module
113
+ * ships on 0.8.7-stable; on an older appliance the module simply doesn't
114
+ * exist and this RPC errors or returns garbage — the renderer below must
115
+ * tolerate that (see `renderProbeFastDiag`).
116
+ *
117
+ * NOTE: same `${...}`-free rule as CUBE_DIAG_RPC_SNAPSHOT above (this one has
118
+ * no Elixir string interpolation at all, but keep the convention documented).
119
+ */
120
+ const CUBE_DIAG_RPC_PROBE_FAST = `Cdrcisco.Cubes.ProbeFastDiag.snapshot() |> Jason.encode!() |> IO.puts()`;
121
+ /**
122
+ * Render the probe_fast snapshot payload into the terse KeyValue-ish text
123
+ * block used by the rest of cube-diag's sections.
124
+ *
125
+ * Payload contract (extend-only; every leaf may independently be the string
126
+ * "unavailable", and the whole payload may be `{"error":"probe_fast_unavailable"}`):
127
+ * - window_seconds: number | "unavailable"
128
+ * - counts: { handled, fallthrough, fault_contained } | "unavailable"
129
+ * - fast_path_enabled: boolean | "unavailable"
130
+ * - budget: { wall_budget_ms, probe_wall_budget_ms,
131
+ * call_control_statement_timeout_ms } | "unavailable"
132
+ * - cube_config_cache_entries: number | "unavailable"
133
+ *
134
+ * On an appliance older than the module's release, the RPC either throws
135
+ * (caught by `section()`) or returns non-JSON / unrecognized JSON — both are
136
+ * treated as "unavailable", never a crash.
137
+ */
138
+ function renderProbeFastDiag(raw) {
139
+ const trimmed = raw.trim();
140
+ const oldApplianceHint = 'probe fast-path diagnostics unavailable (appliance < 0.8.7-rc?)';
141
+ if (!trimmed)
142
+ return oldApplianceHint;
143
+ let payload;
144
+ try {
145
+ payload = JSON.parse(trimmed);
146
+ }
147
+ catch {
148
+ return oldApplianceHint;
149
+ }
150
+ if (typeof payload !== 'object' || payload === null || Array.isArray(payload)) {
151
+ return oldApplianceHint;
152
+ }
153
+ const obj = payload;
154
+ if (typeof obj.error === 'string') {
155
+ return `probe fast-path diagnostics unavailable (${obj.error})`;
156
+ }
157
+ // A recognizable payload needs at least one of the documented top-level
158
+ // keys — otherwise this is "unrecognized JSON", treat like an old appliance.
159
+ const knownKeys = ['window_seconds', 'counts', 'fast_path_enabled', 'budget', 'cube_config_cache_entries'];
160
+ if (!knownKeys.some(k => k in obj)) {
161
+ return oldApplianceHint;
162
+ }
163
+ const fmt = (v) => (v === undefined ? 'unavailable' : typeof v === 'string' ? v : JSON.stringify(v));
164
+ const lines = [];
165
+ lines.push(`window_seconds : ${fmt(obj.window_seconds)}`);
166
+ const counts = obj.counts;
167
+ let handled = 'unavailable';
168
+ let fallthrough = 'unavailable';
169
+ let faultContained = 'unavailable';
170
+ if (counts && typeof counts === 'object' && !Array.isArray(counts)) {
171
+ const c = counts;
172
+ handled = c.handled ?? 'unavailable';
173
+ fallthrough = c.fallthrough ?? 'unavailable';
174
+ faultContained = c.fault_contained ?? 'unavailable';
175
+ }
176
+ else if (counts !== undefined) {
177
+ // counts itself may be the literal string "unavailable".
178
+ handled = counts;
179
+ fallthrough = counts;
180
+ faultContained = counts;
181
+ }
182
+ lines.push(`counts : handled=${fmt(handled)} fallthrough=${fmt(fallthrough)} fault_contained=${fmt(faultContained)}`);
183
+ lines.push(`fast_path : enabled=${fmt(obj.fast_path_enabled)}`);
184
+ const budget = obj.budget;
185
+ let wallBudget = 'unavailable';
186
+ let probeWallBudget = 'unavailable';
187
+ let ccStatementTimeout = 'unavailable';
188
+ if (budget && typeof budget === 'object' && !Array.isArray(budget)) {
189
+ const b = budget;
190
+ wallBudget = b.wall_budget_ms ?? 'unavailable';
191
+ probeWallBudget = b.probe_wall_budget_ms ?? 'unavailable';
192
+ ccStatementTimeout = b.call_control_statement_timeout_ms ?? 'unavailable';
193
+ }
194
+ else if (budget !== undefined) {
195
+ wallBudget = budget;
196
+ probeWallBudget = budget;
197
+ ccStatementTimeout = budget;
198
+ }
199
+ lines.push(`budget_ms : wall=${fmt(wallBudget)} probe_wall=${fmt(probeWallBudget)} call_control_statement_timeout=${fmt(ccStatementTimeout)}`);
200
+ lines.push(`cache_entries : ${fmt(obj.cube_config_cache_entries)}`);
201
+ // Interpretive hint: only fires when we have real (numeric, positive)
202
+ // fallthrough/fault_contained counts — never on "unavailable" leaves — AND
203
+ // the fast path is actually enabled. With the kill switch OFF
204
+ // (fast_path_enabled === false) every probe deliberately takes the budgeted
205
+ // path, so positive fallthrough is EXPECTED, not a "missing the fast path"
206
+ // signal — emit a kill-switch note instead. When fast_path_enabled is
207
+ // unknown/"unavailable", stay silent rather than guess.
208
+ const isPositiveNumber = (v) => typeof v === 'number' && v > 0;
209
+ if (isPositiveNumber(fallthrough) || isPositiveNumber(faultContained)) {
210
+ if (obj.fast_path_enabled === true) {
211
+ lines.push('hint : fallthrough/fault_contained > 0 -> probes are missing the fast path ' +
212
+ '(fallthrough: budgeted-path fallback; fault_contained: fast-path internal error contained)');
213
+ }
214
+ else if (obj.fast_path_enabled === false) {
215
+ lines.push('hint : fast path disabled by kill switch -> fallthrough is expected');
216
+ }
217
+ }
218
+ return lines.join('\n');
219
+ }
99
220
  /**
100
221
  * Compact one-line-per-cube trace sample, called repeatedly by the
101
222
  * `--monitor <secs>` heartbeat trace to capture how the FSM state / learned
@@ -242,11 +363,17 @@ async function containerId(service) {
242
363
  return '';
243
364
  }
244
365
  /**
245
- * Tail container logs and keep only the lines matching the cube/xcc keepalive
246
- * grep. When `opts.since` is given, uses `docker logs --since <window>` instead
247
- * of `--tail`, mirroring the bash reference's 10-minute heartbeat window.
366
+ * Fetch a container's raw combined stdout/stderr log tail, with NO grep
367
+ * filtering and NO truncation. Used both by `logsFiltered()` (which filters
368
+ * this down for display) and by the rejected-keepalive summary, which needs
369
+ * the full unfiltered fetch — parity with the bash/Go collectors, which scan
370
+ * the SAME raw `docker compose logs --tail=6000` fetch for the summary that
371
+ * they display (grep-)filtered and truncated to 800 lines. Reusing the
372
+ * post-filter/post-truncate text for the summary (an earlier version of this
373
+ * port did) could silently undercount rejections squeezed out by the 800-line
374
+ * cap even though they were present in the raw 6000-line fetch.
248
375
  */
249
- async function logsFiltered(service, tailIn, keepTail, grep, opts) {
376
+ async function rawLogsTail(service, tailIn, opts) {
250
377
  const id = await containerId(service);
251
378
  if (!id)
252
379
  return `(no ${service} container found)`;
@@ -257,25 +384,321 @@ async function logsFiltered(service, tailIn, keepTail, grep, opts) {
257
384
  timeout: DEFAULT_TIMEOUT,
258
385
  reject: false,
259
386
  });
260
- const combined = `${(result.stdout ?? '').toString()}\n${(result.stderr ?? '').toString()}`;
387
+ return `${(result.stdout ?? '').toString()}\n${(result.stderr ?? '').toString()}`;
388
+ }
389
+ /**
390
+ * Tail container logs and keep only the lines matching the cube/xcc keepalive
391
+ * grep. When `opts.since` is given, uses `docker logs --since <window>` instead
392
+ * of `--tail`, mirroring the bash reference's 10-minute heartbeat window.
393
+ */
394
+ async function logsFiltered(service, tailIn, keepTail, grep, opts) {
395
+ const combined = await rawLogsTail(service, tailIn, opts);
261
396
  let lines = combined.split('\n');
262
397
  if (grep)
263
398
  lines = lines.filter(l => grep.test(l));
264
399
  return lines.slice(-keepTail).join('\n');
265
400
  }
401
+ // ── XCC wire capture (--pcap) ────────────────────────────────────────────
402
+ //
403
+ // Two legs, parity with the bash _cd_xcc_pcap_* helpers and Go
404
+ // cubeDiagBridgeIface/cubeDiagIngressIface/cubeDiagIngressPcapFilter*:
405
+ // - bridge: the compose `_ct` network's docker bridge iface (br-<id[:12]>),
406
+ // filtered to the plaintext web:4080/CURRI:4000 backend leg.
407
+ // - ingress: the host's primary route-to-internet NIC, filtered to the
408
+ // cube-router TLS/HTTP leg (443/80) scoped to each xcc-enabled
409
+ // cube's hostname (+ resolved IP) and/or last-seen source IP —
410
+ // falling back to a port-only filter when nothing resolves, so
411
+ // the leg still captures traffic with zero DB load.
412
+ /** True when every dot-separated segment is a valid IPv4 octet (0-255, no leading garbage). */
413
+ function isValidIPv4(h) {
414
+ const parts = h.split('.');
415
+ if (parts.length !== 4)
416
+ return false;
417
+ return parts.every(p => /^\d{1,3}$/.test(p) && Number(p) <= 255);
418
+ }
419
+ /** True when `host` is safe to interpolate into a tcpdump BPF `host <x>` clause. */
420
+ function validPcapHost(host) {
421
+ const h = host.trim();
422
+ if (!h)
423
+ return false;
424
+ // IPv4 literal — validated octet-by-octet so e.g. "999.999.999.999" or
425
+ // "1..2.3" (grok T1 P2: the old `/^[0-9.]+$/` + 4-segment check let both
426
+ // through, producing a useless BPF term with no surfaced error) is rejected.
427
+ if (/^[0-9.]+$/.test(h))
428
+ return isValidIPv4(h);
429
+ if (h.includes(':'))
430
+ return /^[0-9a-fA-F:]+$/.test(h);
431
+ return PCAP_HOST_RE.test(h);
432
+ }
433
+ /** Resolve the docker bridge interface backing the compose `_ct` network. */
434
+ async function pcapBridgeIface(net) {
435
+ if (!net)
436
+ return '';
437
+ try {
438
+ const result = await exec('docker', ['network', 'inspect', net, '-f', '{{.Id}}'], {
439
+ timeout: DEFAULT_TIMEOUT,
440
+ reject: false,
441
+ });
442
+ const id = (result.stdout ?? '').toString().trim();
443
+ if (id.length < 12)
444
+ return '';
445
+ const bridge = `br-${id.slice(0, 12)}`;
446
+ const check = await exec('ip', ['link', 'show', bridge], { timeout: DEFAULT_TIMEOUT, reject: false });
447
+ return check.exitCode === 0 ? bridge : '';
448
+ }
449
+ catch (err) {
450
+ debug(`cube-diag: pcap bridge iface resolve failed: ${err instanceof Error ? err.message : String(err)}`);
451
+ return '';
452
+ }
453
+ }
454
+ /** Resolve the primary host NIC used to reach the internet (cube TLS ingress). */
455
+ async function pcapIngressIface() {
456
+ try {
457
+ const result = await exec('ip', ['-o', 'route', 'get', '1.1.1.1'], { timeout: DEFAULT_TIMEOUT, reject: false });
458
+ const out = (result.stdout ?? '').toString().trim();
459
+ const fields = out.split(/\s+/);
460
+ const idx = fields.indexOf('dev');
461
+ return idx >= 0 && idx + 1 < fields.length ? fields[idx + 1] : '';
462
+ }
463
+ catch (err) {
464
+ debug(`cube-diag: pcap ingress iface resolve failed: ${err instanceof Error ? err.message : String(err)}`);
465
+ return '';
466
+ }
467
+ }
266
468
  /**
267
- * Collect the full cube-diag bundle and write it to a single .log file.
469
+ * Build the ingress BPF filter from resolved xcc_hostname values (+ DNS) and
470
+ * per-cube last-seen source IPs. Falls back to a port-only filter when
471
+ * nothing resolves, so the leg still captures every cube's traffic with zero
472
+ * DB load — parity with bash/Go.
473
+ */
474
+ async function pcapIngressFilter() {
475
+ const seen = new Set();
476
+ const parts = [];
477
+ const addHost = (host) => {
478
+ const h = host.trim();
479
+ if (!validPcapHost(h) || seen.has(h))
480
+ return;
481
+ seen.add(h);
482
+ parts.push(`host ${h}`);
483
+ };
484
+ let hostnameRows = '';
485
+ try {
486
+ hostnameRows = await psql(`SELECT DISTINCT xcc_hostname FROM cubes WHERE xcc_enabled = true AND xcc_hostname IS NOT NULL AND xcc_hostname <> '' ORDER BY 1;`);
487
+ }
488
+ catch (err) {
489
+ debug(`cube-diag: pcap hostname query failed: ${err instanceof Error ? err.message : String(err)}`);
490
+ }
491
+ for (const raw of hostnameRows.split('\n')) {
492
+ const h = raw.trim();
493
+ if (!h || h.includes('xcc_hostname') || h.startsWith('-') || h.startsWith('('))
494
+ continue;
495
+ addHost(h);
496
+ try {
497
+ const resolved = await exec('getent', ['hosts', h], { timeout: DEFAULT_TIMEOUT, reject: false });
498
+ const ip = (resolved.stdout ?? '').toString().trim().split(/\s+/)[0];
499
+ if (ip)
500
+ addHost(ip);
501
+ }
502
+ catch {
503
+ // DNS resolution best-effort only.
504
+ }
505
+ }
506
+ // Per-cube NAT/VIP source IPs: index seek (cube_id, inserted_at DESC) LIMIT 1
507
+ // per enabled cube (~dozens of rows), NOT a `SELECT DISTINCT ip_address ...
508
+ // 24h` scan over the whole hot partition (API-1791 lesson — never regress
509
+ // this to a table-wide scan). Routed through psqlLong (15s statement_timeout)
510
+ // rather than the 5s default: it's the same LATERAL-over-cube_event_logs
511
+ // shape as the "24h volume per cube" section, which already needs the long
512
+ // budget under SQL distress (grok T1 P2 — falling back to port-only on a
513
+ // timeout here silently loses host-scoped capture).
514
+ let ipRows = '';
515
+ try {
516
+ ipRows = await psqlLong(`SELECT e.ip_address FROM cubes c CROSS JOIN LATERAL (SELECT ip_address FROM cube_event_logs WHERE cube_id = c.id AND inserted_at > NOW() - INTERVAL '24 hours' AND ip_address IS NOT NULL AND ip_address <> '' ORDER BY inserted_at DESC LIMIT 1) e WHERE c.xcc_enabled = true;`);
517
+ }
518
+ catch (err) {
519
+ debug(`cube-diag: pcap source-ip query failed: ${err instanceof Error ? err.message : String(err)}`);
520
+ }
521
+ for (const raw of ipRows.split('\n')) {
522
+ addHost(raw.trim());
523
+ }
524
+ if (parts.length === 0)
525
+ return PCAP_PORT_CLAUSE;
526
+ return `(${parts.join(' or ')}) and ${PCAP_PORT_CLAUSE}`;
527
+ }
528
+ /** Stat a pcap file for the report (size in bytes, or "not created"). */
529
+ function pcapStatLine(label, path) {
530
+ try {
531
+ const st = statSync(path);
532
+ return ` ${label}: ${path} (${st.size} bytes)`;
533
+ }
534
+ catch {
535
+ return ` ${label}: (not created)`;
536
+ }
537
+ }
538
+ /**
539
+ * Run the two-leg XCC wire capture (bridge + host ingress) for `secs` seconds,
540
+ * writing pcap files alongside the bundle's report and symlinking a
541
+ * `xcc.pcap` compat name to whichever leg actually captured bytes — parity
542
+ * with the bash/Go collectors' `--pcap <secs>` option (a gap this TS port
543
+ * never had until now).
544
+ */
545
+ async function collectPcap(net, secs, bundleDir) {
546
+ if (process.env.CT_CLI_TEST_NO_TCPDUMP === '1') {
547
+ return '(pcap skipped: CT_CLI_TEST_NO_TCPDUMP=1)';
548
+ }
549
+ const tcpdumpCheck = await exec('sh', ['-c', 'command -v tcpdump'], { timeout: DEFAULT_TIMEOUT, reject: false });
550
+ if (tcpdumpCheck.exitCode !== 0) {
551
+ return '(pcap skipped: tcpdump unavailable)';
552
+ }
553
+ // Private-then-publish, same invariant as the report bundle: an interrupted
554
+ // pcap run must never leave a world-readable directory. mkdir's `mode` only
555
+ // applies on CREATE (parity with writeFile's mode caveat elsewhere in this
556
+ // file), so also clamp with chmod in case the directory already existed
557
+ // with broader perms (e.g. a caller re-running against the same --output
558
+ // parent dir).
559
+ await mkdir(bundleDir, { recursive: true, mode: 0o700 });
560
+ await chmod(bundleDir, 0o700).catch(() => { });
561
+ const bridgePath = join(bundleDir, 'xcc-bridge.pcap');
562
+ const ingressPath = join(bundleDir, 'xcc-ingress.pcap');
563
+ const compatPath = join(bundleDir, 'xcc.pcap');
564
+ const lines = [];
565
+ const bridgeIface = await pcapBridgeIface(net);
566
+ const ingressIface = await pcapIngressIface();
567
+ const ingressFilter = await pcapIngressFilter();
568
+ // Returns a `[failed exit=N] stderr` note when tcpdump/sudo/timeout itself
569
+ // failed (permission denied, bad interface, bad filter), so a silent 0-byte
570
+ // capture isn't the only signal (grok T1 P2 — reject:false swallowed
571
+ // exitCode/stderr entirely before this).
572
+ const runTcpdump = async (iface, filter, outPath) => {
573
+ const hostBudget = String(secs + 5);
574
+ const result = await exec('timeout', [hostBudget, 'sudo', 'timeout', String(secs), 'tcpdump', '-i', iface, '-s', '0', '-w', outPath, ...filter.split(' ')], { timeout: (secs + 10) * 1000, reject: false });
575
+ const exitCode = result.exitCode;
576
+ if (exitCode !== undefined && exitCode !== 0) {
577
+ const stderr = (result.stderr ?? '').toString().trim();
578
+ return ` [tcpdump failed: exit=${exitCode}]${stderr ? ` ${stderr}` : ''}`;
579
+ }
580
+ return '';
581
+ };
582
+ if (bridgeIface) {
583
+ lines.push(` bridge ${bridgeIface}: ${PCAP_BRIDGE_FILTER}`);
584
+ const failNote = await runTcpdump(bridgeIface, PCAP_BRIDGE_FILTER, bridgePath);
585
+ if (failNote)
586
+ lines.push(failNote);
587
+ lines.push(pcapStatLine('bridge', bridgePath));
588
+ }
589
+ if (ingressIface && ingressFilter) {
590
+ lines.push(` ingress ${ingressIface}: ${ingressFilter}`);
591
+ const failNote = await runTcpdump(ingressIface, ingressFilter, ingressPath);
592
+ if (failNote)
593
+ lines.push(failNote);
594
+ lines.push(pcapStatLine('ingress', ingressPath));
595
+ }
596
+ if (!bridgeIface && !(ingressIface && ingressFilter)) {
597
+ lines.push(' (no bridge or ingress interface resolved — pcap skipped)');
598
+ }
599
+ // Compat symlink: prefer whichever leg actually captured bytes (pcap header
600
+ // alone is 24 bytes), bridge first — parity with bash/Go.
601
+ const hasBytes = (p) => {
602
+ try {
603
+ return statSync(p).size > 24;
604
+ }
605
+ catch {
606
+ return false;
607
+ }
608
+ };
609
+ try {
610
+ if (hasBytes(bridgePath)) {
611
+ await unlink(compatPath).catch(() => { });
612
+ await symlink('xcc-bridge.pcap', compatPath);
613
+ }
614
+ else if (hasBytes(ingressPath)) {
615
+ await unlink(compatPath).catch(() => { });
616
+ await symlink('xcc-ingress.pcap', compatPath);
617
+ }
618
+ }
619
+ catch (err) {
620
+ debug(`cube-diag: pcap compat symlink failed: ${err instanceof Error ? err.message : String(err)}`);
621
+ }
622
+ lines.push(pcapStatLine('xcc.pcap (compat)', compatPath));
623
+ return lines.join('\n');
624
+ }
625
+ // ── Rejected-keepalive summary (API-1792 parity) ────────────────────────────
626
+ //
627
+ // Distinct api_key_prefix x distinct source_ip x count table, so a stale CUBE
628
+ // key (many events, one key, one/few IPs) is visually distinguishable from a
629
+ // stale CUCM ECC profile (many events, many keys, one router IP). Scoped to
630
+ // keepalive rejections specifically — parity with the bash/Go collectors'
631
+ // `_cd_shell "web XCC/cube logs..."` / cubeDiagRejectedKeepaliveSummary gate.
632
+ const REJECTED_RE = /CURRI edge rejected request|CurriController: Unknown policy|CurriController:.*rejected.*suppressed/i;
633
+ const KEEPALIVE_RE = /endpoint=keepalive|on keepalive|rejected keepalive/i;
634
+ const API_KEY_PREFIX_VAL_RE = /api[_-]?key[_-]?prefix=([^\s,;]+)/i;
635
+ const REMOTE_IP_VAL_RE = /remote_ip=([^\s,;]+)/;
636
+ const SOURCE_IP_VAL_RE = /source_ip=([^\s,;]+)/;
637
+ /**
638
+ * Parse rejected-keepalive / unknown-policy web log lines into a distinct
639
+ * api_key_prefix x distinct source_ip x count table.
640
+ */
641
+ function rejectedKeepaliveSummary(webLog) {
642
+ // Keyed by a JSON-encoded [prefix, ip] tuple rather than a delimited string —
643
+ // both fields come from a `[^\s,;]+` capture so they can never themselves
644
+ // contain the delimiter, but encoding the pair avoids relying on that
645
+ // invariant to safely split the key back apart.
646
+ const counts = new Map();
647
+ const prefixes = new Set();
648
+ const ips = new Set();
649
+ let total = 0;
650
+ for (const line of webLog.split('\n')) {
651
+ if (!REJECTED_RE.test(line) || !KEEPALIVE_RE.test(line))
652
+ continue;
653
+ const prefixMatch = API_KEY_PREFIX_VAL_RE.exec(line);
654
+ const prefix = prefixMatch ? prefixMatch[1] : '(unknown)';
655
+ const ipMatch = REMOTE_IP_VAL_RE.exec(line) ?? SOURCE_IP_VAL_RE.exec(line);
656
+ const ip = ipMatch ? ipMatch[1] : '(unknown)';
657
+ const key = JSON.stringify([prefix, ip]);
658
+ const existing = counts.get(key);
659
+ if (existing) {
660
+ existing.count++;
661
+ }
662
+ else {
663
+ counts.set(key, { prefix, ip, count: 1 });
664
+ }
665
+ prefixes.add(prefix);
666
+ ips.add(ip);
667
+ total++;
668
+ }
669
+ if (total === 0)
670
+ return '(no rejected-keepalive / unknown-policy lines in window)';
671
+ const rows = Array.from(counts.values());
672
+ rows.sort((a, b) => (b.count - a.count) || a.prefix.localeCompare(b.prefix) || a.ip.localeCompare(b.ip));
673
+ const lines = [];
674
+ lines.push(`distinct api_key_prefix: ${prefixes.size} distinct source_ip: ${ips.size} total rejected: ${total}`);
675
+ lines.push('');
676
+ lines.push(`${'api_key_prefix'.padEnd(18)} ${'source_ip'.padEnd(18)} count`);
677
+ for (const r of rows) {
678
+ lines.push(`${r.prefix.padEnd(18)} ${r.ip.padEnd(18)} ${r.count}`);
679
+ }
680
+ return lines.join('\n');
681
+ }
682
+ /**
683
+ * Collect the full cube-diag bundle and write it to a single .log file (plus,
684
+ * when `--pcap` is requested, sibling `xcc-bridge.pcap` / `xcc-ingress.pcap` /
685
+ * `xcc.pcap` files in the same directory as `outputPath`).
268
686
  *
269
687
  * @param opts.outputPath Target log path. Defaults to ./cube-diag-<ISO>.log.
270
688
  * @param opts.captureSecs Seconds to live-capture cube.> / alarms.> subjects (0 = skip).
271
689
  * @param opts.monitorSecs Seconds to RPC-poll each cube's state/interval ~every 5s (0 = skip).
690
+ * @param opts.pcapSecs Seconds to tcpdump the XCC wire (bridge + host TLS ingress legs, 0 = skip).
272
691
  * @returns the path the bundle was written to.
273
692
  */
274
693
  export async function collectCubeDiag(opts) {
275
694
  const captureSecs = opts?.captureSecs ?? 0;
276
695
  const monitorSecs = opts?.monitorSecs ?? 0;
696
+ const pcapSecs = opts?.pcapSecs ?? 0;
277
697
  const iso = new Date().toISOString().replace(/[:.]/g, '-');
278
698
  const outputPath = opts?.outputPath ?? `./cube-diag-${iso}.log`;
699
+ // pcap files land beside the report — same directory as outputPath, so a
700
+ // custom --output still keeps the bundle together on disk.
701
+ const bundleDir = dirname(outputPath) || '.';
279
702
  const parts = [];
280
703
  parts.push('===== CUBE DIAGNOSTIC BUNDLE =====', `Generated: ${new Date().toISOString()}`, 'Bug class: cube stuck REGISTERED, never ONLINE (registered->online requires a healthy SolicitXccProbing keepalive).');
281
704
  // ── Database sections ──────────────────────────────────────────────────
@@ -329,6 +752,14 @@ export async function collectCubeDiag(opts) {
329
752
  // mismatch, DB-vs-KV divergence). This is where a "shows 10s, router is 15s"
330
753
  // stuck cube is caught.
331
754
  parts.push(await section('cube interval + KV health (RPC-decoded)', () => rpcBounded(CUBE_DIAG_RPC_SNAPSHOT)));
755
+ // ── Probe fast-path diagnostics (API-1838) ──────────────────────────────
756
+ // Best-effort like every other section: an old appliance (module doesn't
757
+ // exist yet) or a malformed/errored payload degrades to a one-line hint
758
+ // instead of aborting the bundle.
759
+ parts.push(await section('probe fast-path diagnostics (probe_fast)', async () => {
760
+ const raw = await rpcBounded(CUBE_DIAG_RPC_PROBE_FAST);
761
+ return renderProbeFastDiag(raw);
762
+ }));
332
763
  // ── Optional RPC heartbeat trace ────────────────────────────────────────
333
764
  // Polls the compact per-cube snapshot every ~5s for the window so engineering
334
765
  // can watch the FSM state / learned cadence / freshness window evolve.
@@ -370,15 +801,66 @@ export async function collectCubeDiag(opts) {
370
801
  }));
371
802
  }
372
803
  }
804
+ // ── Optional XCC wire capture (--pcap) ──────────────────────────────────
805
+ // Two legs, overlapping the monitor/capture windows above: bridge (docker
806
+ // backend leg) + host TLS ingress (cube-router leg). Parity gap fix — the
807
+ // bash and Go collectors have shipped this since PR #430; this port never
808
+ // had it until now.
809
+ if (pcapSecs > 0) {
810
+ parts.push(await section(`XCC pcap (${pcapSecs}s, bridge web:4080 + host TLS ingress)`, () => collectPcap(net, pcapSecs, bundleDir)));
811
+ }
373
812
  // ── Log sections ───────────────────────────────────────────────────────
374
813
  // Heartbeat timeline first: the failure_count progression, "did not refresh
375
814
  // healthy" gate hits, and registration rotations are the stuck/flap signature.
376
815
  const heartbeatGrep = /failure_count|did not refresh healthy|registrationID rotated|keepalive/i;
377
816
  parts.push(await section('probe heartbeat timeline (failure_count / refresh / rotation, last 10m)', () => logsFiltered('web', 6000, 200, heartbeatGrep, { since: '10m' })));
817
+ // Fetched ONCE (raw, unfiltered, untruncated) and reused by both sections
818
+ // below — parity with the bash/Go collectors, which scan the SAME raw
819
+ // `docker compose logs --tail=6000` fetch for both the filtered display and
820
+ // the rejected-keepalive summary. Scanning the already-grep-filtered/800-
821
+ // truncated text for the summary (as an earlier version of this port did)
822
+ // could silently undercount rejections squeezed out of the display window
823
+ // even though they were present in the raw 6000-line fetch (grok T1 P2).
824
+ // Captured outside section() so a failure here still degrades both
825
+ // sections independently, matching the original per-section isolation
826
+ // guarantee — rawLogsTail itself never throws (containerId()/exec() are
827
+ // already best-effort internally), but this keeps that invariant explicit
828
+ // rather than assumed.
829
+ let webLogRaw = '';
830
+ let webLogErr = null;
378
831
  const cubeGrep = /xcc|cube|keepalive|solicit|probing|register|authorize|reconnect|liveness|connection_state/i;
379
- parts.push(await section('LOGS: web (filtered, tail 800)', () => logsFiltered('web', 6000, 800, cubeGrep)));
832
+ try {
833
+ webLogRaw = await rawLogsTail('web', 6000);
834
+ }
835
+ catch (err) {
836
+ webLogErr = err instanceof Error ? err.message : String(err);
837
+ }
838
+ parts.push(await section('LOGS: web (filtered, tail 800)', async () => {
839
+ if (webLogErr !== null)
840
+ throw new Error(webLogErr);
841
+ const lines = webLogRaw.split('\n').filter(l => cubeGrep.test(l));
842
+ return lines.slice(-800).join('\n');
843
+ }));
844
+ // API-1792 parity: rejected-keepalive summary — distinct api_key_prefix x
845
+ // distinct source_ip x count. Scans the raw (unfiltered, untruncated) web
846
+ // log tail fetched above rather than a second docker exec.
847
+ parts.push(await section('Rejected-Keepalive Summary (api_key_prefix x source_ip)', async () => {
848
+ if (webLogErr !== null)
849
+ throw new Error(webLogErr);
850
+ return rejectedKeepaliveSummary(webLogRaw);
851
+ }));
380
852
  parts.push(await section('LOGS: nats (tail 300)', () => logsFiltered('nats', 300, 300, null)));
381
853
  const bundle = redactSecrets(parts.join('\n') + '\n');
854
+ // Create the bundle directory private (mkdir handles the case where
855
+ // outputPath's directory doesn't exist yet, e.g. a custom --output nested
856
+ // path) BEFORE writing, mirroring the bash/Go "private then publish"
857
+ // pattern: an interrupted run should never leave a world-readable partial
858
+ // bundle. writeFile's own 0o600 mode covers report.log. When pcapSecs > 0,
859
+ // collectPcap() has already hardened (and re-clamped) this same directory
860
+ // before writing any pcap file, so this call is a no-op in that case.
861
+ if (bundleDir !== '.' && !existsSync(bundleDir)) {
862
+ await mkdir(bundleDir, { recursive: true, mode: 0o700 });
863
+ }
382
864
  await writeFile(outputPath, bundle, { mode: 0o600 });
383
865
  // `mode` only applies when writeFile CREATES the file. If outputPath already
384
866
  // existed with broader perms, the write truncates but leaves the old mode, so