effect-inspect 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/QueryCommands.js +67 -12
- package/dist/cli.js +15 -0
- package/dist/query/Query.d.ts +91 -1
- package/dist/query/Query.js +185 -44
- package/package.json +1 -1
|
@@ -181,6 +181,12 @@ COLLECTOR ADDRESS (live only)
|
|
|
181
181
|
or nothing listens, the result is CollectorUnavailable (exit 8).`;
|
|
182
182
|
const context = `
|
|
183
183
|
CONTEXT FIELDS (every successful per-session response)
|
|
184
|
+
Top-level siblings of result, never inside it (.completeness, not
|
|
185
|
+
.result.completeness). result's own fields are listed above (RESULT FIELDS /
|
|
186
|
+
SPAN ITEM FIELDS); root \`effect-inspect --help\` has the JSON SHAPE overview.
|
|
187
|
+
notices summary only, top level: read it first. { code, message } facts easy
|
|
188
|
+
to miss (open spans, eviction, sampling gaps); run \`summary\` before
|
|
189
|
+
drilling down with other commands.
|
|
184
190
|
query The request as applied, defaults filled in. Check it to confirm
|
|
185
191
|
which filters were used.
|
|
186
192
|
source kind "live"|"file", file (path or null), sessionId (exact), program,
|
|
@@ -194,6 +200,15 @@ CONTEXT FIELDS (every successful per-session response)
|
|
|
194
200
|
microsecond resolution. Wall clock = startedAtEpochMillis + ms. Values
|
|
195
201
|
can be negative (work that began before the client). observedFromMs /
|
|
196
202
|
observedUntilMs: earliest / latest retained timestamp, null when none.
|
|
203
|
+
termination state "active" (still connected at snapshot time; open spans may still
|
|
204
|
+
end), "ended" (the collector recorded a disconnect) or "unknown" (no
|
|
205
|
+
end time on record). lastObservedMs (= observedUntilMs), endedAtMs
|
|
206
|
+
(the disconnect in session ms, from the collector's wall clock, so
|
|
207
|
+
approximate; null unless ended), unobservedTailMs (endedAtMs -
|
|
208
|
+
lastObservedMs: time before the disconnect with nothing retained).
|
|
209
|
+
The protocol has no end-of-session message: a crash, a kill, a clean
|
|
210
|
+
exit and a dropped connection look the same, so open spans at the end
|
|
211
|
+
do not by themselves establish a crash.
|
|
197
212
|
completeness status: "noLossRecorded" (collector counters known, all loss/gap
|
|
198
213
|
counters 0 - still not proof nothing is missing), "lossRecorded" (some
|
|
199
214
|
counter > 0: evidence is partial), or "unknown" (source never kept
|
|
@@ -353,7 +368,8 @@ const summaryCommand = Command.make('summary', {
|
|
|
353
368
|
top: Flag.Int('top').pipe(Flag.optional, Flag.withDescription(`Items per ranked list, 1-${Query.limits.top.max} (default ${Query.limits.top.default}).`)),
|
|
354
369
|
json,
|
|
355
370
|
}, (flags) => Effect.flatMap(answer('summary', flags, given({ top: flags.top })), (response) => emit(response, flags.json))).pipe(Command.withShortDescription('Step 1: counts, failures, longest spans and completeness of one run'), Command.withDescription(text(`
|
|
356
|
-
Overview of one session: span counts by status, failures,
|
|
371
|
+
Overview of one session: notices, span counts by status, failures, unfinished spans
|
|
372
|
+
and where they were last recorded, the longest completed spans, the spans
|
|
357
373
|
with the most time outside recorded children, still-open spans, per-name totals, log
|
|
358
374
|
counts by level and memory samples, plus how complete the evidence is. Start every
|
|
359
375
|
investigation here, then drill down with \`spans\`, \`span\` and \`logs\`.
|
|
@@ -362,9 +378,12 @@ ${sourceRules}
|
|
|
362
378
|
RESULT (abbreviated; lists hold at most --top items)
|
|
363
379
|
{ "ok": true, "apiVersion": 1, "op": "summary",
|
|
364
380
|
"query": { "op": "summary", "sessionId": "failing-run-001", "top": 5 },
|
|
381
|
+
"notices": [],
|
|
365
382
|
"source": { "kind": "live", "sessionId": "failing-run-001", "active": false, ... },
|
|
366
383
|
"time": { "unit": "ms", "reference": "sessionStart",
|
|
367
384
|
"observedFromMs": -1.348, "observedUntilMs": 240.867 },
|
|
385
|
+
"termination": { "state": "ended", "lastObservedMs": 240.867, "endedAtMs": 243,
|
|
386
|
+
"unobservedTailMs": 2.133 },
|
|
368
387
|
"completeness": { "status": "noLossRecorded", "openSpans": 0, ... },
|
|
369
388
|
"conflict": { "count": 0, "detection": "enforced" },
|
|
370
389
|
"result": {
|
|
@@ -374,6 +393,7 @@ RESULT (abbreviated; lists hold at most --top items)
|
|
|
374
393
|
"memory": { "samples": 2, "peakHeapUsedBytes": 7158493, "peakRssBytes": 61489152,
|
|
375
394
|
"lastHeapUsedBytes": 7158493 },
|
|
376
395
|
"failures": { "total": 4, "items": [ SPAN ITEM, ... ] },
|
|
396
|
+
"unfinished": { "open": 0, "innermost": { "total": 0, "items": [] } },
|
|
377
397
|
"longest": [ SPAN ITEM, ... ],
|
|
378
398
|
"largestOutsideChildren": [ SPAN ITEM, ... ],
|
|
379
399
|
"longestOpen": [ SPAN ITEM, ... ],
|
|
@@ -382,17 +402,41 @@ RESULT (abbreviated; lists hold at most --top items)
|
|
|
382
402
|
"maxDurationMs": 241.993, "totalOutsideChildrenMs": 0.658 }, ... ] } } }
|
|
383
403
|
|
|
384
404
|
RESULT FIELDS
|
|
405
|
+
notices Top level, before result: { code, message } facts easy to miss.
|
|
406
|
+
openSpans (spans without a recorded end, the termination state
|
|
407
|
+
and the last recorded position), rankingsCompletedOnly (longest and
|
|
408
|
+
largestOutsideChildren skip open spans), collectorEvicted (oldest
|
|
409
|
+
messages evicted at capacity: data before observedFromMs is
|
|
410
|
+
missing), memorySamplingGap (the largest gap between memory
|
|
411
|
+
samples exceeds 10x the median). Messages state facts;
|
|
412
|
+
explanations are possibilities.
|
|
385
413
|
spans Counts by status. failures lists error and defect spans (not
|
|
386
414
|
interruptions), earliest start first; failures.total counts all.
|
|
387
|
-
|
|
388
|
-
|
|
415
|
+
unfinished open: spans without a recorded end. innermost: open spans with no
|
|
416
|
+
open child - the last recorded position on each open chain, not a
|
|
417
|
+
cause - largest elapsedLowerBoundMs first. Each is a SPAN ITEM plus
|
|
418
|
+
openAncestors: { items: [ { spanId, name, nameTruncated, status,
|
|
419
|
+
startMs, durationMs, elapsedLowerBoundMs } ], truncated }:
|
|
420
|
+
contiguous open ancestors, root-most first, at most 32 nearest;
|
|
421
|
+
truncated marks more above. durationMs is null (no end);
|
|
422
|
+
elapsedLowerBoundMs is observedUntilMs - startMs.
|
|
423
|
+
longest Completed spans only, largest durationMs first.
|
|
424
|
+
largestOutsideChildren Completed spans only, largest outsideChildrenMs first.
|
|
389
425
|
longestOpen Open spans, largest elapsedLowerBoundMs first.
|
|
390
426
|
names Groups by full span name, largest totalDurationMs first. Sums cover
|
|
391
427
|
completed spans; nested and concurrent spans overlap, so totals can
|
|
392
428
|
exceed the run's wall time. failed counts every failure incl.
|
|
393
429
|
interruptions.
|
|
394
|
-
logs.byLevel Only levels that occur.
|
|
395
|
-
|
|
430
|
+
logs.byLevel Only levels that occur.
|
|
431
|
+
memory Process-wide samples (all work in the process), or null without
|
|
432
|
+
samples: { samples, peakHeapUsedBytes, peakHeapAtMs, peakRssBytes,
|
|
433
|
+
lastHeapUsedBytes, firstSampleMs, lastSampleMs, medianIntervalMs,
|
|
434
|
+
maxGapMs, maxGapFromMs, maxGapToMs, spansActiveAtPeak }. Periodic
|
|
435
|
+
samples miss peaks between them; maxGapMs is the longest stretch
|
|
436
|
+
with no sample. spansActiveAtPeak: { total, items: [ { spanId,
|
|
437
|
+
name, nameTruncated, status, startMs, durationMs } ] }, the
|
|
438
|
+
innermost spans active at peakHeapAtMs - active at that time only,
|
|
439
|
+
not shown to be what the heap in use belongs to.
|
|
396
440
|
${spanItem}
|
|
397
441
|
${context}
|
|
398
442
|
${timing}
|
|
@@ -452,9 +496,12 @@ FILTERS (all combine with AND)
|
|
|
452
496
|
|
|
453
497
|
ORDER (--sort)
|
|
454
498
|
start startMs ascending (default).
|
|
455
|
-
duration durationMs descending
|
|
456
|
-
elapsedLowerBoundMs
|
|
457
|
-
outsideChildren outsideChildrenMs descending
|
|
499
|
+
duration durationMs descending. Open spans are interleaved by
|
|
500
|
+
elapsedLowerBoundMs: their true duration is at least that.
|
|
501
|
+
outsideChildren outsideChildrenMs descending. Open spans are interleaved by the
|
|
502
|
+
time so far no recorded child covered (open children count as
|
|
503
|
+
covering up to observedUntilMs): a lower bound, not shown in the
|
|
504
|
+
item (their outsideChildrenMs is null).
|
|
458
505
|
Ties: startMs, then spanId.
|
|
459
506
|
${pagingRules(Query.limits.spans.max, Query.limits.spans.default, 'as --sort')}
|
|
460
507
|
|
|
@@ -462,7 +509,8 @@ RESULT (abbreviated)
|
|
|
462
509
|
{ "ok": true, "apiVersion": 1, "op": "spans",
|
|
463
510
|
"query": { "op": "spans", "sessionId": "failing-run-001", "status": "failed",
|
|
464
511
|
"sort": "start", "limit": 20, "offset": 0 },
|
|
465
|
-
"source": { ... }, "time": { ... }, "
|
|
512
|
+
"source": { ... }, "time": { ... }, "termination": { ... },
|
|
513
|
+
"completeness": { ... }, "conflict": { ... },
|
|
466
514
|
"window": null,
|
|
467
515
|
"result": { "total": 5, "offset": 0, "limit": 20, "nextOffset": null, "items": [
|
|
468
516
|
{ "spanId": "0a40c31fbf88b7db", "traceId": "9d502fd678d8f15c0328b182dc1e3509",
|
|
@@ -486,7 +534,7 @@ ${exitTable(sessionErrors)}
|
|
|
486
534
|
},
|
|
487
535
|
{
|
|
488
536
|
command: 'effect-inspect spans --session failing-run-001 --sort outsideChildren --limit 5 --json',
|
|
489
|
-
description: 'Five spans ranked by elapsed time outside recorded children (
|
|
537
|
+
description: 'Five spans ranked by elapsed time outside recorded children (open spans by a lower bound)',
|
|
490
538
|
},
|
|
491
539
|
{
|
|
492
540
|
command: 'effect-inspect spans --session failing-run-001 --from-ms=0 --to-ms 50 --sort duration --json',
|
|
@@ -516,7 +564,8 @@ RESULT (abbreviated): a SPAN ITEM plus the fields below
|
|
|
516
564
|
{ "ok": true, "apiVersion": 1, "op": "span",
|
|
517
565
|
"query": { "op": "span", "sessionId": "failing-run-001",
|
|
518
566
|
"spanId": "0a40c31fbf88b7db", "children": 20, "events": 20 },
|
|
519
|
-
"source": { ... }, "time": { ... }, "
|
|
567
|
+
"source": { ... }, "time": { ... }, "termination": { ... },
|
|
568
|
+
"completeness": { ... }, "conflict": { ... },
|
|
520
569
|
"result": { "spanId": "0a40c31fbf88b7db", "name": "charge.card", "status": "error",
|
|
521
570
|
...other SPAN ITEM fields...,
|
|
522
571
|
"attributes": { "entries": [], "omittedKeys": 0 },
|
|
@@ -553,6 +602,11 @@ RESULT FIELDS
|
|
|
553
602
|
{ name, nameTruncated, timeMs, attributes }. Effect logs emitted inside
|
|
554
603
|
the span are also recorded as its events (as above); use \`logs --span\`
|
|
555
604
|
for their level and message.
|
|
605
|
+
processMemory Process-wide memory samples within the span's interval (open: up to
|
|
606
|
+
observedUntilMs): { samples, firstSampleMs, lastSampleMs,
|
|
607
|
+
firstHeapUsedBytes, lastHeapUsedBytes, maxHeapUsedBytes }, or null
|
|
608
|
+
when no sample falls in range. Includes all concurrent work in the
|
|
609
|
+
process; not what this span itself used or kept.
|
|
556
610
|
${spanItem}
|
|
557
611
|
${context}
|
|
558
612
|
${timing}
|
|
@@ -609,7 +663,8 @@ RESULT (abbreviated)
|
|
|
609
663
|
{ "ok": true, "apiVersion": 1, "op": "logs",
|
|
610
664
|
"query": { "op": "logs", "sessionId": "failing-run-001",
|
|
611
665
|
"spanId": "0a40c31fbf88b7db", "scope": "subtree", "limit": 50, "offset": 0 },
|
|
612
|
-
"source": { ... }, "time": { ... }, "
|
|
666
|
+
"source": { ... }, "time": { ... }, "termination": { ... },
|
|
667
|
+
"completeness": { ... }, "conflict": { ... },
|
|
613
668
|
"window": null,
|
|
614
669
|
"result": { "total": 1, "offset": 0, "limit": 50, "nextOffset": null, "items": [
|
|
615
670
|
{ "timeMs": -0.506, "level": "Info", "message": "charging card **** 4242",
|
package/dist/cli.js
CHANGED
|
@@ -110,6 +110,21 @@ OUTPUT AND EXIT CODES (all query commands)
|
|
|
110
110
|
8 CollectorUnavailable 9 CollectorError 10 OutputError (export)
|
|
111
111
|
Each command's --help explains its errors and what to do next.
|
|
112
112
|
|
|
113
|
+
JSON SHAPE (success, per-session queries)
|
|
114
|
+
Top-level keys: ok, apiVersion, op, query, notices (summary only), source, time,
|
|
115
|
+
termination, completeness, conflict, result (spans and logs add window).
|
|
116
|
+
Context keys are siblings of result, never inside it: .completeness, not
|
|
117
|
+
.result.completeness. Most-used paths:
|
|
118
|
+
.notices[] summary: read first; { code, message } facts easy to miss
|
|
119
|
+
.result.spans.open summary: span counts by status (also total, ok, error, ...)
|
|
120
|
+
.result.unfinished summary: innermost open spans and their open ancestors
|
|
121
|
+
.result.longest[].durationMs summary: completed spans only, longest first
|
|
122
|
+
.result.memory summary: heap peak, sampling gaps; null without samples
|
|
123
|
+
.result.items[] spans and logs: the page (with total, nextOffset)
|
|
124
|
+
.completeness.status noLossRecorded | lossRecorded | unknown
|
|
125
|
+
.termination.state active | ended | unknown
|
|
126
|
+
Full field lists: \`<command> --help\` (RESULT FIELDS or SPAN ITEM FIELDS, CONTEXT FIELDS).
|
|
127
|
+
|
|
113
128
|
EVIDENCE AND TIMING
|
|
114
129
|
Times are milliseconds since the session's clock origin (when its inspect client
|
|
115
130
|
started) and can be negative. durationMs is elapsed wall time; outsideChildrenMs is
|
package/dist/query/Query.d.ts
CHANGED
|
@@ -339,10 +339,32 @@ export interface ConflictInfo {
|
|
|
339
339
|
readonly count: number | null;
|
|
340
340
|
readonly detection: 'enforced' | 'unavailable' | 'unknown';
|
|
341
341
|
}
|
|
342
|
+
/**
|
|
343
|
+
* How the session ended, as far as the collector knows. The protocol has no
|
|
344
|
+
* end-of-session message: a crash, a kill, a clean exit and a dropped
|
|
345
|
+
* connection all look the same here, so open spans at the end do not by
|
|
346
|
+
* themselves establish a crash. `state`:
|
|
347
|
+
* - `active`: still connected at snapshot time; open spans may still end.
|
|
348
|
+
* - `ended`: the collector recorded a disconnect.
|
|
349
|
+
* - `unknown`: no end time is on record.
|
|
350
|
+
*/
|
|
351
|
+
export interface Termination {
|
|
352
|
+
readonly state: 'active' | 'ended' | 'unknown';
|
|
353
|
+
/** Latest retained timestamp (= `time.observedUntilMs`). */
|
|
354
|
+
readonly lastObservedMs: number | null;
|
|
355
|
+
/**
|
|
356
|
+
* When the collector recorded the disconnect, in session ms. Taken from the
|
|
357
|
+
* collector's wall clock, not the client's, so approximate.
|
|
358
|
+
*/
|
|
359
|
+
readonly endedAtMs: number | null;
|
|
360
|
+
/** `endedAtMs - lastObservedMs`: time before the disconnect with nothing retained. */
|
|
361
|
+
readonly unobservedTailMs: number | null;
|
|
362
|
+
}
|
|
342
363
|
/** Shared context of every successful per-session response. */
|
|
343
364
|
export interface Context {
|
|
344
365
|
readonly source: SourceInfo;
|
|
345
366
|
readonly time: TimeInfo;
|
|
367
|
+
readonly termination: Termination;
|
|
346
368
|
readonly completeness: Completeness;
|
|
347
369
|
readonly conflict: ConflictInfo;
|
|
348
370
|
}
|
|
@@ -410,6 +432,19 @@ export interface SpanRef {
|
|
|
410
432
|
readonly startMs: number;
|
|
411
433
|
readonly durationMs: number | null;
|
|
412
434
|
}
|
|
435
|
+
/**
|
|
436
|
+
* Process-wide memory samples within a span's interval (open: up to
|
|
437
|
+
* `observedUntilMs`). Includes all concurrent work in the process; not what
|
|
438
|
+
* the span itself used or kept.
|
|
439
|
+
*/
|
|
440
|
+
export interface ProcessMemory {
|
|
441
|
+
readonly samples: number;
|
|
442
|
+
readonly firstSampleMs: number;
|
|
443
|
+
readonly lastSampleMs: number;
|
|
444
|
+
readonly firstHeapUsedBytes: number;
|
|
445
|
+
readonly lastHeapUsedBytes: number;
|
|
446
|
+
readonly maxHeapUsedBytes: number;
|
|
447
|
+
}
|
|
413
448
|
export interface SpanDetail extends SpanItem {
|
|
414
449
|
readonly attributes: BoundedAttributes;
|
|
415
450
|
readonly stack: string | null;
|
|
@@ -447,6 +482,8 @@ export interface SpanDetail extends SpanItem {
|
|
|
447
482
|
readonly attributes: BoundedAttributes;
|
|
448
483
|
}>;
|
|
449
484
|
};
|
|
485
|
+
/** `null` when no memory sample falls in the span's interval. */
|
|
486
|
+
readonly processMemory: ProcessMemory | null;
|
|
450
487
|
}
|
|
451
488
|
export interface LogItem {
|
|
452
489
|
readonly timeMs: number;
|
|
@@ -483,6 +520,23 @@ export interface NameGroup {
|
|
|
483
520
|
readonly maxDurationMs: number | null;
|
|
484
521
|
readonly totalOutsideChildrenMs: number;
|
|
485
522
|
}
|
|
523
|
+
/** An open span with no open child: the last recorded position on its open chain. */
|
|
524
|
+
export interface UnfinishedSpan extends SpanItem {
|
|
525
|
+
/** Contiguous open ancestors, root-most first, ending at the direct parent. */
|
|
526
|
+
readonly openAncestors: {
|
|
527
|
+
/** `elapsedLowerBoundMs`: `observedUntilMs - startMs`, as on open span items. */
|
|
528
|
+
readonly items: ReadonlyArray<SpanRef & {
|
|
529
|
+
readonly elapsedLowerBoundMs: number;
|
|
530
|
+
}>;
|
|
531
|
+
/** More open ancestors exist above the first item. */
|
|
532
|
+
readonly truncated: boolean;
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
/** A plain factual statement about the evidence; possible explanations are phrased as such. */
|
|
536
|
+
export interface Notice {
|
|
537
|
+
readonly code: 'openSpans' | 'rankingsCompletedOnly' | 'collectorEvicted' | 'memorySamplingGap';
|
|
538
|
+
readonly message: string;
|
|
539
|
+
}
|
|
486
540
|
export interface Summary {
|
|
487
541
|
readonly spans: {
|
|
488
542
|
readonly total: number;
|
|
@@ -497,17 +551,51 @@ export interface Summary {
|
|
|
497
551
|
readonly total: number;
|
|
498
552
|
readonly byLevel: Partial<Record<Protocol.LogLevel, number>>;
|
|
499
553
|
};
|
|
554
|
+
/**
|
|
555
|
+
* Process-wide memory samples (all work in the process, not per span).
|
|
556
|
+
* Samples are periodic, so peaks between samples are not seen.
|
|
557
|
+
*/
|
|
500
558
|
readonly memory: {
|
|
501
559
|
readonly samples: number;
|
|
502
560
|
readonly peakHeapUsedBytes: number;
|
|
561
|
+
/** Time of the first sample with `peakHeapUsedBytes`. */
|
|
562
|
+
readonly peakHeapAtMs: number;
|
|
503
563
|
readonly peakRssBytes: number;
|
|
504
564
|
readonly lastHeapUsedBytes: number;
|
|
565
|
+
readonly firstSampleMs: number;
|
|
566
|
+
readonly lastSampleMs: number;
|
|
567
|
+
/** Median time between consecutive samples; `null` with one sample. */
|
|
568
|
+
readonly medianIntervalMs: number | null;
|
|
569
|
+
/** Largest time between consecutive samples, from `maxGapFromMs` to `maxGapToMs`; `null` with one sample. */
|
|
570
|
+
readonly maxGapMs: number | null;
|
|
571
|
+
readonly maxGapFromMs: number | null;
|
|
572
|
+
readonly maxGapToMs: number | null;
|
|
573
|
+
/**
|
|
574
|
+
* Innermost spans active at `peakHeapAtMs` (none of their children active
|
|
575
|
+
* then), earliest start first. Active at that time only: they are not
|
|
576
|
+
* shown to be the source of the heap in use.
|
|
577
|
+
*/
|
|
578
|
+
readonly spansActiveAtPeak: {
|
|
579
|
+
readonly total: number;
|
|
580
|
+
readonly items: ReadonlyArray<SpanRef>;
|
|
581
|
+
};
|
|
505
582
|
} | null;
|
|
506
583
|
/** `error`/`defect` spans (not interruptions), earliest start first. */
|
|
507
584
|
readonly failures: {
|
|
508
585
|
readonly total: number;
|
|
509
586
|
readonly items: ReadonlyArray<SpanItem>;
|
|
510
587
|
};
|
|
588
|
+
/**
|
|
589
|
+
* Spans with no recorded end. `innermost`: open spans without an open child,
|
|
590
|
+
* largest `elapsedLowerBoundMs` first, each with its open ancestor chain.
|
|
591
|
+
*/
|
|
592
|
+
readonly unfinished: {
|
|
593
|
+
readonly open: number;
|
|
594
|
+
readonly innermost: {
|
|
595
|
+
readonly total: number;
|
|
596
|
+
readonly items: ReadonlyArray<UnfinishedSpan>;
|
|
597
|
+
};
|
|
598
|
+
};
|
|
511
599
|
/** Completed spans, largest `durationMs` first. */
|
|
512
600
|
readonly longest: ReadonlyArray<SpanItem>;
|
|
513
601
|
/** Completed spans, largest `outsideChildrenMs` first. */
|
|
@@ -547,7 +635,9 @@ export type SessionsResponse = Success<'sessions', SessionsResult> & {
|
|
|
547
635
|
readonly file: string | null;
|
|
548
636
|
};
|
|
549
637
|
};
|
|
550
|
-
export type SummaryResponse = Success<'summary', Summary> &
|
|
638
|
+
export type SummaryResponse = Success<'summary', Summary> & {
|
|
639
|
+
readonly notices: ReadonlyArray<Notice>;
|
|
640
|
+
} & Context;
|
|
551
641
|
/**
|
|
552
642
|
* How `fromMs`/`toMs` were applied to `spans`, or `null` without either:
|
|
553
643
|
* spans whose `[startMs, endMs]` (open: `[startMs, observedUntilMs]`)
|
package/dist/query/Query.js
CHANGED
|
@@ -316,6 +316,8 @@ class Analysis {
|
|
|
316
316
|
outOfOrder;
|
|
317
317
|
clientDropped;
|
|
318
318
|
items = new Map();
|
|
319
|
+
/** Open spans: time so far not covered by a recorded child, a lower bound. */
|
|
320
|
+
openOutside = new Map();
|
|
319
321
|
constructor(source) {
|
|
320
322
|
this.source = source;
|
|
321
323
|
const store = this.store;
|
|
@@ -392,6 +394,8 @@ class Analysis {
|
|
|
392
394
|
}
|
|
393
395
|
const closed = span.end !== undefined;
|
|
394
396
|
const { total, self } = timings(this.store, span, this.now);
|
|
397
|
+
if (!closed)
|
|
398
|
+
this.openOutside.set(span.spanId, ms(self));
|
|
395
399
|
const error = span.outcome?._tag === 'Failure'
|
|
396
400
|
? (() => {
|
|
397
401
|
const { text, truncated } = cut(span.outcome.error, limits.errorChars);
|
|
@@ -443,6 +447,15 @@ class Analysis {
|
|
|
443
447
|
let detection = 'unknown';
|
|
444
448
|
if (capture !== undefined)
|
|
445
449
|
detection = capture.conflictDetection ? 'enforced' : 'unavailable';
|
|
450
|
+
const lastObservedMs = this.observedUntil === undefined ? null : ms(this.observedUntil);
|
|
451
|
+
const endedAtMs = session.endedAtEpochMillis === undefined
|
|
452
|
+
? null
|
|
453
|
+
: ms(session.endedAtEpochMillis - session.clock.wallClockEpochMillis);
|
|
454
|
+
let state = 'unknown';
|
|
455
|
+
if (session.active)
|
|
456
|
+
state = 'active';
|
|
457
|
+
else if (endedAtMs !== null)
|
|
458
|
+
state = 'ended';
|
|
446
459
|
return {
|
|
447
460
|
source: {
|
|
448
461
|
kind: source.kind,
|
|
@@ -458,7 +471,13 @@ class Analysis {
|
|
|
458
471
|
unit: 'ms',
|
|
459
472
|
reference: 'sessionStart',
|
|
460
473
|
observedFromMs: this.observedFrom === undefined ? null : ms(this.observedFrom),
|
|
461
|
-
observedUntilMs:
|
|
474
|
+
observedUntilMs: lastObservedMs,
|
|
475
|
+
},
|
|
476
|
+
termination: {
|
|
477
|
+
state,
|
|
478
|
+
lastObservedMs,
|
|
479
|
+
endedAtMs,
|
|
480
|
+
unobservedTailMs: endedAtMs === null || lastObservedMs === null ? null : ms(endedAtMs - lastObservedMs),
|
|
462
481
|
},
|
|
463
482
|
completeness: {
|
|
464
483
|
status,
|
|
@@ -482,19 +501,50 @@ const page = (all, offset, limit) => ({
|
|
|
482
501
|
items: all.slice(offset, offset + limit),
|
|
483
502
|
});
|
|
484
503
|
const byStart = (a, b) => a.startMs - b.startMs || (a.spanId < b.spanId ? -1 : Number(a.spanId > b.spanId));
|
|
485
|
-
/**
|
|
486
|
-
const ranked = (measure) => (a, b) =>
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
504
|
+
/** `measure` descending, ties by start. */
|
|
505
|
+
const ranked = (measure) => (a, b) => measure(b) - measure(a) || byStart(a, b);
|
|
506
|
+
/**
|
|
507
|
+
* Open spans interleave with completed ones by a lower bound of the measure:
|
|
508
|
+
* `elapsedLowerBoundMs` for duration, and for outside-children time the part
|
|
509
|
+
* of it no recorded child covered so far (open children counted up to now).
|
|
510
|
+
*/
|
|
511
|
+
const sorter = (sort, analysis) => {
|
|
512
|
+
if (sort === 'start')
|
|
513
|
+
return byStart;
|
|
514
|
+
if (sort === 'duration')
|
|
515
|
+
return ranked((item) => item.durationMs ?? item.elapsedLowerBoundMs);
|
|
516
|
+
return ranked((item) => item.outsideChildrenMs ?? analysis.openOutside.get(item.spanId));
|
|
517
|
+
};
|
|
518
|
+
/** Nearest `limits.ancestry` ancestors, root-most first; with `openOnly`, only the contiguous open ones. */
|
|
519
|
+
const ancestry = (analysis, span, openOnly) => {
|
|
520
|
+
const { store } = analysis;
|
|
521
|
+
const items = [];
|
|
522
|
+
const seen = new Set([span.spanId]);
|
|
523
|
+
let parent = span.parentId === undefined ? undefined : store.spans.get(span.parentId);
|
|
524
|
+
let truncated = false;
|
|
525
|
+
while (parent !== undefined &&
|
|
526
|
+
!seen.has(parent.spanId) &&
|
|
527
|
+
!(openOnly && parent.end !== undefined)) {
|
|
528
|
+
if (items.length === limits.ancestry) {
|
|
529
|
+
truncated = true;
|
|
530
|
+
break;
|
|
531
|
+
}
|
|
532
|
+
seen.add(parent.spanId);
|
|
533
|
+
items.push(ref(analysis, parent));
|
|
534
|
+
parent = parent.parentId === undefined ? undefined : store.spans.get(parent.parentId);
|
|
535
|
+
}
|
|
536
|
+
return { items: items.reverse(), truncated };
|
|
493
537
|
};
|
|
494
|
-
const
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
538
|
+
const openAncestors = (analysis, span) => {
|
|
539
|
+
const { items, truncated } = ancestry(analysis, span, true);
|
|
540
|
+
return {
|
|
541
|
+
items: items.map((item) => ({
|
|
542
|
+
...item,
|
|
543
|
+
elapsedLowerBoundMs: analysis.item(analysis.store.spans.get(item.spanId))
|
|
544
|
+
.elapsedLowerBoundMs,
|
|
545
|
+
})),
|
|
546
|
+
truncated,
|
|
547
|
+
};
|
|
498
548
|
};
|
|
499
549
|
const summarize = (analysis, top) => {
|
|
500
550
|
const items = [...analysis.store.spans.values()].map((span) => analysis.item(span));
|
|
@@ -534,30 +584,31 @@ const summarize = (analysis, top) => {
|
|
|
534
584
|
for (const log of analysis.store.logs)
|
|
535
585
|
byLevel[log.level] = (byLevel[log.level] ?? 0) + 1;
|
|
536
586
|
const completed = items.filter((item) => item.endMs !== null);
|
|
587
|
+
const open = items.filter((item) => item.endMs === null).sort(sorter('duration', analysis));
|
|
588
|
+
const innermost = open.filter((item) => item.openChildCount === 0);
|
|
537
589
|
const failures = items
|
|
538
590
|
.filter((item) => item.status === 'error' || item.status === 'defect')
|
|
539
591
|
.sort(byStart);
|
|
540
592
|
const { store } = analysis;
|
|
541
|
-
const last = store.memory.at(-1);
|
|
542
593
|
return {
|
|
543
594
|
spans: counts,
|
|
544
595
|
spanEvents: store.stats().events,
|
|
545
596
|
logs: { total: store.logs.length, byLevel },
|
|
546
|
-
memory:
|
|
547
|
-
? null
|
|
548
|
-
: {
|
|
549
|
-
samples: store.memory.length,
|
|
550
|
-
peakHeapUsedBytes: store.memoryPeak,
|
|
551
|
-
peakRssBytes: store.memoryRssPeak,
|
|
552
|
-
lastHeapUsedBytes: last.heapUsed,
|
|
553
|
-
},
|
|
597
|
+
memory: memorySummary(analysis, top),
|
|
554
598
|
failures: { total: failures.length, items: failures.slice(0, top) },
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
599
|
+
unfinished: {
|
|
600
|
+
open: open.length,
|
|
601
|
+
innermost: {
|
|
602
|
+
total: innermost.length,
|
|
603
|
+
items: innermost.slice(0, top).map((item) => ({
|
|
604
|
+
...item,
|
|
605
|
+
openAncestors: openAncestors(analysis, store.spans.get(item.spanId)),
|
|
606
|
+
})),
|
|
607
|
+
},
|
|
608
|
+
},
|
|
609
|
+
longest: completed.toSorted(sorter('duration', analysis)).slice(0, top),
|
|
610
|
+
largestOutsideChildren: completed.toSorted(sorter('outsideChildren', analysis)).slice(0, top),
|
|
611
|
+
longestOpen: open.slice(0, top),
|
|
561
612
|
names: {
|
|
562
613
|
total: groups.size,
|
|
563
614
|
items: [...groups.values()]
|
|
@@ -572,6 +623,59 @@ const summarize = (analysis, top) => {
|
|
|
572
623
|
},
|
|
573
624
|
};
|
|
574
625
|
};
|
|
626
|
+
const memorySummary = (analysis, top) => {
|
|
627
|
+
const { store } = analysis;
|
|
628
|
+
const samples = store.memory;
|
|
629
|
+
const first = samples[0];
|
|
630
|
+
const last = samples.at(-1);
|
|
631
|
+
if (first === undefined || last === undefined)
|
|
632
|
+
return null;
|
|
633
|
+
const peak = samples.find((sample) => sample.heapUsed === store.memoryPeak);
|
|
634
|
+
const gaps = samples.slice(1).map((sample, i) => sample.time - samples[i].time);
|
|
635
|
+
let gapAt = -1;
|
|
636
|
+
for (let i = 0; i < gaps.length; i++)
|
|
637
|
+
if (gapAt < 0 || gaps[i] > gaps[gapAt])
|
|
638
|
+
gapAt = i;
|
|
639
|
+
const sorted = gaps.toSorted((a, b) => a - b);
|
|
640
|
+
const mid = sorted.length >> 1;
|
|
641
|
+
const median = sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
|
|
642
|
+
const active = [...store.spans.values()].filter((span) => span.start <= peak.time && (span.end ?? Number.POSITIVE_INFINITY) >= peak.time);
|
|
643
|
+
const ids = new Set(active.map((span) => span.spanId));
|
|
644
|
+
const innermost = active
|
|
645
|
+
.filter((span) => !span.children.some((id) => ids.has(id)))
|
|
646
|
+
.map((span) => ref(analysis, span))
|
|
647
|
+
.sort((a, b) => a.startMs - b.startMs || (a.spanId < b.spanId ? -1 : 1));
|
|
648
|
+
return {
|
|
649
|
+
samples: samples.length,
|
|
650
|
+
peakHeapUsedBytes: store.memoryPeak,
|
|
651
|
+
peakHeapAtMs: ms(peak.time),
|
|
652
|
+
peakRssBytes: store.memoryRssPeak,
|
|
653
|
+
lastHeapUsedBytes: last.heapUsed,
|
|
654
|
+
firstSampleMs: ms(first.time),
|
|
655
|
+
lastSampleMs: ms(last.time),
|
|
656
|
+
medianIntervalMs: gaps.length === 0 ? null : ms(median),
|
|
657
|
+
maxGapMs: gapAt < 0 ? null : ms(gaps[gapAt]),
|
|
658
|
+
maxGapFromMs: gapAt < 0 ? null : ms(samples[gapAt].time),
|
|
659
|
+
maxGapToMs: gapAt < 0 ? null : ms(samples[gapAt + 1].time),
|
|
660
|
+
spansActiveAtPeak: { total: innermost.length, items: innermost.slice(0, top) },
|
|
661
|
+
};
|
|
662
|
+
};
|
|
663
|
+
const processMemory = (analysis, span) => {
|
|
664
|
+
const until = span.end ?? analysis.now;
|
|
665
|
+
const inRange = analysis.store.memory.filter((sample) => sample.time >= span.start && sample.time <= until);
|
|
666
|
+
const first = inRange[0];
|
|
667
|
+
const last = inRange.at(-1);
|
|
668
|
+
if (first === undefined || last === undefined)
|
|
669
|
+
return null;
|
|
670
|
+
return {
|
|
671
|
+
samples: inRange.length,
|
|
672
|
+
firstSampleMs: ms(first.time),
|
|
673
|
+
lastSampleMs: ms(last.time),
|
|
674
|
+
firstHeapUsedBytes: first.heapUsed,
|
|
675
|
+
lastHeapUsedBytes: last.heapUsed,
|
|
676
|
+
maxHeapUsedBytes: Math.max(...inRange.map((sample) => sample.heapUsed)),
|
|
677
|
+
};
|
|
678
|
+
};
|
|
575
679
|
const ref = (analysis, span) => {
|
|
576
680
|
const item = analysis.item(span);
|
|
577
681
|
return {
|
|
@@ -585,19 +689,6 @@ const ref = (analysis, span) => {
|
|
|
585
689
|
};
|
|
586
690
|
const detail = (analysis, span, childLimit, eventLimit) => {
|
|
587
691
|
const { store } = analysis;
|
|
588
|
-
const ancestry = [];
|
|
589
|
-
const seen = new Set([span.spanId]);
|
|
590
|
-
let parent = span.parentId === undefined ? undefined : store.spans.get(span.parentId);
|
|
591
|
-
let truncated = false;
|
|
592
|
-
while (parent !== undefined && !seen.has(parent.spanId)) {
|
|
593
|
-
if (ancestry.length === limits.ancestry) {
|
|
594
|
-
truncated = true;
|
|
595
|
-
break;
|
|
596
|
-
}
|
|
597
|
-
seen.add(parent.spanId);
|
|
598
|
-
ancestry.push(ref(analysis, parent));
|
|
599
|
-
parent = parent.parentId === undefined ? undefined : store.spans.get(parent.parentId);
|
|
600
|
-
}
|
|
601
692
|
const external = analysis.externalParents.get(span.spanId);
|
|
602
693
|
let parentInfo = { kind: 'none' };
|
|
603
694
|
if (external !== undefined) {
|
|
@@ -622,7 +713,7 @@ const detail = (analysis, span, childLimit, eventLimit) => {
|
|
|
622
713
|
stack: stack?.text ?? null,
|
|
623
714
|
stackTruncated: stack?.truncated ?? false,
|
|
624
715
|
parent: parentInfo,
|
|
625
|
-
ancestry:
|
|
716
|
+
ancestry: ancestry(analysis, span, false),
|
|
626
717
|
children: { total: children.length, items: children.slice(0, childLimit) },
|
|
627
718
|
events: {
|
|
628
719
|
total: events.length,
|
|
@@ -636,6 +727,7 @@ const detail = (analysis, span, childLimit, eventLimit) => {
|
|
|
636
727
|
};
|
|
637
728
|
}),
|
|
638
729
|
},
|
|
730
|
+
processMemory: processMemory(analysis, span),
|
|
639
731
|
};
|
|
640
732
|
};
|
|
641
733
|
/** Span ids of `root` and every retained descendant. */
|
|
@@ -660,6 +752,53 @@ const spanNotFound = (op, spanId, analysis) => failure(op, 'SpanNotFound', 'The
|
|
|
660
752
|
sessionId: analysis.source.session.sessionId,
|
|
661
753
|
completeness: analysis.context().completeness,
|
|
662
754
|
});
|
|
755
|
+
const plural = (n, word) => `${n} ${word}${n === 1 ? '' : 's'}`;
|
|
756
|
+
/** Facts a reader of `summary` could otherwise miss, most important first. */
|
|
757
|
+
const notices = (context, { unfinished, memory }) => {
|
|
758
|
+
const out = [];
|
|
759
|
+
const { termination, completeness, time } = context;
|
|
760
|
+
if (unfinished.open > 0) {
|
|
761
|
+
const spans = plural(unfinished.open, 'span');
|
|
762
|
+
const first = unfinished.innermost.items[0];
|
|
763
|
+
const outer = first?.openAncestors.items[0];
|
|
764
|
+
const within = outer === undefined
|
|
765
|
+
? ''
|
|
766
|
+
: `, within "${outer.name}" (spanId ${outer.spanId}), open for at least ${outer.elapsedLowerBoundMs} ms`;
|
|
767
|
+
const position = first === undefined
|
|
768
|
+
? ''
|
|
769
|
+
: ` Last recorded position: "${first.name}" (spanId ${first.spanId}), open for at least ${first.elapsedLowerBoundMs} ms${within}, innermost of ${plural(unfinished.innermost.total, 'open chain')}; see result.unfinished.`;
|
|
770
|
+
let message;
|
|
771
|
+
if (termination.state === 'active') {
|
|
772
|
+
message = `${spans} had no recorded end at snapshot time and the program is still connected; they may still end.${position}`;
|
|
773
|
+
}
|
|
774
|
+
else {
|
|
775
|
+
const ended = termination.state === 'ended'
|
|
776
|
+
? `the collector recorded a disconnect at ${termination.endedAtMs} ms${termination.unobservedTailMs === null ? '' : `, ${termination.unobservedTailMs} ms after the last retained message, with nothing retained in between`}`
|
|
777
|
+
: 'no session end time is on record';
|
|
778
|
+
message = `${spans} had no recorded end; ${ended}.${position} The protocol has no end-of-session message, so a crash, a kill, an exit without closing spans and a dropped connection look the same; missing ends alone do not establish which, and an end may also have been sent but lost.`;
|
|
779
|
+
}
|
|
780
|
+
out.push({ code: 'openSpans', message });
|
|
781
|
+
out.push({
|
|
782
|
+
code: 'rankingsCompletedOnly',
|
|
783
|
+
message: `result.longest and result.largestOutsideChildren rank completed spans only; the ${spans} without an end are in result.unfinished and result.longestOpen.`,
|
|
784
|
+
});
|
|
785
|
+
}
|
|
786
|
+
const evicted = completeness.collectorDroppedMessages ?? 0;
|
|
787
|
+
if (evicted > 0) {
|
|
788
|
+
out.push({
|
|
789
|
+
code: 'collectorEvicted',
|
|
790
|
+
message: `The collector evicted the oldest ${plural(evicted, 'message')} of this session at its capacity: data before observedFromMs (${time.observedFromMs} ms) is missing, so earlier spans, logs and memory samples are absent and retained spans may lack their start or parent.`,
|
|
791
|
+
});
|
|
792
|
+
}
|
|
793
|
+
// ponytail: fixed 10x threshold; the sampling interval is not in the protocol.
|
|
794
|
+
if (memory?.maxGapMs != null && memory.maxGapMs > 10 * memory.medianIntervalMs) {
|
|
795
|
+
out.push({
|
|
796
|
+
code: 'memorySamplingGap',
|
|
797
|
+
message: `Memory samples are ${memory.medianIntervalMs} ms apart at the median, but no sample was retained for ${memory.maxGapMs} ms (from ${memory.maxGapFromMs} ms to ${memory.maxGapToMs} ms); process memory in that interval is unknown. The cause is not recorded. Possible explanations: synchronous work blocked the event loop so the sampling timer could not run, or the client's outbound queue was full and dropped samples (see completeness.clientDroppedMessages).`,
|
|
798
|
+
});
|
|
799
|
+
}
|
|
800
|
+
return out;
|
|
801
|
+
};
|
|
663
802
|
// ---------------------------------------------------------------------------
|
|
664
803
|
// Entry points
|
|
665
804
|
// ---------------------------------------------------------------------------
|
|
@@ -716,13 +855,15 @@ const answer = (source, request) => {
|
|
|
716
855
|
switch (request.op) {
|
|
717
856
|
case 'summary': {
|
|
718
857
|
const top = request.top ?? limits.top.default;
|
|
858
|
+
const result = summarize(analysis, top);
|
|
719
859
|
return {
|
|
720
860
|
ok: true,
|
|
721
861
|
apiVersion,
|
|
722
862
|
op: 'summary',
|
|
723
863
|
query: { op, sessionId, top },
|
|
864
|
+
notices: notices(context, result),
|
|
724
865
|
...context,
|
|
725
|
-
result
|
|
866
|
+
result,
|
|
726
867
|
};
|
|
727
868
|
}
|
|
728
869
|
case 'spans': {
|
|
@@ -748,7 +889,7 @@ const answer = (source, request) => {
|
|
|
748
889
|
continue;
|
|
749
890
|
matched.push(item);
|
|
750
891
|
}
|
|
751
|
-
matched.sort(
|
|
892
|
+
matched.sort(sorter(sort, analysis));
|
|
752
893
|
return {
|
|
753
894
|
ok: true,
|
|
754
895
|
apiVersion,
|