opentel-mcp 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +108 -0
- package/README.md +216 -6
- package/package.json +2 -1
- package/src/config.js +10 -0
- package/src/index.d.ts +56 -2
- package/src/instrument.js +319 -7
- package/src/thrash/attributes.js +50 -0
- package/src/thrash/config.js +123 -0
- package/src/thrash/detector.js +316 -0
- package/src/thrash/emitter.js +122 -0
- package/src/thrash/store.js +129 -0
- package/src/thrash/types.d.ts +158 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,113 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.6.0
|
|
4
|
+
|
|
5
|
+
### Added — Agent Thrash Detection
|
|
6
|
+
|
|
7
|
+
Detects when an agent retries the same tool with the same v0.4 failure
|
|
8
|
+
fingerprint repeatedly, and attributes the wasted v0.5 tokens/cost to that
|
|
9
|
+
loop — see the README's new "Agent Thrash Detection" section for the full
|
|
10
|
+
picture, including the sessionId resolution rules below (the one thing
|
|
11
|
+
most likely to be misconfigured).
|
|
12
|
+
|
|
13
|
+
- Metrics, via the same `@opentelemetry/api`-only pattern as the existing
|
|
14
|
+
`mcp.tool.*` instruments: `mcp.tool.loop.detected` (counter),
|
|
15
|
+
`mcp.tool.loop.length` (histogram), `mcp.tool.loop.wasted_tokens`
|
|
16
|
+
(histogram, unit `tokens`), `mcp.tool.loop.wasted_cost_usd` (histogram,
|
|
17
|
+
unit `USD`), `mcp.tool.loop.duration` (histogram, unit `ms`). All five
|
|
18
|
+
carry only `gen_ai.tool.name` as a metric label — `mcp.failure.fingerprint`
|
|
19
|
+
and the new `mcp.loop.session_id` are deliberately excluded from every
|
|
20
|
+
metric (both are unbounded, per-caller values; see
|
|
21
|
+
`METRIC_SAFE_ATTRIBUTES`'s docblock in `src/fingerprint/attributes.js`).
|
|
22
|
+
- One `mcp.loop.detected` span event on the currently active span (never a
|
|
23
|
+
new span), carrying full detail including `mcp.failure.fingerprint` and
|
|
24
|
+
`mcp.loop.session_id` — span events can carry unbounded attributes
|
|
25
|
+
safely, unlike metric labels.
|
|
26
|
+
- `thrashDetection` option on `instrumentMcpServer()` (see `src/config.js`
|
|
27
|
+
and `src/thrash/config.js`): `{ enabled?, threshold?, windowMs?,
|
|
28
|
+
maxTrackedKeys?, entryTtlMs?, reEmitAfter?, assumeSingleSession? }`.
|
|
29
|
+
Every field independently overridable via an `OTEL_MCP_THRASH_*` env var
|
|
30
|
+
(first documented env-var-driven config pattern in this codebase).
|
|
31
|
+
Defaults: enabled, 3 consecutive same-fingerprint failures within 60s,
|
|
32
|
+
1000 max tracked keys (bounded LRU+TTL — see `src/thrash/store.js`), a
|
|
33
|
+
15-minute idle TTL, re-emit every 3 further failures past threshold,
|
|
34
|
+
`assumeSingleSession: false`.
|
|
35
|
+
- **Requires `fingerprinting: true`** (the default): thrash detection keys
|
|
36
|
+
off the same `mcp.failure.fingerprint` fingerprinting computes, so with
|
|
37
|
+
fingerprinting disabled, detection silently never fires regardless of
|
|
38
|
+
`thrashDetection`'s own settings.
|
|
39
|
+
- Safe-by-default session resolution. A generated per-connection fallback
|
|
40
|
+
session id is used only when a transport is structurally confirmed
|
|
41
|
+
single-connection (no `sessionId` property on `server.transport` — e.g.
|
|
42
|
+
stdio) or explicitly opted into via `thrashDetection.assumeSingleSession`
|
|
43
|
+
— **never** merging concurrent HTTP/SSE clients into one shared key,
|
|
44
|
+
which would otherwise fabricate false-positive loops out of unrelated
|
|
45
|
+
clients' failures. Once a server has been observed handing out a real
|
|
46
|
+
session id, a later call with none is skipped entirely rather than
|
|
47
|
+
falling back. A one-time `diag.warn` fires the first time the fallback
|
|
48
|
+
is actually used, naming which of the three conditions triggered it.
|
|
49
|
+
- `bench/thrash-data-benchmark.js`: a runnable data benchmark (distinct
|
|
50
|
+
from the CPU/memory performance benchmark in
|
|
51
|
+
`test/thrash/benchmark.test.js`) measuring detection rate and wasted
|
|
52
|
+
cost against a real in-process MCP `Client`/`Server` pair, with a
|
|
53
|
+
`--sweep` mode across configured broken-tool rates and a closed-form
|
|
54
|
+
sanity check that aborts rather than shipping a row that deviates
|
|
55
|
+
beyond sampling noise. Two published, reproducible runs committed under
|
|
56
|
+
`bench/results/` — see the README for how to read them (they model
|
|
57
|
+
sensitivity to a reader-supplied failure rate, not a measurement of any
|
|
58
|
+
real deployment).
|
|
59
|
+
- In-process summary: `instrumentMcpServer()`'s returned object gets a
|
|
60
|
+
`getThrashSummary()` method (`ThrashDetector.getSummary()` underneath)
|
|
61
|
+
— a zero-infrastructure way to check "is anything thrashing right now"
|
|
62
|
+
without a metrics backend or trace viewer. Returns `activeLoops` (loops
|
|
63
|
+
currently in the bounded store that have crossed `threshold` — bounded
|
|
64
|
+
by `maxTrackedKeys`, not a complete history) and `topOffenders` (up to 5
|
|
65
|
+
by default, configurable via `getThrashSummary({ topOffendersLimit })`,
|
|
66
|
+
sorted by `wastedCostUsd` descending), alongside `totalLoopsDetected` /
|
|
67
|
+
`totalWastedCostUsd` / `totalWastedTokensIn` / `totalWastedTokensOut` —
|
|
68
|
+
cumulative counters that, unlike the two above, survive LRU eviction
|
|
69
|
+
and TTL expiry (incremented at emit time, with delta-based accounting
|
|
70
|
+
so a loop that re-emits multiple times doesn't have its earlier calls'
|
|
71
|
+
cost double-counted). Never throws; an all-zero summary if
|
|
72
|
+
`thrashDetection` is disabled. Pure read — no effect on the hot path.
|
|
73
|
+
`src/thrash/store.js`'s `BoundedTtlMap` gained an `entries()` iterator
|
|
74
|
+
to support this (a minimal, necessary extension of its Phase 2 surface,
|
|
75
|
+
which was previously get/set/delete/size only).
|
|
76
|
+
|
|
77
|
+
### Public API additions
|
|
78
|
+
|
|
79
|
+
- Re-exported from the package root (`src/index.d.ts`, types only —
|
|
80
|
+
`ThrashDetector`/`createThrashEmitter` stay internal): `ThrashConfig`,
|
|
81
|
+
`ThrashDetectedEvent`, `ThrashSummary`, `ThrashOffender` (new
|
|
82
|
+
`src/thrash/types.d.ts`, mirrors the `src/cost/types.d.ts` /
|
|
83
|
+
`src/fingerprint/types.d.ts` pattern).
|
|
84
|
+
- `instrumentMcpServer()`'s return type gained `getThrashSummary?: (options?: { topOffendersLimit?: number }) => ThrashSummary`,
|
|
85
|
+
alongside the existing `shutdown?`.
|
|
86
|
+
- `fingerprinting` and `thrashDetection` (with all of `ThrashConfig`,
|
|
87
|
+
including `assumeSingleSession`) are now declared on `InstrumentOptions`
|
|
88
|
+
in `src/index.d.ts` — both worked at runtime since introduction but were
|
|
89
|
+
missing from the public type declarations until now. Also added:
|
|
90
|
+
`FingerprintContext`, `Classifier`, `ComputeFingerprintOptions` (the
|
|
91
|
+
types needed to type a custom `computeFingerprint()` classifier per the
|
|
92
|
+
README's "Extending it" section — same kind of gap, found in the same
|
|
93
|
+
audit).
|
|
94
|
+
- `npm run typecheck` (`tsc --noEmit`, new `tsconfig.json`): type-checks
|
|
95
|
+
`src/index.d.ts` and the sibling `src/*/types.d.ts` files, plus a new
|
|
96
|
+
type-level test (`test/index.exports.test-d.ts`, using `expectTypeOf`)
|
|
97
|
+
asserting a consumer can construct `InstrumentOptions` with a partial
|
|
98
|
+
`thrashDetection` and a partial `fingerprinting` config. Does not
|
|
99
|
+
type-check the `.js` source itself (`checkJs: false`) — still no build
|
|
100
|
+
step, this only guards the hand-written public type declarations.
|
|
101
|
+
|
|
102
|
+
### Changed (additive, non-breaking)
|
|
103
|
+
|
|
104
|
+
- `applyCostAttribution()` (internal, `src/instrument.js`) now returns
|
|
105
|
+
`{ tokensIn, tokensOut, costUsd } | null` instead of `void`, so thrash
|
|
106
|
+
detection can reuse one call's already-computed cost figures instead of
|
|
107
|
+
re-running the extractor. Not part of the public API and not observable
|
|
108
|
+
from outside `instrument.js`; nothing in v0.5.0 read the previous
|
|
109
|
+
`undefined` return value.
|
|
110
|
+
|
|
3
111
|
## 0.5.0
|
|
4
112
|
|
|
5
113
|
### Added — Cost & Token Attribution
|
package/README.md
CHANGED
|
@@ -420,6 +420,209 @@ convenience default, not a maintained price list.
|
|
|
420
420
|
session-scoped limits are skipped gracefully (not enforced against a
|
|
421
421
|
fallback key) for transports with no session id, like stdio.
|
|
422
422
|
|
|
423
|
+
## Agent Thrash Detection (v0.6.0)
|
|
424
|
+
|
|
425
|
+
Watches for the same tool failing with the same v0.4 failure fingerprint
|
|
426
|
+
several times in a row inside one session — the pattern an LLM agent
|
|
427
|
+
produces when it keeps retrying a call that can't succeed. When that
|
|
428
|
+
crosses a threshold, it attributes the tokens and cost (v0.5) burned by
|
|
429
|
+
the whole retry loop to one event, instead of leaving it scattered across
|
|
430
|
+
N indistinguishable failed-tool-call spans.
|
|
431
|
+
|
|
432
|
+
### Zero-config quick-start
|
|
433
|
+
|
|
434
|
+
```js
|
|
435
|
+
import { instrumentMcpServer } from 'opentel-mcp';
|
|
436
|
+
|
|
437
|
+
instrumentMcpServer(server, {
|
|
438
|
+
serviceName: 'my-mcp-server',
|
|
439
|
+
setupNodeSdk: true,
|
|
440
|
+
});
|
|
441
|
+
```
|
|
442
|
+
|
|
443
|
+
That's it — `thrashDetection` defaults to enabled, same as `fingerprinting`
|
|
444
|
+
and `costTracking`. Any tool that fails 3 times in a row with the same
|
|
445
|
+
fingerprint inside 60 seconds gets flagged automatically. **Requires
|
|
446
|
+
`fingerprinting: true`** (also the default): detection keys off the same
|
|
447
|
+
`mcp.failure.fingerprint` fingerprinting computes, so with fingerprinting
|
|
448
|
+
disabled, thrash detection silently never fires, regardless of
|
|
449
|
+
`thrashDetection`'s own settings.
|
|
450
|
+
|
|
451
|
+
### Metrics
|
|
452
|
+
|
|
453
|
+
Same API-only pattern as every other metric in this README — nothing
|
|
454
|
+
recorded until a `MeterProvider` is registered.
|
|
455
|
+
|
|
456
|
+
| Metric | Type | Unit | Attributes | Emitted when |
|
|
457
|
+
|---|---|---|---|---|
|
|
458
|
+
| `mcp.tool.loop.detected` | Counter | — | `gen_ai.tool.name` | A loop crosses `threshold`, and again every `reEmitAfter` failures past it |
|
|
459
|
+
| `mcp.tool.loop.length` | Histogram | — | `gen_ai.tool.name` | Same |
|
|
460
|
+
| `mcp.tool.loop.wasted_tokens` | Histogram | tokens | `gen_ai.tool.name` | Same |
|
|
461
|
+
| `mcp.tool.loop.wasted_cost_usd` | Histogram | USD | `gen_ai.tool.name` | Same |
|
|
462
|
+
| `mcp.tool.loop.duration` | Histogram | ms | `gen_ai.tool.name` | Same |
|
|
463
|
+
|
|
464
|
+
Every metric above carries **only** `gen_ai.tool.name`. `mcp.failure.fingerprint`
|
|
465
|
+
and `mcp.loop.session_id` are deliberately excluded from every one of them
|
|
466
|
+
— both are unbounded, per-caller values (a new bug is a new fingerprint,
|
|
467
|
+
forever; a new session is a new session id, forever), so putting either on
|
|
468
|
+
a metric label would turn every distinct bug or session into its own
|
|
469
|
+
permanent time series. See `METRIC_SAFE_ATTRIBUTES`'s docblock in
|
|
470
|
+
`src/fingerprint/attributes.js`. Full detail is still available — on the
|
|
471
|
+
span event below, where high-cardinality attributes are safe.
|
|
472
|
+
|
|
473
|
+
### Span event: `mcp.loop.detected`
|
|
474
|
+
|
|
475
|
+
Added to the **currently active span** (never a new one) each time a loop
|
|
476
|
+
metric above fires.
|
|
477
|
+
|
|
478
|
+
| Attribute | Description |
|
|
479
|
+
|---|---|
|
|
480
|
+
| `mcp.loop.length` | Consecutive same-fingerprint failures in the loop, at the moment of this emission |
|
|
481
|
+
| `mcp.loop.wasted_tokens_in` | Cumulative input tokens burned by the loop so far |
|
|
482
|
+
| `mcp.loop.wasted_tokens_out` | Cumulative output tokens burned by the loop so far |
|
|
483
|
+
| `mcp.loop.wasted_cost_usd` | Cumulative estimated USD cost burned by the loop so far |
|
|
484
|
+
| `mcp.loop.duration_ms` | Elapsed ms between the loop's first and most recent failure |
|
|
485
|
+
| `mcp.loop.first_span_id` | Span id of the loop's first failure |
|
|
486
|
+
| `mcp.loop.first_trace_id` | Trace id of the loop's first failure |
|
|
487
|
+
| `mcp.loop.session_id` | The session this loop belongs to |
|
|
488
|
+
| `mcp.failure.fingerprint` | The shared fingerprint (see "Failure Fingerprinting" above) |
|
|
489
|
+
|
|
490
|
+
### Configuration
|
|
491
|
+
|
|
492
|
+
All fields of `thrashDetection`, each independently overridable by its own
|
|
493
|
+
`OTEL_MCP_THRASH_*` env var (first env-var-driven config in this
|
|
494
|
+
codebase) — precedence is explicit option field, then env var, then
|
|
495
|
+
default; an invalid/unparseable env value falls back to the default
|
|
496
|
+
silently, never throws. Source of truth: `src/thrash/config.js`.
|
|
497
|
+
|
|
498
|
+
| Option | Env var | Type | Default | Description |
|
|
499
|
+
|---|---|---|---|---|
|
|
500
|
+
| `enabled` | `OTEL_MCP_THRASH_ENABLED` | boolean | `true` | `false` disables thrash detection entirely |
|
|
501
|
+
| `threshold` | `OTEL_MCP_THRASH_THRESHOLD` | number | `3` | Consecutive same-fingerprint failures required to trigger detection |
|
|
502
|
+
| `windowMs` | `OTEL_MCP_THRASH_WINDOW_MS` | number | `60000` | Failures must fall inside this rolling window to count toward the same loop |
|
|
503
|
+
| `maxTrackedKeys` | `OTEL_MCP_THRASH_MAX_TRACKED_KEYS` | number | `1000` | LRU cap on the bounded store (`src/thrash/store.js`) |
|
|
504
|
+
| `entryTtlMs` | `OTEL_MCP_THRASH_ENTRY_TTL_MS` | number | `900000` | How long an idle tracked key survives before expiry |
|
|
505
|
+
| `reEmitAfter` | `OTEL_MCP_THRASH_RE_EMIT_AFTER` | number | `3` | Re-emit every N further failures past `threshold` (e.g. 3, 6, 9, ...) instead of once |
|
|
506
|
+
| `assumeSingleSession` | `OTEL_MCP_THRASH_ASSUME_SINGLE_SESSION` | boolean | `false` | Force-permits the fallback session id even when the transport can't be determined — see below |
|
|
507
|
+
|
|
508
|
+
### Session id resolution — read this before setting `assumeSingleSession`
|
|
509
|
+
|
|
510
|
+
**This is the one setting most likely to get misconfigured, so this
|
|
511
|
+
section is deliberately explicit.** Thrash detection needs a session
|
|
512
|
+
boundary to group repeated failures under — merge two different clients'
|
|
513
|
+
failures into one bucket and you get a false-positive loop that never
|
|
514
|
+
happened to either client individually.
|
|
515
|
+
|
|
516
|
+
MCP sessions have a transport-provided id (`extra.sessionId`) on
|
|
517
|
+
session-oriented transports, but stdio has none — there's exactly one
|
|
518
|
+
connection for the process's whole lifetime instead. The resolution rules,
|
|
519
|
+
in order:
|
|
520
|
+
|
|
521
|
+
1. **A real `extra.sessionId` always wins**, and permanently marks the
|
|
522
|
+
server as session-aware.
|
|
523
|
+
2. **Once a server has been observed handing out a real session id, a
|
|
524
|
+
later call with none is skipped entirely** — never merged into a
|
|
525
|
+
shared fallback key, even if `assumeSingleSession` is set. A server
|
|
526
|
+
that has proven it hands out real session ids doesn't get to fall back
|
|
527
|
+
just because one particular call lacked one.
|
|
528
|
+
3. **Before any real session id has ever been observed**, a generated
|
|
529
|
+
per-connection fallback id is used only when:
|
|
530
|
+
- the transport is **structurally confirmed single-connection** — no
|
|
531
|
+
`sessionId` property on `server.transport` at all (e.g. stdio's
|
|
532
|
+
`StdioServerTransport`, which has no session concept whatsoever), or
|
|
533
|
+
- **you set `assumeSingleSession: true`** — an explicit opt-in for
|
|
534
|
+
transports the auto-detection can't see (e.g. a custom `Transport`
|
|
535
|
+
implementation), where you already know every connection is 1:1.
|
|
536
|
+
|
|
537
|
+
Otherwise — an undetermined, potentially multi-client transport, with
|
|
538
|
+
`assumeSingleSession` left at its default `false` — detection is
|
|
539
|
+
**skipped silently** for that call rather than guessing.
|
|
540
|
+
|
|
541
|
+
**The risk of getting this wrong:** if you set `assumeSingleSession: true`
|
|
542
|
+
on a transport that's actually serving multiple concurrent clients (a
|
|
543
|
+
typical HTTP/SSE deployment behind a load balancer, for instance), their
|
|
544
|
+
failures get merged into one shared session key. Three unrelated clients
|
|
545
|
+
each failing once looks identical to one client failing three times in a
|
|
546
|
+
row — a false-positive `mcp.loop.detected` event that never happened to
|
|
547
|
+
any real session. Only set `assumeSingleSession: true` when you have
|
|
548
|
+
independent knowledge that the transport is genuinely 1:1 (a custom
|
|
549
|
+
in-process transport, a dedicated single-tenant connection, etc.) — never
|
|
550
|
+
as a blanket "make the warning go away" setting. A one-time `diag.warn`
|
|
551
|
+
fires the first time the fallback is actually used on a given server,
|
|
552
|
+
naming exactly which of the three conditions above triggered it, so you
|
|
553
|
+
have a chance to catch a wrong assumption before it produces bad data.
|
|
554
|
+
|
|
555
|
+
### In-process summary
|
|
556
|
+
|
|
557
|
+
For a zero-infrastructure quick check — no metrics backend, no trace
|
|
558
|
+
viewer, just "is anything thrashing right now" — the object
|
|
559
|
+
`instrumentMcpServer()` returns gets a `getThrashSummary()` method:
|
|
560
|
+
|
|
561
|
+
```js
|
|
562
|
+
const server = instrumentMcpServer(new Server(...), { serviceName: 'my-mcp-server' });
|
|
563
|
+
|
|
564
|
+
// ...later, e.g. in a health-check handler or just to eyeball it:
|
|
565
|
+
console.log(server.getThrashSummary());
|
|
566
|
+
// {
|
|
567
|
+
// activeLoops: 1,
|
|
568
|
+
// totalLoopsDetected: 4,
|
|
569
|
+
// totalWastedCostUsd: 0.09,
|
|
570
|
+
// totalWastedTokensIn: 3600,
|
|
571
|
+
// totalWastedTokensOut: 900,
|
|
572
|
+
// topOffenders: [
|
|
573
|
+
// { toolName: 'lookup_customer', fingerprint: 'a3f4c8e2b1d09f77', loops: 3, wastedCostUsd: 0.03, wastedTokensIn: 900, wastedTokensOut: 225 },
|
|
574
|
+
// ],
|
|
575
|
+
// }
|
|
576
|
+
```
|
|
577
|
+
|
|
578
|
+
No OTel involved — nothing sent anywhere, safe to call from application
|
|
579
|
+
code. Never throws; returns an all-zero summary if `thrashDetection` is
|
|
580
|
+
disabled, or if instrumentation is disabled entirely (in which case
|
|
581
|
+
`getThrashSummary` isn't attached at all — check for its presence, same
|
|
582
|
+
as `shutdown`).
|
|
583
|
+
|
|
584
|
+
**`activeLoops` and `topOffenders` are bounded by `maxTrackedKeys`, and
|
|
585
|
+
are not a complete history.** They reflect only what's currently sitting
|
|
586
|
+
in the bounded LRU+TTL store this instant — a loop that got evicted (past
|
|
587
|
+
`maxTrackedKeys`) or expired (past `entryTtlMs`) since it was last
|
|
588
|
+
detected won't appear in either, even though it really happened.
|
|
589
|
+
`totalLoopsDetected` and `totalWasted*`, by contrast, are cumulative
|
|
590
|
+
counters that survive both eviction and expiry — they answer "how much
|
|
591
|
+
has this process wasted since it started" (or since the last call to an
|
|
592
|
+
internal `reset()`), not "what's currently active." Don't read
|
|
593
|
+
`topOffenders` as an audit log; read the cumulative totals for that.
|
|
594
|
+
|
|
595
|
+
### Benchmarks
|
|
596
|
+
|
|
597
|
+
Two kinds, both under `bench/` and `test/thrash/`:
|
|
598
|
+
|
|
599
|
+
- **Performance** (`test/thrash/benchmark.test.js`, runs as part of
|
|
600
|
+
`npm test`): CPU/memory overhead of the detection code itself.
|
|
601
|
+
- **Data** (`bench/thrash-data-benchmark.js`, a standalone script — its
|
|
602
|
+
own header docblock documents every `--flag`, including `--sweep`):
|
|
603
|
+
how often a *configured* mix of healthy/broken tool calls results in a
|
|
604
|
+
detected loop, and what it would cost. Two runs are committed under
|
|
605
|
+
`bench/results/` as a reproducibility reference —
|
|
606
|
+
[`published-seed-42.json`](bench/results/published-seed-42.json)
|
|
607
|
+
/ [`.txt`](bench/results/published-seed-42.txt) (a single run) and
|
|
608
|
+
[`published-sweep-seed-42.json`](bench/results/published-sweep-seed-42.json)
|
|
609
|
+
/ [`.md`](bench/results/published-sweep-seed-42.md) (a sweep across
|
|
610
|
+
`brokenToolRate` values 0.02–0.25).
|
|
611
|
+
|
|
612
|
+
**Read the sweep table as a model you parameterize with your own
|
|
613
|
+
observed failure rate, not as a measurement of real-world deployments.**
|
|
614
|
+
`brokenToolRate` is an input the table's reader supplies — every row is
|
|
615
|
+
a configured assumption, not something measured from production
|
|
616
|
+
traffic. There is no single headline percentage here to quote as "how
|
|
617
|
+
often agents thrash" — the whole point of the sweep is that the answer
|
|
618
|
+
depends entirely on your own failure rate, which this benchmark cannot
|
|
619
|
+
know. Both committed files carry a full methodology block (how sessions
|
|
620
|
+
were isolated, that retries are scripted rather than driven by a real
|
|
621
|
+
LLM agent loop, the exact retry/token-growth assumptions, the model and
|
|
622
|
+
price used, and every known limitation that could inflate the numbers)
|
|
623
|
+
and an exact `node bench/thrash-data-benchmark.js ...` command to
|
|
624
|
+
reproduce them byte-for-byte.
|
|
625
|
+
|
|
423
626
|
## Configuration
|
|
424
627
|
|
|
425
628
|
All options passed to `instrumentMcpServer(server, options)`. Source of
|
|
@@ -434,6 +637,9 @@ truth: `src/config.js`.
|
|
|
434
637
|
| `enableMetrics` | boolean | `true` | `false` disables `mcp.tool.*` metrics only |
|
|
435
638
|
| `fingerprinting` | boolean | `true` | `false` disables `mcp.failure.*` attributes |
|
|
436
639
|
| `costTracking` | object | see below | Controls cost/token attribution[^9] — see "Cost & Token Attribution" above |
|
|
640
|
+
| `thrashDetection` | object | see below | Controls Agent Thrash Detection[^10] — see "Agent Thrash Detection" above. Requires `fingerprinting: true` |
|
|
641
|
+
|
|
642
|
+
[^10]: `{ enabled?, threshold?, windowMs?, maxTrackedKeys?, entryTtlMs?, reEmitAfter?, assumeSingleSession? }`, all fields optional, individually defaulted, and individually overridable via an `OTEL_MCP_THRASH_*` env var — see "Agent Thrash Detection" → "Configuration" above for the full table.
|
|
437
643
|
|
|
438
644
|
[^9]: `{ enabled?: boolean; pricingTable?: PricingTable; extractor?: UsageExtractor; budget?: { perSessionUsd?: number; perToolUsd?: number } }`, all fields optional and individually defaulted — `{ enabled: true, pricingTable: DEFAULT_PRICING, extractor: defaultExtractor }` with budget tracking off.
|
|
439
645
|
[^2]: Required only when `setupNodeSdk` is `true`. Has no effect otherwise — the host app's registered `TracerProvider` owns the resource; passing it anyway logs a one-time `diag.warn`.
|
|
@@ -485,11 +691,12 @@ moved there from the main `semantic-conventions` repo, where the MCP
|
|
|
485
691
|
conventions are now deprecated) for everything they define, and adds
|
|
486
692
|
namespaces of its own where they don't yet: `mcp.tool.*` (call-count and
|
|
487
693
|
duration metrics, and — as of v0.5.0 — token/cost attribution and budget
|
|
488
|
-
attributes)
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
694
|
+
attributes), `mcp.failure.*` (failure fingerprinting), and — as of
|
|
695
|
+
v0.6.0 — `mcp.tool.loop.*` / `mcp.loop.*` (Agent Thrash Detection). All
|
|
696
|
+
are documented as non-spec at every attribute (`src/attributes.js`,
|
|
697
|
+
`src/fingerprint/attributes.js`, `src/thrash/attributes.js`), and are
|
|
698
|
+
candidates to fold into the spec's own metrics/error vocabulary if it
|
|
699
|
+
grows an equivalent. Full reasoning: ADR 004 in `docs/adr/`.
|
|
493
700
|
|
|
494
701
|
One exception, also in `src/attributes.js`: `gen_ai.response.model` *is*
|
|
495
702
|
a real spec attribute, co-emitted alongside the custom `mcp.tool.model`
|
|
@@ -505,13 +712,16 @@ pragmatic choice rather than a spec-pure one.
|
|
|
505
712
|
- Supports both low-level `Server` and high-level `McpServer` APIs
|
|
506
713
|
- @modelcontextprotocol/sdk ^1.0.0
|
|
507
714
|
- @opentelemetry/api ^1.9.0
|
|
508
|
-
-
|
|
715
|
+
- 369 tests (`npm test`) — see `test/`
|
|
716
|
+
- `npm run typecheck` (`tsc --noEmit`) type-checks the public `.d.ts`
|
|
717
|
+
surface (`src/index.d.ts` and friends) — see CONTRIBUTING.md
|
|
509
718
|
|
|
510
719
|
## Roadmap
|
|
511
720
|
|
|
512
721
|
- v0.4: Deep Failure Fingerprinting ✓ — see "Failure Fingerprinting" above
|
|
513
722
|
and ADR 006.
|
|
514
723
|
- v0.5: Cost & Token Attribution ✓ — see "Cost & Token Attribution" above.
|
|
724
|
+
- v0.6: Agent Thrash Detection ✓ — see "Agent Thrash Detection" above.
|
|
515
725
|
- Future: failure clustering + regression detection; recovery hints;
|
|
516
726
|
root-cause chaining across parent spans; alignment with the OTel GenAI
|
|
517
727
|
SIG's MCP semantic conventions when published
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opentel-mcp",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "One-line OpenTelemetry instrumentation for Model Context Protocol (MCP) servers",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
"test": "vitest run",
|
|
25
25
|
"test:watch": "vitest",
|
|
26
26
|
"test:coverage": "vitest run --coverage",
|
|
27
|
+
"typecheck": "tsc --noEmit",
|
|
27
28
|
"bench": "vitest bench --run",
|
|
28
29
|
"prepack": "node scripts/strip-workspaces.js",
|
|
29
30
|
"postpack": "node scripts/restore-workspaces.js"
|
package/src/config.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { diag } from '@opentelemetry/api';
|
|
7
7
|
import { DEFAULT_PRICING } from './cost/pricing.js';
|
|
8
8
|
import { defaultExtractor } from './cost/extractor.js';
|
|
9
|
+
import { resolveThrashConfig } from './thrash/config.js';
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* @typedef {object} CostTrackingOptions
|
|
@@ -56,6 +57,14 @@ import { defaultExtractor } from './cost/extractor.js';
|
|
|
56
57
|
* src/metrics.js, and src/cost/budget.js). Defaults to `{ enabled: true, pricingTable: DEFAULT_PRICING,
|
|
57
58
|
* extractor: defaultExtractor }` with budget tracking off; any fields you omit from a partial object fall
|
|
58
59
|
* back to those defaults individually, so `{ enabled: false }` alone works.
|
|
60
|
+
* @property {Partial<import('./thrash/config.js').ThrashConfig>} [thrashDetection] - Controls Agent Thrash
|
|
61
|
+
* Detection (v0.6.0): detecting when a tool is retried repeatedly with the same failure fingerprint, and
|
|
62
|
+
* attributing the wasted tokens/cost to that loop (see instrument.js's applyThrashDetection() /
|
|
63
|
+
* applyThrashSuccessClear(), src/thrash/detector.js, and src/thrash/emitter.js). Resolved via
|
|
64
|
+
* resolveThrashConfig() (src/thrash/config.js) — same partial-overrides-individual-defaults behavior as
|
|
65
|
+
* costTracking above. Requires `fingerprinting` to also be enabled (the default): thrash detection keys
|
|
66
|
+
* off the same mcp.failure.fingerprint fingerprinting computes, so with fingerprinting off there is
|
|
67
|
+
* nothing to key off and detection silently never fires, regardless of this option.
|
|
59
68
|
*/
|
|
60
69
|
|
|
61
70
|
// Guards the "serviceName has no effect" diagnostic below so it fires once
|
|
@@ -111,5 +120,6 @@ export function resolveOptions(options) {
|
|
|
111
120
|
extractor: rawCostTracking.extractor ?? defaultExtractor,
|
|
112
121
|
budget: rawCostTracking.budget,
|
|
113
122
|
},
|
|
123
|
+
thrashDetection: resolveThrashConfig(opts.thrashDetection),
|
|
114
124
|
};
|
|
115
125
|
}
|
package/src/index.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
|
2
2
|
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
3
3
|
import type { CostTrackingOptions } from './cost/types.d.ts';
|
|
4
|
+
import type { ThrashConfig, ThrashSummary } from './thrash/types.d.ts';
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* Options for {@link instrumentMcpServer}.
|
|
@@ -68,6 +69,35 @@ export interface InstrumentOptions {
|
|
|
68
69
|
* `{ enabled: false }` alone works. See {@link CostTrackingOptions} (`src/cost/types.d.ts`).
|
|
69
70
|
*/
|
|
70
71
|
costTracking?: CostTrackingOptions;
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Set to `false` to disable deep-failure fingerprinting. When enabled (the default), every thrown error
|
|
75
|
+
* and tool-level failure (`isError: true`) is run through `computeFingerprint()`
|
|
76
|
+
* (`src/fingerprint/compose.js`), adding `mcp.failure.*` span attributes and an `mcp.failure.category`
|
|
77
|
+
* attribute on the `mcp.tool.errors` / `mcp.tool.silent_failures` / `mcp.tool.duration` metrics.
|
|
78
|
+
* `computeFingerprint()` never throws, so this only trades a small amount of per-failure CPU for
|
|
79
|
+
* fingerprinting.
|
|
80
|
+
*
|
|
81
|
+
* `thrashDetection` (below) depends on this: it keys off the same `mcp.failure.fingerprint`
|
|
82
|
+
* fingerprinting computes, so with `fingerprinting: false` thrash detection silently never fires,
|
|
83
|
+
* regardless of `thrashDetection`'s own settings.
|
|
84
|
+
*
|
|
85
|
+
* @default true
|
|
86
|
+
*/
|
|
87
|
+
fingerprinting?: boolean;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Controls Agent Thrash Detection (v0.6.0): detecting when a tool is retried repeatedly with the same
|
|
91
|
+
* failure fingerprint, and attributing the wasted tokens/cost to that loop (the 5 `mcp.tool.loop.*`
|
|
92
|
+
* metrics plus one `mcp.loop.detected` span event — see the README's "Agent Thrash Detection" section).
|
|
93
|
+
* Any fields you omit from a partial object fall back to their defaults individually, same as
|
|
94
|
+
* `costTracking` above. Requires `fingerprinting` to also be enabled (the default) — see that option's
|
|
95
|
+
* doc. See {@link ThrashConfig} (`src/thrash/types.d.ts`) for the full field list and defaults,
|
|
96
|
+
* including `assumeSingleSession`, which you should read carefully before enabling on any transport
|
|
97
|
+
* that might serve more than one client: getting it wrong merges unrelated clients' failures into
|
|
98
|
+
* false-positive loops.
|
|
99
|
+
*/
|
|
100
|
+
thrashDetection?: Partial<ThrashConfig>;
|
|
71
101
|
}
|
|
72
102
|
|
|
73
103
|
/**
|
|
@@ -111,12 +141,22 @@ export type DuckTypedMcpServer = {
|
|
|
111
141
|
* created for it — call it during your process's own shutdown sequence to
|
|
112
142
|
* avoid losing buffered spans. `shutdown` is typed as optional because it
|
|
113
143
|
* is only attached at runtime when `setupNodeSdk` is `true`; check for its
|
|
114
|
-
* presence before calling.
|
|
144
|
+
* presence before calling. The returned object also gets a
|
|
145
|
+
* `getThrashSummary()` method (v0.6.0) returning a point-in-time,
|
|
146
|
+
* in-process {@link ThrashSummary} — no OTel involved, nothing sent
|
|
147
|
+
* anywhere; see the README's "Agent Thrash Detection" section. Unlike
|
|
148
|
+
* `shutdown`, it's attached unconditionally (not gated behind
|
|
149
|
+
* `setupNodeSdk`) — still typed as optional because it, like `shutdown`,
|
|
150
|
+
* is never attached when `options.enabled` is `false` (nothing is
|
|
151
|
+
* instrumented at all in that case).
|
|
115
152
|
*/
|
|
116
153
|
export function instrumentMcpServer<T extends Server | McpServer | DuckTypedMcpServer>(
|
|
117
154
|
server: T,
|
|
118
155
|
options?: InstrumentOptions,
|
|
119
|
-
): T & {
|
|
156
|
+
): T & {
|
|
157
|
+
shutdown?: () => Promise<void>;
|
|
158
|
+
getThrashSummary?: (options?: { topOffendersLimit?: number }) => ThrashSummary;
|
|
159
|
+
};
|
|
120
160
|
|
|
121
161
|
// --- Deep-failure fingerprinting (src/fingerprint/) ---
|
|
122
162
|
//
|
|
@@ -129,6 +169,9 @@ export type {
|
|
|
129
169
|
FailureOrigin,
|
|
130
170
|
FingerprintResult,
|
|
131
171
|
FingerprintInputs,
|
|
172
|
+
FingerprintContext,
|
|
173
|
+
Classifier,
|
|
174
|
+
ComputeFingerprintOptions,
|
|
132
175
|
} from './fingerprint/types.d.ts';
|
|
133
176
|
|
|
134
177
|
export { computeFingerprint } from './fingerprint/compose.js';
|
|
@@ -152,3 +195,14 @@ export type {
|
|
|
152
195
|
export { DEFAULT_PRICING } from './cost/pricing.js';
|
|
153
196
|
export { defaultExtractor } from './cost/extractor.js';
|
|
154
197
|
export { calculateCost } from './cost/calculator.js';
|
|
198
|
+
|
|
199
|
+
// --- Agent Thrash Detection (src/thrash/) ---
|
|
200
|
+
//
|
|
201
|
+
// Re-exported here so TypeScript consumers get these types from the
|
|
202
|
+
// package root instead of reaching into src/thrash/* directly. See
|
|
203
|
+
// src/thrash/types.d.ts for the full shape documentation. Unlike
|
|
204
|
+
// src/cost/ and src/fingerprint/ above, no runtime values are re-exported
|
|
205
|
+
// here yet — ThrashDetector and createThrashEmitter are internal to
|
|
206
|
+
// instrument.js's wiring, not part of the public API.
|
|
207
|
+
|
|
208
|
+
export type { ThrashConfig, ThrashDetectedEvent, ThrashSummary, ThrashOffender } from './thrash/types.d.ts';
|