@reticlehq/server 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -5
- package/SKILL.md +251 -33
- package/dist/bridge/bridge.d.ts +6 -1
- package/dist/bridge/bridge.js +27 -4
- package/dist/capsule/causal-summary.d.ts +20 -0
- package/dist/capsule/causal-summary.js +31 -6
- package/dist/capsule/divergence.js +7 -1
- package/dist/cli/chromium-hint.d.ts +68 -0
- package/dist/cli/chromium-hint.js +145 -0
- package/dist/cli/cli-doctor.js +31 -36
- package/dist/cli/cli-kill.d.ts +58 -0
- package/dist/cli/cli-kill.js +162 -0
- package/dist/cli/cli-launch.d.ts +18 -0
- package/dist/cli/cli-launch.js +32 -1
- package/dist/cli/cli-parse.d.ts +16 -1
- package/dist/cli/cli-parse.js +32 -9
- package/dist/cli/cli-port.d.ts +8 -0
- package/dist/cli/cli-port.js +36 -1
- package/dist/cli/cli-verify.js +2 -2
- package/dist/cli/doctor-daemon-line.js +10 -1
- package/dist/cli/doctor-sessions-line.d.ts +12 -0
- package/dist/cli/doctor-sessions-line.js +66 -0
- package/dist/cli/port-holder.d.ts +10 -0
- package/dist/cli/port-holder.js +26 -2
- package/dist/cli.js +139 -23
- package/dist/crawl/crawl-tools.js +12 -2
- package/dist/crawl/crawl.d.ts +13 -0
- package/dist/crawl/crawl.js +50 -2
- package/dist/daemon/daemon-resilience.js +2 -2
- package/dist/daemon/loopback-alias.d.ts +32 -0
- package/dist/daemon/loopback-alias.js +80 -0
- package/dist/events/contradictions.d.ts +14 -0
- package/dist/events/contradictions.js +26 -17
- package/dist/events/predicate-eval.d.ts +26 -3
- package/dist/events/predicate-eval.js +284 -95
- package/dist/events/predicate-parse.d.ts +1 -1
- package/dist/events/predicate-parse.js +18 -4
- package/dist/events/predicate.js +170 -18
- package/dist/events/ring-buffer.d.ts +8 -0
- package/dist/events/ring-buffer.js +34 -1
- package/dist/flows/flow-classify.js +1 -1
- package/dist/flows/flow-tools.js +12 -4
- package/dist/flows/heal-run.js +1 -1
- package/dist/honesty/blind-spots.js +6 -2
- package/dist/honesty/honesty.d.ts +11 -0
- package/dist/honesty/honesty.js +9 -1
- package/dist/honesty/unsettled.d.ts +40 -0
- package/dist/honesty/unsettled.js +78 -0
- package/dist/honesty/verified.d.ts +7 -0
- package/dist/honesty/verified.js +27 -4
- package/dist/http-server.js +11 -1
- package/dist/index.js +29 -2
- package/dist/init/agent-rules.d.ts +16 -0
- package/dist/init/agent-rules.js +96 -17
- package/dist/init/astro-patch.d.ts +2 -1
- package/dist/init/astro-patch.js +11 -5
- package/dist/init/capabilities.d.ts +22 -0
- package/dist/init/capabilities.js +76 -0
- package/dist/init/confirm.d.ts +96 -0
- package/dist/init/confirm.js +138 -0
- package/dist/init/cra.js +3 -2
- package/dist/init/detect.d.ts +8 -0
- package/dist/init/detect.js +15 -0
- package/dist/init/non-js-project.d.ts +27 -0
- package/dist/init/non-js-project.js +84 -0
- package/dist/init/plan-framework.d.ts +9 -0
- package/dist/init/plan-framework.js +72 -16
- package/dist/init/plan.d.ts +40 -4
- package/dist/init/plan.js +183 -43
- package/dist/init/run.d.ts +24 -2
- package/dist/init/run.js +181 -46
- package/dist/init/slash-command.d.ts +1 -1
- package/dist/init/slash-command.js +46 -6
- package/dist/init/snippets.d.ts +64 -5
- package/dist/init/snippets.js +220 -31
- package/dist/init/vite-config.js +29 -3
- package/dist/input/network-mock-tools.d.ts +19 -0
- package/dist/input/network-mock-tools.js +29 -4
- package/dist/license/license.js +1 -1
- package/dist/mcp/mcp-proxy.d.ts +69 -3
- package/dist/mcp/mcp-proxy.js +215 -23
- package/dist/mcp/mcp.js +58 -7
- package/dist/mcp/proxy-lifecycle.d.ts +20 -1
- package/dist/mcp/proxy-lifecycle.js +22 -1
- package/dist/pool/browser-pool.d.ts +22 -2
- package/dist/pool/browser-pool.js +44 -4
- package/dist/pool/playwright-launcher.js +10 -2
- package/dist/project/learned-routes.d.ts +24 -0
- package/dist/project/learned-routes.js +63 -0
- package/dist/project/project-store.d.ts +6 -0
- package/dist/project/project-store.js +25 -1
- package/dist/session/live-control-tools.js +39 -2
- package/dist/session/live-control.d.ts +7 -9
- package/dist/session/live-control.js +34 -1
- package/dist/session/no-session-diagnosis.d.ts +23 -4
- package/dist/session/no-session-diagnosis.js +128 -44
- package/dist/session/no-session-watch.js +12 -0
- package/dist/session/ref-provenance.d.ts +54 -0
- package/dist/session/ref-provenance.js +61 -0
- package/dist/session/review-store.d.ts +19 -0
- package/dist/session/review-store.js +34 -5
- package/dist/session/session-lease.js +1 -1
- package/dist/session/session-manager.d.ts +9 -0
- package/dist/session/session-manager.js +11 -0
- package/dist/session/session.d.ts +8 -0
- package/dist/session/session.js +12 -0
- package/dist/status-payload.d.ts +13 -1
- package/dist/status-payload.js +6 -1
- package/dist/telemetry/app-instrumented.d.ts +37 -0
- package/dist/telemetry/app-instrumented.js +54 -0
- package/dist/telemetry/automation-hint.d.ts +17 -0
- package/dist/telemetry/automation-hint.js +71 -0
- package/dist/telemetry/bug-found.d.ts +0 -20
- package/dist/telemetry/bug-found.js +38 -21
- package/dist/telemetry/cli-telemetry.js +7 -1
- package/dist/telemetry/daemon-telemetry.d.ts +4 -4
- package/dist/telemetry/daemon-telemetry.js +8 -4
- package/dist/telemetry/feedback.js +1 -1
- package/dist/telemetry/init-telemetry.js +9 -1
- package/dist/telemetry/install-source.d.ts +36 -0
- package/dist/telemetry/install-source.js +40 -0
- package/dist/telemetry/run-telemetry.js +4 -1
- package/dist/telemetry/session-metrics.d.ts +3 -3
- package/dist/telemetry/session-metrics.js +9 -4
- package/dist/telemetry/telemetry.d.ts +7 -1
- package/dist/telemetry/telemetry.js +17 -2
- package/dist/telemetry/tool-refused.d.ts +19 -0
- package/dist/telemetry/tool-refused.js +55 -0
- package/dist/telemetry/verification-of.js +72 -8
- package/dist/temp-dir.d.ts +27 -0
- package/dist/temp-dir.js +46 -0
- package/dist/tools/act-tools.js +77 -11
- package/dist/tools/assert-grade.d.ts +1 -1
- package/dist/tools/assert-grade.js +27 -5
- package/dist/tools/assert-verdict.js +25 -1
- package/dist/tools/contract-tools.js +4 -0
- package/dist/tools/dynamic-tools.js +1 -1
- package/dist/tools/error-recovery.d.ts +10 -0
- package/dist/tools/error-recovery.js +70 -5
- package/dist/tools/feedback-invite.js +1 -1
- package/dist/tools/invoke-tool.js +149 -32
- package/dist/tools/lease-tools.d.ts +29 -1
- package/dist/tools/lease-tools.js +58 -5
- package/dist/tools/navigate-result.js +1 -1
- package/dist/tools/observe-tools.js +1 -1
- package/dist/tools/observed-nothing.js +1 -1
- package/dist/tools/react-grace.d.ts +48 -0
- package/dist/tools/react-grace.js +87 -0
- package/dist/tools/read-tools.js +20 -4
- package/dist/tools/settle-in-flight.d.ts +79 -0
- package/dist/tools/settle-in-flight.js +106 -0
- package/dist/tools/snapshot-delta.d.ts +7 -0
- package/dist/tools/snapshot-delta.js +130 -6
- package/dist/tools/tool-kit.d.ts +52 -5
- package/dist/tools/tool-kit.js +55 -24
- package/dist/tools/tool-surface.js +28 -9
- package/dist/tools/tools.js +38 -4
- package/dist/visual/capture-cleanup.d.ts +16 -0
- package/dist/visual/capture-cleanup.js +39 -0
- package/dist/visual/visual-tools.d.ts +18 -0
- package/dist/visual/visual-tools.js +24 -5
- package/docs/README.md +56 -11
- package/docs/actions.mdx +87 -0
- package/docs/agent-cheatsheet.md +102 -26
- package/docs/architecture.md +46 -35
- package/docs/benchmarks.md +76 -72
- package/docs/best-practices.mdx +211 -0
- package/docs/capabilities.mdx +387 -0
- package/docs/cli/affected.mdx +51 -0
- package/docs/cli/capsules.mdx +69 -0
- package/docs/cli/cloud.mdx +127 -0
- package/docs/cli/doctor.mdx +76 -0
- package/docs/cli/drive.mdx +55 -0
- package/docs/cli/feedback.mdx +67 -0
- package/docs/cli/gate.mdx +64 -0
- package/docs/cli/help.mdx +92 -0
- package/docs/cli/hunt.mdx +57 -0
- package/docs/cli/identify.mdx +62 -0
- package/docs/cli/init.mdx +94 -0
- package/docs/cli/kill.mdx +91 -0
- package/docs/cli/license.mdx +48 -0
- package/docs/cli/mcp.mdx +61 -0
- package/docs/cli/open.mdx +68 -0
- package/docs/cli/restart.mdx +60 -0
- package/docs/cli/rollback.mdx +35 -0
- package/docs/cli/serve.mdx +62 -0
- package/docs/cli/status.mdx +64 -0
- package/docs/cli/stop.mdx +56 -0
- package/docs/cli/telemetry.mdx +57 -0
- package/docs/cli/update.mdx +52 -0
- package/docs/cli/verify.mdx +57 -0
- package/docs/cli/version.mdx +39 -0
- package/docs/cli/watch.mdx +55 -0
- package/docs/cli.mdx +219 -0
- package/docs/debugging.md +13 -9
- package/docs/deploy-checks.md +15 -11
- package/docs/desktop-apps.md +49 -40
- package/docs/desktop.mdx +188 -0
- package/docs/docs.json +363 -0
- package/docs/enterprise.md +20 -16
- package/docs/faq.mdx +88 -0
- package/docs/fixtures.md +11 -7
- package/docs/flows.md +26 -22
- package/docs/for-agents.md +71 -0
- package/docs/frameworks.mdx +199 -0
- package/docs/gate-plan.md +45 -41
- package/docs/gates.md +23 -19
- package/docs/getting-started.md +76 -65
- package/docs/human-control.md +22 -18
- package/docs/index.mdx +101 -0
- package/docs/install-agentic.mdx +215 -0
- package/docs/install-manual.mdx +194 -0
- package/docs/instrumentation.mdx +232 -0
- package/docs/integration-patterns.md +38 -34
- package/docs/local-registry.md +16 -12
- package/docs/multi-agent-testing.md +16 -12
- package/docs/packages/babel-plugin.mdx +70 -0
- package/docs/packages/browser.mdx +111 -0
- package/docs/packages/core.mdx +106 -0
- package/docs/packages/electron.mdx +87 -0
- package/docs/packages/eslint-plugin.mdx +84 -0
- package/docs/packages/next.mdx +91 -0
- package/docs/packages/react.mdx +91 -0
- package/docs/packages/server.mdx +98 -0
- package/docs/packages/tauri.mdx +63 -0
- package/docs/packages/test.mdx +108 -0
- package/docs/packages/vite-plugin.mdx +96 -0
- package/docs/packages.mdx +217 -0
- package/docs/platform-integration.md +33 -26
- package/docs/predicates.mdx +179 -0
- package/docs/quickstart.mdx +211 -0
- package/docs/recipes.mdx +242 -0
- package/docs/skill-file.mdx +105 -0
- package/docs/state-management.mdx +197 -0
- package/docs/style.css +14 -0
- package/docs/system-map.md +47 -32
- package/docs/telemetry-contract.md +101 -46
- package/docs/telemetry-events.mdx +430 -0
- package/docs/telemetry.md +63 -50
- package/docs/testing.md +10 -6
- package/docs/token-efficiency.md +30 -26
- package/docs/tools-act-and-wait.mdx +123 -0
- package/docs/tools-act-sequence.mdx +120 -0
- package/docs/tools-act.mdx +82 -0
- package/docs/tools-assert.mdx +70 -0
- package/docs/tools-console.mdx +59 -0
- package/docs/tools-inspect.mdx +85 -0
- package/docs/tools-navigate.mdx +63 -0
- package/docs/tools-network.mdx +86 -0
- package/docs/tools-observe.mdx +89 -0
- package/docs/tools-overview.mdx +157 -0
- package/docs/tools-query.mdx +82 -0
- package/docs/tools-session-and-feedback.mdx +91 -0
- package/docs/tools-sessions.mdx +65 -0
- package/docs/tools-snapshot.mdx +75 -0
- package/docs/tools-state.mdx +81 -0
- package/docs/tools-tools-and-run.mdx +125 -0
- package/docs/tools-wait-for.mdx +66 -0
- package/docs/troubleshooting.mdx +193 -0
- package/docs/usage.md +175 -153
- package/docs/vs-chrome-devtools-mcp.mdx +82 -0
- package/docs/vs-playwright-mcp.mdx +85 -0
- package/docs/vs-screenshots.mdx +96 -0
- package/docs/what-is-reticle.mdx +63 -0
- package/docs/why-reticle.mdx +107 -0
- package/package.json +2 -2
- package/docs/matrix/2.5.0/claude-code-machine.json +0 -19
- package/docs/matrix/2.5.0/codex-machine.json +0 -20
- package/docs/matrix/2.5.0/cursor-machine.json +0 -22
- package/docs/matrix/2.5.0/gemini-machine.json +0 -22
- package/docs/matrix/2.5.0/opencode-machine.json +0 -22
- package/docs/matrix/2.5.0/vscode-machine.json +0 -22
- package/docs/matrix/2.5.0/windsurf-machine.json +0 -22
- package/docs/matrix/MATRIX.md +0 -26
- package/docs/matrix/README.md +0 -59
- package/docs/mint.json +0 -61
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* anyone's app: that a class of defect was found, never what it was found in.
|
|
17
17
|
*/
|
|
18
18
|
import { ReticleTool } from '../tools/tool-names.js';
|
|
19
|
-
import { BugSource, ContradictionKind } from '@reticlehq/core';
|
|
19
|
+
import { BugAttribution, BugSource, ContradictionKind, isAbsenceDerived, } from '@reticlehq/core';
|
|
20
20
|
/** Cap per tool call — a pathological crawl must not turn one call into a thousand events. */
|
|
21
21
|
const MAX_BUGS_PER_CALL = 25;
|
|
22
22
|
function kindsOf(value) {
|
|
@@ -90,25 +90,35 @@ function couldNotRun(failure) {
|
|
|
90
90
|
true === failure.couldNotRun);
|
|
91
91
|
}
|
|
92
92
|
/**
|
|
93
|
-
*
|
|
93
|
+
* WHOSE fault a contradiction was.
|
|
94
94
|
*
|
|
95
|
-
*
|
|
96
|
-
* `request-never-settled` — which Reticle raises from the ABSENCE of a settle event, so a
|
|
97
|
-
* poll manufactured it with the app doing nothing wrong.
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
95
|
+
* `attribution` shipped twice and was wrong both times. It stamped `app` on every contradiction,
|
|
96
|
+
* including `request-never-settled` — which Reticle raises from the ABSENCE of a settle event, so a
|
|
97
|
+
* dev-overlay poll manufactured it with the app doing nothing wrong. Across two full real drives
|
|
98
|
+
* EVERY `attribution: 'app'` was a misattribution, while the one defect that genuinely was a bad
|
|
99
|
+
* agent predicate carried none. A metric confidently wrong about whose fault a defect is, is worse
|
|
100
|
+
* than no metric: it is the number a founder steers on, and it points at the customer.
|
|
101
101
|
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
102
|
+
* The evidence that was missing then exists now, and it is already drawn in core for the verdict:
|
|
103
|
+
* `ABSENCE_DERIVED_CONTRADICTIONS` separates the kinds inferred from something NOT having happened
|
|
104
|
+
* inside a window Reticle chose the end of, from the kinds positively OBSERVED — a request that came
|
|
105
|
+
* back failed, a signal the app fired that disagrees with its own screen, a written field echoed
|
|
106
|
+
* back changed. Only the second class is evidence against the app, and it is the same line that
|
|
107
|
+
* decides whether a verdict may say `no`. Reusing it rather than inventing a second judgement beside
|
|
108
|
+
* it is the whole point: every historical misattribution was an absence-derived kind, so this rule
|
|
109
|
+
* produces zero of them on the data that broke the last two versions.
|
|
105
110
|
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
+
* Everything else — a failed assertion, a replay regression, a crawl anomaly that is not a
|
|
112
|
+
* contradiction — is `unclassified`, which is a VALUE and not a gap. A failed `element.present`
|
|
113
|
+
* covers "the button is missing", "the API is down" and "the agent mistyped a testid" identically,
|
|
114
|
+
* and a console error can be the app, a browser extension or a framework dev overlay. Guessing there
|
|
115
|
+
* puts a guess into the one number we intend to publish. See issue #122.
|
|
111
116
|
*/
|
|
117
|
+
function attributionOf(kind) {
|
|
118
|
+
return CONTRADICTION_KINDS.has(kind) && !isAbsenceDerived(kind)
|
|
119
|
+
? BugAttribution.APP
|
|
120
|
+
: BugAttribution.UNCLASSIFIED;
|
|
121
|
+
}
|
|
112
122
|
export function bugsInResult(toolName, result) {
|
|
113
123
|
// `reticle_run` is a WRAPPER: its handler calls runTool on the real tool, which already reported
|
|
114
124
|
// that result's defects under the real tool's name — then the outer chokepoint reports the very
|
|
@@ -133,6 +143,7 @@ export function bugsInResult(toolName, result) {
|
|
|
133
143
|
kind,
|
|
134
144
|
falseGreen: passed,
|
|
135
145
|
tool: toolName,
|
|
146
|
+
attribution: attributionOf(kind),
|
|
136
147
|
});
|
|
137
148
|
}
|
|
138
149
|
// 2. Crawl anomalies — found autonomously, with nobody writing a test. Crawl reports its own
|
|
@@ -148,8 +159,9 @@ export function bugsInResult(toolName, result) {
|
|
|
148
159
|
// — even though no assertion was made. See the definition on `falseGreen`.
|
|
149
160
|
falseGreen: CONTRADICTION_KINDS.has(kind),
|
|
150
161
|
tool: toolName,
|
|
151
|
-
//
|
|
152
|
-
// fault
|
|
162
|
+
// Same kind, same owner, wherever it arrives. A `request-never-settled` found by the crawler
|
|
163
|
+
// is no better evidence about whose fault it was than one found by an assertion.
|
|
164
|
+
attribution: attributionOf(kind),
|
|
153
165
|
});
|
|
154
166
|
}
|
|
155
167
|
// 3. A failed assertion — the agent declared a consequence and it did not hold.
|
|
@@ -167,7 +179,9 @@ export function bugsInResult(toolName, result) {
|
|
|
167
179
|
kind,
|
|
168
180
|
falseGreen: false,
|
|
169
181
|
tool: toolName,
|
|
170
|
-
//
|
|
182
|
+
// The assertion label cannot say whose fault it was: `element.present` covers a missing
|
|
183
|
+
// button, a downed API and a mistyped testid identically.
|
|
184
|
+
attribution: BugAttribution.UNCLASSIFIED,
|
|
171
185
|
});
|
|
172
186
|
}
|
|
173
187
|
// 4. Replay failures — a saved flow that used to pass no longer does. That is a regression caught,
|
|
@@ -184,8 +198,11 @@ export function bugsInResult(toolName, result) {
|
|
|
184
198
|
kind: 'flow-regression',
|
|
185
199
|
falseGreen: false,
|
|
186
200
|
tool: toolName,
|
|
187
|
-
// A
|
|
188
|
-
//
|
|
201
|
+
// A flow that used to pass and no longer does is a regression SOMEWHERE, and the row does
|
|
202
|
+
// not say where: the app changed, or a selector strategy of ours did. Rows that never RAN
|
|
203
|
+
// are skipped above, which keeps Reticle's outright failures out of the count entirely, but
|
|
204
|
+
// that is a weaker fact than the positive evidence `app` requires.
|
|
205
|
+
attribution: BugAttribution.UNCLASSIFIED,
|
|
189
206
|
});
|
|
190
207
|
}
|
|
191
208
|
}
|
|
@@ -8,6 +8,7 @@ import { TelemetryEventKind } from '@reticlehq/core';
|
|
|
8
8
|
import { DAEMON_INNER_COMMAND, knownCommand } from '../cli/cli-parse.js';
|
|
9
9
|
import { getTelemetry } from './telemetry.js';
|
|
10
10
|
import { describeCliFlags } from './argument-shape.js';
|
|
11
|
+
import { resolveInstallSource } from './install-source.js';
|
|
11
12
|
/**
|
|
12
13
|
* Report that a person ran a `reticle` subcommand.
|
|
13
14
|
*
|
|
@@ -45,7 +46,12 @@ export function reportCliRun(argv) {
|
|
|
45
46
|
// installed the SDK into nine apps — the top of every funnel, missing.
|
|
46
47
|
// `detach` so a quick command (`version`/`gate`) exits immediately instead of waiting out the POST.
|
|
47
48
|
if (telemetry.firstRun)
|
|
48
|
-
void telemetry.emit(TelemetryEventKind.RETICLE_INSTALLED, {
|
|
49
|
+
void telemetry.emit(TelemetryEventKind.RETICLE_INSTALLED, {
|
|
50
|
+
detach: true,
|
|
51
|
+
// WHICH published route brought this machine in. Self-declared by the channel, never inferred
|
|
52
|
+
// — see install-source.ts for which of the four routes can actually carry the marker.
|
|
53
|
+
installSource: resolveInstallSource(),
|
|
54
|
+
});
|
|
49
55
|
if (!isHumanCliCommand(command))
|
|
50
56
|
return;
|
|
51
57
|
void telemetry.emit(TelemetryEventKind.CLI_COMMAND_RUN, {
|
|
@@ -24,10 +24,10 @@ export declare const SESSION_FLUSH_MS: number;
|
|
|
24
24
|
* tick — a closed laptop, OOM, `kill -9`, a force-quit editor — reaches no shutdown handler and has
|
|
25
25
|
* emitted nothing, so the whole session is invisible.
|
|
26
26
|
*
|
|
27
|
-
*
|
|
28
|
-
* sessions never reported a summary at all.** Every "did anyone use Reticle" number is computed
|
|
29
|
-
* the
|
|
30
|
-
* must not do.
|
|
27
|
+
* In the field `daemon_started` outran `daemon_stopped` by a wide margin: **a meaningful share of
|
|
28
|
+
* sessions never reported a summary at all.** Every "did anyone use Reticle" number is computed only
|
|
29
|
+
* on the sessions that did, and undercounts by an unknown amount, which is the one thing a funnel
|
|
30
|
+
* metric must not do.
|
|
31
31
|
*
|
|
32
32
|
* Ninety seconds is chosen against what it protects: a session that did real work usually does it
|
|
33
33
|
* early (snapshot -> act -> assert is seconds), so one early tick captures nearly all of it. It
|
|
@@ -12,6 +12,7 @@ import { getSessionMetrics } from './session-metrics.js';
|
|
|
12
12
|
import { profileProject } from './project-profile.js';
|
|
13
13
|
import { startUpdateCheck } from '../update/update-nudge.js';
|
|
14
14
|
import { markDaemonStart } from './mcp-connection.js';
|
|
15
|
+
import { markInstrumentationClock } from './app-instrumented.js';
|
|
15
16
|
/** Let the daemon finish coming up before walking the source tree for a profile. */
|
|
16
17
|
const PROJECT_PROFILE_DELAY_MS = 5_000;
|
|
17
18
|
/**
|
|
@@ -31,10 +32,10 @@ export const SESSION_FLUSH_MS = 5 * 60 * 1000;
|
|
|
31
32
|
* tick — a closed laptop, OOM, `kill -9`, a force-quit editor — reaches no shutdown handler and has
|
|
32
33
|
* emitted nothing, so the whole session is invisible.
|
|
33
34
|
*
|
|
34
|
-
*
|
|
35
|
-
* sessions never reported a summary at all.** Every "did anyone use Reticle" number is computed
|
|
36
|
-
* the
|
|
37
|
-
* must not do.
|
|
35
|
+
* In the field `daemon_started` outran `daemon_stopped` by a wide margin: **a meaningful share of
|
|
36
|
+
* sessions never reported a summary at all.** Every "did anyone use Reticle" number is computed only
|
|
37
|
+
* on the sessions that did, and undercounts by an unknown amount, which is the one thing a funnel
|
|
38
|
+
* metric must not do.
|
|
38
39
|
*
|
|
39
40
|
* Ninety seconds is chosen against what it protects: a session that did real work usually does it
|
|
40
41
|
* early (snapshot -> act -> assert is seconds), so one early tick captures nearly all of it. It
|
|
@@ -53,6 +54,9 @@ export function installDaemonTelemetry(cwd, now = () => Date.now()) {
|
|
|
53
54
|
// this, so a published fix was invisible until someone manually ran `reticle update`.
|
|
54
55
|
startUpdateCheck(now);
|
|
55
56
|
markDaemonStart(now());
|
|
57
|
+
// Same clock, different half of the install: one measures how long before an AGENT attached, this
|
|
58
|
+
// one how long before an APP did. The gap between them is the funnel.
|
|
59
|
+
markInstrumentationClock(now());
|
|
56
60
|
// One profile per daemon start: what kind of project this is and how deeply Reticle is used in it.
|
|
57
61
|
// Deferred off the boot path — it walks the source tree, and nothing about a daemon coming up
|
|
58
62
|
// should wait on a filesystem scan.
|
|
@@ -147,7 +147,7 @@ export async function submitFeedback(input, opts = {}) {
|
|
|
147
147
|
// Reticle SOURCE CHECKOUT disables telemetry by cwd, and that is exactly where release runs and
|
|
148
148
|
// contributor sessions happen — so the reports most worth having were the ones silently lost.
|
|
149
149
|
reason: isReticleSourceCheckout(opts.cwd ?? process.cwd())
|
|
150
|
-
? 'this is a Reticle source checkout, where telemetry is disabled by design
|
|
150
|
+
? 'this is a Reticle source checkout, where telemetry is disabled by design, so feedback has nowhere to go. Open an issue at https://github.com/reticlehq/reticle/issues instead, or run from the app you are verifying.'
|
|
151
151
|
: `telemetry is disabled on this machine, so feedback has nowhere to go. Re-enable with \`reticle telemetry enable\`, or open an issue at https://github.com/reticlehq/reticle/issues.`,
|
|
152
152
|
redacted,
|
|
153
153
|
context,
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { TelemetryEventKind } from '@reticlehq/core';
|
|
14
14
|
import { getTelemetry } from './telemetry.js';
|
|
15
|
+
import { resolveInstallSource } from './install-source.js';
|
|
15
16
|
/** Classified init failures — our own vocabulary, never a raw error or a path. */
|
|
16
17
|
export const InitFailure = {
|
|
17
18
|
/** Run outside a project root. The single most common first-run mistake. */
|
|
@@ -25,7 +26,14 @@ export const InitFailure = {
|
|
|
25
26
|
/** Report one init outcome. Best-effort; setup must never fail because a metric did. */
|
|
26
27
|
export function reportInitOutcome(init) {
|
|
27
28
|
try {
|
|
28
|
-
void getTelemetry().emit(TelemetryEventKind.INIT_COMPLETED, {
|
|
29
|
+
void getTelemetry().emit(TelemetryEventKind.INIT_COMPLETED, {
|
|
30
|
+
init,
|
|
31
|
+
detach: true,
|
|
32
|
+
// The other half of attribution: `reticle_installed` says a machine arrived, this says the
|
|
33
|
+
// setup on it finished, and only together do they say which route converts. Same self-declared
|
|
34
|
+
// marker, same refusal to infer — see install-source.ts.
|
|
35
|
+
installSource: resolveInstallSource(),
|
|
36
|
+
});
|
|
29
37
|
}
|
|
30
38
|
catch {
|
|
31
39
|
/* a metric must never break `reticle init` */
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which published route brought an install in.
|
|
3
|
+
*
|
|
4
|
+
* Four install routes ship at once and none of them could be told apart, so every question about
|
|
5
|
+
* where distribution effort is working had no data behind it at all.
|
|
6
|
+
*
|
|
7
|
+
* ONE mechanism: an environment variable a channel sets on itself. Everything else that looked like
|
|
8
|
+
* a signal turned out not to be one — see `InstallSource` in core for why `npm_config_user_agent`,
|
|
9
|
+
* a `.claude-plugin/` directory and an installed skill folder all fail to separate these routes.
|
|
10
|
+
*
|
|
11
|
+
* WHICH CHANNELS ARE GENUINELY DETECTABLE TODAY, so nobody later reads a small `unknown` as success:
|
|
12
|
+
* - `plugin` — real. The Claude Code plugin registers the MCP server itself and can set the marker
|
|
13
|
+
* in that server's `env`, so it is carried by the process without anybody typing anything.
|
|
14
|
+
* - `skill_file` / `npx_skill` / `docs_site` / `readme` — real ONLY where that channel's own copy
|
|
15
|
+
* of the install command carries the marker. Each is a separately published artifact, so a copy
|
|
16
|
+
* that has not been updated is indistinguishable from no channel at all.
|
|
17
|
+
* - `cli_direct` — NOT detectable. Nobody types a marker to say they typed a command. It exists so
|
|
18
|
+
* a wrapper can declare it, and until something does, a direct install reports `unknown`.
|
|
19
|
+
*
|
|
20
|
+
* `unknown` is therefore expected to be the largest bucket for as long as the marker is spreading,
|
|
21
|
+
* and shrinking it is a distribution job, never a classifier job. A guessed attribution is worse
|
|
22
|
+
* than none: it is the number distribution decisions get steered on, and a guess and a measurement
|
|
23
|
+
* are indistinguishable once they are in the same column.
|
|
24
|
+
*/
|
|
25
|
+
import { InstallSource } from '@reticlehq/core';
|
|
26
|
+
/** The one marker. Set by a channel on the process that runs the install. */
|
|
27
|
+
export declare const INSTALL_SOURCE_ENV = "RETICLE_INSTALL_SOURCE";
|
|
28
|
+
/**
|
|
29
|
+
* The declared install source, or `unknown`.
|
|
30
|
+
*
|
|
31
|
+
* Narrowed against the closed list rather than echoed: an echo would forward whatever somebody
|
|
32
|
+
* exported — a path, a URL, a campaign string — straight onto the wire, which is the rule this
|
|
33
|
+
* whole contract is built on. Trimmed and lowercased first, because the marker travels through
|
|
34
|
+
* shell snippets and copy-paste, and `Skill_File` is the same channel as `skill_file`.
|
|
35
|
+
*/
|
|
36
|
+
export declare function resolveInstallSource(env?: NodeJS.ProcessEnv): InstallSource;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which published route brought an install in.
|
|
3
|
+
*
|
|
4
|
+
* Four install routes ship at once and none of them could be told apart, so every question about
|
|
5
|
+
* where distribution effort is working had no data behind it at all.
|
|
6
|
+
*
|
|
7
|
+
* ONE mechanism: an environment variable a channel sets on itself. Everything else that looked like
|
|
8
|
+
* a signal turned out not to be one — see `InstallSource` in core for why `npm_config_user_agent`,
|
|
9
|
+
* a `.claude-plugin/` directory and an installed skill folder all fail to separate these routes.
|
|
10
|
+
*
|
|
11
|
+
* WHICH CHANNELS ARE GENUINELY DETECTABLE TODAY, so nobody later reads a small `unknown` as success:
|
|
12
|
+
* - `plugin` — real. The Claude Code plugin registers the MCP server itself and can set the marker
|
|
13
|
+
* in that server's `env`, so it is carried by the process without anybody typing anything.
|
|
14
|
+
* - `skill_file` / `npx_skill` / `docs_site` / `readme` — real ONLY where that channel's own copy
|
|
15
|
+
* of the install command carries the marker. Each is a separately published artifact, so a copy
|
|
16
|
+
* that has not been updated is indistinguishable from no channel at all.
|
|
17
|
+
* - `cli_direct` — NOT detectable. Nobody types a marker to say they typed a command. It exists so
|
|
18
|
+
* a wrapper can declare it, and until something does, a direct install reports `unknown`.
|
|
19
|
+
*
|
|
20
|
+
* `unknown` is therefore expected to be the largest bucket for as long as the marker is spreading,
|
|
21
|
+
* and shrinking it is a distribution job, never a classifier job. A guessed attribution is worse
|
|
22
|
+
* than none: it is the number distribution decisions get steered on, and a guess and a measurement
|
|
23
|
+
* are indistinguishable once they are in the same column.
|
|
24
|
+
*/
|
|
25
|
+
import { InstallSource } from '@reticlehq/core';
|
|
26
|
+
/** The one marker. Set by a channel on the process that runs the install. */
|
|
27
|
+
export const INSTALL_SOURCE_ENV = 'RETICLE_INSTALL_SOURCE';
|
|
28
|
+
const KNOWN = new Set(Object.values(InstallSource));
|
|
29
|
+
/**
|
|
30
|
+
* The declared install source, or `unknown`.
|
|
31
|
+
*
|
|
32
|
+
* Narrowed against the closed list rather than echoed: an echo would forward whatever somebody
|
|
33
|
+
* exported — a path, a URL, a campaign string — straight onto the wire, which is the rule this
|
|
34
|
+
* whole contract is built on. Trimmed and lowercased first, because the marker travels through
|
|
35
|
+
* shell snippets and copy-paste, and `Skill_File` is the same channel as `skill_file`.
|
|
36
|
+
*/
|
|
37
|
+
export function resolveInstallSource(env = process.env) {
|
|
38
|
+
const declared = (env[INSTALL_SOURCE_ENV] ?? '').trim().toLowerCase();
|
|
39
|
+
return KNOWN.has(declared) ? declared : InstallSource.UNKNOWN;
|
|
40
|
+
}
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* finished, already-classified verdict, so this cannot double-count and does not need to understand
|
|
11
11
|
* how a replay works.
|
|
12
12
|
*/
|
|
13
|
-
import { BugSource, RunFlowStatus, TelemetryActor, TelemetryEventKind, VerdictStatus, } from '@reticlehq/core';
|
|
13
|
+
import { BugAttribution, BugSource, RunFlowStatus, TelemetryActor, TelemetryEventKind, VerdictStatus, } from '@reticlehq/core';
|
|
14
14
|
import { getSessionMetrics } from './session-metrics.js';
|
|
15
15
|
import { getTelemetry } from './telemetry.js';
|
|
16
16
|
/** The tool name reported for this path, so CI runs are distinguishable from agent-driven ones. */
|
|
@@ -52,6 +52,9 @@ export function reportRunTelemetry(run) {
|
|
|
52
52
|
falseGreen: false,
|
|
53
53
|
tool: VERIFY_SURFACE,
|
|
54
54
|
repeat: !first,
|
|
55
|
+
// A regression SOMEWHERE, and the row does not say where — the app changed, or a selector
|
|
56
|
+
// strategy of ours did. `app` needs positive evidence; see bug-found.ts.
|
|
57
|
+
attribution: BugAttribution.UNCLASSIFIED,
|
|
55
58
|
},
|
|
56
59
|
});
|
|
57
60
|
}
|
|
@@ -73,10 +73,10 @@ export declare class SessionMetrics {
|
|
|
73
73
|
/**
|
|
74
74
|
* Ask the agent for a verdict, once, when it has driven the page without asking for one.
|
|
75
75
|
*
|
|
76
|
-
*
|
|
77
|
-
* **
|
|
76
|
+
* In the field most sessions that made a tool call produced no verdict — and
|
|
77
|
+
* **almost all of those never called a verdict-producing tool even once.** They drove the app
|
|
78
78
|
* with `reticle_act` and never asked whether it worked. This counter already existed
|
|
79
|
-
* (`abandonedActions
|
|
79
|
+
* (`abandonedActions`); it was reported to US and never to the agent.
|
|
80
80
|
*
|
|
81
81
|
* Three is the threshold because the median verdict-less session made four tool calls: any higher
|
|
82
82
|
* and it never fires for the sessions that need it. Re-arms after a verdict, so a second
|
|
@@ -70,7 +70,7 @@ export class SessionMetrics {
|
|
|
70
70
|
* whole session in a single event" — reported a 30-minute session containing zero tool calls,
|
|
71
71
|
* because it only ever carried the residue since the last flush.
|
|
72
72
|
*
|
|
73
|
-
*
|
|
73
|
+
* In the field almost every `daemon_stopped` row had `toolCalls: 0`, at a median
|
|
74
74
|
* duration of 30.5 minutes, while `session_progress` for the same daemons carried real tool
|
|
75
75
|
* histograms. Anyone computing "did this session use Reticle" off the event named for the end of
|
|
76
76
|
* the session got `no` essentially always. That is not a small skew — it is the funnel reading
|
|
@@ -332,10 +332,10 @@ export class SessionMetrics {
|
|
|
332
332
|
/**
|
|
333
333
|
* Ask the agent for a verdict, once, when it has driven the page without asking for one.
|
|
334
334
|
*
|
|
335
|
-
*
|
|
336
|
-
* **
|
|
335
|
+
* In the field most sessions that made a tool call produced no verdict — and
|
|
336
|
+
* **almost all of those never called a verdict-producing tool even once.** They drove the app
|
|
337
337
|
* with `reticle_act` and never asked whether it worked. This counter already existed
|
|
338
|
-
* (`abandonedActions
|
|
338
|
+
* (`abandonedActions`); it was reported to US and never to the agent.
|
|
339
339
|
*
|
|
340
340
|
* Three is the threshold because the median verdict-less session made four tool calls: any higher
|
|
341
341
|
* and it never fires for the sessions that need it. Re-arms after a verdict, so a second
|
|
@@ -515,6 +515,11 @@ export class SessionMetrics {
|
|
|
515
515
|
: {}),
|
|
516
516
|
...(this.#surface === undefined ? {} : { surface: this.#surface }),
|
|
517
517
|
...(final ? { endReason: this.#endReason() } : {}),
|
|
518
|
+
// The headline metric, as a FIELD. Read off the lifetime counter, because every other
|
|
519
|
+
// verification number in this payload is windowed and a reader who summed the wrong one got a
|
|
520
|
+
// different question's answer. `false` is sent, not omitted: a session that drove an app and
|
|
521
|
+
// never asked whether it worked is the whole finding.
|
|
522
|
+
...(final ? { endedWithVerdict: this.#lifetimeVerifications > 0 } : {}),
|
|
518
523
|
...(this.#feedbackPrompted > 0 ? { feedbackPrompted: this.#feedbackPrompted } : {}),
|
|
519
524
|
...(this.#errorClasses.size > 0
|
|
520
525
|
? { errorClasses: Object.fromEntries(this.#errorClasses) }
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TelemetryEventKind, ProjectIdSource, type Crash, type Feedback, type Identity, type BugFound, type InitOutcome, type McpConnection, type McpOutage, type ProjectProfile, type SessionSummary, type TelemetryActor, type Verification, type VersionChange } from '@reticlehq/core';
|
|
1
|
+
import { TelemetryEventKind, ProjectIdSource, type Crash, type Feedback, type Identity, type BugFound, type InitOutcome, type InstallSource, type McpConnection, type McpOutage, type ToolRefusal, type AppInstrumentation, type ProjectProfile, type SessionSummary, type TelemetryActor, type Verification, type VersionChange } from '@reticlehq/core';
|
|
2
2
|
/**
|
|
3
3
|
* Feedback gets its own, much larger budget — and a retry.
|
|
4
4
|
*
|
|
@@ -72,8 +72,14 @@ export interface TelemetryExtra {
|
|
|
72
72
|
init?: InitOutcome;
|
|
73
73
|
/** `bug_found`: one defect Reticle found in the app under test. */
|
|
74
74
|
bug?: BugFound;
|
|
75
|
+
/** `tool_refused`: a call Reticle could not serve, and why. */
|
|
76
|
+
refusal?: ToolRefusal;
|
|
75
77
|
/** `mcp_connection_lost`: the agent lost its tools — which stage, and why. */
|
|
76
78
|
outage?: McpOutage;
|
|
79
|
+
/** `app_instrumented`: an app carrying the SDK reached this daemon for the first time. */
|
|
80
|
+
instrumentation?: AppInstrumentation;
|
|
81
|
+
/** `reticle_installed` / `init_completed`: which published route brought this install in. */
|
|
82
|
+
installSource?: InstallSource;
|
|
77
83
|
}
|
|
78
84
|
export interface Telemetry {
|
|
79
85
|
/**
|
|
@@ -71,6 +71,7 @@ const DETACHED_SEND_SCRIPT = "fetch(process.argv[1],{method:'POST',headers:{'con
|
|
|
71
71
|
/** Env var names — mirror cloud-sync's `RETICLE_*` convention. */
|
|
72
72
|
import { isReticleSourceCheckout } from './dev-repo.js';
|
|
73
73
|
import { gitFacts } from './git-facts.js';
|
|
74
|
+
import { currentAutomationHint } from './automation-hint.js';
|
|
74
75
|
const Env = {
|
|
75
76
|
DISABLE: 'RETICLE_TELEMETRY', // "0" / "false" / "off" → disabled
|
|
76
77
|
DO_NOT_TRACK: 'DO_NOT_TRACK', // the cross-tool opt-out convention (any truthy value)
|
|
@@ -200,7 +201,7 @@ export const describeTelemetry = (env = process.env, dir = RETICLE_DIR, cwd = pr
|
|
|
200
201
|
if (isReticleSourceCheckout(cwd)) {
|
|
201
202
|
return {
|
|
202
203
|
enabled: false,
|
|
203
|
-
reason: 'this is a Reticle source checkout
|
|
204
|
+
reason: 'this is a Reticle source checkout; developing it is not using it',
|
|
204
205
|
policyUrl: POLICY_URL,
|
|
205
206
|
};
|
|
206
207
|
}
|
|
@@ -259,6 +260,9 @@ export const createTelemetry = (opts) => {
|
|
|
259
260
|
const doFetch = opts.fetchImpl ?? fetch;
|
|
260
261
|
const url = `${(env[Env.URL] ?? DEFAULT_URL).replace(/\/+$/, '')}${TELEMETRY_PATH}`;
|
|
261
262
|
const ci = env[Env.CI] !== undefined && env[Env.CI] !== '';
|
|
263
|
+
// Read once per process, next to `ci` and never instead of it: this is a second angle on the same
|
|
264
|
+
// question, and the rule that it can never filter a row lives in automation-hint.ts.
|
|
265
|
+
const automation = currentAutomationHint(env);
|
|
262
266
|
const os = platform();
|
|
263
267
|
// Negated so it reads the way people say it: UTC+2 is +120, not -120.
|
|
264
268
|
const tzOffsetMin = -new Date().getTimezoneOffset();
|
|
@@ -300,6 +304,9 @@ export const createTelemetry = (opts) => {
|
|
|
300
304
|
ts: now(),
|
|
301
305
|
version: opts.version,
|
|
302
306
|
ci,
|
|
307
|
+
// A SCALAR, so it needs the event build and the wire schema and no `blocks` entry — the same
|
|
308
|
+
// three-place trap `outage` fell into, and `installSource` documents below.
|
|
309
|
+
...(automation !== undefined ? { automation } : {}),
|
|
303
310
|
os,
|
|
304
311
|
tzOffsetMin,
|
|
305
312
|
projectIdSource,
|
|
@@ -316,14 +323,20 @@ export const createTelemetry = (opts) => {
|
|
|
316
323
|
...(extra?.connection !== undefined ? { connection: extra.connection } : {}),
|
|
317
324
|
...(extra?.init !== undefined ? { init: extra.init } : {}),
|
|
318
325
|
...(extra?.bug !== undefined ? { bug: extra.bug } : {}),
|
|
326
|
+
...(extra?.refusal !== undefined ? { refusal: extra.refusal } : {}),
|
|
319
327
|
...(extra?.outage !== undefined ? { outage: extra.outage } : {}),
|
|
328
|
+
...(extra?.instrumentation !== undefined ? { instrumentation: extra.instrumentation } : {}),
|
|
329
|
+
// A SCALAR, so it needs the event build and the wire schema but no entry in `blocks` below —
|
|
330
|
+
// there is nothing to flatten. It still needs both of those, which is the trap `outage` fell
|
|
331
|
+
// into: a field declared on TelemetryExtra and missing from either one is dropped in silence.
|
|
332
|
+
...(extra?.installSource !== undefined ? { installSource: extra.installSource } : {}),
|
|
320
333
|
};
|
|
321
334
|
// Map the core contract onto PostHog's capture shape: id/name/time move up, the rest are properties.
|
|
322
335
|
// The feedback body is FLATTENED into `feedback_*` properties rather than sent as a nested object:
|
|
323
336
|
// PostHog filters and breakdowns operate on top-level properties, so a nested `feedback.stack`
|
|
324
337
|
// would be invisible to the exact "which stacks report the most bugs" question this exists to
|
|
325
338
|
// answer. One prefix keeps them grouped in the UI without a nested path.
|
|
326
|
-
const { anonymousId: distinctId, event: name, ts, sessionId: eventSessionId, feedback, session, project, verification, versionChange, crash, identity, connection, init, bug, outage, ...rest } = event;
|
|
339
|
+
const { anonymousId: distinctId, event: name, ts, sessionId: eventSessionId, feedback, session, project, verification, versionChange, crash, identity, connection, init, bug, refusal, outage, instrumentation, ...rest } = event;
|
|
327
340
|
// `$session_id` is PostHog's OWN session property, so sending ours under that name lights up
|
|
328
341
|
// its built-in session views and funnels for free rather than requiring custom HogQL everywhere.
|
|
329
342
|
// `sessionId` rides only on events that happened inside a daemon run. A one-shot CLI command
|
|
@@ -351,7 +364,9 @@ export const createTelemetry = (opts) => {
|
|
|
351
364
|
connection,
|
|
352
365
|
init,
|
|
353
366
|
bug,
|
|
367
|
+
refusal,
|
|
354
368
|
outage,
|
|
369
|
+
instrumentation,
|
|
355
370
|
};
|
|
356
371
|
for (const [prefix, block] of Object.entries(blocks)) {
|
|
357
372
|
for (const [key, value] of Object.entries(block ?? {})) {
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reporting the calls Reticle could not serve.
|
|
3
|
+
*
|
|
4
|
+
* The refusal path is the largest thing this product does and the least measured one. It computes a
|
|
5
|
+
* precise diagnosis — which of three no-session situations this is, that a ref went stale, that a
|
|
6
|
+
* predicate did not parse — hands it to the agent as prose, and throws it away. So a user who
|
|
7
|
+
* attaches an agent, hits a wall on the first call and never comes back emits nothing at all, and
|
|
8
|
+
* the whole cohort is visible only as the gap between two other numbers. See issue #172.
|
|
9
|
+
*
|
|
10
|
+
* Emitted from `runTool`, which is the one place both dispatch paths cross, so a tool added later is
|
|
11
|
+
* covered without anybody remembering to cover it.
|
|
12
|
+
*/
|
|
13
|
+
import { type RefusalReason } from '@reticlehq/core';
|
|
14
|
+
/** A call that was served. Breaks the retry chain: what follows it is not a retry of anything. */
|
|
15
|
+
export declare function noteToolServed(): void;
|
|
16
|
+
/** One refused call, reported with its cause and whether it was itself a retry. */
|
|
17
|
+
export declare function reportToolRefused(tool: string, reason: RefusalReason): void;
|
|
18
|
+
/** Tests only — the counters are process-lifetime, so each case has to start from zero. */
|
|
19
|
+
export declare function resetToolRefusals(): void;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reporting the calls Reticle could not serve.
|
|
3
|
+
*
|
|
4
|
+
* The refusal path is the largest thing this product does and the least measured one. It computes a
|
|
5
|
+
* precise diagnosis — which of three no-session situations this is, that a ref went stale, that a
|
|
6
|
+
* predicate did not parse — hands it to the agent as prose, and throws it away. So a user who
|
|
7
|
+
* attaches an agent, hits a wall on the first call and never comes back emits nothing at all, and
|
|
8
|
+
* the whole cohort is visible only as the gap between two other numbers. See issue #172.
|
|
9
|
+
*
|
|
10
|
+
* Emitted from `runTool`, which is the one place both dispatch paths cross, so a tool added later is
|
|
11
|
+
* covered without anybody remembering to cover it.
|
|
12
|
+
*/
|
|
13
|
+
import { TelemetryActor, TelemetryEventKind } from '@reticlehq/core';
|
|
14
|
+
import { getTelemetry } from './telemetry.js';
|
|
15
|
+
/**
|
|
16
|
+
* Refusal events one daemon run will send.
|
|
17
|
+
*
|
|
18
|
+
* Volume is part of this taxonomy's design — the per-tool-call event was removed for exactly this
|
|
19
|
+
* reason — and a stuck agent retrying one failing call is the shape most likely to produce hundreds.
|
|
20
|
+
* The cap means the bill cannot scale with the pathology while the SHAPE of it still arrives: the
|
|
21
|
+
* first fifty carry the tool, the cause and the retry flag, and `consecutiveRepeats` on the session
|
|
22
|
+
* summary already reports how long the loop ran.
|
|
23
|
+
*/
|
|
24
|
+
const MAX_REFUSALS_PER_SESSION = 50;
|
|
25
|
+
let sent = 0;
|
|
26
|
+
/**
|
|
27
|
+
* The tool whose call refused immediately before this one, if the last call refused at all.
|
|
28
|
+
*
|
|
29
|
+
* This is what `retried` is computed from, and it is why the event is sent at the moment of the
|
|
30
|
+
* refusal rather than held back until the next call reveals whether one came. Holding it back would
|
|
31
|
+
* lose the event entirely for an agent that gives up — which is precisely the agent this exists to
|
|
32
|
+
* describe — so the flag lands on the RETRY instead of on the first refusal.
|
|
33
|
+
*/
|
|
34
|
+
let lastRefusedTool;
|
|
35
|
+
/** A call that was served. Breaks the retry chain: what follows it is not a retry of anything. */
|
|
36
|
+
export function noteToolServed() {
|
|
37
|
+
lastRefusedTool = undefined;
|
|
38
|
+
}
|
|
39
|
+
/** One refused call, reported with its cause and whether it was itself a retry. */
|
|
40
|
+
export function reportToolRefused(tool, reason) {
|
|
41
|
+
const retried = lastRefusedTool === tool;
|
|
42
|
+
lastRefusedTool = tool;
|
|
43
|
+
if (sent >= MAX_REFUSALS_PER_SESSION)
|
|
44
|
+
return;
|
|
45
|
+
sent += 1;
|
|
46
|
+
void getTelemetry().emit(TelemetryEventKind.TOOL_REFUSED, {
|
|
47
|
+
actor: TelemetryActor.AGENT,
|
|
48
|
+
refusal: { tool, reason, retried },
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
/** Tests only — the counters are process-lifetime, so each case has to start from zero. */
|
|
52
|
+
export function resetToolRefusals() {
|
|
53
|
+
sent = 0;
|
|
54
|
+
lastRefusedTool = undefined;
|
|
55
|
+
}
|