@reticlehq/server 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -5
- package/SKILL.md +64 -29
- package/dist/capsule/causal-summary.d.ts +20 -0
- package/dist/capsule/causal-summary.js +31 -6
- package/dist/capsule/divergence.js +7 -1
- package/dist/cli/chromium-hint.d.ts +33 -5
- package/dist/cli/chromium-hint.js +82 -12
- package/dist/cli/cli-doctor.js +28 -25
- package/dist/cli/cli-kill.d.ts +58 -0
- package/dist/cli/cli-kill.js +162 -0
- package/dist/cli/cli-launch.d.ts +17 -0
- package/dist/cli/cli-launch.js +28 -0
- package/dist/cli/cli-parse.d.ts +16 -1
- package/dist/cli/cli-parse.js +32 -9
- package/dist/cli/cli-port.d.ts +8 -0
- package/dist/cli/cli-port.js +16 -0
- package/dist/cli/cli-verify.js +2 -2
- package/dist/cli/doctor-sessions-line.d.ts +12 -0
- package/dist/cli/doctor-sessions-line.js +66 -0
- package/dist/cli/port-holder.d.ts +10 -0
- package/dist/cli/port-holder.js +26 -2
- package/dist/cli.js +99 -14
- package/dist/crawl/crawl-tools.js +12 -2
- package/dist/crawl/crawl.d.ts +13 -0
- package/dist/crawl/crawl.js +50 -2
- package/dist/events/contradictions.d.ts +14 -0
- package/dist/events/contradictions.js +26 -17
- package/dist/events/predicate-eval.d.ts +20 -1
- package/dist/events/predicate-eval.js +125 -3
- package/dist/events/predicate.js +170 -18
- package/dist/flows/flow-classify.js +1 -1
- package/dist/flows/flow-tools.js +12 -4
- package/dist/flows/heal-run.js +1 -1
- package/dist/honesty/blind-spots.js +6 -2
- package/dist/honesty/unsettled.d.ts +40 -0
- package/dist/honesty/unsettled.js +78 -0
- package/dist/honesty/verified.d.ts +7 -0
- package/dist/honesty/verified.js +3 -3
- package/dist/index.js +3 -0
- package/dist/init/agent-rules.d.ts +16 -0
- package/dist/init/agent-rules.js +96 -22
- package/dist/init/astro-patch.d.ts +2 -1
- package/dist/init/astro-patch.js +11 -5
- package/dist/init/capabilities.d.ts +22 -0
- package/dist/init/capabilities.js +76 -0
- package/dist/init/confirm.d.ts +96 -0
- package/dist/init/confirm.js +138 -0
- package/dist/init/cra.js +3 -2
- package/dist/init/non-js-project.js +29 -5
- package/dist/init/plan-framework.js +53 -15
- package/dist/init/plan.d.ts +40 -4
- package/dist/init/plan.js +163 -40
- package/dist/init/run.d.ts +24 -2
- package/dist/init/run.js +148 -42
- package/dist/init/slash-command.d.ts +1 -1
- package/dist/init/slash-command.js +46 -6
- package/dist/init/snippets.d.ts +44 -4
- package/dist/init/snippets.js +161 -37
- package/dist/init/vite-config.js +29 -3
- package/dist/input/network-mock-tools.d.ts +19 -0
- package/dist/input/network-mock-tools.js +29 -4
- package/dist/license/license.js +1 -1
- package/dist/mcp/mcp-proxy.d.ts +69 -3
- package/dist/mcp/mcp-proxy.js +205 -20
- package/dist/mcp/mcp.js +23 -7
- package/dist/mcp/proxy-lifecycle.d.ts +1 -1
- package/dist/mcp/proxy-lifecycle.js +1 -1
- package/dist/pool/browser-pool.d.ts +22 -2
- package/dist/pool/browser-pool.js +44 -4
- package/dist/project/learned-routes.d.ts +24 -0
- package/dist/project/learned-routes.js +63 -0
- package/dist/project/project-store.d.ts +6 -0
- package/dist/project/project-store.js +25 -1
- package/dist/session/no-session-diagnosis.d.ts +19 -0
- package/dist/session/no-session-diagnosis.js +71 -18
- package/dist/session/no-session-watch.js +12 -0
- package/dist/session/ref-provenance.d.ts +54 -0
- package/dist/session/ref-provenance.js +61 -0
- package/dist/session/session-lease.js +1 -1
- package/dist/telemetry/automation-hint.d.ts +17 -0
- package/dist/telemetry/automation-hint.js +71 -0
- package/dist/telemetry/bug-found.d.ts +0 -20
- package/dist/telemetry/bug-found.js +38 -21
- package/dist/telemetry/cli-telemetry.js +7 -1
- package/dist/telemetry/feedback.js +1 -1
- package/dist/telemetry/init-telemetry.js +9 -1
- package/dist/telemetry/install-source.d.ts +36 -0
- package/dist/telemetry/install-source.js +40 -0
- package/dist/telemetry/run-telemetry.js +4 -1
- package/dist/telemetry/session-metrics.js +5 -0
- package/dist/telemetry/telemetry.d.ts +5 -1
- package/dist/telemetry/telemetry.js +15 -2
- package/dist/telemetry/tool-refused.d.ts +19 -0
- package/dist/telemetry/tool-refused.js +55 -0
- package/dist/telemetry/verification-of.js +46 -7
- package/dist/temp-dir.d.ts +27 -0
- package/dist/temp-dir.js +46 -0
- package/dist/tools/act-tools.js +45 -1
- package/dist/tools/assert-grade.d.ts +1 -1
- package/dist/tools/assert-grade.js +27 -5
- package/dist/tools/assert-verdict.js +18 -1
- package/dist/tools/contract-tools.js +4 -0
- package/dist/tools/dynamic-tools.js +1 -1
- package/dist/tools/error-recovery.d.ts +10 -0
- package/dist/tools/error-recovery.js +64 -4
- package/dist/tools/feedback-invite.js +1 -1
- package/dist/tools/invoke-tool.js +139 -23
- package/dist/tools/lease-tools.d.ts +29 -1
- package/dist/tools/lease-tools.js +58 -5
- package/dist/tools/navigate-result.js +1 -1
- package/dist/tools/observe-tools.js +1 -1
- package/dist/tools/observed-nothing.js +1 -1
- package/dist/tools/react-grace.d.ts +48 -0
- package/dist/tools/react-grace.js +87 -0
- package/dist/tools/read-tools.js +20 -4
- package/dist/tools/settle-in-flight.d.ts +79 -0
- package/dist/tools/settle-in-flight.js +106 -0
- package/dist/tools/snapshot-delta.d.ts +7 -0
- package/dist/tools/snapshot-delta.js +130 -6
- package/dist/tools/tool-surface.js +19 -0
- package/dist/tools/tools.js +16 -4
- package/docs/README.md +56 -11
- package/docs/actions.mdx +87 -0
- package/docs/agent-cheatsheet.md +93 -29
- package/docs/architecture.md +46 -35
- package/docs/benchmarks.md +76 -72
- package/docs/best-practices.mdx +211 -0
- package/docs/capabilities.mdx +387 -0
- package/docs/cli/affected.mdx +51 -0
- package/docs/cli/capsules.mdx +69 -0
- package/docs/cli/cloud.mdx +127 -0
- package/docs/cli/doctor.mdx +76 -0
- package/docs/cli/drive.mdx +55 -0
- package/docs/cli/feedback.mdx +67 -0
- package/docs/cli/gate.mdx +64 -0
- package/docs/cli/help.mdx +92 -0
- package/docs/cli/hunt.mdx +57 -0
- package/docs/cli/identify.mdx +62 -0
- package/docs/cli/init.mdx +94 -0
- package/docs/cli/kill.mdx +91 -0
- package/docs/cli/license.mdx +48 -0
- package/docs/cli/mcp.mdx +61 -0
- package/docs/cli/open.mdx +68 -0
- package/docs/cli/restart.mdx +60 -0
- package/docs/cli/rollback.mdx +35 -0
- package/docs/cli/serve.mdx +62 -0
- package/docs/cli/status.mdx +64 -0
- package/docs/cli/stop.mdx +56 -0
- package/docs/cli/telemetry.mdx +57 -0
- package/docs/cli/update.mdx +52 -0
- package/docs/cli/verify.mdx +57 -0
- package/docs/cli/version.mdx +39 -0
- package/docs/cli/watch.mdx +55 -0
- package/docs/cli.mdx +219 -0
- package/docs/debugging.md +13 -9
- package/docs/deploy-checks.md +15 -11
- package/docs/desktop-apps.md +49 -40
- package/docs/desktop.mdx +188 -0
- package/docs/docs.json +363 -0
- package/docs/enterprise.md +20 -16
- package/docs/faq.mdx +88 -0
- package/docs/fixtures.md +11 -7
- package/docs/flows.md +26 -22
- package/docs/for-agents.md +71 -0
- package/docs/frameworks.mdx +199 -0
- package/docs/gate-plan.md +45 -41
- package/docs/gates.md +22 -18
- package/docs/getting-started.md +76 -65
- package/docs/human-control.md +22 -18
- package/docs/index.mdx +101 -0
- package/docs/install-agentic.mdx +215 -0
- package/docs/install-manual.mdx +194 -0
- package/docs/instrumentation.mdx +232 -0
- package/docs/integration-patterns.md +38 -34
- package/docs/local-registry.md +16 -12
- package/docs/multi-agent-testing.md +16 -12
- package/docs/packages/babel-plugin.mdx +70 -0
- package/docs/packages/browser.mdx +111 -0
- package/docs/packages/core.mdx +106 -0
- package/docs/packages/electron.mdx +87 -0
- package/docs/packages/eslint-plugin.mdx +84 -0
- package/docs/packages/next.mdx +91 -0
- package/docs/packages/react.mdx +91 -0
- package/docs/packages/server.mdx +98 -0
- package/docs/packages/tauri.mdx +63 -0
- package/docs/packages/test.mdx +108 -0
- package/docs/packages/vite-plugin.mdx +96 -0
- package/docs/packages.mdx +217 -0
- package/docs/platform-integration.md +33 -26
- package/docs/predicates.mdx +179 -0
- package/docs/quickstart.mdx +211 -0
- package/docs/recipes.mdx +242 -0
- package/docs/skill-file.mdx +105 -0
- package/docs/state-management.mdx +197 -0
- package/docs/style.css +14 -0
- package/docs/system-map.md +47 -32
- package/docs/telemetry-contract.md +93 -50
- package/docs/telemetry-events.mdx +430 -0
- package/docs/telemetry.md +57 -52
- package/docs/testing.md +10 -6
- package/docs/token-efficiency.md +30 -26
- package/docs/tools-act-and-wait.mdx +123 -0
- package/docs/tools-act-sequence.mdx +120 -0
- package/docs/tools-act.mdx +82 -0
- package/docs/tools-assert.mdx +70 -0
- package/docs/tools-console.mdx +59 -0
- package/docs/tools-inspect.mdx +85 -0
- package/docs/tools-navigate.mdx +63 -0
- package/docs/tools-network.mdx +86 -0
- package/docs/tools-observe.mdx +89 -0
- package/docs/tools-overview.mdx +157 -0
- package/docs/tools-query.mdx +82 -0
- package/docs/tools-session-and-feedback.mdx +91 -0
- package/docs/tools-sessions.mdx +65 -0
- package/docs/tools-snapshot.mdx +75 -0
- package/docs/tools-state.mdx +81 -0
- package/docs/tools-tools-and-run.mdx +125 -0
- package/docs/tools-wait-for.mdx +66 -0
- package/docs/troubleshooting.mdx +193 -0
- package/docs/usage.md +175 -153
- package/docs/vs-chrome-devtools-mcp.mdx +82 -0
- package/docs/vs-playwright-mcp.mdx +85 -0
- package/docs/vs-screenshots.mdx +96 -0
- package/docs/what-is-reticle.mdx +63 -0
- package/docs/why-reticle.mdx +107 -0
- package/package.json +2 -2
- package/docs/matrix/2.5.0/claude-code-machine.json +0 -19
- package/docs/matrix/2.5.0/codex-machine.json +0 -20
- package/docs/matrix/2.5.0/cursor-machine.json +0 -22
- package/docs/matrix/2.5.0/gemini-machine.json +0 -22
- package/docs/matrix/2.5.0/opencode-machine.json +0 -22
- package/docs/matrix/2.5.0/vscode-machine.json +0 -22
- package/docs/matrix/2.5.0/windsurf-machine.json +0 -22
- package/docs/matrix/MATRIX.md +0 -26
- package/docs/matrix/README.md +0 -59
- package/docs/mint.json +0 -61
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which published route brought an install in.
|
|
3
|
+
*
|
|
4
|
+
* Four install routes ship at once and none of them could be told apart, so every question about
|
|
5
|
+
* where distribution effort is working had no data behind it at all.
|
|
6
|
+
*
|
|
7
|
+
* ONE mechanism: an environment variable a channel sets on itself. Everything else that looked like
|
|
8
|
+
* a signal turned out not to be one — see `InstallSource` in core for why `npm_config_user_agent`,
|
|
9
|
+
* a `.claude-plugin/` directory and an installed skill folder all fail to separate these routes.
|
|
10
|
+
*
|
|
11
|
+
* WHICH CHANNELS ARE GENUINELY DETECTABLE TODAY, so nobody later reads a small `unknown` as success:
|
|
12
|
+
* - `plugin` — real. The Claude Code plugin registers the MCP server itself and can set the marker
|
|
13
|
+
* in that server's `env`, so it is carried by the process without anybody typing anything.
|
|
14
|
+
* - `skill_file` / `npx_skill` / `docs_site` / `readme` — real ONLY where that channel's own copy
|
|
15
|
+
* of the install command carries the marker. Each is a separately published artifact, so a copy
|
|
16
|
+
* that has not been updated is indistinguishable from no channel at all.
|
|
17
|
+
* - `cli_direct` — NOT detectable. Nobody types a marker to say they typed a command. It exists so
|
|
18
|
+
* a wrapper can declare it, and until something does, a direct install reports `unknown`.
|
|
19
|
+
*
|
|
20
|
+
* `unknown` is therefore expected to be the largest bucket for as long as the marker is spreading,
|
|
21
|
+
* and shrinking it is a distribution job, never a classifier job. A guessed attribution is worse
|
|
22
|
+
* than none: it is the number distribution decisions get steered on, and a guess and a measurement
|
|
23
|
+
* are indistinguishable once they are in the same column.
|
|
24
|
+
*/
|
|
25
|
+
import { InstallSource } from '@reticlehq/core';
|
|
26
|
+
/** The one marker. Set by a channel on the process that runs the install. */
|
|
27
|
+
export declare const INSTALL_SOURCE_ENV = "RETICLE_INSTALL_SOURCE";
|
|
28
|
+
/**
|
|
29
|
+
* The declared install source, or `unknown`.
|
|
30
|
+
*
|
|
31
|
+
* Narrowed against the closed list rather than echoed: an echo would forward whatever somebody
|
|
32
|
+
* exported — a path, a URL, a campaign string — straight onto the wire, which is the rule this
|
|
33
|
+
* whole contract is built on. Trimmed and lowercased first, because the marker travels through
|
|
34
|
+
* shell snippets and copy-paste, and `Skill_File` is the same channel as `skill_file`.
|
|
35
|
+
*/
|
|
36
|
+
export declare function resolveInstallSource(env?: NodeJS.ProcessEnv): InstallSource;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which published route brought an install in.
|
|
3
|
+
*
|
|
4
|
+
* Four install routes ship at once and none of them could be told apart, so every question about
|
|
5
|
+
* where distribution effort is working had no data behind it at all.
|
|
6
|
+
*
|
|
7
|
+
* ONE mechanism: an environment variable a channel sets on itself. Everything else that looked like
|
|
8
|
+
* a signal turned out not to be one — see `InstallSource` in core for why `npm_config_user_agent`,
|
|
9
|
+
* a `.claude-plugin/` directory and an installed skill folder all fail to separate these routes.
|
|
10
|
+
*
|
|
11
|
+
* WHICH CHANNELS ARE GENUINELY DETECTABLE TODAY, so nobody later reads a small `unknown` as success:
|
|
12
|
+
* - `plugin` — real. The Claude Code plugin registers the MCP server itself and can set the marker
|
|
13
|
+
* in that server's `env`, so it is carried by the process without anybody typing anything.
|
|
14
|
+
* - `skill_file` / `npx_skill` / `docs_site` / `readme` — real ONLY where that channel's own copy
|
|
15
|
+
* of the install command carries the marker. Each is a separately published artifact, so a copy
|
|
16
|
+
* that has not been updated is indistinguishable from no channel at all.
|
|
17
|
+
* - `cli_direct` — NOT detectable. Nobody types a marker to say they typed a command. It exists so
|
|
18
|
+
* a wrapper can declare it, and until something does, a direct install reports `unknown`.
|
|
19
|
+
*
|
|
20
|
+
* `unknown` is therefore expected to be the largest bucket for as long as the marker is spreading,
|
|
21
|
+
* and shrinking it is a distribution job, never a classifier job. A guessed attribution is worse
|
|
22
|
+
* than none: it is the number distribution decisions get steered on, and a guess and a measurement
|
|
23
|
+
* are indistinguishable once they are in the same column.
|
|
24
|
+
*/
|
|
25
|
+
import { InstallSource } from '@reticlehq/core';
|
|
26
|
+
/** The one marker. Set by a channel on the process that runs the install. */
|
|
27
|
+
export const INSTALL_SOURCE_ENV = 'RETICLE_INSTALL_SOURCE';
|
|
28
|
+
const KNOWN = new Set(Object.values(InstallSource));
|
|
29
|
+
/**
|
|
30
|
+
* The declared install source, or `unknown`.
|
|
31
|
+
*
|
|
32
|
+
* Narrowed against the closed list rather than echoed: an echo would forward whatever somebody
|
|
33
|
+
* exported — a path, a URL, a campaign string — straight onto the wire, which is the rule this
|
|
34
|
+
* whole contract is built on. Trimmed and lowercased first, because the marker travels through
|
|
35
|
+
* shell snippets and copy-paste, and `Skill_File` is the same channel as `skill_file`.
|
|
36
|
+
*/
|
|
37
|
+
export function resolveInstallSource(env = process.env) {
|
|
38
|
+
const declared = (env[INSTALL_SOURCE_ENV] ?? '').trim().toLowerCase();
|
|
39
|
+
return KNOWN.has(declared) ? declared : InstallSource.UNKNOWN;
|
|
40
|
+
}
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* finished, already-classified verdict, so this cannot double-count and does not need to understand
|
|
11
11
|
* how a replay works.
|
|
12
12
|
*/
|
|
13
|
-
import { BugSource, RunFlowStatus, TelemetryActor, TelemetryEventKind, VerdictStatus, } from '@reticlehq/core';
|
|
13
|
+
import { BugAttribution, BugSource, RunFlowStatus, TelemetryActor, TelemetryEventKind, VerdictStatus, } from '@reticlehq/core';
|
|
14
14
|
import { getSessionMetrics } from './session-metrics.js';
|
|
15
15
|
import { getTelemetry } from './telemetry.js';
|
|
16
16
|
/** The tool name reported for this path, so CI runs are distinguishable from agent-driven ones. */
|
|
@@ -52,6 +52,9 @@ export function reportRunTelemetry(run) {
|
|
|
52
52
|
falseGreen: false,
|
|
53
53
|
tool: VERIFY_SURFACE,
|
|
54
54
|
repeat: !first,
|
|
55
|
+
// A regression SOMEWHERE, and the row does not say where — the app changed, or a selector
|
|
56
|
+
// strategy of ours did. `app` needs positive evidence; see bug-found.ts.
|
|
57
|
+
attribution: BugAttribution.UNCLASSIFIED,
|
|
55
58
|
},
|
|
56
59
|
});
|
|
57
60
|
}
|
|
@@ -515,6 +515,11 @@ export class SessionMetrics {
|
|
|
515
515
|
: {}),
|
|
516
516
|
...(this.#surface === undefined ? {} : { surface: this.#surface }),
|
|
517
517
|
...(final ? { endReason: this.#endReason() } : {}),
|
|
518
|
+
// The headline metric, as a FIELD. Read off the lifetime counter, because every other
|
|
519
|
+
// verification number in this payload is windowed and a reader who summed the wrong one got a
|
|
520
|
+
// different question's answer. `false` is sent, not omitted: a session that drove an app and
|
|
521
|
+
// never asked whether it worked is the whole finding.
|
|
522
|
+
...(final ? { endedWithVerdict: this.#lifetimeVerifications > 0 } : {}),
|
|
518
523
|
...(this.#feedbackPrompted > 0 ? { feedbackPrompted: this.#feedbackPrompted } : {}),
|
|
519
524
|
...(this.#errorClasses.size > 0
|
|
520
525
|
? { errorClasses: Object.fromEntries(this.#errorClasses) }
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TelemetryEventKind, ProjectIdSource, type Crash, type Feedback, type Identity, type BugFound, type InitOutcome, type McpConnection, type McpOutage, type AppInstrumentation, type ProjectProfile, type SessionSummary, type TelemetryActor, type Verification, type VersionChange } from '@reticlehq/core';
|
|
1
|
+
import { TelemetryEventKind, ProjectIdSource, type Crash, type Feedback, type Identity, type BugFound, type InitOutcome, type InstallSource, type McpConnection, type McpOutage, type ToolRefusal, type AppInstrumentation, type ProjectProfile, type SessionSummary, type TelemetryActor, type Verification, type VersionChange } from '@reticlehq/core';
|
|
2
2
|
/**
|
|
3
3
|
* Feedback gets its own, much larger budget — and a retry.
|
|
4
4
|
*
|
|
@@ -72,10 +72,14 @@ export interface TelemetryExtra {
|
|
|
72
72
|
init?: InitOutcome;
|
|
73
73
|
/** `bug_found`: one defect Reticle found in the app under test. */
|
|
74
74
|
bug?: BugFound;
|
|
75
|
+
/** `tool_refused`: a call Reticle could not serve, and why. */
|
|
76
|
+
refusal?: ToolRefusal;
|
|
75
77
|
/** `mcp_connection_lost`: the agent lost its tools — which stage, and why. */
|
|
76
78
|
outage?: McpOutage;
|
|
77
79
|
/** `app_instrumented`: an app carrying the SDK reached this daemon for the first time. */
|
|
78
80
|
instrumentation?: AppInstrumentation;
|
|
81
|
+
/** `reticle_installed` / `init_completed`: which published route brought this install in. */
|
|
82
|
+
installSource?: InstallSource;
|
|
79
83
|
}
|
|
80
84
|
export interface Telemetry {
|
|
81
85
|
/**
|
|
@@ -71,6 +71,7 @@ const DETACHED_SEND_SCRIPT = "fetch(process.argv[1],{method:'POST',headers:{'con
|
|
|
71
71
|
/** Env var names — mirror cloud-sync's `RETICLE_*` convention. */
|
|
72
72
|
import { isReticleSourceCheckout } from './dev-repo.js';
|
|
73
73
|
import { gitFacts } from './git-facts.js';
|
|
74
|
+
import { currentAutomationHint } from './automation-hint.js';
|
|
74
75
|
const Env = {
|
|
75
76
|
DISABLE: 'RETICLE_TELEMETRY', // "0" / "false" / "off" → disabled
|
|
76
77
|
DO_NOT_TRACK: 'DO_NOT_TRACK', // the cross-tool opt-out convention (any truthy value)
|
|
@@ -200,7 +201,7 @@ export const describeTelemetry = (env = process.env, dir = RETICLE_DIR, cwd = pr
|
|
|
200
201
|
if (isReticleSourceCheckout(cwd)) {
|
|
201
202
|
return {
|
|
202
203
|
enabled: false,
|
|
203
|
-
reason: 'this is a Reticle source checkout
|
|
204
|
+
reason: 'this is a Reticle source checkout; developing it is not using it',
|
|
204
205
|
policyUrl: POLICY_URL,
|
|
205
206
|
};
|
|
206
207
|
}
|
|
@@ -259,6 +260,9 @@ export const createTelemetry = (opts) => {
|
|
|
259
260
|
const doFetch = opts.fetchImpl ?? fetch;
|
|
260
261
|
const url = `${(env[Env.URL] ?? DEFAULT_URL).replace(/\/+$/, '')}${TELEMETRY_PATH}`;
|
|
261
262
|
const ci = env[Env.CI] !== undefined && env[Env.CI] !== '';
|
|
263
|
+
// Read once per process, next to `ci` and never instead of it: this is a second angle on the same
|
|
264
|
+
// question, and the rule that it can never filter a row lives in automation-hint.ts.
|
|
265
|
+
const automation = currentAutomationHint(env);
|
|
262
266
|
const os = platform();
|
|
263
267
|
// Negated so it reads the way people say it: UTC+2 is +120, not -120.
|
|
264
268
|
const tzOffsetMin = -new Date().getTimezoneOffset();
|
|
@@ -300,6 +304,9 @@ export const createTelemetry = (opts) => {
|
|
|
300
304
|
ts: now(),
|
|
301
305
|
version: opts.version,
|
|
302
306
|
ci,
|
|
307
|
+
// A SCALAR, so it needs the event build and the wire schema and no `blocks` entry — the same
|
|
308
|
+
// three-place trap `outage` fell into, and `installSource` documents below.
|
|
309
|
+
...(automation !== undefined ? { automation } : {}),
|
|
303
310
|
os,
|
|
304
311
|
tzOffsetMin,
|
|
305
312
|
projectIdSource,
|
|
@@ -316,15 +323,20 @@ export const createTelemetry = (opts) => {
|
|
|
316
323
|
...(extra?.connection !== undefined ? { connection: extra.connection } : {}),
|
|
317
324
|
...(extra?.init !== undefined ? { init: extra.init } : {}),
|
|
318
325
|
...(extra?.bug !== undefined ? { bug: extra.bug } : {}),
|
|
326
|
+
...(extra?.refusal !== undefined ? { refusal: extra.refusal } : {}),
|
|
319
327
|
...(extra?.outage !== undefined ? { outage: extra.outage } : {}),
|
|
320
328
|
...(extra?.instrumentation !== undefined ? { instrumentation: extra.instrumentation } : {}),
|
|
329
|
+
// A SCALAR, so it needs the event build and the wire schema but no entry in `blocks` below —
|
|
330
|
+
// there is nothing to flatten. It still needs both of those, which is the trap `outage` fell
|
|
331
|
+
// into: a field declared on TelemetryExtra and missing from either one is dropped in silence.
|
|
332
|
+
...(extra?.installSource !== undefined ? { installSource: extra.installSource } : {}),
|
|
321
333
|
};
|
|
322
334
|
// Map the core contract onto PostHog's capture shape: id/name/time move up, the rest are properties.
|
|
323
335
|
// The feedback body is FLATTENED into `feedback_*` properties rather than sent as a nested object:
|
|
324
336
|
// PostHog filters and breakdowns operate on top-level properties, so a nested `feedback.stack`
|
|
325
337
|
// would be invisible to the exact "which stacks report the most bugs" question this exists to
|
|
326
338
|
// answer. One prefix keeps them grouped in the UI without a nested path.
|
|
327
|
-
const { anonymousId: distinctId, event: name, ts, sessionId: eventSessionId, feedback, session, project, verification, versionChange, crash, identity, connection, init, bug, outage, instrumentation, ...rest } = event;
|
|
339
|
+
const { anonymousId: distinctId, event: name, ts, sessionId: eventSessionId, feedback, session, project, verification, versionChange, crash, identity, connection, init, bug, refusal, outage, instrumentation, ...rest } = event;
|
|
328
340
|
// `$session_id` is PostHog's OWN session property, so sending ours under that name lights up
|
|
329
341
|
// its built-in session views and funnels for free rather than requiring custom HogQL everywhere.
|
|
330
342
|
// `sessionId` rides only on events that happened inside a daemon run. A one-shot CLI command
|
|
@@ -352,6 +364,7 @@ export const createTelemetry = (opts) => {
|
|
|
352
364
|
connection,
|
|
353
365
|
init,
|
|
354
366
|
bug,
|
|
367
|
+
refusal,
|
|
355
368
|
outage,
|
|
356
369
|
instrumentation,
|
|
357
370
|
};
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reporting the calls Reticle could not serve.
|
|
3
|
+
*
|
|
4
|
+
* The refusal path is the largest thing this product does and the least measured one. It computes a
|
|
5
|
+
* precise diagnosis — which of three no-session situations this is, that a ref went stale, that a
|
|
6
|
+
* predicate did not parse — hands it to the agent as prose, and throws it away. So a user who
|
|
7
|
+
* attaches an agent, hits a wall on the first call and never comes back emits nothing at all, and
|
|
8
|
+
* the whole cohort is visible only as the gap between two other numbers. See issue #172.
|
|
9
|
+
*
|
|
10
|
+
* Emitted from `runTool`, which is the one place both dispatch paths cross, so a tool added later is
|
|
11
|
+
* covered without anybody remembering to cover it.
|
|
12
|
+
*/
|
|
13
|
+
import { type RefusalReason } from '@reticlehq/core';
|
|
14
|
+
/** A call that was served. Breaks the retry chain: what follows it is not a retry of anything. */
|
|
15
|
+
export declare function noteToolServed(): void;
|
|
16
|
+
/** One refused call, reported with its cause and whether it was itself a retry. */
|
|
17
|
+
export declare function reportToolRefused(tool: string, reason: RefusalReason): void;
|
|
18
|
+
/** Tests only — the counters are process-lifetime, so each case has to start from zero. */
|
|
19
|
+
export declare function resetToolRefusals(): void;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reporting the calls Reticle could not serve.
|
|
3
|
+
*
|
|
4
|
+
* The refusal path is the largest thing this product does and the least measured one. It computes a
|
|
5
|
+
* precise diagnosis — which of three no-session situations this is, that a ref went stale, that a
|
|
6
|
+
* predicate did not parse — hands it to the agent as prose, and throws it away. So a user who
|
|
7
|
+
* attaches an agent, hits a wall on the first call and never comes back emits nothing at all, and
|
|
8
|
+
* the whole cohort is visible only as the gap between two other numbers. See issue #172.
|
|
9
|
+
*
|
|
10
|
+
* Emitted from `runTool`, which is the one place both dispatch paths cross, so a tool added later is
|
|
11
|
+
* covered without anybody remembering to cover it.
|
|
12
|
+
*/
|
|
13
|
+
import { TelemetryActor, TelemetryEventKind } from '@reticlehq/core';
|
|
14
|
+
import { getTelemetry } from './telemetry.js';
|
|
15
|
+
/**
|
|
16
|
+
* Refusal events one daemon run will send.
|
|
17
|
+
*
|
|
18
|
+
* Volume is part of this taxonomy's design — the per-tool-call event was removed for exactly this
|
|
19
|
+
* reason — and a stuck agent retrying one failing call is the shape most likely to produce hundreds.
|
|
20
|
+
* The cap means the bill cannot scale with the pathology while the SHAPE of it still arrives: the
|
|
21
|
+
* first fifty carry the tool, the cause and the retry flag, and `consecutiveRepeats` on the session
|
|
22
|
+
* summary already reports how long the loop ran.
|
|
23
|
+
*/
|
|
24
|
+
const MAX_REFUSALS_PER_SESSION = 50;
|
|
25
|
+
let sent = 0;
|
|
26
|
+
/**
|
|
27
|
+
* The tool whose call refused immediately before this one, if the last call refused at all.
|
|
28
|
+
*
|
|
29
|
+
* This is what `retried` is computed from, and it is why the event is sent at the moment of the
|
|
30
|
+
* refusal rather than held back until the next call reveals whether one came. Holding it back would
|
|
31
|
+
* lose the event entirely for an agent that gives up — which is precisely the agent this exists to
|
|
32
|
+
* describe — so the flag lands on the RETRY instead of on the first refusal.
|
|
33
|
+
*/
|
|
34
|
+
let lastRefusedTool;
|
|
35
|
+
/** A call that was served. Breaks the retry chain: what follows it is not a retry of anything. */
|
|
36
|
+
export function noteToolServed() {
|
|
37
|
+
lastRefusedTool = undefined;
|
|
38
|
+
}
|
|
39
|
+
/** One refused call, reported with its cause and whether it was itself a retry. */
|
|
40
|
+
export function reportToolRefused(tool, reason) {
|
|
41
|
+
const retried = lastRefusedTool === tool;
|
|
42
|
+
lastRefusedTool = tool;
|
|
43
|
+
if (sent >= MAX_REFUSALS_PER_SESSION)
|
|
44
|
+
return;
|
|
45
|
+
sent += 1;
|
|
46
|
+
void getTelemetry().emit(TelemetryEventKind.TOOL_REFUSED, {
|
|
47
|
+
actor: TelemetryActor.AGENT,
|
|
48
|
+
refusal: { tool, reason, retried },
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
/** Tests only — the counters are process-lifetime, so each case has to start from zero. */
|
|
52
|
+
export function resetToolRefusals() {
|
|
53
|
+
sent = 0;
|
|
54
|
+
lastRefusedTool = undefined;
|
|
55
|
+
}
|
|
@@ -53,6 +53,38 @@ const SuiteStatus = {
|
|
|
53
53
|
PASS: 'pass',
|
|
54
54
|
FAIL: 'fail',
|
|
55
55
|
};
|
|
56
|
+
/**
|
|
57
|
+
* Did the assertion itself pass, across three tool shapes that spell it three ways.
|
|
58
|
+
*
|
|
59
|
+
* `reticle_assert` spreads its verdict at the top level, so `pass` is right there. `flow_verify`
|
|
60
|
+
* reports a suite `status`. **`reticle_act_and_wait` nests it under `verdict`**, and that omission is
|
|
61
|
+
* why `falseGreenCaught` had never once been true in the field: `passed` came back `undefined` for
|
|
62
|
+
* every act_and_wait, so `true === passed` was false whatever the app did, on the one tool the
|
|
63
|
+
* product tells agents to reach for first.
|
|
64
|
+
*
|
|
65
|
+
* The metric it feeds is the product's central claim reduced to a boolean, and it was structurally
|
|
66
|
+
* unable to fire on the flagship path. Worse, the test file pinned that: one case used the real
|
|
67
|
+
* nested shape and asserted `falseGreenCaught: false`, directly beneath a comment calling this case
|
|
68
|
+
* "the thesis", while the case that asserted `true` used a flat shape act_and_wait never emits.
|
|
69
|
+
*
|
|
70
|
+
* A permanently-zero metric is indistinguishable from a working one that has nothing to report, so
|
|
71
|
+
* nobody could have noticed this from the data. It took reading the emit path.
|
|
72
|
+
*/
|
|
73
|
+
function passedOf(result, status) {
|
|
74
|
+
if ('boolean' === typeof result['pass'])
|
|
75
|
+
return result['pass'];
|
|
76
|
+
const verdict = result['verdict'];
|
|
77
|
+
if (null !== verdict && 'object' === typeof verdict) {
|
|
78
|
+
const nested = verdict['pass'];
|
|
79
|
+
if ('boolean' === typeof nested)
|
|
80
|
+
return nested;
|
|
81
|
+
}
|
|
82
|
+
if (status === SuiteStatus.PASS)
|
|
83
|
+
return true;
|
|
84
|
+
if (status === SuiteStatus.FAIL)
|
|
85
|
+
return false;
|
|
86
|
+
return undefined;
|
|
87
|
+
}
|
|
56
88
|
/**
|
|
57
89
|
* The verification payload for a result, or undefined when nothing was verified.
|
|
58
90
|
*
|
|
@@ -73,13 +105,7 @@ export function verificationOf(toolName, result, durationMs, brand) {
|
|
|
73
105
|
// metric's convenience — but a status that is neither pass nor fail is NOT a verdict, and an empty
|
|
74
106
|
// suite reports exactly that.
|
|
75
107
|
const status = result['status'];
|
|
76
|
-
const passed =
|
|
77
|
-
? result['pass']
|
|
78
|
-
: status === SuiteStatus.PASS
|
|
79
|
-
? true
|
|
80
|
-
: status === SuiteStatus.FAIL
|
|
81
|
-
? false
|
|
82
|
-
: undefined;
|
|
108
|
+
const passed = passedOf(result, status);
|
|
83
109
|
if (verified === undefined && passed === undefined)
|
|
84
110
|
return undefined;
|
|
85
111
|
const reason = reasonOf(result);
|
|
@@ -88,6 +114,19 @@ export function verificationOf(toolName, result, durationMs, brand) {
|
|
|
88
114
|
via: toolName,
|
|
89
115
|
verified: verified ?? (true === passed ? 'yes' : 'no'),
|
|
90
116
|
passed: passed ?? 'yes' === verified,
|
|
117
|
+
// The assertion passed and Reticle answered `no` anyway: a green actively refuted.
|
|
118
|
+
//
|
|
119
|
+
// Deliberately NOT widened to every refusal of a passing assertion, though several of them are
|
|
120
|
+
// arguably false greens caught too. `decideVerified` refuses a green seven ways and only
|
|
121
|
+
// `contradicted` answers `no`; the rest answer `unknown` on purpose, because UNKNOWN must never
|
|
122
|
+
// collapse into NO. `already_true` is the tempting one: a green that proved nothing because the
|
|
123
|
+
// condition already held before the action, found by a check built for exactly that.
|
|
124
|
+
//
|
|
125
|
+
// It stays out because `reason` is on this same payload. Anyone who wants the wider number can
|
|
126
|
+
// count it without a definition change, and a definition that shifts underneath a metric is
|
|
127
|
+
// worse than one that is narrow, because the old and new series look comparable and are not.
|
|
128
|
+
// Widening it would also have landed in the same release as the fix above, making it impossible
|
|
129
|
+
// to tell how much of any rise was the bug and how much was the redefinition.
|
|
91
130
|
falseGreenCaught: true === passed && 'no' === verified,
|
|
92
131
|
durationMs,
|
|
93
132
|
// Headless CI, a human watching, or somebody's own dev server — three different products behind
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Removing a temp directory on Windows, without inventing a product failure.
|
|
3
|
+
*
|
|
4
|
+
* `rm(dir, { recursive: true, force: true })` is the obvious teardown and it is not enough on
|
|
5
|
+
* Windows. Deletion there is not immediate: a handle the OS, the indexer or a virus scanner still
|
|
6
|
+
* holds keeps the entry alive for a moment after the last close, so the directory is observed
|
|
7
|
+
* non-empty at the instant `rmdir` runs and Node throws `ENOTEMPTY`. `force` does not cover it —
|
|
8
|
+
* that suppresses "missing", not "busy".
|
|
9
|
+
*
|
|
10
|
+
* The failure lands in `afterEach`, so it is reported against the test that just PASSED, and the
|
|
11
|
+
* leftover directory then bleeds state into what runs next. On CI this presented as five failures in
|
|
12
|
+
* one file, four of them consequential and one of them the wonderfully misleading "expected length 1
|
|
13
|
+
* but got 2" — a fake production bug manufactured entirely by teardown. This repo already has that
|
|
14
|
+
* exact story written up in `project-store.test.ts` about a timed-out test's stray writes; this is the
|
|
15
|
+
* same failure arriving through the filesystem instead of through the clock.
|
|
16
|
+
*
|
|
17
|
+
* `maxRetries`/`retryDelay` exist in Node for precisely this, and are documented as Windows-oriented.
|
|
18
|
+
* They are a no-op on POSIX, where the first attempt already succeeds.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* Delete a temp directory tree, tolerating Windows' delayed handle release.
|
|
22
|
+
*
|
|
23
|
+
* Never throws: a temp directory that outlives the test is the operating system's problem, not a
|
|
24
|
+
* reason to fail a run or to report a defect the product does not have. What must not happen is the
|
|
25
|
+
* opposite — a red that says nothing about the code under test.
|
|
26
|
+
*/
|
|
27
|
+
export declare function removeTempDir(dir: string): Promise<void>;
|
package/dist/temp-dir.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Removing a temp directory on Windows, without inventing a product failure.
|
|
3
|
+
*
|
|
4
|
+
* `rm(dir, { recursive: true, force: true })` is the obvious teardown and it is not enough on
|
|
5
|
+
* Windows. Deletion there is not immediate: a handle the OS, the indexer or a virus scanner still
|
|
6
|
+
* holds keeps the entry alive for a moment after the last close, so the directory is observed
|
|
7
|
+
* non-empty at the instant `rmdir` runs and Node throws `ENOTEMPTY`. `force` does not cover it —
|
|
8
|
+
* that suppresses "missing", not "busy".
|
|
9
|
+
*
|
|
10
|
+
* The failure lands in `afterEach`, so it is reported against the test that just PASSED, and the
|
|
11
|
+
* leftover directory then bleeds state into what runs next. On CI this presented as five failures in
|
|
12
|
+
* one file, four of them consequential and one of them the wonderfully misleading "expected length 1
|
|
13
|
+
* but got 2" — a fake production bug manufactured entirely by teardown. This repo already has that
|
|
14
|
+
* exact story written up in `project-store.test.ts` about a timed-out test's stray writes; this is the
|
|
15
|
+
* same failure arriving through the filesystem instead of through the clock.
|
|
16
|
+
*
|
|
17
|
+
* `maxRetries`/`retryDelay` exist in Node for precisely this, and are documented as Windows-oriented.
|
|
18
|
+
* They are a no-op on POSIX, where the first attempt already succeeds.
|
|
19
|
+
*/
|
|
20
|
+
import { rm } from 'node:fs/promises';
|
|
21
|
+
/**
|
|
22
|
+
* Retries and backoff. Small and bounded: this is covering a millisecond-scale handle release, so if
|
|
23
|
+
* it has not gone in half a second the cause is not a race and a slower retry would only hide it.
|
|
24
|
+
*/
|
|
25
|
+
const MAX_RETRIES = 10;
|
|
26
|
+
const RETRY_DELAY_MS = 50;
|
|
27
|
+
/**
|
|
28
|
+
* Delete a temp directory tree, tolerating Windows' delayed handle release.
|
|
29
|
+
*
|
|
30
|
+
* Never throws: a temp directory that outlives the test is the operating system's problem, not a
|
|
31
|
+
* reason to fail a run or to report a defect the product does not have. What must not happen is the
|
|
32
|
+
* opposite — a red that says nothing about the code under test.
|
|
33
|
+
*/
|
|
34
|
+
export async function removeTempDir(dir) {
|
|
35
|
+
try {
|
|
36
|
+
await rm(dir, {
|
|
37
|
+
recursive: true,
|
|
38
|
+
force: true,
|
|
39
|
+
maxRetries: MAX_RETRIES,
|
|
40
|
+
retryDelay: RETRY_DELAY_MS,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// Deliberately swallowed — see above. The OS reclaims the temp directory regardless.
|
|
45
|
+
}
|
|
46
|
+
}
|
package/dist/tools/act-tools.js
CHANGED
|
@@ -14,8 +14,11 @@ import { buildReactionReport, summarizeReaction } from '../events/reaction.js';
|
|
|
14
14
|
import { parsePredicate } from '../events/predicate-parse.js';
|
|
15
15
|
import { causalSummary } from '../capsule/causal-summary.js';
|
|
16
16
|
import { findContradictions } from '../events/contradictions.js';
|
|
17
|
+
import { inFlightRequestLabels, waitForInFlight } from './settle-in-flight.js';
|
|
18
|
+
import { waitForReaction } from './react-grace.js';
|
|
17
19
|
import { decideVerified } from '../honesty/verified.js';
|
|
18
20
|
import { readsDomState } from '../honesty/already-true.js';
|
|
21
|
+
import { describeWaitTarget } from '../honesty/unsettled.js';
|
|
19
22
|
import { saveFailedAssertCapsule } from './act-capsule.js';
|
|
20
23
|
import { buildDivergenceCapsule } from '../capsule/capsule.js';
|
|
21
24
|
import { predicateToExpectedLinks } from '../capsule/predicate-to-links.js';
|
|
@@ -295,7 +298,7 @@ export const ACT_TOOLS = [
|
|
|
295
298
|
.describe('Reaction digest: { window_ms, summary } of what the app did (DOM/network/route/console/signal counts). The full per-event timeline is one reticle_observe { since } away.'),
|
|
296
299
|
summary: z
|
|
297
300
|
.unknown()
|
|
298
|
-
.describe('Bounded causal summary: net {total,errors,headline}, consoleErrors, statePathsChanged, storageKeysChanged, stateDiffs [{path,from,to}], storageDiffs [{key,from,to}], route, signals, layoutShift, longTasks — real before→after diffs (capped), not just readings.'),
|
|
301
|
+
.describe('Bounded causal summary: net {total,errors,headline}, consoleErrors, statePathsChanged, storageKeysChanged, stateDiffs [{path,from,to}], storageDiffs [{key,from,to}], route, signals, layoutShift, longTasks — real before→after diffs (capped), not just readings. Every list is length-capped so the verdict can never be the field a client truncates; `elided` says which lists lost entries and how many.'),
|
|
299
302
|
// Promoted out of `effect` on RED only — the file:line the failure came from, the first thing a
|
|
300
303
|
// repair wants. Undeclared, it was stripped on the validating profile exactly like the structured
|
|
301
304
|
// cause above was, losing the highest-value pointer on the one path (a failed verdict) that has it.
|
|
@@ -380,9 +383,39 @@ export const ACT_TOOLS = [
|
|
|
380
383
|
// stops the spread below from inheriting an earlier reticle_act's action and mutation count.
|
|
381
384
|
session.lastAct.markActed(since, asString(args['action']), mutatedWithin(asRecord(actResult.result)));
|
|
382
385
|
// Honesty: floor the predicate at this act's cursor so a stale buffered event can't satisfy it.
|
|
386
|
+
const predicateStarted = session.elapsed();
|
|
383
387
|
const verdict = timeout > 0
|
|
384
388
|
? await waitForPredicate(session, until, timeout, since)
|
|
385
389
|
: await evaluatePredicate(session, until, since);
|
|
390
|
+
// The predicate resolves the INSTANT it holds, which on an optimistically-navigating app is
|
|
391
|
+
// while the write is still in flight — so the verdict was taken over a window the app had not
|
|
392
|
+
// finished, and came back `unknown / unsettled` asking the CALLER to re-check. A login that
|
|
393
|
+
// genuinely succeeded returned at 492ms of an 8000ms budget with every corroborating channel
|
|
394
|
+
// agreeing. Spend the rest of the budget the caller already granted.
|
|
395
|
+
//
|
|
396
|
+
// This decides nothing: it waits for the answer instead of guessing at it. A response that
|
|
397
|
+
// arrives as a FAILURE still contradicts — and now arrives INSIDE the window, where the
|
|
398
|
+
// detector can see it at all — while a request still open when the budget genuinely runs out
|
|
399
|
+
// reports unsettled exactly as before, an honest limit rather than an early exit. Costs
|
|
400
|
+
// nothing on the common path, where no request is in flight.
|
|
401
|
+
if (verdict.pass && timeout > 0) {
|
|
402
|
+
const sleep = {
|
|
403
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
404
|
+
};
|
|
405
|
+
const spent = () => timeout - (session.elapsed() - predicateStarted);
|
|
406
|
+
await waitForInFlight(session, since, spent(), sleep);
|
|
407
|
+
// Waiting for the response moved the hazard rather than removing it: the response now lands
|
|
408
|
+
// inside the window BY DESIGN, and the app's re-render happens a task or two later. Close
|
|
409
|
+
// the window in that gap and every channel agrees the app took a successful write and did
|
|
410
|
+
// nothing — `response-ignored`, i.e. `verified:"no"` on a correct app, produced entirely by
|
|
411
|
+
// where we stopped looking. A false accusation is the more damaging direction of error for
|
|
412
|
+
// a verification tool: it sends someone to fix code that is not broken.
|
|
413
|
+
//
|
|
414
|
+
// So the response is not the end of the window; the app's REACTION to it is. Paid only in
|
|
415
|
+
// the shape that would otherwise be accused — a successful mutating write with nothing
|
|
416
|
+
// moved after it — and short enough that a genuinely dropped response is still reported.
|
|
417
|
+
await waitForReaction(session, since, spent(), sleep);
|
|
418
|
+
}
|
|
386
419
|
const r = asRecord(actResult.result);
|
|
387
420
|
if ('boolean' === typeof r['settled'])
|
|
388
421
|
settledOutcome = r['settled'];
|
|
@@ -467,9 +500,13 @@ export const ACT_TOOLS = [
|
|
|
467
500
|
// Learning material only. Attribution is unchanged — findings still come exclusively from
|
|
468
501
|
// `windowEvents`, exactly as on the assert path.
|
|
469
502
|
const prior = since > 0 ? (await session.queryEvents({ since: 0 })).filter((e) => e.t < since) : [];
|
|
503
|
+
//
|
|
504
|
+
// `actionSince` is the window's own floor here, which is the tightest attribution there is —
|
|
505
|
+
// and it is what lets `duplicate-request` claim "one user action was performed" and mean it.
|
|
470
506
|
const contradictions = findContradictions(windowEvents, {
|
|
471
507
|
action: acted,
|
|
472
508
|
prior,
|
|
509
|
+
actionSince: since,
|
|
473
510
|
...session.lastAct.effect(),
|
|
474
511
|
});
|
|
475
512
|
// The single field an agent reads. Everything below it is the evidence it was derived from;
|
|
@@ -492,6 +529,13 @@ export const ACT_TOOLS = [
|
|
|
492
529
|
// `settled` is genuinely optional: a wait that declared no predicate never measured it, and
|
|
493
530
|
// passing `false` there would report "never settled" about something never asked to settle.
|
|
494
531
|
...(settledOutcome === undefined ? {} : { settled: settledOutcome }),
|
|
532
|
+
// What the wait was for and what was still outstanding when it ended. Read only by the
|
|
533
|
+
// UNSETTLED clauses; supplied always because both of them are reachable from here and the
|
|
534
|
+
// window is already in hand.
|
|
535
|
+
unsettled: {
|
|
536
|
+
waitedFor: describeWaitTarget(until),
|
|
537
|
+
stillInFlight: inFlightRequestLabels(windowEvents),
|
|
538
|
+
},
|
|
495
539
|
});
|
|
496
540
|
return withControl(session, {
|
|
497
541
|
...decision,
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { HonestyGrade } from '../honesty/honesty.js';
|
|
14
14
|
import type { Predicate } from '../events/predicate.js';
|
|
15
|
-
export declare const PRESENCE_ONLY_ADVICE = "This predicate only checks element/text presence, not an observable consequence. A locator healed to the wrong element (or a stale render) can satisfy it while the feature is broken. Prefer a { signal } or { net } assertion
|
|
15
|
+
export declare const PRESENCE_ONLY_ADVICE = "This predicate only checks element/text presence, not an observable consequence. A locator healed to the wrong element (or a stale render) can satisfy it while the feature is broken. Prefer a { signal } or { net } assertion (or allOf it with one) so green means the feature actually worked.";
|
|
16
16
|
/**
|
|
17
17
|
* True when the predicate asserts ONLY element/text presence with no signal/net consequence anywhere
|
|
18
18
|
* — the weak pattern worth nudging. A predicate that mixes in a consequence, or that checks something
|
|
@@ -12,24 +12,46 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { isConsequenceKind, isPresenceKind, PredicateKind } from '@reticlehq/core';
|
|
14
14
|
import { HonestyGrade } from '../honesty/honesty.js';
|
|
15
|
-
export const PRESENCE_ONLY_ADVICE = 'This predicate only checks element/text presence, not an observable consequence. A locator healed to the wrong element (or a stale render) can satisfy it while the feature is broken. Prefer a { signal } or { net } assertion
|
|
16
|
-
function walk(predicate) {
|
|
15
|
+
export const PRESENCE_ONLY_ADVICE = 'This predicate only checks element/text presence, not an observable consequence. A locator healed to the wrong element (or a stale render) can satisfy it while the feature is broken. Prefer a { signal } or { net } assertion (or allOf it with one) so green means the feature actually worked.';
|
|
16
|
+
function walk(predicate, negated = false) {
|
|
17
17
|
if (PredicateKind.ALL_OF === predicate.kind || PredicateKind.ANY_OF === predicate.kind) {
|
|
18
|
-
const subs = predicate.predicates.map(walk);
|
|
18
|
+
const subs = predicate.predicates.map((p) => walk(p, negated));
|
|
19
19
|
return {
|
|
20
20
|
consequence: subs.some((s) => s.consequence),
|
|
21
21
|
presence: subs.some((s) => s.presence),
|
|
22
22
|
};
|
|
23
23
|
}
|
|
24
|
+
// Under a NOT, checking content strengthens nothing, so the exemption below must not apply. The
|
|
25
|
+
// nudge warns that a healed or wrong locator can satisfy the predicate, and for an ABSENCE claim a
|
|
26
|
+
// wrong locator satisfies it trivially: "the error message is gone" is true of a selector that
|
|
27
|
+
// never matched anything in the first place. Negation makes a presence check weaker, not stronger.
|
|
24
28
|
if (PredicateKind.NOT === predicate.kind)
|
|
25
|
-
return walk(predicate.predicate);
|
|
29
|
+
return walk(predicate.predicate, true);
|
|
26
30
|
// Leaf: classify against the single source of truth in core. Kinds that are neither consequence
|
|
27
31
|
// nor presence (route/console/settled/animation) correctly return false for both.
|
|
28
32
|
return {
|
|
29
33
|
consequence: isConsequenceKind(predicate.kind),
|
|
30
|
-
presence: isPresenceKind(predicate.kind),
|
|
34
|
+
presence: isPresenceKind(predicate.kind) && (negated || !assertsElementContent(predicate)),
|
|
31
35
|
};
|
|
32
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* An element predicate that checks the element's CONTENT, not merely that it is there.
|
|
39
|
+
*
|
|
40
|
+
* The presence nudge exists because "a locator healed to the wrong element, or a stale render, can
|
|
41
|
+
* satisfy this while the feature is broken". That is exactly true of `{ element, role, name }` and
|
|
42
|
+
* exactly false of `{ element, role, name, value: "274.58" }`: a wrong element now has to carry the
|
|
43
|
+
* same value to pass, which is the thing the nudge is warning cannot be relied on.
|
|
44
|
+
*
|
|
45
|
+
* Kept as a nudge-level distinction rather than a new honesty grade. The grade still reads
|
|
46
|
+
* `presence`, which UNDERSTATES what was proved, and understating is the safe direction for a field
|
|
47
|
+
* whose whole job is to qualify a green. Telling an agent that a value check "only checks presence"
|
|
48
|
+
* is the direction that costs something: it argues for replacing an assertion that is already sound.
|
|
49
|
+
*/
|
|
50
|
+
function assertsElementContent(predicate) {
|
|
51
|
+
if (predicate.kind !== PredicateKind.ELEMENT)
|
|
52
|
+
return false;
|
|
53
|
+
return predicate.query?.value !== undefined || predicate.query?.text !== undefined;
|
|
54
|
+
}
|
|
33
55
|
/**
|
|
34
56
|
* True when the predicate asserts ONLY element/text presence with no signal/net consequence anywhere
|
|
35
57
|
* — the weak pattern worth nudging. A predicate that mixes in a consequence, or that checks something
|
|
@@ -5,6 +5,8 @@ import { buildHonestyBlock } from '../honesty/honesty.js';
|
|
|
5
5
|
import { hasAcceptedWrite } from '../honesty/accepted-write.js';
|
|
6
6
|
import { hasUnreadWriteOutcome } from '../honesty/unread-outcome.js';
|
|
7
7
|
import { decideVerified } from '../honesty/verified.js';
|
|
8
|
+
import { describeWaitTarget } from '../honesty/unsettled.js';
|
|
9
|
+
import { inFlightRequestLabels } from './settle-in-flight.js';
|
|
8
10
|
import { gradeOfPredicate } from './assert-grade.js';
|
|
9
11
|
/**
|
|
10
12
|
* The honesty verdict for a plain `reticle_assert`.
|
|
@@ -49,7 +51,16 @@ observationLost) {
|
|
|
49
51
|
// measured live, `observe` over a wide window reported `unit-mismatch` while `assert` on the same
|
|
50
52
|
// session reported none. Attribution is unchanged: findings still come only from windowEvents.
|
|
51
53
|
const prior = since > 0 ? (await session.queryEvents({ since: 0 })).filter((e) => e.t < since) : [];
|
|
52
|
-
|
|
54
|
+
// The last act, when it falls inside the window being judged. Without it `duplicate-request` counted
|
|
55
|
+
// `method + url` across whatever window the caller asked for, so two legitimate separate saves to one
|
|
56
|
+
// endpoint were reported as a double submit — the instrument accusing the app of something nobody
|
|
57
|
+
// could show it did. An assert taken over a window that predates the act attributes nothing, and the
|
|
58
|
+
// rule then stays quiet.
|
|
59
|
+
const actCursor = session.lastAct.cursor();
|
|
60
|
+
const contradictions = findContradictions(windowEvents, {
|
|
61
|
+
prior,
|
|
62
|
+
...(actCursor !== undefined && actCursor >= since ? { actionSince: actCursor } : {}),
|
|
63
|
+
});
|
|
53
64
|
// Only a spot that IMPEACHES the capture downgrades a verdict. A structural boundary (virtualized
|
|
54
65
|
// rows, a cross-origin frame) is reported as coverage and must not impugn what WAS observed.
|
|
55
66
|
const impeaching = buildCoverageStatement(spots.filter((sp) => impeachesCapture(sp.kind)));
|
|
@@ -79,6 +90,12 @@ observationLost) {
|
|
|
79
90
|
contradictions,
|
|
80
91
|
...(outcomePending ? { outcomePending } : {}),
|
|
81
92
|
...(outcomeUnread ? { outcomeUnread } : {}),
|
|
93
|
+
// Same detail the act path supplies: this route reaches UNSETTLED through an absence-derived
|
|
94
|
+
// contradiction, and "the window closed before the app finished" is no more actionable here.
|
|
95
|
+
unsettled: {
|
|
96
|
+
waitedFor: describeWaitTarget(predicate),
|
|
97
|
+
stillInFlight: inFlightRequestLabels(windowEvents),
|
|
98
|
+
},
|
|
82
99
|
});
|
|
83
100
|
return { decision: decision, contradictions, coverage };
|
|
84
101
|
}
|
|
@@ -38,6 +38,10 @@ export const CONTRACT_TOOLS = [
|
|
|
38
38
|
{
|
|
39
39
|
name: ReticleTool.CAPABILITIES,
|
|
40
40
|
description: 'The app-advertised testable surface (reticle.describe): testids, signals, stores, and named flows. Call this first to learn what to assert on without reading source. Pass { fromDisk:true } to read the git-checked .reticle/contract.json instead of the live session (a fresh agent can learn the surface with no browser attached).',
|
|
41
|
+
// Every core tool carries one, so an agent reading the surface never has to guess the shape.
|
|
42
|
+
// `fromDisk` is the example deliberately: it is the call that works with nothing attached, which
|
|
43
|
+
// is the state a fresh agent is actually in when it first reads this list.
|
|
44
|
+
example: { [FROM_DISK_ARG]: true },
|
|
41
45
|
inputSchema: { [FROM_DISK_ARG]: z.boolean().optional(), ...sessionIdShape },
|
|
42
46
|
outputSchema: {
|
|
43
47
|
testids: z.array(z.string()),
|