@reticlehq/server 3.5.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/SKILL.md +16 -8
- package/dist/chromium-launch-options.d.ts +3 -1
- package/dist/chromium-launch-options.js +2 -1
- package/dist/command/cli/adhoc-verdict.d.ts +1 -0
- package/dist/command/cli/adhoc-verdict.js +6 -3
- package/dist/command/cli/answers/open-note.d.ts +2 -0
- package/dist/command/cli/answers/open-note.js +22 -0
- package/dist/command/cli/cli-flow-commands.js +4 -8
- package/dist/command/cli/cli-parse.d.ts +1 -1
- package/dist/command/cli/cli-parse.js +10 -10
- package/dist/command/cli/cli-verify.d.ts +1 -1
- package/dist/command/cli/cli-verify.js +35 -25
- package/dist/command/cli/cloud-cli.js +51 -12
- package/dist/command/cli/cloud-login.d.ts +8 -0
- package/dist/command/cli/cloud-login.js +36 -2
- package/dist/command/cli/demo/demo-page.js +11 -3
- package/dist/command/cli/doctor/browser/chromium-hint.d.ts +46 -0
- package/dist/command/cli/doctor/browser/chromium-hint.js +99 -1
- package/dist/command/cli/drive/drive-attach.d.ts +27 -2
- package/dist/command/cli/drive/drive-attach.js +71 -4
- package/dist/command/cli/drive/drive-command.d.ts +2 -1
- package/dist/command/cli/drive/drive-command.js +13 -7
- package/dist/command/cli/lifecycle/daemon-lifecycle.js +4 -1
- package/dist/command/cli/ports/port-holder.d.ts +10 -0
- package/dist/command/cli/ports/port-holder.js +24 -0
- package/dist/command/cli/setup-mcp-cli.js +6 -4
- package/dist/command/cli/tutorial.d.ts +2 -2
- package/dist/command/cli/tutorial.js +6 -9
- package/dist/command/cli.d.ts +6 -0
- package/dist/command/cli.js +28 -22
- package/dist/command/daemon/binding/daemon-status-probe.d.ts +2 -0
- package/dist/command/daemon/binding/daemon-status-probe.js +1 -0
- package/dist/command/daemon/daemon-resolve.d.ts +14 -0
- package/dist/command/daemon/daemon-resolve.js +33 -2
- package/dist/command/setup/agent-writer.d.ts +1 -1
- package/dist/command/setup/agent-writer.js +3 -2
- package/dist/command/setup/bringup/connect-lease.d.ts +1 -0
- package/dist/command/setup/bringup/connect-lease.js +1 -0
- package/dist/command/setup/bringup/dev-server-wait.d.ts +8 -0
- package/dist/command/setup/bringup/dev-server-wait.js +19 -0
- package/dist/command/setup/init/init-port.d.ts +4 -0
- package/dist/command/setup/init/init-port.js +10 -0
- package/dist/command/setup/init/init-runtime.d.ts +10 -0
- package/dist/command/setup/init/init-runtime.js +18 -1
- package/dist/command/setup/node-effects.js +2 -1
- package/dist/command/setup/remaining-steps.js +2 -1
- package/dist/command/setup/run-setup.d.ts +5 -0
- package/dist/command/setup/run-setup.js +35 -27
- package/dist/command/setup/setup-command.d.ts +2 -1
- package/dist/command/setup/setup-command.js +2 -0
- package/dist/command/version/version-skew.js +11 -2
- package/dist/features/crawl/crawl-tools.js +4 -0
- package/dist/features/crawl/crawl.d.ts +6 -0
- package/dist/features/crawl/crawl.js +9 -1
- package/dist/features/exhaust/ledger.d.ts +7 -0
- package/dist/features/exhaust/ledger.js +35 -9
- package/dist/features/harness/app-findings.d.ts +2 -0
- package/dist/features/harness/app-findings.js +52 -0
- package/dist/features/harness/drive-report.d.ts +27 -1
- package/dist/features/harness/drive-report.js +162 -11
- package/dist/features/harness/drivers.d.ts +10 -14
- package/dist/features/harness/drivers.js +10 -14
- package/dist/features/harness/fill-values.d.ts +3 -73
- package/dist/features/harness/fill-values.js +3 -169
- package/dist/features/harness/goals.d.ts +21 -0
- package/dist/features/harness/goals.js +77 -0
- package/dist/features/harness/harness.d.ts +21 -0
- package/dist/features/harness/harness.js +69 -2
- package/dist/features/harness/platform/drive-target.d.ts +30 -0
- package/dist/features/harness/platform/drive-target.js +81 -0
- package/dist/features/harness/platform/local-apps.d.ts +94 -0
- package/dist/features/harness/platform/local-apps.js +204 -0
- package/dist/features/harness/platform/remote-drive.d.ts +180 -0
- package/dist/features/harness/platform/remote-drive.js +365 -0
- package/dist/features/harness/platform/script.d.ts +48 -0
- package/dist/features/harness/platform/script.js +128 -0
- package/dist/features/harness/platform/server-driver.d.ts +41 -0
- package/dist/features/harness/platform/server-driver.js +179 -0
- package/dist/features/harness/platform-config.d.ts +7 -0
- package/dist/features/harness/platform-config.js +14 -2
- package/dist/features/harness/script/app-controls.d.ts +5 -0
- package/dist/features/harness/script/app-controls.js +40 -0
- package/dist/features/harness/script/lane-ids.d.ts +14 -0
- package/dist/features/harness/script/lane-ids.js +16 -0
- package/dist/features/harness/script/script-run.d.ts +41 -0
- package/dist/features/harness/script/script-run.js +299 -0
- package/dist/features/license/license.d.ts +2 -2
- package/dist/features/visual/visual-tools.d.ts +10 -1
- package/dist/features/visual/visual-tools.js +40 -3
- package/dist/hooks/driven-by.d.ts +4 -0
- package/dist/hooks/driven-by.js +16 -0
- package/dist/hooks/hook-commands.js +15 -4
- package/dist/hooks/tool-log.d.ts +9 -0
- package/dist/hooks/tool-log.js +29 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +76 -82
- package/dist/judgement/runs/artifact/build-verification-run.js +6 -0
- package/dist/judgement/runs/artifact/replay-mapping.d.ts +1 -1
- package/dist/judgement/runs/artifact/replay-mapping.js +16 -6
- package/dist/judgement/runs/artifact/run-store.d.ts +1 -1
- package/dist/judgement/runs/artifact/run-store.js +26 -1
- package/dist/judgement/runs/drive-run.d.ts +22 -0
- package/dist/judgement/runs/drive-run.js +99 -1
- package/dist/judgement/runs/repair-prompt.js +1 -1
- package/dist/judgement/runs/reticle-runner.js +8 -1
- package/dist/judgement/runs/verification-sync.d.ts +3 -2
- package/dist/judgement/runs/verification-sync.js +4 -3
- package/dist/language/flows/change/gate.d.ts +10 -0
- package/dist/language/flows/change/gate.js +22 -0
- package/dist/language/flows/decision.js +34 -0
- package/dist/language/flows/flow-anchor.d.ts +54 -8
- package/dist/language/flows/flow-anchor.js +88 -3
- package/dist/language/flows/flow-cross-step.d.ts +15 -0
- package/dist/language/flows/flow-cross-step.js +21 -0
- package/dist/language/flows/flow-learning.js +8 -11
- package/dist/language/flows/flow-mutate-tools.js +5 -3
- package/dist/language/flows/flow-replay-run.d.ts +23 -9
- package/dist/language/flows/flow-replay-run.js +101 -94
- package/dist/language/flows/flow-replay.d.ts +23 -1
- package/dist/language/flows/flow-replay.js +92 -19
- package/dist/language/flows/flow-scope.js +5 -1
- package/dist/language/flows/flow-step-runners.d.ts +20 -2
- package/dist/language/flows/flow-step-runners.js +236 -42
- package/dist/language/flows/flow-store-for-session.d.ts +2 -2
- package/dist/language/flows/flow-store-for-session.js +2 -2
- package/dist/language/flows/flow-tools.js +58 -34
- package/dist/language/flows/flows.js +5 -2
- package/dist/language/flows/heal-run.js +6 -4
- package/dist/language/flows/recording/flake.d.ts +2 -2
- package/dist/language/flows/recording/tape/recordings.d.ts +30 -4
- package/dist/language/flows/recording/tape/recordings.js +40 -8
- package/dist/language/flows/replay-from-hud.d.ts +20 -0
- package/dist/language/flows/replay-from-hud.js +51 -0
- package/dist/language/flows/replay.d.ts +9 -5
- package/dist/language/flows/replay.js +27 -9
- package/dist/language/flows/suite/suite-flakes.js +2 -0
- package/dist/language/flows/suite-selection.d.ts +2 -15
- package/dist/language/flows/suite-selection.js +4 -3
- package/dist/language/flows/verify-change-tools.js +8 -4
- package/dist/launch-chromium.d.ts +27 -5
- package/dist/launch-chromium.js +38 -8
- package/dist/memory/cloud/cloud-config.d.ts +12 -0
- package/dist/memory/cloud/cloud-config.js +15 -0
- package/dist/memory/cloud/cloud-sync.d.ts +17 -14
- package/dist/memory/cloud/cloud-sync.js +12 -1
- package/dist/memory/cloud/harness-config.d.ts +26 -1
- package/dist/memory/cloud/harness-config.js +99 -4
- package/dist/memory/cloud/hud-notices-source.d.ts +21 -0
- package/dist/memory/cloud/hud-notices-source.js +128 -0
- package/dist/memory/cloud/memory-scope.d.ts +144 -0
- package/dist/memory/cloud/memory-scope.js +205 -0
- package/dist/memory/cloud/sync-cycle.d.ts +12 -3
- package/dist/memory/cloud/sync-cycle.js +16 -3
- package/dist/memory/cloud/sync-daemon.js +7 -5
- package/dist/memory/cloud/sync-disk.d.ts +1 -1
- package/dist/memory/cloud/sync-disk.js +5 -8
- package/dist/memory/impact/account-watch.d.ts +19 -0
- package/dist/memory/impact/account-watch.js +34 -0
- package/dist/memory/impact/impact-recorder.d.ts +8 -0
- package/dist/memory/impact/impact-recorder.js +12 -2
- package/dist/memory/impact/impact-store.d.ts +16 -0
- package/dist/memory/impact/impact-store.js +142 -26
- package/dist/memory/intent/intent-shard.d.ts +2 -2
- package/dist/memory/intent/intent-tools.js +80 -2
- package/dist/memory/journal/envelope.d.ts +8 -8
- package/dist/memory/journal/journal-recorder.js +7 -1
- package/dist/memory/journal/on-disk/retention.d.ts +3 -0
- package/dist/memory/journal/on-disk/retention.js +5 -0
- package/dist/memory/journal/on-disk/startup-maintenance.js +3 -1
- package/dist/memory/journal/on-disk/workspace-tiers.js +7 -4
- package/dist/memory/journal/session-end.js +6 -4
- package/dist/memory/project/artifact-root-resolver.d.ts +26 -1
- package/dist/memory/project/artifact-root-resolver.js +94 -13
- package/dist/memory/project/artifact-root.d.ts +42 -5
- package/dist/memory/project/artifact-root.js +87 -8
- package/dist/memory/project/dir/reticle-dir.d.ts +4 -2
- package/dist/memory/project/dir/reticle-dir.js +3 -1
- package/dist/memory/project/project-tools.js +22 -3
- package/dist/memory/project/session-root.d.ts +24 -1
- package/dist/memory/project/session-root.js +41 -5
- package/dist/memory/project/sync-status.d.ts +39 -0
- package/dist/memory/project/sync-status.js +90 -0
- package/dist/memory/recall/memory-tools.js +2 -0
- package/dist/memory/recall/project-memory.d.ts +19 -0
- package/dist/memory/recall/project-memory.js +45 -6
- package/dist/portal/bridge/bridge.d.ts +3 -1
- package/dist/portal/bridge/bridge.js +19 -2
- package/dist/portal/bridge/pairing-token.d.ts +9 -0
- package/dist/portal/bridge/pairing-token.js +17 -0
- package/dist/portal/input/real-input.d.ts +81 -10
- package/dist/portal/input/real-input.js +207 -22
- package/dist/portal/input/viewport-tools.js +3 -3
- package/dist/portal/pool/browser-pool.d.ts +22 -9
- package/dist/portal/pool/browser-pool.js +70 -35
- package/dist/portal/pool/playwright-launcher.d.ts +13 -0
- package/dist/portal/pool/playwright-launcher.js +17 -1
- package/dist/portal/pool/pool-contract.d.ts +7 -4
- package/dist/portal/session/command-payload.d.ts +8 -0
- package/dist/portal/session/command-payload.js +19 -0
- package/dist/portal/session/dev-server/dev-command.d.ts +12 -0
- package/dist/portal/session/dev-server/dev-command.js +61 -12
- package/dist/portal/session/facts/departure-note.d.ts +22 -0
- package/dist/portal/session/facts/departure-note.js +47 -0
- package/dist/portal/session/facts/session-replaced.d.ts +5 -0
- package/dist/portal/session/facts/session-replaced.js +5 -0
- package/dist/portal/session/first-run-wiring.d.ts +42 -0
- package/dist/portal/session/first-run-wiring.js +132 -0
- package/dist/portal/session/human/notes-ledger.d.ts +20 -0
- package/dist/portal/session/human/notes-ledger.js +55 -0
- package/dist/portal/session/human/review-store.d.ts +12 -0
- package/dist/portal/session/human/review-store.js +18 -10
- package/dist/portal/session/live-control-tools.d.ts +0 -13
- package/dist/portal/session/live-control-tools.js +28 -3
- package/dist/portal/session/no-session-diagnosis.d.ts +7 -0
- package/dist/portal/session/no-session-diagnosis.js +15 -0
- package/dist/portal/session/no-session-next-action.d.ts +11 -0
- package/dist/portal/session/no-session-next-action.js +31 -5
- package/dist/portal/session/no-session-watch.d.ts +14 -1
- package/dist/portal/session/no-session-watch.js +63 -10
- package/dist/portal/session/presence/session-mirror.js +3 -0
- package/dist/portal/session/serving-directory.d.ts +27 -0
- package/dist/portal/session/serving-directory.js +45 -0
- package/dist/portal/session/session-health.d.ts +7 -0
- package/dist/portal/session/session-health.js +6 -0
- package/dist/portal/session/session-info.d.ts +8 -1
- package/dist/portal/session/session-info.js +6 -0
- package/dist/portal/session/session-manager.d.ts +21 -0
- package/dist/portal/session/session-manager.js +44 -8
- package/dist/portal/session/session-reaper.js +4 -1
- package/dist/portal/session/session-successor.d.ts +5 -0
- package/dist/portal/session/session-tools.js +26 -3
- package/dist/portal/session/session.d.ts +7 -2
- package/dist/portal/session/session.js +21 -18
- package/dist/start-options.d.ts +6 -0
- package/dist/status-payload.d.ts +10 -0
- package/dist/status-payload.js +23 -0
- package/dist/surface/http-server.js +11 -1
- package/dist/surface/mcp/mcp-proxy.d.ts +5 -0
- package/dist/surface/mcp/mcp-proxy.js +36 -5
- package/dist/surface/mcp/mcp.d.ts +11 -6
- package/dist/surface/mcp/mcp.js +27 -13
- package/dist/surface/mcp/server-instructions.d.ts +4 -2
- package/dist/surface/mcp/server-instructions.js +14 -4
- package/dist/surface/mcp/shared-params.js +2 -2
- package/dist/surface/tools/act/navigation/navigate-arrival.d.ts +8 -0
- package/dist/surface/tools/act/navigation/navigate-arrival.js +56 -2
- package/dist/surface/tools/act/settle-in-flight.js +25 -3
- package/dist/surface/tools/act-sequence-tool.js +2 -2
- package/dist/surface/tools/act-tools.js +23 -7
- package/dist/surface/tools/assert/assert-verdict.d.ts +1 -1
- package/dist/surface/tools/assert/assert-verdict.js +17 -2
- package/dist/surface/tools/browser-tools.js +8 -1
- package/dist/surface/tools/chat-drives.d.ts +26 -0
- package/dist/surface/tools/chat-drives.js +165 -0
- package/dist/surface/tools/empty-session-list.d.ts +41 -0
- package/dist/surface/tools/empty-session-list.js +80 -0
- package/dist/surface/tools/error-recovery.d.ts +6 -0
- package/dist/surface/tools/error-recovery.js +37 -4
- package/dist/surface/tools/explore-tools.d.ts +10 -1
- package/dist/surface/tools/explore-tools.js +82 -13
- package/dist/surface/tools/feedback-tools.js +14 -0
- package/dist/surface/tools/harness-explore.d.ts +96 -14
- package/dist/surface/tools/harness-explore.js +227 -206
- package/dist/surface/tools/harness-plan.d.ts +12 -0
- package/dist/surface/tools/harness-plan.js +31 -0
- package/dist/surface/tools/harness-script.d.ts +8 -0
- package/dist/surface/tools/harness-script.js +342 -0
- package/dist/surface/tools/harness-toolset.d.ts +14 -1
- package/dist/surface/tools/harness-toolset.js +77 -6
- package/dist/surface/tools/invoke-tool.js +52 -1
- package/dist/surface/tools/lease-seed.d.ts +29 -0
- package/dist/surface/tools/lease-seed.js +123 -0
- package/dist/surface/tools/lease-tools.d.ts +3 -23
- package/dist/surface/tools/lease-tools.js +30 -131
- package/dist/surface/tools/live-call-text.js +2 -2
- package/dist/surface/tools/next-step.d.ts +19 -0
- package/dist/surface/tools/next-step.js +159 -0
- package/dist/surface/tools/observe-tools.js +2 -4
- package/dist/surface/tools/platform-moment.d.ts +12 -0
- package/dist/surface/tools/platform-moment.js +168 -0
- package/dist/surface/tools/query-hint-schema.d.ts +7 -0
- package/dist/surface/tools/query-hint-schema.js +10 -0
- package/dist/surface/tools/read-tools.js +8 -2
- package/dist/surface/tools/real-input-attempt.js +96 -13
- package/dist/surface/tools/tool-kit.d.ts +15 -1
- package/dist/surface/tools/tool-kit.js +4 -6
- package/dist/surface/tools/tools.js +20 -48
- package/dist/telemetry/hud-metrics.d.ts +2 -0
- package/dist/telemetry/hud-metrics.js +17 -0
- package/dist/telemetry/telemetry.d.ts +12 -0
- package/dist/telemetry/telemetry.js +43 -14
- package/dist/telemetry/verification-of.js +1 -0
- package/dist/wire-journal.js +4 -1
- package/docs/README.md +1 -1
- package/docs/agent-cheatsheet.md +1 -1
- package/docs/autodrive.md +29 -25
- package/docs/cli/doctor.mdx +6 -0
- package/docs/cli/drive.mdx +8 -2
- package/docs/cli/init.mdx +11 -13
- package/docs/cli/status.mdx +5 -1
- package/docs/cli/version.mdx +3 -3
- package/docs/cli.mdx +1 -1
- package/docs/desktop.mdx +5 -1
- package/docs/environment.mdx +3 -7
- package/docs/getting-started.md +13 -11
- package/docs/human-control.md +1 -2
- package/docs/index.mdx +4 -3
- package/docs/install-manual.mdx +2 -2
- package/docs/packages/tauri.mdx +2 -2
- package/docs/packages/vite-plugin.mdx +22 -1
- package/docs/packages.mdx +1 -1
- package/docs/quickstart.mdx +17 -7
- package/docs/telemetry.md +1 -1
- package/docs/tools/session-and-feedback.mdx +9 -9
- package/docs/usage.md +5 -2
- package/docs/what-is-recorded.md +13 -3
- package/docs/what-is-reticle.mdx +3 -1
- package/package.json +8 -12
- package/dist/features/harness/driver.d.ts +0 -50
- package/dist/features/harness/driver.js +0 -236
- package/dist/features/harness/jev-driver.d.ts +0 -188
- package/dist/features/harness/jev-driver.js +0 -678
- package/dist/features/harness/openai-driver.d.ts +0 -58
- package/dist/features/harness/openai-driver.js +0 -206
- package/dist/memory/project/dir/fill-value-store.d.ts +0 -32
- package/dist/memory/project/dir/fill-value-store.js +0 -88
- package/docs/local-registry.md +0 -143
|
@@ -87,6 +87,10 @@ export const CRAWL_TOOLS = [
|
|
|
87
87
|
.optional()
|
|
88
88
|
.describe('Present only when the page exceeded one snapshot, so controls past the cap were never listed. When present, a zero-anomaly result does NOT mean the app is clean.'),
|
|
89
89
|
truncated: z.boolean(),
|
|
90
|
+
notJudged: z
|
|
91
|
+
.array(z.object({ ref: z.string(), desc: z.string(), reason: z.string() }))
|
|
92
|
+
.optional()
|
|
93
|
+
.describe('Controls clicked whose silence could not be judged, e.g. on a background tab. They are NOT reported as dead, and they are not known to work either.'),
|
|
90
94
|
// Only on `exhaustive` — see features/exhaust/exhaust-run.ts. Loose: it carries the levels.
|
|
91
95
|
exhaustive: z.record(z.unknown()).optional(),
|
|
92
96
|
},
|
|
@@ -81,6 +81,12 @@ export interface CrawlReport {
|
|
|
81
81
|
truncated: boolean;
|
|
82
82
|
/** Present only when the snapshot itself was capped, i.e. controls exist that were never listed. */
|
|
83
83
|
coverageNote?: string;
|
|
84
|
+
/** Controls whose silence could not be judged, present only when non-empty. */
|
|
85
|
+
notJudged?: {
|
|
86
|
+
ref: string;
|
|
87
|
+
desc: string;
|
|
88
|
+
reason: string;
|
|
89
|
+
}[];
|
|
84
90
|
}
|
|
85
91
|
export interface CrawlOptions {
|
|
86
92
|
maxSteps?: number;
|
|
@@ -33,6 +33,8 @@ function revealedControlsNote(count) {
|
|
|
33
33
|
'started were never visited — interactiveFound is a floor, not a total; re-run the crawl from ' +
|
|
34
34
|
'the state you are now in to cover them');
|
|
35
35
|
}
|
|
36
|
+
/** Why a silent control on a throttled page is not reported as dead. */
|
|
37
|
+
const BACKGROUND_TAB_NOT_JUDGED = 'not judged: the page is throttled (its tab is in the background, or it has not been heard from recently), so a reaction may not show inside the sample window; bring the tab to the front and crawl again';
|
|
36
38
|
/** Any buffer event that proves the app reacted to a click (vs a dead/no-op control). */
|
|
37
39
|
function isActivity(e) {
|
|
38
40
|
return (e.type === EventType.NET_REQUEST ||
|
|
@@ -255,6 +257,7 @@ export async function crawl(session, opts, sleep) {
|
|
|
255
257
|
// `truncated:false` would positively assert that nothing was cut.
|
|
256
258
|
const coverageCapped = true === snapshot.truncated;
|
|
257
259
|
const anomalies = [];
|
|
260
|
+
const notJudged = [];
|
|
258
261
|
const visited = [];
|
|
259
262
|
const counts = { consoleErrors: 0, failedRequests: 0, deadControls: 0, contradictions: 0 };
|
|
260
263
|
const seen = new Set();
|
|
@@ -401,7 +404,11 @@ export async function crawl(session, opts, sleep) {
|
|
|
401
404
|
//
|
|
402
405
|
// A second click costs one round trip on the rare control that looked dead, and nothing at all
|
|
403
406
|
// on every control that did not. A genuinely dead control is silent twice; the flake is not.
|
|
404
|
-
|
|
407
|
+
// A throttled page may defer DOM observer flushes, so silence cannot prove a dead control.
|
|
408
|
+
if (true === session.throttled?.()) {
|
|
409
|
+
notJudged.push({ ref: item.ref, desc: item.desc, reason: BACKGROUND_TAB_NOT_JUDGED });
|
|
410
|
+
}
|
|
411
|
+
else if (await stillSilent(session, item, settleMs, confirm, sleep)) {
|
|
405
412
|
counts.deadControls += 1;
|
|
406
413
|
anomalies.push({
|
|
407
414
|
kind: CrawlAnomalyKind.DEAD_CONTROL,
|
|
@@ -435,6 +442,7 @@ export async function crawl(session, opts, sleep) {
|
|
|
435
442
|
anomalies,
|
|
436
443
|
counts,
|
|
437
444
|
visited,
|
|
445
|
+
...(0 === notJudged.length ? {} : { notJudged }),
|
|
438
446
|
// True when coverage was bounded for EITHER reason: the step budget ran out, or the page was
|
|
439
447
|
// bigger than one snapshot. Conflating "I stopped early" with "I saw everything" is what turns a
|
|
440
448
|
// partial sweep into "all controls healthy".
|
|
@@ -158,6 +158,13 @@ export interface LevelReport {
|
|
|
158
158
|
/** How many more are missing than `missing` names. */
|
|
159
159
|
missingOverflow?: number;
|
|
160
160
|
}
|
|
161
|
+
/**
|
|
162
|
+
* Each level's percentage for the project at `reticleRoot`, for the HUD: numbers only, and only the
|
|
163
|
+
* levels with an honest denominator. Undefined when nothing has been measured yet.
|
|
164
|
+
*
|
|
165
|
+
* Synchronous because the impact snapshot it feeds is: one small file read per snapshot.
|
|
166
|
+
*/
|
|
167
|
+
export declare function coveragePercents(reticleRoot: string): Record<string, number> | undefined;
|
|
161
168
|
export declare function emptyLedger(): AppLedger;
|
|
162
169
|
export declare function mergeLedger(into: AppLedger, delta: LedgerDelta): AppLedger;
|
|
163
170
|
export declare function levelsOf(ledger: AppLedger): LevelReport[];
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
* Union-only. A later drive can add coverage and never remove it, so the number only moves down when
|
|
19
19
|
* the app grows something nobody has covered yet — which is precisely what the ratchet is for.
|
|
20
20
|
*/
|
|
21
|
+
import { readFileSync } from 'node:fs';
|
|
21
22
|
import { dirname } from 'node:path';
|
|
22
23
|
import { z } from 'zod';
|
|
23
24
|
import { writeFileAtomic } from '../../memory/project/fs/write-atomic.js';
|
|
@@ -45,6 +46,39 @@ const LedgerSchema = z.object({
|
|
|
45
46
|
codeTaken: z.record(z.number()).default({}),
|
|
46
47
|
best: z.record(z.number()),
|
|
47
48
|
});
|
|
49
|
+
/** A ledger file's text, or an empty ledger for anything unreadable: coverage never throws. */
|
|
50
|
+
function parseLedger(text) {
|
|
51
|
+
try {
|
|
52
|
+
const parsed = z
|
|
53
|
+
.object({ version: z.literal(LEDGER_VERSION), ledger: LedgerSchema })
|
|
54
|
+
.safeParse(JSON.parse(text));
|
|
55
|
+
return parsed.success ? parsed.data.ledger : emptyLedger();
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
return emptyLedger();
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Each level's percentage for the project at `reticleRoot`, for the HUD: numbers only, and only the
|
|
63
|
+
* levels with an honest denominator. Undefined when nothing has been measured yet.
|
|
64
|
+
*
|
|
65
|
+
* Synchronous because the impact snapshot it feeds is: one small file read per snapshot.
|
|
66
|
+
*/
|
|
67
|
+
export function coveragePercents(reticleRoot) {
|
|
68
|
+
let text;
|
|
69
|
+
try {
|
|
70
|
+
text = readFileSync(reticleDirPaths(reticleRoot).coverage, 'utf8');
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
const out = {};
|
|
76
|
+
for (const report of levelsOf(parseLedger(text))) {
|
|
77
|
+
if (report.pct !== undefined)
|
|
78
|
+
out[report.level] = report.pct;
|
|
79
|
+
}
|
|
80
|
+
return 0 < Object.keys(out).length ? out : undefined;
|
|
81
|
+
}
|
|
48
82
|
export function emptyLedger() {
|
|
49
83
|
return {
|
|
50
84
|
routes: { discovered: [], reached: [] },
|
|
@@ -213,15 +247,7 @@ export class LedgerStore {
|
|
|
213
247
|
return emptyLedger();
|
|
214
248
|
throw error;
|
|
215
249
|
}
|
|
216
|
-
|
|
217
|
-
const parsed = z
|
|
218
|
-
.object({ version: z.literal(LEDGER_VERSION), ledger: LedgerSchema })
|
|
219
|
-
.safeParse(JSON.parse(text));
|
|
220
|
-
return parsed.success ? parsed.data.ledger : emptyLedger();
|
|
221
|
-
}
|
|
222
|
-
catch {
|
|
223
|
-
return emptyLedger();
|
|
224
|
-
}
|
|
250
|
+
return parseLedger(text);
|
|
225
251
|
}
|
|
226
252
|
/**
|
|
227
253
|
* Fold a delta in, raise the best, persist, and hand back the result. Under the file's lock: two
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a drive found that no single click's check could: the page rendering something other than
|
|
3
|
+
* what the API sent (`reticle_reconcile`, run on each page the Harness reaches), and the crawl's
|
|
4
|
+
* anomalies (blank routes, dead controls, console errors, contradictions).
|
|
5
|
+
*
|
|
6
|
+
* Listed in the drive's report beside the verdict, because the agent reading it reports what it is
|
|
7
|
+
* shown: a defect the drive saw and the report left out is a defect nobody hears about.
|
|
8
|
+
*/
|
|
9
|
+
import { ReticleTool, asRecord, asString } from '@reticlehq/core';
|
|
10
|
+
/** Enough to act on without burying the verdict. */
|
|
11
|
+
const MAX_FINDINGS = 20;
|
|
12
|
+
export function appFindings(toolCalls) {
|
|
13
|
+
const seen = new Set();
|
|
14
|
+
const lines = [];
|
|
15
|
+
const add = (key, line) => {
|
|
16
|
+
if (seen.has(key) || MAX_FINDINGS <= lines.length)
|
|
17
|
+
return;
|
|
18
|
+
seen.add(key);
|
|
19
|
+
lines.push(line);
|
|
20
|
+
};
|
|
21
|
+
for (const call of toolCalls) {
|
|
22
|
+
if (call.isError)
|
|
23
|
+
continue;
|
|
24
|
+
const result = asRecord(call.result);
|
|
25
|
+
if (ReticleTool.RECONCILE === call.name) {
|
|
26
|
+
const mismatches = result['mismatches'];
|
|
27
|
+
for (const raw of Array.isArray(mismatches) ? mismatches : []) {
|
|
28
|
+
const m = asRecord(raw);
|
|
29
|
+
const entity = asString(m['entity']) ?? '?';
|
|
30
|
+
const field = asString(m['field']) ?? '?';
|
|
31
|
+
const shown = asString(m['rendered']);
|
|
32
|
+
add(`r:${entity}:${field}`, ` RENDERED ≠ API: ${entity} ${field} — the API says ${String(m['api'])}${shown === undefined ? '' : `, the page shows ${shown}`} (${asString(m['why']) ?? ''})`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
if (ReticleTool.CRAWL === call.name) {
|
|
36
|
+
const anomalies = result['anomalies'];
|
|
37
|
+
for (const raw of Array.isArray(anomalies) ? anomalies : []) {
|
|
38
|
+
const a = asRecord(raw);
|
|
39
|
+
const kind = asString(a['kind']) ?? 'anomaly';
|
|
40
|
+
const desc = asString(a['desc']) ?? '';
|
|
41
|
+
const detail = asString(a['detail']);
|
|
42
|
+
add(`c:${kind}:${desc}`, ` CRAWL ${kind}: ${desc}${detail === undefined ? '' : ` — ${detail}`}`);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return 0 === lines.length
|
|
47
|
+
? []
|
|
48
|
+
: [
|
|
49
|
+
'Found beyond the clicks (each page compared with the API, and a crawl of the app):',
|
|
50
|
+
...lines,
|
|
51
|
+
];
|
|
52
|
+
}
|
|
@@ -37,7 +37,15 @@ export interface DrivenStep {
|
|
|
37
37
|
* noise, and the report is read every drive.
|
|
38
38
|
*/
|
|
39
39
|
claimed: string | undefined;
|
|
40
|
+
/**
|
|
41
|
+
* What the engine saw that decided it: each channel that disagreed, and any write the app sent
|
|
42
|
+
* that failed. The act's own result carries all of it; a report that kept only the verdict word
|
|
43
|
+
* handed the caller "unit-mismatch" and left it unable to say what was wrong.
|
|
44
|
+
*/
|
|
45
|
+
evidence: string[];
|
|
40
46
|
}
|
|
47
|
+
/** Driving calls that ran with the destructive-action permission: these really changed the app. */
|
|
48
|
+
export declare function confirmedDestructive(toolCalls: readonly ToolOutcome[]): number;
|
|
41
49
|
/**
|
|
42
50
|
* Replays this drive ran, and what each came back with.
|
|
43
51
|
*
|
|
@@ -54,6 +62,8 @@ export declare const ReplayOutcome: {
|
|
|
54
62
|
readonly OK: "ok";
|
|
55
63
|
readonly DRIFT: "drift";
|
|
56
64
|
readonly ERROR: "error";
|
|
65
|
+
/** Stopped at a step the flow marks destructive: not run, which is not a regression. */
|
|
66
|
+
readonly GUARDED: "guarded";
|
|
57
67
|
};
|
|
58
68
|
export declare function replayedFlows(toolCalls: readonly ToolOutcome[]): {
|
|
59
69
|
name: string;
|
|
@@ -61,4 +71,20 @@ export declare function replayedFlows(toolCalls: readonly ToolOutcome[]): {
|
|
|
61
71
|
}[];
|
|
62
72
|
/** Reduce the raw call log to the actions that actually drove the app. */
|
|
63
73
|
export declare function drivenSteps(toolCalls: readonly ToolOutcome[]): DrivenStep[];
|
|
64
|
-
|
|
74
|
+
/** How the drive's checks came out: act_and_wait with an `until`, and asserts. */
|
|
75
|
+
export declare function checkTally(toolCalls: readonly ToolOutcome[]): {
|
|
76
|
+
held: number;
|
|
77
|
+
failed: number;
|
|
78
|
+
undecided: number;
|
|
79
|
+
};
|
|
80
|
+
/**
|
|
81
|
+
* The first line a caller reads, and the one the HUD ends on. A drive whose only passing check was
|
|
82
|
+
* a page change, beside a refund that failed, is NOT "proved its checks"; that line ended a run
|
|
83
|
+
* where the journey it was asked to prove had failed.
|
|
84
|
+
*/
|
|
85
|
+
export declare function verdictLine(tally: {
|
|
86
|
+
held: number;
|
|
87
|
+
failed: number;
|
|
88
|
+
undecided: number;
|
|
89
|
+
}): string;
|
|
90
|
+
export declare function describeDrive(toolCalls: readonly ToolOutcome[], savedFlows: readonly string[], unverifiedFlows?: readonly string[]): string;
|
|
@@ -17,9 +17,72 @@
|
|
|
17
17
|
* that is what a coding agent has to gate on, and it says "not proved" in those words rather than
|
|
18
18
|
* leaving a verdict of `unknown` to be read as a pass.
|
|
19
19
|
*/
|
|
20
|
+
import { appFindings } from './app-findings.js';
|
|
20
21
|
import { asRecord, asString, ReticleTool, Verified } from '@reticlehq/core';
|
|
22
|
+
function refusedAsDestructive(call) {
|
|
23
|
+
const error = asString(asRecord(call.result)['error']) ?? '';
|
|
24
|
+
return call.isError && error.includes(DESTRUCTIVE_REFUSAL);
|
|
25
|
+
}
|
|
26
|
+
/** Driving calls that ran with the destructive-action permission: these really changed the app. */
|
|
27
|
+
export function confirmedDestructive(toolCalls) {
|
|
28
|
+
return toolCalls.filter((call) => DRIVING_TOOLS.has(call.name) &&
|
|
29
|
+
!call.isError &&
|
|
30
|
+
true === asRecord(asRecord(call.args)['args'])[DESTRUCTIVE_REFUSAL]).length;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* A failed claim whose page then showed a dialog: the control opens a confirmation, and the drive
|
|
34
|
+
* guessed it would send the request itself. Reported as a defect, a row's "Refund" button that opens
|
|
35
|
+
* the "Refund now" dialog sent the caller looking for a dead control.
|
|
36
|
+
*/
|
|
37
|
+
function openedADialog(result, after) {
|
|
38
|
+
if (Verified.NO !== result['verified'])
|
|
39
|
+
return [];
|
|
40
|
+
const next = after.find((call) => ReticleTool.SNAPSHOT === call.name);
|
|
41
|
+
const dialogs = asRecord(asRecord(next?.result)['status'])['visibleDialogs'];
|
|
42
|
+
return Array.isArray(dialogs) && 0 < dialogs.length
|
|
43
|
+
? [
|
|
44
|
+
'it opened a dialog instead: likely the drive guessed the consequence wrong, not an app defect',
|
|
45
|
+
]
|
|
46
|
+
: [];
|
|
47
|
+
}
|
|
48
|
+
/** Characters of a response body quoted in the evidence: enough for an error message. */
|
|
49
|
+
const MAX_BODY_CHARS = 200;
|
|
50
|
+
/** Contradictions quoted per step. */
|
|
51
|
+
const MAX_EVIDENCE = 3;
|
|
52
|
+
/** The deciding evidence an act result carries, as short lines. */
|
|
53
|
+
function evidenceOf(result) {
|
|
54
|
+
const lines = [];
|
|
55
|
+
const contradictions = result['contradictions'];
|
|
56
|
+
if (Array.isArray(contradictions)) {
|
|
57
|
+
for (const raw of contradictions.slice(0, MAX_EVIDENCE)) {
|
|
58
|
+
const c = asRecord(raw);
|
|
59
|
+
const parts = [asString(c['counter']), asString(c['detail'])].filter((part) => part !== undefined && 0 < part.length);
|
|
60
|
+
lines.push(`${asString(c['kind']) ?? 'contradiction'}: ${parts.join(' — ')}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const verdict = asRecord(result['verdict']);
|
|
64
|
+
const seen = asRecord(verdict['evidence']);
|
|
65
|
+
const status = seen['status'];
|
|
66
|
+
if ('number' === typeof status && 400 <= status) {
|
|
67
|
+
const body = asString(seen['responseBody']);
|
|
68
|
+
lines.push(`${asString(seen['method']) ?? ''} ${asString(seen['url']) ?? ''} → ${String(status)}` +
|
|
69
|
+
(body === undefined ? '' : ` ${body.slice(0, MAX_BODY_CHARS)}`));
|
|
70
|
+
}
|
|
71
|
+
const observed = asString(verdict['observed']);
|
|
72
|
+
if (false === verdict['pass'] && observed !== undefined)
|
|
73
|
+
lines.push(`observed: ${observed}`);
|
|
74
|
+
return lines;
|
|
75
|
+
}
|
|
76
|
+
/** The permission a destructive action needs, named in the gate's refusal and carried on a retry. */
|
|
77
|
+
const DESTRUCTIVE_REFUSAL = 'confirmDangerous';
|
|
21
78
|
/** How many steps are listed before the account starts counting instead of naming. */
|
|
22
79
|
const MAX_LISTED = 12;
|
|
80
|
+
/**
|
|
81
|
+
* Failures are listed first, and up to this many of them however long the run. Listed in drive
|
|
82
|
+
* order, a whole-app run named 12 of 70 actions and left its failures in "… and 58 more", so the
|
|
83
|
+
* agent reading it could not say what any of them was.
|
|
84
|
+
*/
|
|
85
|
+
const MAX_FAILED = 40;
|
|
23
86
|
/** Actions that change the app. Reads are not part of the story of what was driven. */
|
|
24
87
|
const DRIVING_TOOLS = new Set([
|
|
25
88
|
ReticleTool.ACT,
|
|
@@ -61,16 +124,27 @@ function targetOf(args, result) {
|
|
|
61
124
|
* resolving — the app moved under a recording, which is a finding about the RECORDING rather than
|
|
62
125
|
* proof the feature broke, and collapsing it into "failed" would send somebody to fix working code.
|
|
63
126
|
*/
|
|
64
|
-
export const ReplayOutcome = {
|
|
127
|
+
export const ReplayOutcome = {
|
|
128
|
+
OK: 'ok',
|
|
129
|
+
DRIFT: 'drift',
|
|
130
|
+
ERROR: 'error',
|
|
131
|
+
/** Stopped at a step the flow marks destructive: not run, which is not a regression. */
|
|
132
|
+
GUARDED: 'guarded',
|
|
133
|
+
};
|
|
65
134
|
export function replayedFlows(toolCalls) {
|
|
66
135
|
const out = [];
|
|
67
136
|
for (const call of toolCalls) {
|
|
68
137
|
if (ReticleTool.FLOW_REPLAY !== call.name)
|
|
69
138
|
continue;
|
|
70
139
|
const name = asString(asRecord(call.args)['flowName']) ?? 'a flow';
|
|
140
|
+
const result = asRecord(call.result);
|
|
141
|
+
const message = asString(asRecord(result['error'])['message']) ?? asString(result['error']) ?? '';
|
|
142
|
+
const status = call.isError ? ReplayOutcome.ERROR : asString(result['status']);
|
|
71
143
|
out.push({
|
|
72
144
|
name,
|
|
73
|
-
status:
|
|
145
|
+
status: ReplayOutcome.ERROR === status && message.includes(DESTRUCTIVE_REFUSAL)
|
|
146
|
+
? ReplayOutcome.GUARDED
|
|
147
|
+
: status,
|
|
74
148
|
});
|
|
75
149
|
}
|
|
76
150
|
return out;
|
|
@@ -109,11 +183,15 @@ function describeClaim(until) {
|
|
|
109
183
|
/** Reduce the raw call log to the actions that actually drove the app. */
|
|
110
184
|
export function drivenSteps(toolCalls) {
|
|
111
185
|
const steps = [];
|
|
112
|
-
for (const call of toolCalls) {
|
|
186
|
+
for (const [index, call] of toolCalls.entries()) {
|
|
113
187
|
if (!DRIVING_TOOLS.has(call.name))
|
|
114
188
|
continue;
|
|
115
189
|
const args = asRecord(call.args);
|
|
116
190
|
const result = asRecord(call.result);
|
|
191
|
+
// The gate's refusal never reached the app; listed beside the confirmed retry, it read as "the
|
|
192
|
+
// refund was blocked" to the agent reading this, about a refund that had gone through.
|
|
193
|
+
if (refusedAsDestructive(call))
|
|
194
|
+
continue;
|
|
117
195
|
const action = ReticleTool.NAVIGATE === call.name ? 'navigate' : (asString(args['action']) ?? 'act');
|
|
118
196
|
steps.push({
|
|
119
197
|
action,
|
|
@@ -121,6 +199,9 @@ export function drivenSteps(toolCalls) {
|
|
|
121
199
|
verified: call.isError ? 'error' : asString(result['verified']),
|
|
122
200
|
because: call.isError ? asString(result['error']) : asString(result['because']),
|
|
123
201
|
claimed: describeClaim(args['until']),
|
|
202
|
+
evidence: call.isError
|
|
203
|
+
? []
|
|
204
|
+
: [...evidenceOf(result), ...openedADialog(result, toolCalls.slice(index + 1))],
|
|
124
205
|
});
|
|
125
206
|
}
|
|
126
207
|
return steps;
|
|
@@ -134,19 +215,66 @@ export function drivenSteps(toolCalls) {
|
|
|
134
215
|
* rather than as a failure — it calls for a better check, not a code change, and collapsing the two
|
|
135
216
|
* would send an agent to rewrite working code.
|
|
136
217
|
*/
|
|
218
|
+
/** Every `__reticle_*` query parameter, wherever it sits in the route. */
|
|
219
|
+
const RETICLE_PARAMS = /[?&]__reticle_[^&#]*/g;
|
|
137
220
|
/** The pages the drive's own snapshots reported, in order, each distinct page once. */
|
|
138
221
|
function pagesReached(toolCalls) {
|
|
139
222
|
const pages = [];
|
|
140
223
|
for (const call of toolCalls) {
|
|
141
224
|
if (ReticleTool.SNAPSHOT !== call.name || call.isError)
|
|
142
225
|
continue;
|
|
143
|
-
const
|
|
226
|
+
const raw = asString(asRecord(asRecord(call.result)['status'])['route']);
|
|
227
|
+
// Reticle's own parameters (a leased tab's session and project) are not part of the page.
|
|
228
|
+
const route = raw?.replace(RETICLE_PARAMS, '').replace(/^\/\?(?=#|$)/, '/');
|
|
144
229
|
if (route !== undefined && !pages.includes(route))
|
|
145
230
|
pages.push(route);
|
|
146
231
|
}
|
|
147
232
|
return pages;
|
|
148
233
|
}
|
|
149
|
-
|
|
234
|
+
/** How the drive's checks came out: act_and_wait with an `until`, and asserts. */
|
|
235
|
+
export function checkTally(toolCalls) {
|
|
236
|
+
// One check per control and claim, at its worst: a live drive pressed "Sign in" fifteen times and
|
|
237
|
+
// reported "15 of 15 check(s) held" for one fact proved fifteen times over.
|
|
238
|
+
const worst = new Map();
|
|
239
|
+
for (const call of toolCalls) {
|
|
240
|
+
const args = asRecord(call.args);
|
|
241
|
+
const isCheck = ReticleTool.ASSERT === call.name ||
|
|
242
|
+
(ReticleTool.ACT_AND_WAIT === call.name && args['until'] !== undefined);
|
|
243
|
+
if (!isCheck || call.isError)
|
|
244
|
+
continue;
|
|
245
|
+
const result = asRecord(call.result);
|
|
246
|
+
const effect = asRecord(result['effect']);
|
|
247
|
+
const control = asString(effect['testid']) ??
|
|
248
|
+
`${asString(effect['role']) ?? ''} ${asString(effect['name']) ?? asString(args['ref']) ?? ''}`;
|
|
249
|
+
const key = `${call.name}|${control}|${JSON.stringify(args['until'] ?? args['predicate'] ?? null)}`;
|
|
250
|
+
const verified = asString(result['verified']);
|
|
251
|
+
const now = Verified.YES === verified ? 'held' : Verified.NO === verified ? 'failed' : 'undecided';
|
|
252
|
+
const before = worst.get(key);
|
|
253
|
+
if (before === undefined || 'failed' === now || ('undecided' === now && 'held' === before))
|
|
254
|
+
worst.set(key, now);
|
|
255
|
+
}
|
|
256
|
+
const tally = { held: 0, failed: 0, undecided: 0 };
|
|
257
|
+
for (const verdict of worst.values())
|
|
258
|
+
tally[verdict] += 1;
|
|
259
|
+
return tally;
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* The first line a caller reads, and the one the HUD ends on. A drive whose only passing check was
|
|
263
|
+
* a page change, beside a refund that failed, is NOT "proved its checks"; that line ended a run
|
|
264
|
+
* where the journey it was asked to prove had failed.
|
|
265
|
+
*/
|
|
266
|
+
export function verdictLine(tally) {
|
|
267
|
+
const total = tally.held + tally.failed + tally.undecided;
|
|
268
|
+
if (0 === total)
|
|
269
|
+
return 'NOT PROVED: the drive ran no check.';
|
|
270
|
+
const counts = `${String(tally.held)} of ${String(total)} check(s) held, ${String(tally.failed)} failed, ${String(tally.undecided)} undecided`;
|
|
271
|
+
if (0 < tally.failed)
|
|
272
|
+
return `NOT PROVED — ${counts}. The failures below are findings.`;
|
|
273
|
+
if (0 < tally.undecided)
|
|
274
|
+
return `NOT PROVED — ${counts}.`;
|
|
275
|
+
return `PROVED — ${counts}.`;
|
|
276
|
+
}
|
|
277
|
+
export function describeDrive(toolCalls, savedFlows, unverifiedFlows = []) {
|
|
150
278
|
const steps = drivenSteps(toolCalls);
|
|
151
279
|
const replays = replayedFlows(toolCalls);
|
|
152
280
|
/*
|
|
@@ -162,12 +290,16 @@ export function describeDrive(toolCalls, savedFlows) {
|
|
|
162
290
|
const held = replays.filter((r) => ReplayOutcome.OK === r.status);
|
|
163
291
|
const failed = replays.filter((r) => ReplayOutcome.ERROR === r.status);
|
|
164
292
|
const drifted = replays.filter((r) => ReplayOutcome.DRIFT === r.status);
|
|
293
|
+
const guarded = replays.filter((r) => ReplayOutcome.GUARDED === r.status);
|
|
165
294
|
const lines = [
|
|
166
295
|
`Replayed ${String(replays.length)} recorded journey(s) with NO model in the loop: ` +
|
|
167
|
-
`${String(held.length)} still hold, ${String(failed.length)} failed, ${String(drifted.length)} drifted
|
|
296
|
+
`${String(held.length)} still hold, ${String(failed.length)} failed, ${String(drifted.length)} drifted` +
|
|
297
|
+
`${0 === guarded.length ? '' : `, ${String(guarded.length)} not run`}.`,
|
|
168
298
|
];
|
|
169
299
|
if (0 < failed.length)
|
|
170
300
|
lines.push(` FAILED: ${failed.map((r) => r.name).join(', ')} — regressions in journeys that used to pass.`);
|
|
301
|
+
if (0 < guarded.length)
|
|
302
|
+
lines.push(` NOT RUN: ${guarded.map((r) => r.name).join(', ')} — stopped at a step marked destructive. Not a regression: replay with confirmDangerous to run it.`);
|
|
171
303
|
if (0 < drifted.length)
|
|
172
304
|
lines.push(` DRIFTED: ${drifted.map((r) => r.name).join(', ')} — the app moved under these recordings; the flow needs re-anchoring, not the app fixing.`);
|
|
173
305
|
return lines.join('\n');
|
|
@@ -180,6 +312,7 @@ export function describeDrive(toolCalls, savedFlows) {
|
|
|
180
312
|
const undecided = steps.filter((step) => Verified.YES !== step.verified && Verified.NO !== step.verified);
|
|
181
313
|
const pages = pagesReached(toolCalls);
|
|
182
314
|
const lines = [
|
|
315
|
+
verdictLine(checkTally(toolCalls)),
|
|
183
316
|
...(replayLine === undefined ? [] : [replayLine]),
|
|
184
317
|
`Drove ${String(steps.length)} action(s): ${String(proved.length)} proved, ` +
|
|
185
318
|
`${String(failed.length)} failed, ${String(undecided.length)} not decided.`,
|
|
@@ -188,8 +321,10 @@ export function describeDrive(toolCalls, savedFlows) {
|
|
|
188
321
|
...(0 === pages.length
|
|
189
322
|
? []
|
|
190
323
|
: [`Reached ${String(pages.length)} page(s): ${pages.join(' → ')}.`]),
|
|
324
|
+
...appFindings(toolCalls),
|
|
191
325
|
];
|
|
192
|
-
|
|
326
|
+
const listed = [...failed, ...proved, ...undecided].slice(0, Math.max(MAX_LISTED, Math.min(failed.length, MAX_FAILED)));
|
|
327
|
+
for (const step of listed) {
|
|
193
328
|
const verdict = step.verified ?? 'nothing declared';
|
|
194
329
|
const because = step.because === undefined ? '' : ` — ${step.because}`;
|
|
195
330
|
// The claim, on failures only: it is what separates "the app is broken" from "the drive guessed
|
|
@@ -201,14 +336,30 @@ export function describeDrive(toolCalls, savedFlows) {
|
|
|
201
336
|
? ` (claimed ${step.claimed})`
|
|
202
337
|
: '';
|
|
203
338
|
lines.push(` ${step.action} ${step.target}: ${verdict}${claimed}${because}`);
|
|
339
|
+
for (const line of step.evidence)
|
|
340
|
+
lines.push(` ${line}`);
|
|
341
|
+
}
|
|
342
|
+
if (listed.length < steps.length) {
|
|
343
|
+
const unlistedFailed = Math.max(0, failed.length - MAX_FAILED);
|
|
344
|
+
lines.push(` … and ${String(steps.length - listed.length)} more` +
|
|
345
|
+
(0 === unlistedFailed
|
|
346
|
+
? ', none of them failed.'
|
|
347
|
+
: `, ${String(unlistedFailed)} of them failed.`));
|
|
204
348
|
}
|
|
205
|
-
|
|
206
|
-
|
|
349
|
+
const confirmed = confirmedDestructive(toolCalls);
|
|
350
|
+
if (0 < confirmed)
|
|
351
|
+
lines.push(`${String(confirmed)} destructive action(s) were confirmed and REALLY RAN against the app (the drive confirms one only after the gate refuses it; the refusal itself changed nothing).`);
|
|
207
352
|
if (0 < failed.length)
|
|
208
353
|
lines.push(`${String(failed.length)} action(s) FAILED — the app did not do what the drive declared it would. These are findings.`);
|
|
209
354
|
if (0 < undecided.length)
|
|
210
355
|
lines.push(`${String(undecided.length)} action(s) were NOT PROVED. That is undecided evidence, not a failure: it calls for a better check, not a code change.`);
|
|
211
|
-
|
|
212
|
-
|
|
356
|
+
// A flow with no step that asserts anything replays as "verified nothing": offered as a replay it
|
|
357
|
+
// reads as evidence, so it is named for what it is instead.
|
|
358
|
+
const empty = new Set(unverifiedFlows);
|
|
359
|
+
const replayable = savedFlows.filter((name) => !empty.has(name));
|
|
360
|
+
if (0 < replayable.length)
|
|
361
|
+
lines.push(`Replay any of this without a model: reticle_verify { action: "flows" } — saved ${replayable.join(', ')}.`);
|
|
362
|
+
for (const name of unverifiedFlows)
|
|
363
|
+
lines.push(`Saved ${name}, but ${name} checks nothing: no step asserts a consequence, so its replay will verify nothing. Drive again naming the end state, or add an expect to its last step.`);
|
|
213
364
|
return lines.join('\n');
|
|
214
365
|
}
|
|
@@ -1,19 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The
|
|
2
|
+
* The Harness's driver names.
|
|
3
3
|
*
|
|
4
|
-
* A leaf module on purpose
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
4
|
+
* A leaf module on purpose: the tool table reads these while the tool surface is still loading, and
|
|
5
|
+
* a module that imports nothing cannot be read half-initialised.
|
|
6
|
+
*
|
|
7
|
+
* There is one driver. Every Harness decision, its models and its planning run on the Reticle
|
|
8
|
+
* platform, on the workspace's monthly Harness credits; this machine executes what it is told.
|
|
9
9
|
*/
|
|
10
|
-
/**
|
|
11
|
-
export declare const
|
|
12
|
-
/** Answers typed questions; picks from the elements on the page and cannot invent one. */
|
|
13
|
-
export declare const JEV_DRIVER_NAME = "jev";
|
|
14
|
-
/** Generates its tool calls, over Chat Completions. Here so an A/B has a third arm. */
|
|
15
|
-
export declare const OPENAI_DRIVER_NAME = "openai";
|
|
16
|
-
/** What a caller may name. Order is the order a reader sees them in the tool description. */
|
|
17
|
-
export declare const DRIVER_NAMES: readonly ["anthropic", "jev", "openai"];
|
|
10
|
+
/** The platform's Harness: it decides on its side, this machine executes. See server-driver.ts. */
|
|
11
|
+
export declare const SERVER_DRIVER = "server";
|
|
18
12
|
/** An injected driver: not selectable, because only a caller in-process can supply one. */
|
|
19
13
|
export declare const CUSTOM_DRIVER_NAME = "custom";
|
|
14
|
+
/** What a drive needs, said once for every place that tells somebody. */
|
|
15
|
+
export declare const EXPLORE_NEEDS = "Runs on the Reticle platform: needs a linked project (`reticle connect`); Free has credits.";
|
|
@@ -1,19 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The
|
|
2
|
+
* The Harness's driver names.
|
|
3
3
|
*
|
|
4
|
-
* A leaf module on purpose
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
4
|
+
* A leaf module on purpose: the tool table reads these while the tool surface is still loading, and
|
|
5
|
+
* a module that imports nothing cannot be read half-initialised.
|
|
6
|
+
*
|
|
7
|
+
* There is one driver. Every Harness decision, its models and its planning run on the Reticle
|
|
8
|
+
* platform, on the workspace's monthly Harness credits; this machine executes what it is told.
|
|
9
9
|
*/
|
|
10
|
-
/**
|
|
11
|
-
export const
|
|
12
|
-
/** Answers typed questions; picks from the elements on the page and cannot invent one. */
|
|
13
|
-
export const JEV_DRIVER_NAME = 'jev';
|
|
14
|
-
/** Generates its tool calls, over Chat Completions. Here so an A/B has a third arm. */
|
|
15
|
-
export const OPENAI_DRIVER_NAME = 'openai';
|
|
16
|
-
/** What a caller may name. Order is the order a reader sees them in the tool description. */
|
|
17
|
-
export const DRIVER_NAMES = [ANTHROPIC_DRIVER_NAME, JEV_DRIVER_NAME, OPENAI_DRIVER_NAME];
|
|
10
|
+
/** The platform's Harness: it decides on its side, this machine executes. See server-driver.ts. */
|
|
11
|
+
export const SERVER_DRIVER = 'server';
|
|
18
12
|
/** An injected driver: not selectable, because only a caller in-process can supply one. */
|
|
19
13
|
export const CUSTOM_DRIVER_NAME = 'custom';
|
|
14
|
+
/** What a drive needs, said once for every place that tells somebody. */
|
|
15
|
+
export const EXPLORE_NEEDS = 'Runs on the Reticle platform: needs a linked project (`reticle connect`); Free has credits.';
|