@reticlehq/server 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -5
- package/SKILL.md +130 -778
- package/dist/bridge/bridge.d.ts +23 -0
- package/dist/bridge/bridge.js +80 -2
- package/dist/bridge/bridge.test-harness.d.ts +5 -0
- package/dist/bridge/bridge.test-harness.js +6 -4
- package/dist/capsule/causal-summary.d.ts +36 -1
- package/dist/capsule/causal-summary.js +33 -7
- package/dist/capsule/divergence.js +7 -1
- package/dist/cli/chromium-hint.d.ts +33 -5
- package/dist/cli/chromium-hint.js +82 -12
- package/dist/cli/cli-doctor.js +65 -34
- package/dist/cli/cli-flow-commands.js +8 -8
- package/dist/cli/cli-kill.d.ts +58 -0
- package/dist/cli/cli-kill.js +162 -0
- package/dist/cli/cli-parse.d.ts +16 -1
- package/dist/cli/cli-parse.js +32 -9
- package/dist/cli/cli-port.d.ts +32 -0
- package/dist/cli/cli-port.js +82 -35
- package/dist/cli/cli-verify.d.ts +21 -0
- package/dist/cli/cli-verify.js +47 -3
- package/dist/cli/config-discovery.d.ts +35 -0
- package/dist/cli/config-discovery.js +163 -0
- package/dist/cli/doctor-daemon-line.js +4 -1
- package/dist/cli/doctor-project-line.d.ts +37 -0
- package/dist/cli/doctor-project-line.js +35 -0
- package/dist/cli/doctor-rows.d.ts +51 -0
- package/dist/cli/doctor-rows.js +54 -0
- package/dist/cli/doctor-sessions-line.d.ts +12 -0
- package/dist/cli/doctor-sessions-line.js +69 -0
- package/dist/cli/drive-attach.d.ts +99 -0
- package/dist/cli/drive-attach.js +203 -0
- package/dist/cli/drive-command.d.ts +13 -0
- package/dist/cli/drive-command.js +103 -0
- package/dist/cli/open-note.d.ts +20 -0
- package/dist/cli/open-note.js +37 -0
- package/dist/cli/port-holder.d.ts +10 -0
- package/dist/cli/port-holder.js +26 -2
- package/dist/cli/status-next-action.d.ts +37 -0
- package/dist/cli/status-next-action.js +61 -0
- package/dist/cli.js +119 -39
- package/dist/crawl/crawl-tools.js +12 -2
- package/dist/crawl/crawl.d.ts +13 -0
- package/dist/crawl/crawl.js +50 -2
- package/dist/events/contradictions.d.ts +46 -0
- package/dist/events/contradictions.js +35 -22
- package/dist/events/declared.d.ts +45 -0
- package/dist/events/declared.js +73 -0
- package/dist/events/echo-mismatch.d.ts +8 -2
- package/dist/events/echo-mismatch.js +33 -5
- package/dist/events/event-filters.d.ts +1 -1
- package/dist/events/event-filters.js +7 -4
- package/dist/events/predicate-eval.d.ts +17 -72
- package/dist/events/predicate-eval.js +117 -236
- package/dist/events/predicate-parse.js +31 -3
- package/dist/events/predicate-route.d.ts +32 -0
- package/dist/events/predicate-route.js +102 -0
- package/dist/events/predicate-schema.d.ts +99 -0
- package/dist/events/predicate-schema.js +309 -0
- package/dist/events/predicate.d.ts +8 -0
- package/dist/events/predicate.js +296 -22
- package/dist/flows/annotate-tools.js +3 -0
- package/dist/flows/annotate.js +18 -4
- package/dist/flows/coverage.d.ts +38 -5
- package/dist/flows/coverage.js +30 -4
- package/dist/flows/decision.d.ts +15 -0
- package/dist/flows/decision.js +1 -1
- package/dist/flows/flow-classify.js +1 -1
- package/dist/flows/flow-replay-run.js +8 -1
- package/dist/flows/flow-tools.js +21 -5
- package/dist/flows/heal-run.js +1 -1
- package/dist/flows/replay.js +3 -0
- package/dist/honesty/accepted-write.d.ts +3 -0
- package/dist/honesty/accepted-write.js +8 -3
- package/dist/honesty/already-true.js +15 -0
- package/dist/honesty/blind-spots.js +10 -2
- package/dist/honesty/honesty.d.ts +11 -0
- package/dist/honesty/honesty.js +1 -0
- package/dist/honesty/uncaptured-bodies.d.ts +4 -2
- package/dist/honesty/uncaptured-bodies.js +9 -2
- package/dist/honesty/unread-outcome.d.ts +8 -0
- package/dist/honesty/unread-outcome.js +36 -9
- package/dist/honesty/unsettled.d.ts +49 -0
- package/dist/honesty/unsettled.js +82 -0
- package/dist/honesty/verified.d.ts +36 -3
- package/dist/honesty/verified.js +54 -11
- package/dist/http-server.d.ts +19 -0
- package/dist/http-server.js +202 -4
- package/dist/impact/impact-recorder.d.ts +23 -0
- package/dist/impact/impact-recorder.js +76 -0
- package/dist/impact/impact-store.d.ts +47 -0
- package/dist/impact/impact-store.js +171 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +47 -10
- package/dist/init/agent-rules.d.ts +36 -0
- package/dist/init/agent-rules.js +132 -22
- package/dist/init/astro-patch.d.ts +2 -1
- package/dist/init/astro-patch.js +38 -6
- package/dist/init/capabilities.d.ts +22 -0
- package/dist/init/capabilities.js +76 -0
- package/dist/init/confirm.d.ts +114 -0
- package/dist/init/confirm.js +165 -0
- package/dist/init/connect-steps.d.ts +2 -0
- package/dist/init/connect-steps.js +50 -0
- package/dist/init/cra.js +3 -2
- package/dist/init/csp-check.d.ts +38 -0
- package/dist/init/csp-check.js +92 -0
- package/dist/init/csp-doctor.d.ts +25 -0
- package/dist/init/csp-doctor.js +49 -0
- package/dist/init/cursor.js +6 -9
- package/dist/init/detect.d.ts +8 -0
- package/dist/init/detect.js +33 -3
- package/dist/init/mcp-clients.d.ts +1 -0
- package/dist/init/mcp-clients.js +51 -7
- package/dist/init/mcp.d.ts +15 -0
- package/dist/init/mcp.js +19 -0
- package/dist/init/non-js-project.js +29 -5
- package/dist/init/plan-framework.d.ts +10 -0
- package/dist/init/plan-framework.js +78 -16
- package/dist/init/plan.d.ts +47 -6
- package/dist/init/plan.js +228 -69
- package/dist/init/run.d.ts +24 -2
- package/dist/init/run.js +157 -45
- package/dist/init/slash-command.d.ts +1 -1
- package/dist/init/slash-command.js +54 -6
- package/dist/init/snippets.d.ts +45 -5
- package/dist/init/snippets.js +194 -42
- package/dist/init/vite-config.js +53 -5
- package/dist/input/network-mock-tools.d.ts +19 -0
- package/dist/input/network-mock-tools.js +37 -5
- package/dist/input/scroll-find.d.ts +6 -0
- package/dist/input/scroll-find.js +17 -2
- package/dist/input/scroll-tools.js +4 -0
- package/dist/journal/attach-journal.d.ts +9 -1
- package/dist/journal/attach-journal.js +13 -2
- package/dist/journal/journal-query.d.ts +13 -0
- package/dist/journal/journal-query.js +28 -3
- package/dist/journal/workspace-gitignore.d.ts +10 -0
- package/dist/journal/workspace-gitignore.js +59 -0
- package/dist/license/license.js +1 -1
- package/dist/mcp/mcp-proxy.d.ts +35 -30
- package/dist/mcp/mcp-proxy.js +306 -104
- package/dist/mcp/mcp.d.ts +10 -1
- package/dist/mcp/mcp.js +70 -34
- package/dist/mcp/proxy-handshake.d.ts +82 -1
- package/dist/mcp/proxy-handshake.js +109 -2
- package/dist/mcp/proxy-lifecycle.d.ts +11 -2
- package/dist/mcp/proxy-lifecycle.js +12 -3
- package/dist/mcp/proxy-log.d.ts +63 -0
- package/dist/mcp/proxy-log.js +132 -0
- package/dist/mcp/server-instructions.d.ts +17 -0
- package/dist/mcp/server-instructions.js +56 -0
- package/dist/mcp/shared-params.d.ts +27 -0
- package/dist/mcp/shared-params.js +36 -0
- package/dist/pool/browser-pool.d.ts +22 -2
- package/dist/pool/browser-pool.js +44 -4
- package/dist/project/learned-routes.d.ts +24 -0
- package/dist/project/learned-routes.js +63 -0
- package/dist/project/project-store.d.ts +6 -0
- package/dist/project/project-store.js +25 -1
- package/dist/project/reticle-dir.d.ts +2 -0
- package/dist/project/reticle-dir.js +1 -0
- package/dist/session/connection-memory.d.ts +59 -0
- package/dist/session/connection-memory.js +108 -0
- package/dist/session/dev-command.d.ts +26 -0
- package/dist/session/dev-command.js +89 -0
- package/dist/session/dev-server-probe.d.ts +68 -0
- package/dist/session/dev-server-probe.js +102 -7
- package/dist/session/last-act.d.ts +26 -0
- package/dist/session/last-act.js +35 -1
- package/dist/session/live-control-tools.js +42 -5
- package/dist/session/no-session-diagnosis.d.ts +67 -2
- package/dist/session/no-session-diagnosis.js +236 -27
- package/dist/session/no-session-next-action.d.ts +46 -0
- package/dist/session/no-session-next-action.js +85 -0
- package/dist/session/no-session-watch.d.ts +40 -1
- package/dist/session/no-session-watch.js +174 -8
- package/dist/session/pending-commands.d.ts +12 -0
- package/dist/session/pending-commands.js +13 -1
- package/dist/session/ref-provenance.d.ts +54 -0
- package/dist/session/ref-provenance.js +61 -0
- package/dist/session/session-lease.js +1 -1
- package/dist/session/session-manager.d.ts +4 -0
- package/dist/session/session-manager.js +35 -5
- package/dist/session/session.d.ts +10 -1
- package/dist/session/session.js +146 -5
- package/dist/telemetry/app-instrumented.d.ts +8 -0
- package/dist/telemetry/app-instrumented.js +10 -0
- package/dist/telemetry/automation-hint.d.ts +17 -0
- package/dist/telemetry/automation-hint.js +71 -0
- package/dist/telemetry/bug-found.d.ts +15 -17
- package/dist/telemetry/bug-found.js +52 -21
- package/dist/telemetry/cli-telemetry.js +7 -1
- package/dist/telemetry/daemon-telemetry.js +24 -0
- package/dist/telemetry/feedback.js +1 -1
- package/dist/telemetry/init-telemetry.js +9 -1
- package/dist/telemetry/install-source.d.ts +56 -0
- package/dist/telemetry/install-source.js +74 -0
- package/dist/telemetry/instrumentation-stall.d.ts +50 -0
- package/dist/telemetry/instrumentation-stall.js +90 -0
- package/dist/telemetry/run-telemetry.js +4 -1
- package/dist/telemetry/session-metrics.d.ts +8 -0
- package/dist/telemetry/session-metrics.js +15 -0
- package/dist/telemetry/telemetry.d.ts +7 -1
- package/dist/telemetry/telemetry.js +26 -2
- package/dist/telemetry/tool-refused.d.ts +19 -0
- package/dist/telemetry/tool-refused.js +55 -0
- package/dist/telemetry/verification-of.js +46 -7
- package/dist/temp-dir.d.ts +27 -0
- package/dist/temp-dir.js +46 -0
- package/dist/tools/act-tools.js +209 -28
- package/dist/tools/assert-grade.d.ts +1 -1
- package/dist/tools/assert-grade.js +27 -5
- package/dist/tools/assert-verdict.js +49 -7
- package/dist/tools/browser-tools.js +10 -0
- package/dist/tools/contract-tools.js +4 -0
- package/dist/tools/dynamic-tools.js +16 -7
- package/dist/tools/error-recovery.d.ts +10 -0
- package/dist/tools/error-recovery.js +73 -9
- package/dist/tools/feedback-invite.js +1 -1
- package/dist/tools/invoke-tool.js +159 -26
- package/dist/tools/lease-hint.d.ts +30 -8
- package/dist/tools/lease-hint.js +65 -14
- package/dist/tools/lease-tools.d.ts +40 -1
- package/dist/tools/lease-tools.js +181 -83
- package/dist/tools/navigate-result.js +1 -1
- package/dist/tools/observe-tools.js +10 -4
- package/dist/tools/observed-nothing.js +1 -1
- package/dist/tools/react-grace.d.ts +52 -0
- package/dist/tools/react-grace.js +90 -0
- package/dist/tools/read-tools.js +67 -5
- package/dist/tools/resolve-target.d.ts +33 -0
- package/dist/tools/resolve-target.js +53 -0
- package/dist/tools/settle-in-flight.d.ts +94 -0
- package/dist/tools/settle-in-flight.js +146 -0
- package/dist/tools/snapshot-delta.d.ts +10 -0
- package/dist/tools/snapshot-delta.js +194 -9
- package/dist/tools/tool-kit.d.ts +8 -0
- package/dist/tools/tool-surface.d.ts +60 -0
- package/dist/tools/tool-surface.js +86 -1
- package/dist/tools/tools.js +61 -8
- package/dist/tools/unadvertised-help.js +9 -6
- package/docs/README.md +55 -11
- package/docs/actions.mdx +120 -0
- package/docs/agent-cheatsheet.md +95 -29
- package/docs/architecture.md +52 -35
- package/docs/benchmarks.md +127 -75
- package/docs/best-practices.mdx +216 -0
- package/docs/capabilities.mdx +534 -0
- package/docs/cli/affected.mdx +53 -0
- package/docs/cli/capsules.mdx +70 -0
- package/docs/cli/cloud.mdx +153 -0
- package/docs/cli/doctor.mdx +109 -0
- package/docs/cli/drive.mdx +74 -0
- package/docs/cli/feedback.mdx +67 -0
- package/docs/cli/gate.mdx +64 -0
- package/docs/cli/help.mdx +94 -0
- package/docs/cli/hunt.mdx +73 -0
- package/docs/cli/identify.mdx +76 -0
- package/docs/cli/init.mdx +121 -0
- package/docs/cli/kill.mdx +91 -0
- package/docs/cli/license.mdx +48 -0
- package/docs/cli/mcp.mdx +61 -0
- package/docs/cli/open.mdx +74 -0
- package/docs/cli/restart.mdx +60 -0
- package/docs/cli/rollback.mdx +35 -0
- package/docs/cli/serve.mdx +69 -0
- package/docs/cli/status.mdx +88 -0
- package/docs/cli/stop.mdx +64 -0
- package/docs/cli/telemetry.mdx +57 -0
- package/docs/cli/update.mdx +52 -0
- package/docs/cli/verify.mdx +76 -0
- package/docs/cli/version.mdx +39 -0
- package/docs/cli/watch.mdx +62 -0
- package/docs/cli.mdx +241 -0
- package/docs/debugging.md +16 -10
- package/docs/deploy-checks.md +45 -12
- package/docs/desktop.mdx +384 -0
- package/docs/docs.json +364 -0
- package/docs/enterprise.md +22 -16
- package/docs/faq.mdx +102 -0
- package/docs/fixtures.md +13 -7
- package/docs/flows.md +29 -23
- package/docs/for-agents.md +71 -0
- package/docs/frameworks.mdx +238 -0
- package/docs/gate-plan.md +48 -42
- package/docs/gates.md +28 -22
- package/docs/getting-started.md +129 -77
- package/docs/human-control.md +24 -18
- package/docs/index.mdx +143 -0
- package/docs/install-agentic.mdx +224 -0
- package/docs/install-manual.mdx +196 -0
- package/docs/instrumentation.mdx +238 -0
- package/docs/integration-patterns.md +41 -35
- package/docs/local-registry.md +19 -13
- package/docs/multi-agent-testing.md +19 -13
- package/docs/packages/babel-plugin.mdx +79 -0
- package/docs/packages/browser.mdx +111 -0
- package/docs/packages/core.mdx +116 -0
- package/docs/packages/electron.mdx +85 -0
- package/docs/packages/eslint-plugin.mdx +86 -0
- package/docs/packages/next.mdx +89 -0
- package/docs/packages/react.mdx +91 -0
- package/docs/packages/server.mdx +98 -0
- package/docs/packages/tauri.mdx +63 -0
- package/docs/packages/test.mdx +119 -0
- package/docs/packages/vite-plugin.mdx +94 -0
- package/docs/packages.mdx +218 -0
- package/docs/platform-integration.md +35 -26
- package/docs/predicates.mdx +248 -0
- package/docs/quickstart.mdx +257 -0
- package/docs/recipes.mdx +282 -0
- package/docs/skill-file.mdx +107 -0
- package/docs/state-management.mdx +199 -0
- package/docs/style.css +14 -0
- package/docs/system-map.md +50 -33
- package/docs/telemetry-contract.md +106 -52
- package/docs/telemetry-events.mdx +475 -0
- package/docs/telemetry.md +63 -55
- package/docs/testing.md +44 -6
- package/docs/token-efficiency.md +37 -27
- package/docs/tools-act-and-wait.mdx +173 -0
- package/docs/tools-act-sequence.mdx +137 -0
- package/docs/tools-act.mdx +95 -0
- package/docs/tools-assert.mdx +129 -0
- package/docs/tools-console.mdx +63 -0
- package/docs/tools-inspect.mdx +88 -0
- package/docs/tools-navigate.mdx +99 -0
- package/docs/tools-network.mdx +119 -0
- package/docs/tools-observe.mdx +145 -0
- package/docs/tools-overview.mdx +160 -0
- package/docs/tools-query.mdx +126 -0
- package/docs/tools-session-and-feedback.mdx +112 -0
- package/docs/tools-sessions.mdx +94 -0
- package/docs/tools-snapshot.mdx +129 -0
- package/docs/tools-state.mdx +104 -0
- package/docs/tools-tools-and-run.mdx +156 -0
- package/docs/tools-wait-for.mdx +85 -0
- package/docs/troubleshooting.mdx +263 -0
- package/docs/usage.md +209 -165
- package/docs/vs-chrome-devtools-mcp.mdx +123 -0
- package/docs/vs-playwright-mcp.mdx +125 -0
- package/docs/vs-screenshots.mdx +137 -0
- package/docs/what-is-reticle.mdx +63 -0
- package/docs/why-reticle.mdx +109 -0
- package/package.json +5 -5
- package/docs/desktop-apps.md +0 -256
- package/docs/matrix/2.5.0/claude-code-machine.json +0 -19
- package/docs/matrix/2.5.0/codex-machine.json +0 -20
- package/docs/matrix/2.5.0/cursor-machine.json +0 -22
- package/docs/matrix/2.5.0/gemini-machine.json +0 -22
- package/docs/matrix/2.5.0/opencode-machine.json +0 -22
- package/docs/matrix/2.5.0/vscode-machine.json +0 -22
- package/docs/matrix/2.5.0/windsurf-machine.json +0 -22
- package/docs/matrix/MATRIX.md +0 -26
- package/docs/matrix/README.md +0 -59
- package/docs/mint.json +0 -61
package/dist/events/predicate.js
CHANGED
|
@@ -5,8 +5,21 @@ import { selectPath, capDepth } from '../session/state-select.js';
|
|
|
5
5
|
import { describeTestidMiss } from './testid-near-miss.js';
|
|
6
6
|
import { predicateToExpectedLinks } from '../capsule/predicate-to-links.js';
|
|
7
7
|
import { isAmbient, ambientKeyOf } from '../journal/ambient.js';
|
|
8
|
-
import {
|
|
8
|
+
import { evalRoute } from './predicate-route.js';
|
|
9
|
+
import { PredicateSchema, matchValue, evalNet, evalConsole, evalAnimation, evalSignal, evalSettled, residualQueryChecks, satisfiesResiduals, describeResidual, } from './predicate-eval.js';
|
|
9
10
|
export { PredicateSchema, PredicateKind };
|
|
11
|
+
/**
|
|
12
|
+
* A composite result that carries a child's "nobody could evaluate this" up to the verdict rule.
|
|
13
|
+
*
|
|
14
|
+
* `decideVerified` already treats `inconclusive` correctly and does it ahead of the failure clause,
|
|
15
|
+
* so all a composite has to do is stop dropping the field on its way out. `pass` stays false because
|
|
16
|
+
* nothing was proven; what changes is that the verdict reads UNKNOWN rather than blaming the app for
|
|
17
|
+
* a clause the CALL under-specified.
|
|
18
|
+
*/
|
|
19
|
+
function unreadableComposite(child, evidence) {
|
|
20
|
+
const reason = child.inconclusive ?? 'a sub-predicate could not be evaluated';
|
|
21
|
+
return { pass: false, failureReason: reason, inconclusive: reason, evidence };
|
|
22
|
+
}
|
|
10
23
|
async function matchOnce(session, query, state) {
|
|
11
24
|
const res = await session.command(ReticleCommand.MATCH, { query, state });
|
|
12
25
|
if (!res.ok)
|
|
@@ -14,8 +27,44 @@ async function matchOnce(session, query, state) {
|
|
|
14
27
|
return (res.result ?? { matched: false, count: 0, elements: [] });
|
|
15
28
|
}
|
|
16
29
|
async function evalElement(session, query, state, absent, diagnose) {
|
|
17
|
-
|
|
30
|
+
// Fields the browser's locator would have DROPPED, enforced back here — see residualQueryChecks.
|
|
31
|
+
// Checked before the round-trip when nothing can enforce them: a predicate that cannot be evaluated
|
|
32
|
+
// must say so rather than resolve to whatever the surviving half of it happened to match.
|
|
33
|
+
const residual = residualQueryChecks(query);
|
|
34
|
+
if (residual.unusable.length > 0) {
|
|
35
|
+
const reason = `the element locator ignores ${residual.unusable.map((f) => `\`${f}\``).join(', ')} ` +
|
|
36
|
+
`in ${JSON.stringify(query)} — it resolves by the first of by+value, component/source, role, ` +
|
|
37
|
+
'text, label, placeholder, testid, alt that is present, and nothing here can check the rest. ' +
|
|
38
|
+
'Assert them one locator at a time, or move the extra field into the locator';
|
|
39
|
+
return { pass: false, failureReason: reason, inconclusive: reason };
|
|
40
|
+
}
|
|
41
|
+
let match = await matchOnce(session, query, state);
|
|
18
42
|
const subject = JSON.stringify(query);
|
|
43
|
+
// A residual narrows the SET; `count` is every match while `elements` is only the described prefix,
|
|
44
|
+
// so a locator broad enough to be truncated cannot be narrowed honestly. Say so instead of guessing.
|
|
45
|
+
if (residual.checks.length > 0 && match.count > match.elements.length) {
|
|
46
|
+
const reason = `${String(match.count)} elements matched ${subject} and only ${String(match.elements.length)} were described, so ${residual.checks.map(([f]) => `\`${f}\``).join(', ')} could not be checked against all of them — narrow the locator`;
|
|
47
|
+
return { pass: false, failureReason: reason, inconclusive: reason };
|
|
48
|
+
}
|
|
49
|
+
const kept = match.elements.filter((element) => satisfiesResiduals(element, residual.checks));
|
|
50
|
+
// The locator found something and the dropped fields disagree with it. Reported separately from a
|
|
51
|
+
// plain miss because the fixes are opposite: the element IS there, its value is not what was claimed.
|
|
52
|
+
if (residual.checks.length > 0 && match.matched && 0 === kept.length && !absent) {
|
|
53
|
+
const wanted = residual.checks.map(([f, want]) => `${f}=${JSON.stringify(want)}`).join(', ');
|
|
54
|
+
return {
|
|
55
|
+
pass: false,
|
|
56
|
+
failureReason: `element matching ${subject} is present but ${wanted} does not hold`,
|
|
57
|
+
observed: match.elements
|
|
58
|
+
.map((element) => residual.checks.map(([f]) => describeResidual(element, f)).join(', '))
|
|
59
|
+
.join('; '),
|
|
60
|
+
expected: `an element matching ${subject} with ${wanted}`,
|
|
61
|
+
assertion: `element.${residual.checks[0]?.[0] ?? 'residual'}`,
|
|
62
|
+
evidence: match.elements,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
if (residual.checks.length > 0) {
|
|
66
|
+
match = { ...match, matched: kept.length > 0, count: kept.length, elements: kept };
|
|
67
|
+
}
|
|
19
68
|
// A given-but-missing scope is handled ASYMMETRICALLY, because "absent" and "present" ask different
|
|
20
69
|
// questions of a scope that no longer exists:
|
|
21
70
|
// - ABSENT: an element is trivially absent from a container that isn't there. This is also the
|
|
@@ -114,8 +163,134 @@ async function evalElement(session, query, state, absent, diagnose) {
|
|
|
114
163
|
...(present.length > 0 ? { evidence: { presentTestids: present } } : {}),
|
|
115
164
|
};
|
|
116
165
|
}
|
|
166
|
+
/** The single element of a list, or undefined when there is not exactly one. */
|
|
167
|
+
function oneOf(names) {
|
|
168
|
+
return 1 === names.length ? names[0] : undefined;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* `readState`'s truncation report, when the caps actually fired.
|
|
172
|
+
*
|
|
173
|
+
* Its PRESENCE is the warning — the field is omitted entirely on an intact read, which is what lets
|
|
174
|
+
* "the list is short" stay distinguishable from "I shortened the list".
|
|
175
|
+
*/
|
|
176
|
+
function truncationOf(result) {
|
|
177
|
+
if ('object' !== typeof result || null === result)
|
|
178
|
+
return undefined;
|
|
179
|
+
const report = result.truncation;
|
|
180
|
+
return 'object' === typeof report && null !== report
|
|
181
|
+
? report
|
|
182
|
+
: undefined;
|
|
183
|
+
}
|
|
184
|
+
/** A scoped `readState` reply, narrowed to the shape `selectPath` would have produced. */
|
|
185
|
+
function asSelection(result) {
|
|
186
|
+
if ('object' !== typeof result || null === result)
|
|
187
|
+
return undefined;
|
|
188
|
+
const record = result;
|
|
189
|
+
return 'boolean' === typeof record.found
|
|
190
|
+
? { found: record.found, value: record.value }
|
|
191
|
+
: undefined;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* A NAMED store needs no whole-store read. `reticle_state`'s scoped mode selects the path out of the
|
|
195
|
+
* RAW store IN-PAGE and returns only it, so the assertion resolves in one round trip against a payload
|
|
196
|
+
* the size of the value — not the store. The unnamed case below still needs the wide read, because
|
|
197
|
+
* that is how it discovers WHICH store carries the path; only the named branch changes.
|
|
198
|
+
*
|
|
199
|
+
* A scoped read answers `found: false` for both "no such store" and "no such path", so the two are
|
|
200
|
+
* kept distinct here using the `storeNames` list the reply always carries — otherwise the payload gets
|
|
201
|
+
* cheaper while the message gets worse.
|
|
202
|
+
*/
|
|
203
|
+
async function evalStateNamed(session, p, storeName) {
|
|
204
|
+
const scoped = await session.command(ReticleCommand.STATE_READ, {
|
|
205
|
+
store: storeName,
|
|
206
|
+
path: p.path,
|
|
207
|
+
});
|
|
208
|
+
if (!scoped.ok) {
|
|
209
|
+
return {
|
|
210
|
+
pass: false,
|
|
211
|
+
failureReason: 'state read failed',
|
|
212
|
+
observed: 'the store could not be read',
|
|
213
|
+
expected: 'a readable registered store',
|
|
214
|
+
assertion: 'state.unreadable',
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
const reply = (scoped.result ?? {});
|
|
218
|
+
const names = Array.isArray(reply.storeNames) ? reply.storeNames : [];
|
|
219
|
+
// Resolve the value. A CURRENT SDK honours the scoped read and answers `{ found, value }` selected
|
|
220
|
+
// in-page — the whole point of this path. An OLDER SDK (version skew) or any transport that ignores
|
|
221
|
+
// `path` answers the whole-store shape `{ stores }`; walk the path server-side there so the verdict
|
|
222
|
+
// stays correct across SDK versions. The scoped win is simply unavailable on the old ones.
|
|
223
|
+
const scopedSel = asSelection(scoped.result);
|
|
224
|
+
const wholeStores = reply.stores;
|
|
225
|
+
const scopedKeys = Array.isArray(reply.availableKeys)
|
|
226
|
+
? reply.availableKeys
|
|
227
|
+
: undefined;
|
|
228
|
+
let selection;
|
|
229
|
+
if (scopedSel !== undefined) {
|
|
230
|
+
selection =
|
|
231
|
+
scopedKeys === undefined
|
|
232
|
+
? { found: scopedSel.found, value: scopedSel.value }
|
|
233
|
+
: { found: scopedSel.found, value: scopedSel.value, availableKeys: scopedKeys };
|
|
234
|
+
}
|
|
235
|
+
else if (wholeStores !== undefined) {
|
|
236
|
+
selection = selectPath(wholeStores[storeName], p.path);
|
|
237
|
+
}
|
|
238
|
+
else {
|
|
239
|
+
selection = { found: false };
|
|
240
|
+
}
|
|
241
|
+
// "no store named X" and "X has no such path" both surface as found:false; keep them distinct using
|
|
242
|
+
// the store list the reply carries, or the message regresses while the payload improves.
|
|
243
|
+
const storeAbsent = wholeStores !== undefined
|
|
244
|
+
? !(storeName in wholeStores)
|
|
245
|
+
: names.length > 0 && !names.includes(storeName);
|
|
246
|
+
if (!selection.found && storeAbsent) {
|
|
247
|
+
return {
|
|
248
|
+
pass: false,
|
|
249
|
+
failureReason: `no store named '${storeName}' is registered (${names.join(', ')})`,
|
|
250
|
+
observed: `store '${storeName}' is not registered`,
|
|
251
|
+
expected: `a registered store named '${storeName}'`,
|
|
252
|
+
assertion: 'state.store-missing',
|
|
253
|
+
evidence: { searchedStores: names },
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
// A scoped sub-tree can itself be too big for the caps; a comparison against a value known to be
|
|
257
|
+
// incomplete is an unanswered question, not a failure. Same rule the whole-store path applies.
|
|
258
|
+
if (truncationOf(scoped.result) !== undefined) {
|
|
259
|
+
const reason = `state '${p.path}' could not be read intact — the transport caps truncated it, so the ` +
|
|
260
|
+
'value was never compared. Assert a narrower path, or a smaller field inside it';
|
|
261
|
+
return { pass: false, failureReason: reason, inconclusive: reason };
|
|
262
|
+
}
|
|
263
|
+
if (!selection.found) {
|
|
264
|
+
return {
|
|
265
|
+
pass: false,
|
|
266
|
+
failureReason: `state path '${p.path}' not found in store '${storeName}'`,
|
|
267
|
+
observed: `no path '${p.path}' in store '${storeName}'`,
|
|
268
|
+
expected: `store '${storeName}' to expose '${p.path}'`,
|
|
269
|
+
assertion: 'state.path-missing',
|
|
270
|
+
evidence: { availableKeys: selection.availableKeys },
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
const want = p.equals === undefined ? '*' : p.equals;
|
|
274
|
+
if (matchValue(selection.value, want)) {
|
|
275
|
+
return {
|
|
276
|
+
pass: true,
|
|
277
|
+
evidence: { store: storeName, path: p.path, value: capDepth(selection.value, 1) },
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
return {
|
|
281
|
+
pass: false,
|
|
282
|
+
failureReason: `state '${p.path}' is ${JSON.stringify(capDepth(selection.value, 0))}, expected ${JSON.stringify(want)}`,
|
|
283
|
+
observed: `${p.path} = ${JSON.stringify(capDepth(selection.value, 0))}`,
|
|
284
|
+
expected: `${p.path} = ${JSON.stringify(want)}`,
|
|
285
|
+
assertion: 'state.equals',
|
|
286
|
+
evidence: { store: storeName, path: p.path, value: capDepth(selection.value, 1) },
|
|
287
|
+
};
|
|
288
|
+
}
|
|
117
289
|
async function evalState(session, p) {
|
|
118
|
-
|
|
290
|
+
// Named store: one scoped read, no whole-store payload (issue #336).
|
|
291
|
+
if (p.store !== undefined)
|
|
292
|
+
return await evalStateNamed(session, p, p.store);
|
|
293
|
+
const res = await session.command(ReticleCommand.STATE_READ, {});
|
|
119
294
|
if (!res.ok) {
|
|
120
295
|
return {
|
|
121
296
|
pass: false,
|
|
@@ -127,17 +302,75 @@ async function evalState(session, p) {
|
|
|
127
302
|
}
|
|
128
303
|
const stores = (res.result ?? {}).stores ?? {};
|
|
129
304
|
const names = Object.keys(stores);
|
|
130
|
-
|
|
305
|
+
// Ambiguity is about the PATH, not the store count.
|
|
306
|
+
//
|
|
307
|
+
// Registering more than one store is normal — an app store, a query cache, and the render meter
|
|
308
|
+
// Reticle registers itself — and asking "which of these three?" when only one of them HAS the
|
|
309
|
+
// path is a question with one possible answer. It was costing a real verdict: a bench-app drive
|
|
310
|
+
// asserting `{path:'view'}` returned `unknown` while the same response body carried the matching
|
|
311
|
+
// `view` state diff. `unknown` is not a pass, so that is a verification that did not happen.
|
|
312
|
+
//
|
|
313
|
+
// So narrow to the stores that actually carry the path, and refuse only when THOSE collide.
|
|
314
|
+
const candidates = p.store === undefined ? names.filter((n) => selectPath(stores[n], p.path).found) : [];
|
|
315
|
+
const storeName = p.store ?? (1 === names.length ? names[0] : oneOf(candidates));
|
|
131
316
|
if (storeName === undefined) {
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
//
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
317
|
+
// With no store registered there is nothing to read, and with two stores that both carry the
|
|
318
|
+
// path there is no way to pick — neither is a finding about the app, no assertion was evaluated,
|
|
319
|
+
// so both are inconclusive rather than failed. See honesty/inconclusive.
|
|
320
|
+
//
|
|
321
|
+
// Zero candidates is the one case that is NOT a question: every registered store was searched
|
|
322
|
+
// and none exposes the path, so the assertion cannot hold anywhere. That is the same verdict a
|
|
323
|
+
// named store has always produced for a missing path, and it falls through to it below.
|
|
324
|
+
if (0 === names.length) {
|
|
325
|
+
const reason = 'no registered store to read state from';
|
|
326
|
+
return { pass: false, failureReason: reason, inconclusive: reason };
|
|
327
|
+
}
|
|
328
|
+
if (candidates.length > 1) {
|
|
329
|
+
// Names the stores that actually collide. Listing all of them made the reader weigh
|
|
330
|
+
// candidates that could never have matched.
|
|
331
|
+
const reason = `multiple stores (${candidates.join(', ')}) expose '${p.path}'; name one with \`store\``;
|
|
332
|
+
return { pass: false, failureReason: reason, inconclusive: reason };
|
|
333
|
+
}
|
|
334
|
+
return {
|
|
335
|
+
pass: false,
|
|
336
|
+
failureReason: `state path '${p.path}' not found in any registered store (${names.join(', ')})`,
|
|
337
|
+
observed: `no path '${p.path}' in ${names.join(', ')}`,
|
|
338
|
+
expected: `some registered store to expose '${p.path}'`,
|
|
339
|
+
assertion: 'state.path-missing',
|
|
340
|
+
evidence: { searchedStores: names },
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
let selection = selectPath(stores[storeName], p.path);
|
|
344
|
+
// The whole-store read walks into a value the transport caps may already have mangled.
|
|
345
|
+
//
|
|
346
|
+
// `readState` has a SCOPED mode that selects the path out of the RAW store before sanitising, added
|
|
347
|
+
// precisely so a large or deep path still resolves. This path was not using it, so a store with one
|
|
348
|
+
// big collection in it truncated the small value sitting beside it, and the comparison then ran
|
|
349
|
+
// against the literal string "[TRUNCATED]" and returned a confident `no`. Measured on the Atlas
|
|
350
|
+
// fixture: a one-element array reported as a failed assertion while the same payload's state diffs
|
|
351
|
+
// showed the assertion holding.
|
|
352
|
+
//
|
|
353
|
+
// Re-read only when a cap actually fired, so an intact read costs exactly what it did before.
|
|
354
|
+
if (truncationOf(res.result) !== undefined) {
|
|
355
|
+
const scoped = await session.command(ReticleCommand.STATE_READ, {
|
|
356
|
+
store: storeName,
|
|
357
|
+
path: p.path,
|
|
358
|
+
});
|
|
359
|
+
const result = scoped.ok ? scoped.result : undefined;
|
|
360
|
+
const scopedTruncation = truncationOf(result);
|
|
361
|
+
if (scopedTruncation !== undefined) {
|
|
362
|
+
// Even the raw sub-tree was too big. Nothing here knows what the value IS, so nothing here can
|
|
363
|
+
// say the assertion failed — that would be an accusation the evidence does not support.
|
|
364
|
+
const reason = `state '${p.path}' could not be read intact — the transport caps truncated it, so the ` +
|
|
365
|
+
'value was never compared. Assert a narrower path, or a smaller field inside it';
|
|
366
|
+
return { pass: false, failureReason: reason, inconclusive: reason };
|
|
367
|
+
}
|
|
368
|
+
if (result !== undefined) {
|
|
369
|
+
const scopedSelection = asSelection(result);
|
|
370
|
+
if (scopedSelection !== undefined)
|
|
371
|
+
selection = scopedSelection;
|
|
372
|
+
}
|
|
139
373
|
}
|
|
140
|
-
const selection = selectPath(stores[storeName], p.path);
|
|
141
374
|
if (!selection.found) {
|
|
142
375
|
return {
|
|
143
376
|
pass: false,
|
|
@@ -183,11 +416,16 @@ diagnose = true) {
|
|
|
183
416
|
case PredicateKind.ELEMENT:
|
|
184
417
|
return evalElement(session, predicate.query, predicate.state, predicate.absent ?? false, diagnose);
|
|
185
418
|
case PredicateKind.TEXT:
|
|
186
|
-
return evalElement(session,
|
|
419
|
+
return evalElement(session,
|
|
420
|
+
// `scope` passes straight through: the text predicate has always been an element query
|
|
421
|
+
// with only `text` filled in, so scoping it needs the field, not a second code path.
|
|
422
|
+
undefined === predicate.scope
|
|
423
|
+
? { text: predicate.contains }
|
|
424
|
+
: { text: predicate.contains, scope: predicate.scope }, true === predicate.visible ? ElementState.VISIBLE : undefined, predicate.absent ?? false, diagnose);
|
|
187
425
|
case PredicateKind.NET:
|
|
188
426
|
return evalNet(events, predicate);
|
|
189
427
|
case PredicateKind.ROUTE:
|
|
190
|
-
return evalRoute(events, predicate);
|
|
428
|
+
return evalRoute(events, predicate, session.url);
|
|
191
429
|
case PredicateKind.CONSOLE:
|
|
192
430
|
return evalConsole(events, predicate);
|
|
193
431
|
case PredicateKind.ANIMATION:
|
|
@@ -206,27 +444,55 @@ diagnose = true) {
|
|
|
206
444
|
}
|
|
207
445
|
case PredicateKind.ALL_OF: {
|
|
208
446
|
const results = await Promise.all(predicate.predicates.map((p) => evaluatePredicate(session, p, since, diagnose)));
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
447
|
+
// A clause that genuinely failed OUTRANKS one nobody could read. Softening a real failure to
|
|
448
|
+
// UNKNOWN would hide the defect the agent came for, which is the more expensive of the two
|
|
449
|
+
// mistakes; the reverse — grading an unreadable clause as a defect in the app — is the one
|
|
450
|
+
// that was happening.
|
|
451
|
+
const failed = results.find((r) => !r.pass && r.inconclusive === undefined);
|
|
452
|
+
if (failed !== undefined) {
|
|
453
|
+
return {
|
|
213
454
|
pass: false,
|
|
214
455
|
failureReason: failed.failureReason ?? 'a sub-predicate of allOf failed',
|
|
456
|
+
// A conjunction is decided as soon as ONE clause is: nothing the others do later can
|
|
457
|
+
// rescue it. This is what makes the early exit reach real calls, since an exact count is
|
|
458
|
+
// usually asserted alongside the UI change it is meant to accompany.
|
|
459
|
+
...(true === failed.decided ? { decided: true } : {}),
|
|
215
460
|
evidence: results,
|
|
216
461
|
};
|
|
462
|
+
}
|
|
463
|
+
const unreadable = results.find((r) => r.inconclusive !== undefined);
|
|
464
|
+
if (unreadable !== undefined)
|
|
465
|
+
return unreadableComposite(unreadable, results);
|
|
466
|
+
return { pass: true, evidence: results.map((r) => r.evidence) };
|
|
217
467
|
}
|
|
218
468
|
case PredicateKind.ANY_OF: {
|
|
219
469
|
const results = await Promise.all(predicate.predicates.map((p) => evaluatePredicate(session, p, since, diagnose)));
|
|
220
470
|
const passed = results.find((r) => r.pass);
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
471
|
+
if (passed !== undefined)
|
|
472
|
+
return { pass: true, evidence: passed.evidence };
|
|
473
|
+
// "No sub-predicate matched" is a claim anyOf is not entitled to make while one of them was
|
|
474
|
+
// never read: the unreadable clause might have been the one that would have matched.
|
|
475
|
+
const unreadable = results.find((r) => r.inconclusive !== undefined);
|
|
476
|
+
if (unreadable !== undefined)
|
|
477
|
+
return unreadableComposite(unreadable, results);
|
|
478
|
+
return { pass: false, failureReason: 'no sub-predicate of anyOf matched', evidence: results };
|
|
224
479
|
}
|
|
225
480
|
case PredicateKind.NOT: {
|
|
226
481
|
const inner = await evaluatePredicate(session, predicate.predicate, since, diagnose);
|
|
482
|
+
// The sharpest case of the three, and the only one that produced a GREEN. `not` read the
|
|
483
|
+
// child's `pass: false` as "the inner predicate did not hold" and passed — so an assertion
|
|
484
|
+
// nobody could evaluate became a verdict of verified, manufactured out of a missing reading.
|
|
485
|
+
// You cannot negate an answer nobody had.
|
|
486
|
+
if (inner.inconclusive !== undefined)
|
|
487
|
+
return unreadableComposite(inner, inner);
|
|
488
|
+
// A green negation used to return a bare `{ pass: true }`, so the payload fell back to
|
|
489
|
+
// whatever the caller had — every element matching the OUTER locator. Three clauses negating
|
|
490
|
+
// three different names then produced byte-identical responses, which reads as a checker that
|
|
491
|
+
// dropped the name rather than one that correctly found all three absent. The evidence is the
|
|
492
|
+
// whole answer here: what was looked for, and that it was not there.
|
|
227
493
|
return inner.pass
|
|
228
494
|
? { pass: false, failureReason: 'negated predicate unexpectedly held', evidence: inner }
|
|
229
|
-
: { pass: true };
|
|
495
|
+
: { pass: true, evidence: { negated: predicate.predicate, held: false, saw: inner } };
|
|
230
496
|
}
|
|
231
497
|
default:
|
|
232
498
|
return { pass: false, failureReason: 'unknown predicate' };
|
|
@@ -342,6 +608,14 @@ export function waitForPredicate(session, predicate, timeoutMs, since = 0) {
|
|
|
342
608
|
void evaluatePredicate(session, predicate, since, false)
|
|
343
609
|
.then((r) => {
|
|
344
610
|
if (!r.pass) {
|
|
611
|
+
// Final already: stop rather than spend a budget that cannot change the answer. Only
|
|
612
|
+
// where the evaluator could PROVE it (see EvalResult.decided) — an ordinary miss keeps
|
|
613
|
+
// waiting, because "it has not happened yet" and "it will not happen" are the same
|
|
614
|
+
// reading until the budget ends.
|
|
615
|
+
if (true === r.decided) {
|
|
616
|
+
finish(r);
|
|
617
|
+
return;
|
|
618
|
+
}
|
|
345
619
|
// A time-based failure knows when it could stop being one — re-check THEN rather than on
|
|
346
620
|
// the next blind tick. Without this, every `settled` wait paid up to a full poll interval
|
|
347
621
|
// of dead time after the quiet window had already closed: measured at 566–627ms across
|
|
@@ -118,6 +118,9 @@ export const ANNOTATE_TOOLS = [
|
|
|
118
118
|
recovery: z.string().optional(),
|
|
119
119
|
target: z.string().optional(),
|
|
120
120
|
compiled: z.string().optional(),
|
|
121
|
+
// Set when the compiled annotation would overstate what a replay can check — e.g. a
|
|
122
|
+
// success-state that flow_save will grade assertion-free (#395).
|
|
123
|
+
note: z.string().optional(),
|
|
121
124
|
code: z.string().optional(),
|
|
122
125
|
},
|
|
123
126
|
handler: (deps, args) => {
|
package/dist/flows/annotate.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AnnotationErrorCode, AnnotationKind, AnnotationTarget, COMPILED_PREDICATE_PREFIX, } from '@reticlehq/core';
|
|
1
|
+
import { AnnotationErrorCode, AnnotationKind, AnnotationTarget, COMPILED_PREDICATE_PREFIX, flowExpectHasConsequence, flowExpectIsPresenceOnly, } from '@reticlehq/core';
|
|
2
2
|
/**
|
|
3
3
|
* The PURE compiler at the heart of the annotation facet. A structured annotation +
|
|
4
4
|
* the current captured-step count → an AnnotateOutcome (a result envelope + the patch to apply to
|
|
@@ -96,11 +96,25 @@ function stepPatch(a, stepCount, stepExpect) {
|
|
|
96
96
|
patch: { stepIndex: stepCount - 1, stepExpect },
|
|
97
97
|
};
|
|
98
98
|
}
|
|
99
|
+
// A success end-condition that is not an observable CONSEQUENCE (signal / net / state) cannot lift
|
|
100
|
+
// the flow above the grade flow_save will give it. Saying "will succeed when ..." without that
|
|
101
|
+
// caveat is the #395 contradiction: annotate promises a check that flow_save then grades
|
|
102
|
+
// assertion-free, so a flow that only checks the console replays green through any regression. Name
|
|
103
|
+
// the grade here, using the same consequence/presence classification every grader shares.
|
|
104
|
+
const SUCCESS_ASSERTION_FREE_NOTE = 'This success-state is not an observable consequence, so flow_save will grade the flow assertion-free — it passes even when the feature is broken. Use a signal / net / state success-state to give the flow something that can fail.';
|
|
105
|
+
const SUCCESS_PRESENCE_ONLY_NOTE = 'This success-state only checks element presence, so flow_save will grade the flow presence-only — a locator healed to the wrong element still passes it. Use a signal / net / state success-state to assert an observable consequence.';
|
|
99
106
|
function flowSuccess(a, success) {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
107
|
+
const result = {
|
|
108
|
+
ok: true,
|
|
109
|
+
target: AnnotationTarget.FLOW,
|
|
110
|
+
compiled: describeCompiled(a),
|
|
103
111
|
};
|
|
112
|
+
if (!flowExpectHasConsequence(success)) {
|
|
113
|
+
result.note = flowExpectIsPresenceOnly(success)
|
|
114
|
+
? SUCCESS_PRESENCE_ONLY_NOTE
|
|
115
|
+
: SUCCESS_ASSERTION_FREE_NOTE;
|
|
116
|
+
}
|
|
117
|
+
return { result, patch: { success } };
|
|
104
118
|
}
|
|
105
119
|
/**
|
|
106
120
|
* Human-readable confirmation text for the recorder strip / tool result,
|
package/dist/flows/coverage.d.ts
CHANGED
|
@@ -9,11 +9,18 @@ interface DeclaredSurface {
|
|
|
9
9
|
signals: readonly string[];
|
|
10
10
|
flows: readonly string[];
|
|
11
11
|
}
|
|
12
|
-
interface CoverageDimension {
|
|
12
|
+
export interface CoverageDimension {
|
|
13
13
|
total: number;
|
|
14
14
|
covered: number;
|
|
15
|
-
/**
|
|
16
|
-
|
|
15
|
+
/**
|
|
16
|
+
* Percent covered, rounded. PRESENT ONLY WHEN IT WAS MEASURED — absent when nothing is declared,
|
|
17
|
+
* same rule as the honesty layer's `{ pct?: number; partial: boolean }`.
|
|
18
|
+
*
|
|
19
|
+
* This used to report 100 for an empty surface ("vacuously complete"), which made
|
|
20
|
+
* `covered: 0, total: 0, pct: 100` — a contradiction the caller had to notice unaided, in output an
|
|
21
|
+
* agent reads to decide whether its change is covered.
|
|
22
|
+
*/
|
|
23
|
+
pct?: number;
|
|
17
24
|
/** Declared-but-never-exercised members — what "everything" is silently missing. */
|
|
18
25
|
uncovered: string[];
|
|
19
26
|
}
|
|
@@ -21,8 +28,34 @@ export interface Coverage {
|
|
|
21
28
|
testids: CoverageDimension;
|
|
22
29
|
signals: CoverageDimension;
|
|
23
30
|
flows: CoverageDimension;
|
|
24
|
-
/** Overall percent across all three dimensions combined. */
|
|
25
|
-
overallPct
|
|
31
|
+
/** Overall percent across all three dimensions combined; absent when nothing is declared. */
|
|
32
|
+
overallPct?: number;
|
|
26
33
|
}
|
|
34
|
+
/** The gate's outcome when the project has recorded no flows at all — not a pass, and not a failure. */
|
|
35
|
+
export declare const NO_FLOWS = "no_flows";
|
|
36
|
+
/**
|
|
37
|
+
* What to do about it, since the empty state is otherwise indistinguishable from success.
|
|
38
|
+
*
|
|
39
|
+
* Names the TOOLS, not a `reticle record` CLI verb: this CLI does not dispatch one, and a suggestion
|
|
40
|
+
* that errors is a second dead end on top of the first (see suggested-commands-exist.test.ts). Same
|
|
41
|
+
* wording as the empty-suite verdict, which is the same situation reached from `reticle verify`.
|
|
42
|
+
*/
|
|
43
|
+
export declare const NO_FLOWS_NOTE = "no flows recorded \u2014 nothing was checked. Record one with reticle_record { action: \"start\" }, then reticle_flow_save.";
|
|
44
|
+
/** The gate's flow-coverage line. `pct` and `outcome` are mutually exclusive by construction. */
|
|
45
|
+
export interface FlowCoverageReport {
|
|
46
|
+
covered: number;
|
|
47
|
+
total: number;
|
|
48
|
+
pct?: number;
|
|
49
|
+
outcome?: typeof NO_FLOWS;
|
|
50
|
+
note?: string;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Report flow coverage so that an EMPTY suite cannot be read as a passing one.
|
|
54
|
+
*
|
|
55
|
+
* Same rule `buildSuiteVerdict` already applies to `reticle verify`, where zero flows reports
|
|
56
|
+
* `unverifiable` rather than "all 0 flows pass": a green that cannot go red is not a pass. Callers
|
|
57
|
+
* must treat a present `outcome` as not-passing.
|
|
58
|
+
*/
|
|
59
|
+
export declare function flowCoverageReport(flows: CoverageDimension): FlowCoverageReport;
|
|
27
60
|
export declare function computeCoverage(declared: DeclaredSurface, exercised: DeclaredSurface): Coverage;
|
|
28
61
|
export {};
|
package/dist/flows/coverage.js
CHANGED
|
@@ -4,11 +4,36 @@
|
|
|
4
4
|
* (contract testids/signals/flows) actually touched by passing verification, so "everything" is honest
|
|
5
5
|
* about what everything means. Pure set math; the deviation report + gate carry the number.
|
|
6
6
|
*/
|
|
7
|
+
/** The gate's outcome when the project has recorded no flows at all — not a pass, and not a failure. */
|
|
8
|
+
export const NO_FLOWS = 'no_flows';
|
|
9
|
+
/**
|
|
10
|
+
* What to do about it, since the empty state is otherwise indistinguishable from success.
|
|
11
|
+
*
|
|
12
|
+
* Names the TOOLS, not a `reticle record` CLI verb: this CLI does not dispatch one, and a suggestion
|
|
13
|
+
* that errors is a second dead end on top of the first (see suggested-commands-exist.test.ts). Same
|
|
14
|
+
* wording as the empty-suite verdict, which is the same situation reached from `reticle verify`.
|
|
15
|
+
*/
|
|
16
|
+
export const NO_FLOWS_NOTE = 'no flows recorded — nothing was checked. Record one with reticle_record { action: "start" }, then reticle_flow_save.';
|
|
17
|
+
/**
|
|
18
|
+
* Report flow coverage so that an EMPTY suite cannot be read as a passing one.
|
|
19
|
+
*
|
|
20
|
+
* Same rule `buildSuiteVerdict` already applies to `reticle verify`, where zero flows reports
|
|
21
|
+
* `unverifiable` rather than "all 0 flows pass": a green that cannot go red is not a pass. Callers
|
|
22
|
+
* must treat a present `outcome` as not-passing.
|
|
23
|
+
*/
|
|
24
|
+
export function flowCoverageReport(flows) {
|
|
25
|
+
const { covered, total } = flows;
|
|
26
|
+
if (0 === total)
|
|
27
|
+
return { covered, total, outcome: NO_FLOWS, note: NO_FLOWS_NOTE };
|
|
28
|
+
return { covered, total, ...(flows.pct === undefined ? {} : { pct: flows.pct }) };
|
|
29
|
+
}
|
|
7
30
|
function dimension(declared, exercised) {
|
|
8
31
|
const uncovered = declared.filter((d) => !exercised.has(d));
|
|
9
32
|
const covered = declared.length - uncovered.length;
|
|
10
|
-
const
|
|
11
|
-
|
|
33
|
+
const base = { total: declared.length, covered, uncovered };
|
|
34
|
+
if (0 === declared.length)
|
|
35
|
+
return base;
|
|
36
|
+
return { ...base, pct: Math.round((covered / declared.length) * 100) };
|
|
12
37
|
}
|
|
13
38
|
export function computeCoverage(declared, exercised) {
|
|
14
39
|
const testids = dimension(declared.testids, new Set(exercised.testids));
|
|
@@ -16,6 +41,7 @@ export function computeCoverage(declared, exercised) {
|
|
|
16
41
|
const flows = dimension(declared.flows, new Set(exercised.flows));
|
|
17
42
|
const total = testids.total + signals.total + flows.total;
|
|
18
43
|
const covered = testids.covered + signals.covered + flows.covered;
|
|
19
|
-
|
|
20
|
-
|
|
44
|
+
if (0 === total)
|
|
45
|
+
return { testids, signals, flows };
|
|
46
|
+
return { testids, signals, flows, overallPct: Math.round((covered / total) * 100) };
|
|
21
47
|
}
|
package/dist/flows/decision.d.ts
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
import { type FlowFile, type FlowReplayResult, type ReplayDecision, type SuiteVerdict } from '@reticlehq/core';
|
|
2
2
|
export declare function buildDecision(result: FlowReplayResult, flow?: FlowFile): ReplayDecision;
|
|
3
|
+
/**
|
|
4
|
+
* Aggregate per-flow replays into one suite verdict — the autonomous loop's consolidated answer
|
|
5
|
+
* after a build: did anything break, and what's the prioritized fix list. Pure: pass the already
|
|
6
|
+
* decision-annotated replay results + their flows. Passing flows are counted; only failures carry
|
|
7
|
+
* detail (token-cheap), each with the actionable decision so the agent fixes without re-querying.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Why a green replay still verified nothing, or undefined when it genuinely did.
|
|
11
|
+
*
|
|
12
|
+
* A flow with no steps, or one that asserts no observable consequence, replays green no matter what
|
|
13
|
+
* the app does — the grader already knows this at save time and says so. Counting it as `passed`
|
|
14
|
+
* turned that warning into "all 1 flow pass", which is a false green in the exact feature sold as
|
|
15
|
+
* the regression suite. Classified only when the flow file is available; never guessed.
|
|
16
|
+
*/
|
|
17
|
+
export declare function unverifiableReason(flow: FlowFile | undefined): string | undefined;
|
|
3
18
|
export declare function buildSuiteVerdict(runs: ReadonlyArray<{
|
|
4
19
|
replay: FlowReplayResult;
|
|
5
20
|
flow?: FlowFile;
|
package/dist/flows/decision.js
CHANGED
|
@@ -100,7 +100,7 @@ function suiteVerdictOf(status) {
|
|
|
100
100
|
* turned that warning into "all 1 flow pass", which is a false green in the exact feature sold as
|
|
101
101
|
* the regression suite. Classified only when the flow file is available; never guessed.
|
|
102
102
|
*/
|
|
103
|
-
function unverifiableReason(flow) {
|
|
103
|
+
export function unverifiableReason(flow) {
|
|
104
104
|
if (flow === undefined)
|
|
105
105
|
return undefined;
|
|
106
106
|
if (0 === flow.steps.length) {
|
|
@@ -27,7 +27,7 @@ export const FlowAssertionGrade = {
|
|
|
27
27
|
/** Performs actions but asserts nothing observable — passes even if the feature is broken. */
|
|
28
28
|
ASSERTION_FREE: 'assertion-free',
|
|
29
29
|
};
|
|
30
|
-
const ASSERTION_FREE_WARNING = 'This flow performs actions but asserts no observable consequence
|
|
30
|
+
const ASSERTION_FREE_WARNING = 'This flow performs actions but asserts no observable consequence, so it will pass even if the feature is broken. Add a consequence assertion with reticle_annotate (assert-signal / assert-net) or a success-state.';
|
|
31
31
|
const PRESENCE_ONLY_WARNING = 'This flow only checks element presence, not an observable consequence (signal/network). A locator healed to the wrong element can still pass it. Add a consequence assertion (assert-signal / assert-net / success-state).';
|
|
32
32
|
const INTENT_WITHOUT_OUTCOME_WARNING = 'This flow declares a business intent but asserts no observable outcome (signal/network) — it claims to verify a goal it cannot actually check. Add a success-state consequence so the flow fails when the goal stops being met.';
|
|
33
33
|
// Consequence vs presence-only classification lives in @reticlehq/core (one source for all graders).
|
|
@@ -2,7 +2,7 @@ import { EventType, FLOW_SIGNAL_TIMEOUT_MS, FlowErrorCode, RecordedFlowSchema, R
|
|
|
2
2
|
import { asString } from '../tools/tools-helpers.js';
|
|
3
3
|
import { replayFlow } from './flow-replay.js';
|
|
4
4
|
import { assertSuccess, dynamicTestids, successLabel, SUCCESS_STEP_TOOL } from './flow-success.js';
|
|
5
|
-
import { buildDecision } from './decision.js';
|
|
5
|
+
import { buildDecision, unverifiableReason } from './decision.js';
|
|
6
6
|
import { waitForPredicate } from '../events/predicate.js';
|
|
7
7
|
import { computeSegments } from '../journal/rollups.js';
|
|
8
8
|
import { AssertionTiersStore } from './assertion-tiers-store.js';
|
|
@@ -174,6 +174,13 @@ export async function replayNamedFlow(deps, args) {
|
|
|
174
174
|
return errored;
|
|
175
175
|
}
|
|
176
176
|
const result = { name, status, steps };
|
|
177
|
+
// A green that cannot go red is not a pass. `reticle_flow_verify` already refuses to count these,
|
|
178
|
+
// via this same function -- a single-flow caller saw a bare `ok` and had no way to learn the flow
|
|
179
|
+
// asserts nothing. Derived from the same helper on purpose: two copies of this judgement would
|
|
180
|
+
// drift, and the sibling tools would then disagree about the same flow.
|
|
181
|
+
const cannotFail = status === ReplayStatus.OK ? unverifiableReason(loaded.value) : undefined;
|
|
182
|
+
if (cannotFail !== undefined)
|
|
183
|
+
result.unverifiable = { reason: cannotFail };
|
|
177
184
|
if (status !== ReplayStatus.OK)
|
|
178
185
|
result.decision = buildDecision(result, loaded.value);
|
|
179
186
|
applyStartPathHint(result, startPathHint);
|