@buildanagent/harness 0.0.0-stage → 0.1.0-alpha.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +54 -0
- package/LICENSE-APACHE +202 -0
- package/LICENSE-MIT +21 -0
- package/README.md +822 -2
- package/dist/ablation.d.ts +2 -0
- package/dist/ablation.js +3 -0
- package/dist/ablation.js.map +1 -0
- package/dist/completion/check.d.ts +64 -0
- package/dist/completion/check.js +152 -0
- package/dist/completion/check.js.map +1 -0
- package/dist/completion/checks.d.ts +45 -0
- package/dist/completion/checks.js +173 -0
- package/dist/completion/checks.js.map +1 -0
- package/dist/completion/flags.d.ts +25 -0
- package/dist/completion/flags.js +87 -0
- package/dist/completion/flags.js.map +1 -0
- package/dist/completion/index.d.ts +8 -0
- package/dist/completion/index.js +9 -0
- package/dist/completion/index.js.map +1 -0
- package/dist/completion/snapshot.d.ts +33 -0
- package/dist/completion/snapshot.js +83 -0
- package/dist/completion/snapshot.js.map +1 -0
- package/dist/context/cap.d.ts +32 -0
- package/dist/context/cap.js +50 -0
- package/dist/context/cap.js.map +1 -0
- package/dist/context/estimate.d.ts +18 -0
- package/dist/context/estimate.js +20 -0
- package/dist/context/estimate.js.map +1 -0
- package/dist/context/file-spill.d.ts +15 -0
- package/dist/context/file-spill.js +40 -0
- package/dist/context/file-spill.js.map +1 -0
- package/dist/context/index.d.ts +10 -0
- package/dist/context/index.js +10 -0
- package/dist/context/index.js.map +1 -0
- package/dist/context/mask.d.ts +27 -0
- package/dist/context/mask.js +71 -0
- package/dist/context/mask.js.map +1 -0
- package/dist/context/mid-turn.d.ts +40 -0
- package/dist/context/mid-turn.js +82 -0
- package/dist/context/mid-turn.js.map +1 -0
- package/dist/context/policy.d.ts +9 -0
- package/dist/context/policy.js +140 -0
- package/dist/context/policy.js.map +1 -0
- package/dist/context/spill-ledger.d.ts +25 -0
- package/dist/context/spill-ledger.js +54 -0
- package/dist/context/spill-ledger.js.map +1 -0
- package/dist/context/spill.d.ts +10 -0
- package/dist/context/spill.js +18 -0
- package/dist/context/spill.js.map +1 -0
- package/dist/context/summarize.d.ts +53 -0
- package/dist/context/summarize.js +204 -0
- package/dist/context/summarize.js.map +1 -0
- package/dist/context/summary-template.d.ts +29 -0
- package/dist/context/summary-template.js +69 -0
- package/dist/context/summary-template.js.map +1 -0
- package/dist/context/types.d.ts +150 -0
- package/dist/context/types.js +2 -0
- package/dist/context/types.js.map +1 -0
- package/dist/context/write-cap.d.ts +19 -0
- package/dist/context/write-cap.js +41 -0
- package/dist/context/write-cap.js.map +1 -0
- package/dist/create-harness.d.ts +7 -0
- package/dist/create-harness.js +21 -0
- package/dist/create-harness.js.map +1 -0
- package/dist/credentials/addresses.d.ts +14 -0
- package/dist/credentials/addresses.js +144 -0
- package/dist/credentials/addresses.js.map +1 -0
- package/dist/credentials/allowlist.d.ts +54 -0
- package/dist/credentials/allowlist.js +134 -0
- package/dist/credentials/allowlist.js.map +1 -0
- package/dist/credentials/egress-policy.d.ts +69 -0
- package/dist/credentials/egress-policy.js +170 -0
- package/dist/credentials/egress-policy.js.map +1 -0
- package/dist/credentials/egress-proxy.d.ts +82 -0
- package/dist/credentials/egress-proxy.js +650 -0
- package/dist/credentials/egress-proxy.js.map +1 -0
- package/dist/credentials/git-route.d.ts +68 -0
- package/dist/credentials/git-route.js +288 -0
- package/dist/credentials/git-route.js.map +1 -0
- package/dist/credentials/index.d.ts +13 -0
- package/dist/credentials/index.js +14 -0
- package/dist/credentials/index.js.map +1 -0
- package/dist/credentials/scrub.d.ts +23 -0
- package/dist/credentials/scrub.js +115 -0
- package/dist/credentials/scrub.js.map +1 -0
- package/dist/errors.d.ts +60 -0
- package/dist/errors.js +67 -0
- package/dist/errors.js.map +1 -0
- package/dist/extensions/compose.d.ts +27 -0
- package/dist/extensions/compose.js +83 -0
- package/dist/extensions/compose.js.map +1 -0
- package/dist/extensions/context-files-node.d.ts +28 -0
- package/dist/extensions/context-files-node.js +74 -0
- package/dist/extensions/context-files-node.js.map +1 -0
- package/dist/extensions/context-files.d.ts +68 -0
- package/dist/extensions/context-files.js +214 -0
- package/dist/extensions/context-files.js.map +1 -0
- package/dist/extensions/digest.d.ts +6 -0
- package/dist/extensions/digest.js +68 -0
- package/dist/extensions/digest.js.map +1 -0
- package/dist/extensions/index.d.ts +10 -0
- package/dist/extensions/index.js +10 -0
- package/dist/extensions/index.js.map +1 -0
- package/dist/extensions/load.d.ts +42 -0
- package/dist/extensions/load.js +211 -0
- package/dist/extensions/load.js.map +1 -0
- package/dist/extensions/lock.d.ts +8 -0
- package/dist/extensions/lock.js +95 -0
- package/dist/extensions/lock.js.map +1 -0
- package/dist/extensions/node.d.ts +10 -0
- package/dist/extensions/node.js +11 -0
- package/dist/extensions/node.js.map +1 -0
- package/dist/extensions/project-dirs.d.ts +24 -0
- package/dist/extensions/project-dirs.js +97 -0
- package/dist/extensions/project-dirs.js.map +1 -0
- package/dist/extensions/skills-node.d.ts +34 -0
- package/dist/extensions/skills-node.js +123 -0
- package/dist/extensions/skills-node.js.map +1 -0
- package/dist/extensions/skills.d.ts +51 -0
- package/dist/extensions/skills.js +175 -0
- package/dist/extensions/skills.js.map +1 -0
- package/dist/extensions/trust-file.d.ts +3 -0
- package/dist/extensions/trust-file.js +62 -0
- package/dist/extensions/trust-file.js.map +1 -0
- package/dist/extensions/trust-store-node.d.ts +10 -0
- package/dist/extensions/trust-store-node.js +33 -0
- package/dist/extensions/trust-store-node.js.map +1 -0
- package/dist/extensions/types.d.ts +42 -0
- package/dist/extensions/types.js +2 -0
- package/dist/extensions/types.js.map +1 -0
- package/dist/harness.d.ts +156 -0
- package/dist/harness.js +2 -0
- package/dist/harness.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp/connection.d.ts +22 -0
- package/dist/mcp/connection.js +267 -0
- package/dist/mcp/connection.js.map +1 -0
- package/dist/mcp/content.d.ts +27 -0
- package/dist/mcp/content.js +177 -0
- package/dist/mcp/content.js.map +1 -0
- package/dist/mcp/errors.d.ts +19 -0
- package/dist/mcp/errors.js +21 -0
- package/dist/mcp/errors.js.map +1 -0
- package/dist/mcp/index.d.ts +10 -0
- package/dist/mcp/index.js +10 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/manager.d.ts +7 -0
- package/dist/mcp/manager.js +173 -0
- package/dist/mcp/manager.js.map +1 -0
- package/dist/mcp/record.d.ts +7 -0
- package/dist/mcp/record.js +20 -0
- package/dist/mcp/record.js.map +1 -0
- package/dist/mcp/stdio.d.ts +39 -0
- package/dist/mcp/stdio.js +382 -0
- package/dist/mcp/stdio.js.map +1 -0
- package/dist/mcp/types.d.ts +172 -0
- package/dist/mcp/types.js +2 -0
- package/dist/mcp/types.js.map +1 -0
- package/dist/memory/index.d.ts +2 -0
- package/dist/memory/index.js +3 -0
- package/dist/memory/index.js.map +1 -0
- package/dist/permissions/default-rules.d.ts +7 -0
- package/dist/permissions/default-rules.js +42 -0
- package/dist/permissions/default-rules.js.map +1 -0
- package/dist/permissions/errors.d.ts +11 -0
- package/dist/permissions/errors.js +14 -0
- package/dist/permissions/errors.js.map +1 -0
- package/dist/permissions/index.d.ts +9 -0
- package/dist/permissions/index.js +9 -0
- package/dist/permissions/index.js.map +1 -0
- package/dist/permissions/match.d.ts +33 -0
- package/dist/permissions/match.js +287 -0
- package/dist/permissions/match.js.map +1 -0
- package/dist/permissions/pipeline.d.ts +9 -0
- package/dist/permissions/pipeline.js +235 -0
- package/dist/permissions/pipeline.js.map +1 -0
- package/dist/permissions/rules.d.ts +22 -0
- package/dist/permissions/rules.js +74 -0
- package/dist/permissions/rules.js.map +1 -0
- package/dist/permissions/shadow.d.ts +5 -0
- package/dist/permissions/shadow.js +22 -0
- package/dist/permissions/shadow.js.map +1 -0
- package/dist/permissions/types.d.ts +104 -0
- package/dist/permissions/types.js +2 -0
- package/dist/permissions/types.js.map +1 -0
- package/dist/sandbox/base.d.ts +45 -0
- package/dist/sandbox/base.js +74 -0
- package/dist/sandbox/base.js.map +1 -0
- package/dist/sandbox/docker/index.d.ts +41 -0
- package/dist/sandbox/docker/index.js +497 -0
- package/dist/sandbox/docker/index.js.map +1 -0
- package/dist/sandbox/errors.d.ts +44 -0
- package/dist/sandbox/errors.js +57 -0
- package/dist/sandbox/errors.js.map +1 -0
- package/dist/sandbox/host-files.d.ts +65 -0
- package/dist/sandbox/host-files.js +258 -0
- package/dist/sandbox/host-files.js.map +1 -0
- package/dist/sandbox/index.d.ts +10 -0
- package/dist/sandbox/index.js +10 -0
- package/dist/sandbox/index.js.map +1 -0
- package/dist/sandbox/none.d.ts +17 -0
- package/dist/sandbox/none.js +108 -0
- package/dist/sandbox/none.js.map +1 -0
- package/dist/sandbox/policy.d.ts +49 -0
- package/dist/sandbox/policy.js +149 -0
- package/dist/sandbox/policy.js.map +1 -0
- package/dist/sandbox/process.d.ts +61 -0
- package/dist/sandbox/process.js +253 -0
- package/dist/sandbox/process.js.map +1 -0
- package/dist/sandbox/spawn-env.d.ts +20 -0
- package/dist/sandbox/spawn-env.js +128 -0
- package/dist/sandbox/spawn-env.js.map +1 -0
- package/dist/sandbox/srt/index.d.ts +38 -0
- package/dist/sandbox/srt/index.js +283 -0
- package/dist/sandbox/srt/index.js.map +1 -0
- package/dist/sandbox/srt/settings.d.ts +60 -0
- package/dist/sandbox/srt/settings.js +84 -0
- package/dist/sandbox/srt/settings.js.map +1 -0
- package/dist/sandbox/types.d.ts +125 -0
- package/dist/sandbox/types.js +6 -0
- package/dist/sandbox/types.js.map +1 -0
- package/dist/server/answers.d.ts +16 -0
- package/dist/server/answers.js +43 -0
- package/dist/server/answers.js.map +1 -0
- package/dist/server/connection.d.ts +99 -0
- package/dist/server/connection.js +309 -0
- package/dist/server/connection.js.map +1 -0
- package/dist/server/dispatcher.d.ts +86 -0
- package/dist/server/dispatcher.js +800 -0
- package/dist/server/dispatcher.js.map +1 -0
- package/dist/server/index.d.ts +13 -0
- package/dist/server/index.js +12 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/local-transport.d.ts +44 -0
- package/dist/server/local-transport.js +103 -0
- package/dist/server/local-transport.js.map +1 -0
- package/dist/server/outbox.d.ts +31 -0
- package/dist/server/outbox.js +130 -0
- package/dist/server/outbox.js.map +1 -0
- package/dist/server/params.d.ts +25 -0
- package/dist/server/params.js +112 -0
- package/dist/server/params.js.map +1 -0
- package/dist/server/protocol.schema.json +2448 -0
- package/dist/server/rpc-session.d.ts +43 -0
- package/dist/server/rpc-session.js +167 -0
- package/dist/server/rpc-session.js.map +1 -0
- package/dist/server/stdio-streams.d.ts +67 -0
- package/dist/server/stdio-streams.js +218 -0
- package/dist/server/stdio-streams.js.map +1 -0
- package/dist/server/stdio.d.ts +59 -0
- package/dist/server/stdio.js +189 -0
- package/dist/server/stdio.js.map +1 -0
- package/dist/server/websocket.d.ts +106 -0
- package/dist/server/websocket.js +327 -0
- package/dist/server/websocket.js.map +1 -0
- package/dist/session/approvals.d.ts +81 -0
- package/dist/session/approvals.js +217 -0
- package/dist/session/approvals.js.map +1 -0
- package/dist/session/code-call-log.d.ts +42 -0
- package/dist/session/code-call-log.js +63 -0
- package/dist/session/code-call-log.js.map +1 -0
- package/dist/session/code-call-result.d.ts +19 -0
- package/dist/session/code-call-result.js +95 -0
- package/dist/session/code-call-result.js.map +1 -0
- package/dist/session/code-call.d.ts +49 -0
- package/dist/session/code-call.js +345 -0
- package/dist/session/code-call.js.map +1 -0
- package/dist/session/effect-scope.d.ts +11 -0
- package/dist/session/effect-scope.js +10 -0
- package/dist/session/effect-scope.js.map +1 -0
- package/dist/session/errors.d.ts +19 -0
- package/dist/session/errors.js +27 -0
- package/dist/session/errors.js.map +1 -0
- package/dist/session/file-index.d.ts +8 -0
- package/dist/session/file-index.js +70 -0
- package/dist/session/file-index.js.map +1 -0
- package/dist/session/file-key-store.d.ts +16 -0
- package/dist/session/file-key-store.js +169 -0
- package/dist/session/file-key-store.js.map +1 -0
- package/dist/session/file-lock.d.ts +52 -0
- package/dist/session/file-lock.js +324 -0
- package/dist/session/file-lock.js.map +1 -0
- package/dist/session/index.d.ts +22 -0
- package/dist/session/index.js +20 -0
- package/dist/session/index.js.map +1 -0
- package/dist/session/jsonl-store.d.ts +37 -0
- package/dist/session/jsonl-store.js +386 -0
- package/dist/session/jsonl-store.js.map +1 -0
- package/dist/session/jsonl.d.ts +9 -0
- package/dist/session/jsonl.js +10 -0
- package/dist/session/jsonl.js.map +1 -0
- package/dist/session/lock.d.ts +19 -0
- package/dist/session/lock.js +2 -0
- package/dist/session/lock.js.map +1 -0
- package/dist/session/memory-store.d.ts +12 -0
- package/dist/session/memory-store.js +60 -0
- package/dist/session/memory-store.js.map +1 -0
- package/dist/session/migrate.d.ts +41 -0
- package/dist/session/migrate.js +86 -0
- package/dist/session/migrate.js.map +1 -0
- package/dist/session/pointer.d.ts +15 -0
- package/dist/session/pointer.js +22 -0
- package/dist/session/pointer.js.map +1 -0
- package/dist/session/rebuild.d.ts +53 -0
- package/dist/session/rebuild.js +230 -0
- package/dist/session/rebuild.js.map +1 -0
- package/dist/session/resume.d.ts +26 -0
- package/dist/session/resume.js +128 -0
- package/dist/session/resume.js.map +1 -0
- package/dist/session/retention.d.ts +97 -0
- package/dist/session/retention.js +702 -0
- package/dist/session/retention.js.map +1 -0
- package/dist/session/runner.d.ts +156 -0
- package/dist/session/runner.js +656 -0
- package/dist/session/runner.js.map +1 -0
- package/dist/session/scheduler.d.ts +47 -0
- package/dist/session/scheduler.js +85 -0
- package/dist/session/scheduler.js.map +1 -0
- package/dist/session/secret-key-store.d.ts +47 -0
- package/dist/session/secret-key-store.js +79 -0
- package/dist/session/secret-key-store.js.map +1 -0
- package/dist/session/session.d.ts +106 -0
- package/dist/session/session.js +353 -0
- package/dist/session/session.js.map +1 -0
- package/dist/session/sticky.d.ts +48 -0
- package/dist/session/sticky.js +232 -0
- package/dist/session/sticky.js.map +1 -0
- package/dist/session/store.d.ts +108 -0
- package/dist/session/store.js +163 -0
- package/dist/session/store.js.map +1 -0
- package/dist/telemetry/index.d.ts +2 -0
- package/dist/telemetry/index.js +3 -0
- package/dist/telemetry/index.js.map +1 -0
- package/dist/testing/fake-sandbox.d.ts +52 -0
- package/dist/testing/fake-sandbox.js +370 -0
- package/dist/testing/fake-sandbox.js.map +1 -0
- package/dist/testing/index.d.ts +10 -0
- package/dist/testing/index.js +11 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/key-store-conformance.d.ts +22 -0
- package/dist/testing/key-store-conformance.js +97 -0
- package/dist/testing/key-store-conformance.js.map +1 -0
- package/dist/testing/run-cases.d.ts +21 -0
- package/dist/testing/run-cases.js +20 -0
- package/dist/testing/run-cases.js.map +1 -0
- package/dist/testing/sandbox-conformance.d.ts +64 -0
- package/dist/testing/sandbox-conformance.js +468 -0
- package/dist/testing/sandbox-conformance.js.map +1 -0
- package/dist/testing/scheduler-conformance.d.ts +32 -0
- package/dist/testing/scheduler-conformance.js +72 -0
- package/dist/testing/scheduler-conformance.js.map +1 -0
- package/dist/testing/store-conformance.d.ts +21 -0
- package/dist/testing/store-conformance.js +320 -0
- package/dist/testing/store-conformance.js.map +1 -0
- package/dist/tools/builtins/bash.d.ts +9 -0
- package/dist/tools/builtins/bash.js +75 -0
- package/dist/tools/builtins/bash.js.map +1 -0
- package/dist/tools/builtins/builtins.d.ts +38 -0
- package/dist/tools/builtins/builtins.js +48 -0
- package/dist/tools/builtins/builtins.js.map +1 -0
- package/dist/tools/builtins/descriptions.d.ts +69 -0
- package/dist/tools/builtins/descriptions.js +94 -0
- package/dist/tools/builtins/descriptions.js.map +1 -0
- package/dist/tools/builtins/fetch.d.ts +12 -0
- package/dist/tools/builtins/fetch.js +157 -0
- package/dist/tools/builtins/fetch.js.map +1 -0
- package/dist/tools/builtins/files.d.ts +27 -0
- package/dist/tools/builtins/files.js +274 -0
- package/dist/tools/builtins/files.js.map +1 -0
- package/dist/tools/builtins/paths.d.ts +30 -0
- package/dist/tools/builtins/paths.js +62 -0
- package/dist/tools/builtins/paths.js.map +1 -0
- package/dist/tools/builtins/results.d.ts +15 -0
- package/dist/tools/builtins/results.js +31 -0
- package/dist/tools/builtins/results.js.map +1 -0
- package/dist/tools/builtins/search.d.ts +18 -0
- package/dist/tools/builtins/search.js +217 -0
- package/dist/tools/builtins/search.js.map +1 -0
- package/dist/tools/errors.d.ts +13 -0
- package/dist/tools/errors.js +17 -0
- package/dist/tools/errors.js.map +1 -0
- package/dist/tools/index.d.ts +12 -0
- package/dist/tools/index.js +11 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/registry.d.ts +3 -0
- package/dist/tools/registry.js +234 -0
- package/dist/tools/registry.js.map +1 -0
- package/dist/tools/tool-search.d.ts +30 -0
- package/dist/tools/tool-search.js +118 -0
- package/dist/tools/tool-search.js.map +1 -0
- package/dist/tools/types.d.ts +106 -0
- package/dist/tools/types.js +2 -0
- package/dist/tools/types.js.map +1 -0
- package/llms.txt +469 -0
- package/package.json +118 -3
package/dist/ablation.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ablation.js","sourceRoot":"","sources":["../src/ablation.ts"],"names":[],"mappings":"AAAA,0GAA0G;AAC1G,OAAO,EAAE,CAAC","sourcesContent":["/** `@buildanagent/harness/ablation`: a stub that holds the entry's place until its component is built. */\nexport {};\n"]}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The completion check: after a turn ends `done`, the harness runs the project's checks inside the sandbox and diffs the
|
|
3
|
+
* workspace against the turn's start state. The result is recorded as a `completion_check` entry and never changes the
|
|
4
|
+
* turn's stop reason, because the model's claim is checked in code but the person decides what a failure means.
|
|
5
|
+
*/
|
|
6
|
+
import type { CompletionCheckEntry } from "@buildanagent/protocol";
|
|
7
|
+
import type { Sandbox } from "../sandbox/types.js";
|
|
8
|
+
import { type CheckConfig, type ResolvedChecks } from "./checks.js";
|
|
9
|
+
import { type WorkspaceSnapshot } from "./flags.js";
|
|
10
|
+
import { type Snapshotter } from "./snapshot.js";
|
|
11
|
+
/** How long one check may run before it is stopped with exit code 124. */
|
|
12
|
+
export declare const DEFAULT_CHECK_TIMEOUT_MS: number;
|
|
13
|
+
/** The exit code recorded for a check that the sandbox's policy refused to run. */
|
|
14
|
+
export declare const DENIED_EXIT_CODE = 126;
|
|
15
|
+
/** The exit code recorded for a check that never finished because the turn's signal aborted (130, as for SIGINT). */
|
|
16
|
+
export declare const ABORTED_EXIT_CODE = 130;
|
|
17
|
+
/** What a turn needs to check its own completion. */
|
|
18
|
+
export interface CompletionSettings {
|
|
19
|
+
/** The session's sandbox: the checks run in it, and its files are diffed. */
|
|
20
|
+
readonly sandbox: Sandbox;
|
|
21
|
+
/** The whole environment of each check, built with `spawnEnv()`. */
|
|
22
|
+
readonly env: Readonly<Record<string, string>>;
|
|
23
|
+
/** The longest one check may run, in milliseconds. Default 10 minutes. */
|
|
24
|
+
readonly timeoutMs?: number;
|
|
25
|
+
/** How the workspace is snapshotted for the diff. Default `gitSnapshotter`. */
|
|
26
|
+
readonly snapshotter?: Snapshotter;
|
|
27
|
+
}
|
|
28
|
+
/** What a check found: the body of the `completion_check` entry. */
|
|
29
|
+
export type CompletionOutcome = Omit<CompletionCheckEntry, "type">;
|
|
30
|
+
/**
|
|
31
|
+
* What a turn recorded before the model ran. The checks are resolved here, so a turn that edits `.agents/harness.json`
|
|
32
|
+
* can't change which commands run, and the check config is kept so an edit to it is flagged.
|
|
33
|
+
*/
|
|
34
|
+
export interface CompletionStart {
|
|
35
|
+
/** The workspace, or `undefined` when it can't be diffed. */
|
|
36
|
+
readonly workspace: WorkspaceSnapshot | undefined;
|
|
37
|
+
/** The checks that will run, or `undefined` when they could not be resolved; `failures` then says why. */
|
|
38
|
+
readonly checks: ResolvedChecks | undefined;
|
|
39
|
+
/** The check config's text, or `undefined` when it could not be read. */
|
|
40
|
+
readonly config: CheckConfig | undefined;
|
|
41
|
+
/** One flag for each part of the start state that could not be taken. */
|
|
42
|
+
readonly failures: readonly string[];
|
|
43
|
+
}
|
|
44
|
+
/** The flag for a completion check that could not do its work, for whatever reason `error` gives. */
|
|
45
|
+
export declare function completionFailure(error: unknown): string;
|
|
46
|
+
/**
|
|
47
|
+
* Takes the turn's start state, once, before the run: it resolves the checks, reads the check config and snapshots the
|
|
48
|
+
* workspace. It never throws. What it could not take is recorded in `failures`, and the turn still runs.
|
|
49
|
+
*/
|
|
50
|
+
export declare function snapshotStart(settings: CompletionSettings, signal: AbortSignal): Promise<CompletionStart>;
|
|
51
|
+
/**
|
|
52
|
+
* Runs the checks `start` resolved before the turn, in the sandbox, one after another and all of them even when one
|
|
53
|
+
* fails, and compares the workspace and the check config with `start`. `passed` is true when every check exited 0, and
|
|
54
|
+
* so it is also true when the project has no checks; `checks` is then empty. `passed` is false when the checks could
|
|
55
|
+
* not be resolved before the turn. A check that timed out has exit code 124, and one the sandbox's policy refused has
|
|
56
|
+
* `DENIED_EXIT_CODE` and a flag that names the refusal. When `signal` aborts, the check running and each one after it
|
|
57
|
+
* has `ABORTED_EXIT_CODE` and a flag, no diff is made, and the call still returns. `flagged` holds, in order, the
|
|
58
|
+
* start's failures, the refusals and aborts, a `check config changed` flag for each part of the check config the turn
|
|
59
|
+
* edited, and the diff patterns.
|
|
60
|
+
*
|
|
61
|
+
* With `start` `undefined`, the checks are resolved now and nothing is compared; that throws a `HarnessError` with code
|
|
62
|
+
* `invalid_completion_config` when `.agents/harness.json` is unusable. A sandbox error from a check throws.
|
|
63
|
+
*/
|
|
64
|
+
export declare function runCompletionCheck(settings: CompletionSettings, start: CompletionStart | undefined, signal: AbortSignal): Promise<CompletionOutcome>;
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { configChanges, readCheckConfig, resolveChecks, scriptNamesOf, } from "./checks.js";
|
|
2
|
+
import { flagChanges } from "./flags.js";
|
|
3
|
+
import { gitSnapshotter } from "./snapshot.js";
|
|
4
|
+
/** How long one check may run before it is stopped with exit code 124. */
|
|
5
|
+
export const DEFAULT_CHECK_TIMEOUT_MS = 10 * 60 * 1000;
|
|
6
|
+
/** The exit code recorded for a check that the sandbox's policy refused to run. */
|
|
7
|
+
export const DENIED_EXIT_CODE = 126;
|
|
8
|
+
/** The exit code recorded for a check that never finished because the turn's signal aborted (130, as for SIGINT). */
|
|
9
|
+
export const ABORTED_EXIT_CODE = 130;
|
|
10
|
+
/**
|
|
11
|
+
* Takes a snapshot. `undefined` means the workspace can't be diffed, which also covers a signal that aborted the
|
|
12
|
+
* snapshot, because a completion check must never throw for an abort. `earlier` is the start state when this is the end.
|
|
13
|
+
*/
|
|
14
|
+
async function snapshotWorkspace(settings, signal, earlier) {
|
|
15
|
+
const snapshotter = settings.snapshotter ?? gitSnapshotter;
|
|
16
|
+
try {
|
|
17
|
+
return await snapshotter(settings.sandbox, {
|
|
18
|
+
env: settings.env,
|
|
19
|
+
signal,
|
|
20
|
+
...(earlier === undefined ? {} : { known: new Set(earlier.files.keys()), skipped: earlier.skipped }),
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
if (signal.aborted)
|
|
25
|
+
return undefined;
|
|
26
|
+
throw error;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/** The flag for a completion check that could not do its work, for whatever reason `error` gives. */
|
|
30
|
+
export function completionFailure(error) {
|
|
31
|
+
return `completion check failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Takes the turn's start state, once, before the run: it resolves the checks, reads the check config and snapshots the
|
|
35
|
+
* workspace. It never throws. What it could not take is recorded in `failures`, and the turn still runs.
|
|
36
|
+
*/
|
|
37
|
+
export async function snapshotStart(settings, signal) {
|
|
38
|
+
const failures = [];
|
|
39
|
+
let checks;
|
|
40
|
+
let config;
|
|
41
|
+
let workspace;
|
|
42
|
+
try {
|
|
43
|
+
checks = await resolveChecks(settings.sandbox);
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
failures.push(completionFailure(error));
|
|
47
|
+
}
|
|
48
|
+
try {
|
|
49
|
+
config = await readCheckConfig(settings.sandbox, scriptNamesOf(checks?.commands ?? []));
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
failures.push(completionFailure(error));
|
|
53
|
+
}
|
|
54
|
+
try {
|
|
55
|
+
workspace = await snapshotWorkspace(settings, signal);
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
failures.push(completionFailure(error));
|
|
59
|
+
}
|
|
60
|
+
return { workspace, checks, config, failures };
|
|
61
|
+
}
|
|
62
|
+
function abortedCheck(text, checks, denials) {
|
|
63
|
+
checks.push({ command: text, exitCode: ABORTED_EXIT_CODE });
|
|
64
|
+
denials.push(`check aborted before it finished: ${text}`);
|
|
65
|
+
}
|
|
66
|
+
/** Records one check's result. A check the signal aborted counts as aborted, whatever the sandbox returned for it. */
|
|
67
|
+
function recordResult(text, result, signal, checks, denials) {
|
|
68
|
+
if (signal.aborted)
|
|
69
|
+
abortedCheck(text, checks, denials);
|
|
70
|
+
else if ("denied" in result) {
|
|
71
|
+
checks.push({ command: text, exitCode: DENIED_EXIT_CODE });
|
|
72
|
+
denials.push(`check refused by the sandbox: ${text}: ${result.denied.reason}`);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
checks.push({ command: text, exitCode: result.exitCode });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
async function runChecks(settings, commands, signal) {
|
|
79
|
+
const checks = [];
|
|
80
|
+
const denials = [];
|
|
81
|
+
for (const command of commands) {
|
|
82
|
+
const text = command.join(" ");
|
|
83
|
+
// A check after the abort is recorded without being started, whatever the sandbox does with an aborted signal.
|
|
84
|
+
if (signal.aborted) {
|
|
85
|
+
abortedCheck(text, checks, denials);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
const result = await settings.sandbox.exec(command, {
|
|
90
|
+
env: settings.env,
|
|
91
|
+
signal,
|
|
92
|
+
timeoutMs: settings.timeoutMs ?? DEFAULT_CHECK_TIMEOUT_MS,
|
|
93
|
+
});
|
|
94
|
+
recordResult(text, result, signal, checks, denials);
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
if (!signal.aborted)
|
|
98
|
+
throw error;
|
|
99
|
+
abortedCheck(text, checks, denials);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return { checks, denials };
|
|
103
|
+
}
|
|
104
|
+
/** The flags for what the turn changed in the check config, or a failure flag when it can't be read again. */
|
|
105
|
+
async function configFlags(sandbox, before) {
|
|
106
|
+
if (before === undefined)
|
|
107
|
+
return [];
|
|
108
|
+
try {
|
|
109
|
+
return configChanges(before, await readCheckConfig(sandbox, [...before.scripts.keys()]));
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
return [completionFailure(error)];
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/** The diff flags against the start's workspace, none after an abort, or a failure flag when the end snapshot throws. */
|
|
116
|
+
async function diffFlags(settings, start, signal) {
|
|
117
|
+
if (start === undefined || signal.aborted)
|
|
118
|
+
return [];
|
|
119
|
+
try {
|
|
120
|
+
const end = await snapshotWorkspace(settings, signal, start);
|
|
121
|
+
return end === undefined ? [] : flagChanges(start, end);
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
return [completionFailure(error)];
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Runs the checks `start` resolved before the turn, in the sandbox, one after another and all of them even when one
|
|
129
|
+
* fails, and compares the workspace and the check config with `start`. `passed` is true when every check exited 0, and
|
|
130
|
+
* so it is also true when the project has no checks; `checks` is then empty. `passed` is false when the checks could
|
|
131
|
+
* not be resolved before the turn. A check that timed out has exit code 124, and one the sandbox's policy refused has
|
|
132
|
+
* `DENIED_EXIT_CODE` and a flag that names the refusal. When `signal` aborts, the check running and each one after it
|
|
133
|
+
* has `ABORTED_EXIT_CODE` and a flag, no diff is made, and the call still returns. `flagged` holds, in order, the
|
|
134
|
+
* start's failures, the refusals and aborts, a `check config changed` flag for each part of the check config the turn
|
|
135
|
+
* edited, and the diff patterns.
|
|
136
|
+
*
|
|
137
|
+
* With `start` `undefined`, the checks are resolved now and nothing is compared; that throws a `HarnessError` with code
|
|
138
|
+
* `invalid_completion_config` when `.agents/harness.json` is unusable. A sandbox error from a check throws.
|
|
139
|
+
*/
|
|
140
|
+
export async function runCompletionCheck(settings, start, signal) {
|
|
141
|
+
const resolved = start === undefined ? await resolveChecks(settings.sandbox) : start.checks;
|
|
142
|
+
const { checks, denials } = resolved === undefined ? { checks: [], denials: [] } : await runChecks(settings, resolved.commands, signal);
|
|
143
|
+
// The checks ran first, so what they wrote (a build, a coverage report) is in the end state; git ignores most of it.
|
|
144
|
+
const changed = await configFlags(settings.sandbox, start?.config);
|
|
145
|
+
const diff = await diffFlags(settings, start?.workspace, signal);
|
|
146
|
+
return {
|
|
147
|
+
checks,
|
|
148
|
+
passed: resolved !== undefined && checks.every((check) => check.exitCode === 0),
|
|
149
|
+
flagged: [...(start?.failures ?? []), ...denials, ...changed, ...diff],
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=check.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"check.js","sourceRoot":"","sources":["../../src/completion/check.ts"],"names":[],"mappings":"AAOA,OAAO,EAEL,aAAa,EAEb,eAAe,EACf,aAAa,EACb,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,WAAW,EAA0B,MAAM,YAAY,CAAC;AACjE,OAAO,EAAE,cAAc,EAAoB,MAAM,eAAe,CAAC;AAEjE,0EAA0E;AAC1E,MAAM,CAAC,MAAM,wBAAwB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvD,mFAAmF;AACnF,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAEpC,qHAAqH;AACrH,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAiBrC;;;GAGG;AACH,KAAK,UAAU,iBAAiB,CAC9B,QAA4B,EAC5B,MAAmB,EACnB,OAA2B;IAE3B,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAW,IAAI,cAAc,CAAC;IAC3D,IAAI,CAAC;QACH,OAAO,MAAM,WAAW,CAAC,QAAQ,CAAC,OAAO,EAAE;YACzC,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,MAAM;YACN,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;SACrG,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,CAAC,OAAO;YAAE,OAAO,SAAS,CAAC;QACrC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAiBD,qGAAqG;AACrG,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,OAAO,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAC9F,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAA4B,EAAE,MAAmB;IACnF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,MAAkC,CAAC;IACvC,IAAI,MAA+B,CAAC;IACpC,IAAI,SAAwC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,MAAM,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;IAC1F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,MAAmC,EAAE,OAAiB;IACxF,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,IAAI,CAAC,qCAAqC,IAAI,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED,sHAAsH;AACtH,SAAS,YAAY,CACnB,IAAY,EACZ,MAA4C,EAC5C,MAAmB,EACnB,MAAmC,EACnC,OAAiB;IAEjB,IAAI,MAAM,CAAC,OAAO;QAAE,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;SACnD,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,iCAAiC,IAAI,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACjF,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAA4B,EAC5B,QAAwC,EACxC,MAAmB;IAEnB,MAAM,MAAM,GAAgC,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/B,+GAA+G;QAC/G,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YACpC,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE;gBAClD,GAAG,EAAE,QAAQ,CAAC,GAAG;gBACjB,MAAM;gBACN,SAAS,EAAE,QAAQ,CAAC,SAAS,IAAI,wBAAwB;aAC1D,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,OAAO;gBAAE,MAAM,KAAK,CAAC;YACjC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC7B,CAAC;AAED,8GAA8G;AAC9G,KAAK,UAAU,WAAW,CAAC,OAAgB,EAAE,MAA+B;IAC1E,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,OAAO,aAAa,CAAC,MAAM,EAAE,MAAM,eAAe,CAAC,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACpC,CAAC;AACH,CAAC;AAED,yHAAyH;AACzH,KAAK,UAAU,SAAS,CAAC,QAA4B,EAAE,KAAoC,EAAE,MAAmB;IAC9G,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACrD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7D,OAAO,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACpC,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAA4B,EAC5B,KAAkC,EAClC,MAAmB;IAEnB,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;IAC5F,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GACvB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC9G,qHAAqH;IACrH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IACjE,OAAO;QACL,MAAM;QACN,MAAM,EAAE,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,CAAC,CAAC;QAC/E,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,EAAE,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC;KACvE,CAAC;AACJ,CAAC","sourcesContent":["/**\n * The completion check: after a turn ends `done`, the harness runs the project's checks inside the sandbox and diffs the\n * workspace against the turn's start state. The result is recorded as a `completion_check` entry and never changes the\n * turn's stop reason, because the model's claim is checked in code but the person decides what a failure means.\n */\nimport type { CompletionCheckEntry } from \"@buildanagent/protocol\";\nimport type { Sandbox } from \"../sandbox/types.js\";\nimport {\n type CheckConfig,\n configChanges,\n type ResolvedChecks,\n readCheckConfig,\n resolveChecks,\n scriptNamesOf,\n} from \"./checks.js\";\nimport { flagChanges, type WorkspaceSnapshot } from \"./flags.js\";\nimport { gitSnapshotter, type Snapshotter } from \"./snapshot.js\";\n\n/** How long one check may run before it is stopped with exit code 124. */\nexport const DEFAULT_CHECK_TIMEOUT_MS = 10 * 60 * 1000;\n\n/** The exit code recorded for a check that the sandbox's policy refused to run. */\nexport const DENIED_EXIT_CODE = 126;\n\n/** The exit code recorded for a check that never finished because the turn's signal aborted (130, as for SIGINT). */\nexport const ABORTED_EXIT_CODE = 130;\n\n/** What a turn needs to check its own completion. */\nexport interface CompletionSettings {\n /** The session's sandbox: the checks run in it, and its files are diffed. */\n readonly sandbox: Sandbox;\n /** The whole environment of each check, built with `spawnEnv()`. */\n readonly env: Readonly<Record<string, string>>;\n /** The longest one check may run, in milliseconds. Default 10 minutes. */\n readonly timeoutMs?: number;\n /** How the workspace is snapshotted for the diff. Default `gitSnapshotter`. */\n readonly snapshotter?: Snapshotter;\n}\n\n/** What a check found: the body of the `completion_check` entry. */\nexport type CompletionOutcome = Omit<CompletionCheckEntry, \"type\">;\n\n/**\n * Takes a snapshot. `undefined` means the workspace can't be diffed, which also covers a signal that aborted the\n * snapshot, because a completion check must never throw for an abort. `earlier` is the start state when this is the end.\n */\nasync function snapshotWorkspace(\n settings: CompletionSettings,\n signal: AbortSignal,\n earlier?: WorkspaceSnapshot,\n): Promise<WorkspaceSnapshot | undefined> {\n const snapshotter = settings.snapshotter ?? gitSnapshotter;\n try {\n return await snapshotter(settings.sandbox, {\n env: settings.env,\n signal,\n ...(earlier === undefined ? {} : { known: new Set(earlier.files.keys()), skipped: earlier.skipped }),\n });\n } catch (error) {\n if (signal.aborted) return undefined;\n throw error;\n }\n}\n\n/**\n * What a turn recorded before the model ran. The checks are resolved here, so a turn that edits `.agents/harness.json`\n * can't change which commands run, and the check config is kept so an edit to it is flagged.\n */\nexport interface CompletionStart {\n /** The workspace, or `undefined` when it can't be diffed. */\n readonly workspace: WorkspaceSnapshot | undefined;\n /** The checks that will run, or `undefined` when they could not be resolved; `failures` then says why. */\n readonly checks: ResolvedChecks | undefined;\n /** The check config's text, or `undefined` when it could not be read. */\n readonly config: CheckConfig | undefined;\n /** One flag for each part of the start state that could not be taken. */\n readonly failures: readonly string[];\n}\n\n/** The flag for a completion check that could not do its work, for whatever reason `error` gives. */\nexport function completionFailure(error: unknown): string {\n return `completion check failed: ${error instanceof Error ? error.message : String(error)}`;\n}\n\n/**\n * Takes the turn's start state, once, before the run: it resolves the checks, reads the check config and snapshots the\n * workspace. It never throws. What it could not take is recorded in `failures`, and the turn still runs.\n */\nexport async function snapshotStart(settings: CompletionSettings, signal: AbortSignal): Promise<CompletionStart> {\n const failures: string[] = [];\n let checks: ResolvedChecks | undefined;\n let config: CheckConfig | undefined;\n let workspace: WorkspaceSnapshot | undefined;\n try {\n checks = await resolveChecks(settings.sandbox);\n } catch (error) {\n failures.push(completionFailure(error));\n }\n try {\n config = await readCheckConfig(settings.sandbox, scriptNamesOf(checks?.commands ?? []));\n } catch (error) {\n failures.push(completionFailure(error));\n }\n try {\n workspace = await snapshotWorkspace(settings, signal);\n } catch (error) {\n failures.push(completionFailure(error));\n }\n return { workspace, checks, config, failures };\n}\n\nfunction abortedCheck(text: string, checks: CompletionOutcome[\"checks\"], denials: string[]): void {\n checks.push({ command: text, exitCode: ABORTED_EXIT_CODE });\n denials.push(`check aborted before it finished: ${text}`);\n}\n\n/** Records one check's result. A check the signal aborted counts as aborted, whatever the sandbox returned for it. */\nfunction recordResult(\n text: string,\n result: Awaited<ReturnType<Sandbox[\"exec\"]>>,\n signal: AbortSignal,\n checks: CompletionOutcome[\"checks\"],\n denials: string[],\n): void {\n if (signal.aborted) abortedCheck(text, checks, denials);\n else if (\"denied\" in result) {\n checks.push({ command: text, exitCode: DENIED_EXIT_CODE });\n denials.push(`check refused by the sandbox: ${text}: ${result.denied.reason}`);\n } else {\n checks.push({ command: text, exitCode: result.exitCode });\n }\n}\n\nasync function runChecks(\n settings: CompletionSettings,\n commands: readonly (readonly string[])[],\n signal: AbortSignal,\n): Promise<{ checks: CompletionOutcome[\"checks\"]; denials: string[] }> {\n const checks: CompletionOutcome[\"checks\"] = [];\n const denials: string[] = [];\n for (const command of commands) {\n const text = command.join(\" \");\n // A check after the abort is recorded without being started, whatever the sandbox does with an aborted signal.\n if (signal.aborted) {\n abortedCheck(text, checks, denials);\n continue;\n }\n try {\n const result = await settings.sandbox.exec(command, {\n env: settings.env,\n signal,\n timeoutMs: settings.timeoutMs ?? DEFAULT_CHECK_TIMEOUT_MS,\n });\n recordResult(text, result, signal, checks, denials);\n } catch (error) {\n if (!signal.aborted) throw error;\n abortedCheck(text, checks, denials);\n }\n }\n return { checks, denials };\n}\n\n/** The flags for what the turn changed in the check config, or a failure flag when it can't be read again. */\nasync function configFlags(sandbox: Sandbox, before: CheckConfig | undefined): Promise<string[]> {\n if (before === undefined) return [];\n try {\n return configChanges(before, await readCheckConfig(sandbox, [...before.scripts.keys()]));\n } catch (error) {\n return [completionFailure(error)];\n }\n}\n\n/** The diff flags against the start's workspace, none after an abort, or a failure flag when the end snapshot throws. */\nasync function diffFlags(settings: CompletionSettings, start: WorkspaceSnapshot | undefined, signal: AbortSignal) {\n if (start === undefined || signal.aborted) return [];\n try {\n const end = await snapshotWorkspace(settings, signal, start);\n return end === undefined ? [] : flagChanges(start, end);\n } catch (error) {\n return [completionFailure(error)];\n }\n}\n\n/**\n * Runs the checks `start` resolved before the turn, in the sandbox, one after another and all of them even when one\n * fails, and compares the workspace and the check config with `start`. `passed` is true when every check exited 0, and\n * so it is also true when the project has no checks; `checks` is then empty. `passed` is false when the checks could\n * not be resolved before the turn. A check that timed out has exit code 124, and one the sandbox's policy refused has\n * `DENIED_EXIT_CODE` and a flag that names the refusal. When `signal` aborts, the check running and each one after it\n * has `ABORTED_EXIT_CODE` and a flag, no diff is made, and the call still returns. `flagged` holds, in order, the\n * start's failures, the refusals and aborts, a `check config changed` flag for each part of the check config the turn\n * edited, and the diff patterns.\n *\n * With `start` `undefined`, the checks are resolved now and nothing is compared; that throws a `HarnessError` with code\n * `invalid_completion_config` when `.agents/harness.json` is unusable. A sandbox error from a check throws.\n */\nexport async function runCompletionCheck(\n settings: CompletionSettings,\n start: CompletionStart | undefined,\n signal: AbortSignal,\n): Promise<CompletionOutcome> {\n const resolved = start === undefined ? await resolveChecks(settings.sandbox) : start.checks;\n const { checks, denials } =\n resolved === undefined ? { checks: [], denials: [] } : await runChecks(settings, resolved.commands, signal);\n // The checks ran first, so what they wrote (a build, a coverage report) is in the end state; git ignores most of it.\n const changed = await configFlags(settings.sandbox, start?.config);\n const diff = await diffFlags(settings, start?.workspace, signal);\n return {\n checks,\n passed: resolved !== undefined && checks.every((check) => check.exitCode === 0),\n flagged: [...(start?.failures ?? []), ...denials, ...changed, ...diff],\n };\n}\n"]}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { Sandbox } from "../sandbox/types.js";
|
|
2
|
+
/** Where a project lists its checks, relative to the workspace root. */
|
|
3
|
+
export declare const HARNESS_CONFIG_PATH = ".agents/harness.json";
|
|
4
|
+
/** The `package.json` scripts discovery runs, in the order it runs them. */
|
|
5
|
+
export declare const DISCOVERED_SCRIPTS: readonly string[];
|
|
6
|
+
/** The checks to run, and whether the project named them or the harness found them. */
|
|
7
|
+
export interface ResolvedChecks {
|
|
8
|
+
readonly source: "configured" | "discovered";
|
|
9
|
+
/** Each check as an executable and its arguments, run with no shell. */
|
|
10
|
+
readonly commands: readonly (readonly string[])[];
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Finds what checks the task. `completionChecks` in `.agents/harness.json` wins: each entry is a command string, split
|
|
14
|
+
* on spaces, or a list of an executable and its arguments, and neither goes through a shell. Without it, the harness
|
|
15
|
+
* runs the `package.json` scripts `typecheck`, `test` and `check`, in that order, when they exist, with the runner the
|
|
16
|
+
* lockfile names (`pnpm`, `yarn`, else `npm`), and a `Makefile` `test` target when no script matched.
|
|
17
|
+
* Throws a `HarnessError` with code `invalid_completion_config` for a config file that can't be read as described.
|
|
18
|
+
*/
|
|
19
|
+
export declare function resolveChecks(sandbox: Sandbox): Promise<ResolvedChecks>;
|
|
20
|
+
/**
|
|
21
|
+
* The parts of the workspace that decide what a check does, as text: the config file, the bodies of the `package.json`
|
|
22
|
+
* scripts discovery names and the resolved checks run (with their `pre` and `post` hooks), and the `Makefile` `test` rule. Each is `undefined` when absent.
|
|
23
|
+
*/
|
|
24
|
+
export interface CheckConfig {
|
|
25
|
+
readonly harnessConfig: string | undefined;
|
|
26
|
+
readonly scripts: ReadonlyMap<string, string | undefined>;
|
|
27
|
+
readonly makeTestRule: string | undefined;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* The `package.json` scripts a set of commands runs through a package runner (`npm run e2e`, `pnpm lint`), each with its
|
|
31
|
+
* `pre` and `post` hooks, because npm and yarn run those hooks too. Flags are skipped, and a command that is not a
|
|
32
|
+
* package runner names none.
|
|
33
|
+
*/
|
|
34
|
+
export declare function scriptNamesOf(commands: readonly (readonly string[])[]): string[];
|
|
35
|
+
/**
|
|
36
|
+
* Reads the check config. The scripts it tracks are the discovered ones plus `extra`, the scripts the resolved checks
|
|
37
|
+
* run. Throws what the sandbox throws for a file it can't read, except a missing one.
|
|
38
|
+
*/
|
|
39
|
+
export declare function readCheckConfig(sandbox: Sandbox, extra?: readonly string[]): Promise<CheckConfig>;
|
|
40
|
+
/**
|
|
41
|
+
* One flag for each part of the check config that differs between `before` and `after`. The harness runs the checks it
|
|
42
|
+
* resolved before the turn, but a package runner reads a script's body when it runs, and `make` reads the `Makefile`,
|
|
43
|
+
* so an edit to either changes what a check does even though the command is the same.
|
|
44
|
+
*/
|
|
45
|
+
export declare function configChanges(before: CheckConfig, after: CheckConfig): string[];
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which commands check a finished task: the project's own list in `.agents/harness.json`, or else the ones the harness
|
|
3
|
+
* finds. A configured list replaces discovery, so a project can name a different set, or an empty one to run nothing.
|
|
4
|
+
*/
|
|
5
|
+
import { HarnessError } from "../errors.js";
|
|
6
|
+
/** Where a project lists its checks, relative to the workspace root. */
|
|
7
|
+
export const HARNESS_CONFIG_PATH = ".agents/harness.json";
|
|
8
|
+
/** The `package.json` scripts discovery runs, in the order it runs them. */
|
|
9
|
+
export const DISCOVERED_SCRIPTS = ["typecheck", "test", "check"];
|
|
10
|
+
// `test:` or `all test:`, but not a variable assignment such as `test := x` or `test ::= x`.
|
|
11
|
+
const MAKE_TEST_TARGET = /^(?:[\w.-]+[ \t]+)*test[ \t]*:(?!=|:=)/m;
|
|
12
|
+
const DECODER = new TextDecoder();
|
|
13
|
+
/** Reads a file as text, or returns `undefined` when it does not exist. Any other failure is thrown. */
|
|
14
|
+
async function readText(sandbox, path) {
|
|
15
|
+
try {
|
|
16
|
+
return DECODER.decode(await sandbox.readFile(path));
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
if (error instanceof HarnessError && error.code === "sandbox_not_found")
|
|
20
|
+
return undefined;
|
|
21
|
+
throw error;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function invalid(message) {
|
|
25
|
+
return new HarnessError("invalid_completion_config", `${HARNESS_CONFIG_PATH}: ${message}`);
|
|
26
|
+
}
|
|
27
|
+
/** One configured check: a string split on spaces, or a list that is the executable and its arguments. */
|
|
28
|
+
function commandOf(entry, index) {
|
|
29
|
+
if (typeof entry === "string") {
|
|
30
|
+
const words = entry.trim().split(/\s+/);
|
|
31
|
+
if (words[0] === "")
|
|
32
|
+
throw invalid(`completionChecks[${index}] is an empty string`);
|
|
33
|
+
return words;
|
|
34
|
+
}
|
|
35
|
+
if (Array.isArray(entry) && entry.length > 0 && entry.every((word) => typeof word === "string" && word !== "")) {
|
|
36
|
+
return [...entry];
|
|
37
|
+
}
|
|
38
|
+
throw invalid(`completionChecks[${index}] must be a command string or a non-empty list of non-empty strings`);
|
|
39
|
+
}
|
|
40
|
+
/** The configured checks, or `undefined` when the file or the key is absent. */
|
|
41
|
+
async function configuredChecks(sandbox) {
|
|
42
|
+
const text = await readText(sandbox, HARNESS_CONFIG_PATH);
|
|
43
|
+
if (text === undefined)
|
|
44
|
+
return undefined;
|
|
45
|
+
let config;
|
|
46
|
+
try {
|
|
47
|
+
config = JSON.parse(text);
|
|
48
|
+
}
|
|
49
|
+
catch (cause) {
|
|
50
|
+
throw new HarnessError("invalid_completion_config", `${HARNESS_CONFIG_PATH} is not valid JSON`, { cause });
|
|
51
|
+
}
|
|
52
|
+
if (typeof config !== "object" || config === null || Array.isArray(config))
|
|
53
|
+
throw invalid("it must hold an object");
|
|
54
|
+
const list = config.completionChecks;
|
|
55
|
+
if (list === undefined)
|
|
56
|
+
return undefined;
|
|
57
|
+
if (!Array.isArray(list))
|
|
58
|
+
throw invalid("completionChecks must be a list");
|
|
59
|
+
return list.map(commandOf);
|
|
60
|
+
}
|
|
61
|
+
async function packageRunner(sandbox) {
|
|
62
|
+
if ((await readText(sandbox, "pnpm-lock.yaml")) !== undefined)
|
|
63
|
+
return "pnpm";
|
|
64
|
+
if ((await readText(sandbox, "yarn.lock")) !== undefined)
|
|
65
|
+
return "yarn";
|
|
66
|
+
return "npm";
|
|
67
|
+
}
|
|
68
|
+
/** The `scripts` object of a `package.json`, or `undefined` when the text is not JSON or holds no such object. */
|
|
69
|
+
function scriptsOf(text) {
|
|
70
|
+
try {
|
|
71
|
+
const scripts = JSON.parse(text).scripts;
|
|
72
|
+
return typeof scripts === "object" && scripts !== null ? scripts : undefined;
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
// A package.json that is not JSON has no scripts to run; the project's own tools will say what is wrong with it.
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/** The scripts of a `package.json` that discovery looks for, or none when the file is missing or unreadable as JSON. */
|
|
80
|
+
async function discoveredScripts(sandbox) {
|
|
81
|
+
const text = await readText(sandbox, "package.json");
|
|
82
|
+
const scripts = text === undefined ? undefined : scriptsOf(text);
|
|
83
|
+
if (scripts === undefined)
|
|
84
|
+
return [];
|
|
85
|
+
return DISCOVERED_SCRIPTS.filter((name) => typeof scripts[name] === "string");
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Finds what checks the task. `completionChecks` in `.agents/harness.json` wins: each entry is a command string, split
|
|
89
|
+
* on spaces, or a list of an executable and its arguments, and neither goes through a shell. Without it, the harness
|
|
90
|
+
* runs the `package.json` scripts `typecheck`, `test` and `check`, in that order, when they exist, with the runner the
|
|
91
|
+
* lockfile names (`pnpm`, `yarn`, else `npm`), and a `Makefile` `test` target when no script matched.
|
|
92
|
+
* Throws a `HarnessError` with code `invalid_completion_config` for a config file that can't be read as described.
|
|
93
|
+
*/
|
|
94
|
+
export async function resolveChecks(sandbox) {
|
|
95
|
+
const configured = await configuredChecks(sandbox);
|
|
96
|
+
if (configured !== undefined)
|
|
97
|
+
return { source: "configured", commands: configured };
|
|
98
|
+
const scripts = await discoveredScripts(sandbox);
|
|
99
|
+
if (scripts.length > 0) {
|
|
100
|
+
const runner = await packageRunner(sandbox);
|
|
101
|
+
return { source: "discovered", commands: scripts.map((name) => [runner, "run", name]) };
|
|
102
|
+
}
|
|
103
|
+
const makefile = await readText(sandbox, "Makefile");
|
|
104
|
+
if (makefile !== undefined && MAKE_TEST_TARGET.test(makefile)) {
|
|
105
|
+
return { source: "discovered", commands: [["make", "test"]] };
|
|
106
|
+
}
|
|
107
|
+
return { source: "discovered", commands: [] };
|
|
108
|
+
}
|
|
109
|
+
/** The `test` rule of a `Makefile`: its target line and the tab-indented recipe lines after it. */
|
|
110
|
+
function makeTestRule(makefile) {
|
|
111
|
+
const match = MAKE_TEST_TARGET.exec(makefile);
|
|
112
|
+
if (match === null)
|
|
113
|
+
return undefined;
|
|
114
|
+
const lines = makefile.slice(match.index).split("\n");
|
|
115
|
+
const end = lines.findIndex((line, index) => index > 0 && !line.startsWith("\t"));
|
|
116
|
+
return lines.slice(0, end === -1 ? lines.length : end).join("\n");
|
|
117
|
+
}
|
|
118
|
+
const PACKAGE_RUNNERS = new Set(["npm", "pnpm", "yarn", "bun"]);
|
|
119
|
+
/**
|
|
120
|
+
* The `package.json` scripts a set of commands runs through a package runner (`npm run e2e`, `pnpm lint`), each with its
|
|
121
|
+
* `pre` and `post` hooks, because npm and yarn run those hooks too. Flags are skipped, and a command that is not a
|
|
122
|
+
* package runner names none.
|
|
123
|
+
*/
|
|
124
|
+
export function scriptNamesOf(commands) {
|
|
125
|
+
const names = new Set();
|
|
126
|
+
for (const [runner, ...rest] of commands) {
|
|
127
|
+
if (runner === undefined || !PACKAGE_RUNNERS.has(runner))
|
|
128
|
+
continue;
|
|
129
|
+
const words = rest.filter((word) => !word.startsWith("-"));
|
|
130
|
+
const name = words[0] === "run" || words[0] === "run-script" ? words[1] : words[0];
|
|
131
|
+
if (name !== undefined)
|
|
132
|
+
for (const each of [name, `pre${name}`, `post${name}`])
|
|
133
|
+
names.add(each);
|
|
134
|
+
}
|
|
135
|
+
return [...names];
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Reads the check config. The scripts it tracks are the discovered ones plus `extra`, the scripts the resolved checks
|
|
139
|
+
* run. Throws what the sandbox throws for a file it can't read, except a missing one.
|
|
140
|
+
*/
|
|
141
|
+
export async function readCheckConfig(sandbox, extra = []) {
|
|
142
|
+
const harnessConfig = await readText(sandbox, HARNESS_CONFIG_PATH);
|
|
143
|
+
const packageJson = await readText(sandbox, "package.json");
|
|
144
|
+
const makefile = await readText(sandbox, "Makefile");
|
|
145
|
+
const scripts = packageJson === undefined ? undefined : scriptsOf(packageJson);
|
|
146
|
+
return {
|
|
147
|
+
harnessConfig,
|
|
148
|
+
scripts: new Map([...new Set([...DISCOVERED_SCRIPTS, ...extra])].map((name) => {
|
|
149
|
+
const body = scripts?.[name];
|
|
150
|
+
return [name, typeof body === "string" ? body : undefined];
|
|
151
|
+
})),
|
|
152
|
+
makeTestRule: makefile === undefined ? undefined : makeTestRule(makefile),
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* One flag for each part of the check config that differs between `before` and `after`. The harness runs the checks it
|
|
157
|
+
* resolved before the turn, but a package runner reads a script's body when it runs, and `make` reads the `Makefile`,
|
|
158
|
+
* so an edit to either changes what a check does even though the command is the same.
|
|
159
|
+
*/
|
|
160
|
+
export function configChanges(before, after) {
|
|
161
|
+
const flags = [];
|
|
162
|
+
if (before.harnessConfig !== after.harnessConfig)
|
|
163
|
+
flags.push(`check config changed: ${HARNESS_CONFIG_PATH}`);
|
|
164
|
+
for (const name of before.scripts.keys()) {
|
|
165
|
+
if (before.scripts.get(name) !== after.scripts.get(name)) {
|
|
166
|
+
flags.push(`check config changed: package.json script ${name}`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
if (before.makeTestRule !== after.makeTestRule)
|
|
170
|
+
flags.push("check config changed: Makefile test target");
|
|
171
|
+
return flags;
|
|
172
|
+
}
|
|
173
|
+
//# sourceMappingURL=checks.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checks.js","sourceRoot":"","sources":["../../src/completion/checks.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,wEAAwE;AACxE,MAAM,CAAC,MAAM,mBAAmB,GAAG,sBAAsB,CAAC;AAE1D,4EAA4E;AAC5E,MAAM,CAAC,MAAM,kBAAkB,GAAsB,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AASpF,6FAA6F;AAC7F,MAAM,gBAAgB,GAAG,yCAAyC,CAAC;AAEnE,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAElC,wGAAwG;AACxG,KAAK,UAAU,QAAQ,CAAC,OAAgB,EAAE,IAAY;IACpD,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,mBAAmB;YAAE,OAAO,SAAS,CAAC;QAC1F,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,OAAe;IAC9B,OAAO,IAAI,YAAY,CAAC,2BAA2B,EAAE,GAAG,mBAAmB,KAAK,OAAO,EAAE,CAAC,CAAC;AAC7F,CAAC;AAED,0GAA0G;AAC1G,SAAS,SAAS,CAAC,KAAc,EAAE,KAAa;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE;YAAE,MAAM,OAAO,CAAC,oBAAoB,KAAK,sBAAsB,CAAC,CAAC;QACpF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,EAAE,CAAC,EAAE,CAAC;QAC/G,OAAO,CAAC,GAAI,KAAkB,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,OAAO,CAAC,oBAAoB,KAAK,qEAAqE,CAAC,CAAC;AAChH,CAAC;AAED,gFAAgF;AAChF,KAAK,UAAU,gBAAgB,CAAC,OAAgB;IAC9C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IAC1D,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACzC,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,YAAY,CAAC,2BAA2B,EAAE,GAAG,mBAAmB,oBAAoB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7G,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,MAAM,OAAO,CAAC,wBAAwB,CAAC,CAAC;IACpH,MAAM,IAAI,GAAI,MAAkC,CAAC,gBAAgB,CAAC;IAClE,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,OAAO,CAAC,iCAAiC,CAAC,CAAC;IAC3E,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,OAAgB;IAC3C,IAAI,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IAC7E,IAAI,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IACxE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kHAAkH;AAClH,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,MAAM,OAAO,GAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAA2B,CAAC,OAAO,CAAC;QACpE,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,CAAC,CAAC,CAAE,OAAmC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5G,CAAC;IAAC,MAAM,CAAC;QACP,iHAAiH;QACjH,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,wHAAwH;AACxH,KAAK,UAAU,iBAAiB,CAAC,OAAgB;IAC/C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACjE,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACrC,OAAO,kBAAkB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,CAAC;AAChF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAgB;IAClD,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACnD,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC;IACpF,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;QAC5C,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;IAC1F,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACrD,IAAI,QAAQ,KAAK,SAAS,IAAI,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;IAChE,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;AAChD,CAAC;AAYD,mGAAmG;AACnG,SAAS,YAAY,CAAC,QAAgB;IACpC,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC9C,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACtD,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IAClF,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;AAErF;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,QAAwC;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzC,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,SAAS;QACnE,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,IAAI,KAAK,SAAS;YAAE,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE,MAAM,IAAI,EAAE,EAAE,OAAO,IAAI,EAAE,CAAC;gBAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAgB,EAAE,KAAK,GAAsB,EAAE;IACnF,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACnE,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAC/E,OAAO;QACL,aAAa;QACb,OAAO,EAAE,IAAI,GAAG,CACd,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,kBAAkB,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3D,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO,CAAC,IAAI,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7D,CAAC,CAAC,CACH;QACD,YAAY,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC;KAC1E,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,MAAmB,EAAE,KAAkB;IACnE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,aAAa,KAAK,KAAK,CAAC,aAAa;QAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,mBAAmB,EAAE,CAAC,CAAC;IAC7G,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QACzC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC,6CAA6C,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,KAAK,KAAK,CAAC,YAAY;QAAE,KAAK,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IACzG,OAAO,KAAK,CAAC;AACf,CAAC","sourcesContent":["/**\n * Which commands check a finished task: the project's own list in `.agents/harness.json`, or else the ones the harness\n * finds. A configured list replaces discovery, so a project can name a different set, or an empty one to run nothing.\n */\nimport { HarnessError } from \"../errors.js\";\nimport type { Sandbox } from \"../sandbox/types.js\";\n\n/** Where a project lists its checks, relative to the workspace root. */\nexport const HARNESS_CONFIG_PATH = \".agents/harness.json\";\n\n/** The `package.json` scripts discovery runs, in the order it runs them. */\nexport const DISCOVERED_SCRIPTS: readonly string[] = [\"typecheck\", \"test\", \"check\"];\n\n/** The checks to run, and whether the project named them or the harness found them. */\nexport interface ResolvedChecks {\n readonly source: \"configured\" | \"discovered\";\n /** Each check as an executable and its arguments, run with no shell. */\n readonly commands: readonly (readonly string[])[];\n}\n\n// `test:` or `all test:`, but not a variable assignment such as `test := x` or `test ::= x`.\nconst MAKE_TEST_TARGET = /^(?:[\\w.-]+[ \\t]+)*test[ \\t]*:(?!=|:=)/m;\n\nconst DECODER = new TextDecoder();\n\n/** Reads a file as text, or returns `undefined` when it does not exist. Any other failure is thrown. */\nasync function readText(sandbox: Sandbox, path: string): Promise<string | undefined> {\n try {\n return DECODER.decode(await sandbox.readFile(path));\n } catch (error) {\n if (error instanceof HarnessError && error.code === \"sandbox_not_found\") return undefined;\n throw error;\n }\n}\n\nfunction invalid(message: string): HarnessError {\n return new HarnessError(\"invalid_completion_config\", `${HARNESS_CONFIG_PATH}: ${message}`);\n}\n\n/** One configured check: a string split on spaces, or a list that is the executable and its arguments. */\nfunction commandOf(entry: unknown, index: number): string[] {\n if (typeof entry === \"string\") {\n const words = entry.trim().split(/\\s+/);\n if (words[0] === \"\") throw invalid(`completionChecks[${index}] is an empty string`);\n return words;\n }\n if (Array.isArray(entry) && entry.length > 0 && entry.every((word) => typeof word === \"string\" && word !== \"\")) {\n return [...(entry as string[])];\n }\n throw invalid(`completionChecks[${index}] must be a command string or a non-empty list of non-empty strings`);\n}\n\n/** The configured checks, or `undefined` when the file or the key is absent. */\nasync function configuredChecks(sandbox: Sandbox): Promise<string[][] | undefined> {\n const text = await readText(sandbox, HARNESS_CONFIG_PATH);\n if (text === undefined) return undefined;\n let config: unknown;\n try {\n config = JSON.parse(text);\n } catch (cause) {\n throw new HarnessError(\"invalid_completion_config\", `${HARNESS_CONFIG_PATH} is not valid JSON`, { cause });\n }\n if (typeof config !== \"object\" || config === null || Array.isArray(config)) throw invalid(\"it must hold an object\");\n const list = (config as Record<string, unknown>).completionChecks;\n if (list === undefined) return undefined;\n if (!Array.isArray(list)) throw invalid(\"completionChecks must be a list\");\n return list.map(commandOf);\n}\n\nasync function packageRunner(sandbox: Sandbox): Promise<string> {\n if ((await readText(sandbox, \"pnpm-lock.yaml\")) !== undefined) return \"pnpm\";\n if ((await readText(sandbox, \"yarn.lock\")) !== undefined) return \"yarn\";\n return \"npm\";\n}\n\n/** The `scripts` object of a `package.json`, or `undefined` when the text is not JSON or holds no such object. */\nfunction scriptsOf(text: string): Record<string, unknown> | undefined {\n try {\n const scripts = (JSON.parse(text) as { scripts?: unknown }).scripts;\n return typeof scripts === \"object\" && scripts !== null ? (scripts as Record<string, unknown>) : undefined;\n } catch {\n // A package.json that is not JSON has no scripts to run; the project's own tools will say what is wrong with it.\n return undefined;\n }\n}\n\n/** The scripts of a `package.json` that discovery looks for, or none when the file is missing or unreadable as JSON. */\nasync function discoveredScripts(sandbox: Sandbox): Promise<string[]> {\n const text = await readText(sandbox, \"package.json\");\n const scripts = text === undefined ? undefined : scriptsOf(text);\n if (scripts === undefined) return [];\n return DISCOVERED_SCRIPTS.filter((name) => typeof scripts[name] === \"string\");\n}\n\n/**\n * Finds what checks the task. `completionChecks` in `.agents/harness.json` wins: each entry is a command string, split\n * on spaces, or a list of an executable and its arguments, and neither goes through a shell. Without it, the harness\n * runs the `package.json` scripts `typecheck`, `test` and `check`, in that order, when they exist, with the runner the\n * lockfile names (`pnpm`, `yarn`, else `npm`), and a `Makefile` `test` target when no script matched.\n * Throws a `HarnessError` with code `invalid_completion_config` for a config file that can't be read as described.\n */\nexport async function resolveChecks(sandbox: Sandbox): Promise<ResolvedChecks> {\n const configured = await configuredChecks(sandbox);\n if (configured !== undefined) return { source: \"configured\", commands: configured };\n const scripts = await discoveredScripts(sandbox);\n if (scripts.length > 0) {\n const runner = await packageRunner(sandbox);\n return { source: \"discovered\", commands: scripts.map((name) => [runner, \"run\", name]) };\n }\n const makefile = await readText(sandbox, \"Makefile\");\n if (makefile !== undefined && MAKE_TEST_TARGET.test(makefile)) {\n return { source: \"discovered\", commands: [[\"make\", \"test\"]] };\n }\n return { source: \"discovered\", commands: [] };\n}\n\n/**\n * The parts of the workspace that decide what a check does, as text: the config file, the bodies of the `package.json`\n * scripts discovery names and the resolved checks run (with their `pre` and `post` hooks), and the `Makefile` `test` rule. Each is `undefined` when absent.\n */\nexport interface CheckConfig {\n readonly harnessConfig: string | undefined;\n readonly scripts: ReadonlyMap<string, string | undefined>;\n readonly makeTestRule: string | undefined;\n}\n\n/** The `test` rule of a `Makefile`: its target line and the tab-indented recipe lines after it. */\nfunction makeTestRule(makefile: string): string | undefined {\n const match = MAKE_TEST_TARGET.exec(makefile);\n if (match === null) return undefined;\n const lines = makefile.slice(match.index).split(\"\\n\");\n const end = lines.findIndex((line, index) => index > 0 && !line.startsWith(\"\\t\"));\n return lines.slice(0, end === -1 ? lines.length : end).join(\"\\n\");\n}\n\nconst PACKAGE_RUNNERS: ReadonlySet<string> = new Set([\"npm\", \"pnpm\", \"yarn\", \"bun\"]);\n\n/**\n * The `package.json` scripts a set of commands runs through a package runner (`npm run e2e`, `pnpm lint`), each with its\n * `pre` and `post` hooks, because npm and yarn run those hooks too. Flags are skipped, and a command that is not a\n * package runner names none.\n */\nexport function scriptNamesOf(commands: readonly (readonly string[])[]): string[] {\n const names = new Set<string>();\n for (const [runner, ...rest] of commands) {\n if (runner === undefined || !PACKAGE_RUNNERS.has(runner)) continue;\n const words = rest.filter((word) => !word.startsWith(\"-\"));\n const name = words[0] === \"run\" || words[0] === \"run-script\" ? words[1] : words[0];\n if (name !== undefined) for (const each of [name, `pre${name}`, `post${name}`]) names.add(each);\n }\n return [...names];\n}\n\n/**\n * Reads the check config. The scripts it tracks are the discovered ones plus `extra`, the scripts the resolved checks\n * run. Throws what the sandbox throws for a file it can't read, except a missing one.\n */\nexport async function readCheckConfig(sandbox: Sandbox, extra: readonly string[] = []): Promise<CheckConfig> {\n const harnessConfig = await readText(sandbox, HARNESS_CONFIG_PATH);\n const packageJson = await readText(sandbox, \"package.json\");\n const makefile = await readText(sandbox, \"Makefile\");\n const scripts = packageJson === undefined ? undefined : scriptsOf(packageJson);\n return {\n harnessConfig,\n scripts: new Map(\n [...new Set([...DISCOVERED_SCRIPTS, ...extra])].map((name) => {\n const body = scripts?.[name];\n return [name, typeof body === \"string\" ? body : undefined];\n }),\n ),\n makeTestRule: makefile === undefined ? undefined : makeTestRule(makefile),\n };\n}\n\n/**\n * One flag for each part of the check config that differs between `before` and `after`. The harness runs the checks it\n * resolved before the turn, but a package runner reads a script's body when it runs, and `make` reads the `Makefile`,\n * so an edit to either changes what a check does even though the command is the same.\n */\nexport function configChanges(before: CheckConfig, after: CheckConfig): string[] {\n const flags: string[] = [];\n if (before.harnessConfig !== after.harnessConfig) flags.push(`check config changed: ${HARNESS_CONFIG_PATH}`);\n for (const name of before.scripts.keys()) {\n if (before.scripts.get(name) !== after.scripts.get(name)) {\n flags.push(`check config changed: package.json script ${name}`);\n }\n }\n if (before.makeTestRule !== after.makeTestRule) flags.push(\"check config changed: Makefile test target\");\n return flags;\n}\n"]}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The diff flags: patterns in what a turn changed that suggest the checks were gamed instead of met. A flag is a
|
|
3
|
+
* sentence for a person to read, never a verdict, because every pattern has honest causes.
|
|
4
|
+
*/
|
|
5
|
+
/** What the workspace held at one moment. */
|
|
6
|
+
export interface WorkspaceSnapshot {
|
|
7
|
+
/** The text of each file, by path relative to the workspace root. */
|
|
8
|
+
readonly files: ReadonlyMap<string, string>;
|
|
9
|
+
/** Paths left out for their size, their encoding or the file cap. The diff ignores them, so none counts as deleted. */
|
|
10
|
+
readonly skipped: ReadonlySet<string>;
|
|
11
|
+
}
|
|
12
|
+
/** The most flags one check records. The last entry says how many more there were. */
|
|
13
|
+
export declare const MAX_FLAGS = 100;
|
|
14
|
+
/**
|
|
15
|
+
* Whether a path looks like a test file: a code file (by its extension, such as `.ts`, `.py` or `.go`) in a `test`,
|
|
16
|
+
* `tests` or `__tests__` folder, or a file named as a test anywhere. A data file in a test folder is not one.
|
|
17
|
+
*/
|
|
18
|
+
export declare function isTestFile(path: string): boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Compares the workspace at the start of a turn with the end, and returns one sentence for each pattern the plan names:
|
|
21
|
+
* an added skip marker (`.skip(`, `xit(` or `@pytest.mark.skip`), an added placeholder (`TODO` or `FIXME` in capitals, or
|
|
22
|
+
* "not implemented" in any case), a new file whose name has `mock`, `stub` or `fake` in it, a deleted test file, and a test file with
|
|
23
|
+
* fewer assertion lines than before. Paths come out in sorted order, so the same change gives the same flags.
|
|
24
|
+
*/
|
|
25
|
+
export declare function flagChanges(before: WorkspaceSnapshot, after: WorkspaceSnapshot): string[];
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The diff flags: patterns in what a turn changed that suggest the checks were gamed instead of met. A flag is a
|
|
3
|
+
* sentence for a person to read, never a verdict, because every pattern has honest causes.
|
|
4
|
+
*/
|
|
5
|
+
/** The most flags one check records. The last entry says how many more there were. */
|
|
6
|
+
export const MAX_FLAGS = 100;
|
|
7
|
+
const SKIP_MARKER = /\.skip\(|\bxit\(|@pytest\.mark\.skip/;
|
|
8
|
+
const PLACEHOLDER_MARKER = /\b(?:TODO|FIXME)\b/;
|
|
9
|
+
const NOT_IMPLEMENTED = /not implemented/i;
|
|
10
|
+
const FAKE_NAME = /mock|stub|fake/i;
|
|
11
|
+
const TEST_DIRECTORY = /(?:^|\/)(?:tests?|__tests__)\//;
|
|
12
|
+
// Only code in a test folder counts, so a fixture or snapshot there (`test/fixtures/sample.json`) is not a test file.
|
|
13
|
+
const CODE_FILE = /\.(?:[cm]?[jt]sx?|py|go|rs|java|kt|rb|php|cs|swift|scala|c|cc|cpp|h|hpp)$/;
|
|
14
|
+
const TEST_NAME = /(?:\.(?:test|spec)\.[a-z]+|_test\.[a-z]+)$|^test_.*\.py$/;
|
|
15
|
+
// An assertion line: `expect(`, `assert(`, `assert.x(`, `assertEqual(`, `self.assertX(` or a Python `assert` statement.
|
|
16
|
+
const ASSERTION = /\bexpect\s*\(|\bassert\w*\s*[.(]|^\s*assert\s/;
|
|
17
|
+
const LINE_LIMIT = 80;
|
|
18
|
+
const baseName = (path) => path.slice(path.lastIndexOf("/") + 1);
|
|
19
|
+
/**
|
|
20
|
+
* Whether a path looks like a test file: a code file (by its extension, such as `.ts`, `.py` or `.go`) in a `test`,
|
|
21
|
+
* `tests` or `__tests__` folder, or a file named as a test anywhere. A data file in a test folder is not one.
|
|
22
|
+
*/
|
|
23
|
+
export function isTestFile(path) {
|
|
24
|
+
return (TEST_DIRECTORY.test(path) && CODE_FILE.test(path)) || TEST_NAME.test(baseName(path));
|
|
25
|
+
}
|
|
26
|
+
const clip = (line) => {
|
|
27
|
+
const text = line.trim();
|
|
28
|
+
return text.length > LINE_LIMIT ? `${text.slice(0, LINE_LIMIT)}...` : text;
|
|
29
|
+
};
|
|
30
|
+
/** The lines of `after` that `before` lacks, counting repeats, so a moved line is not an added one. */
|
|
31
|
+
function addedLines(before, after) {
|
|
32
|
+
const seen = new Map();
|
|
33
|
+
for (const line of before.split("\n"))
|
|
34
|
+
seen.set(line, (seen.get(line) ?? 0) + 1);
|
|
35
|
+
const added = [];
|
|
36
|
+
for (const line of after.split("\n")) {
|
|
37
|
+
const left = seen.get(line) ?? 0;
|
|
38
|
+
if (left > 0)
|
|
39
|
+
seen.set(line, left - 1);
|
|
40
|
+
else
|
|
41
|
+
added.push(line);
|
|
42
|
+
}
|
|
43
|
+
return added;
|
|
44
|
+
}
|
|
45
|
+
const countAssertions = (text) => text.split("\n").filter((line) => ASSERTION.test(line)).length;
|
|
46
|
+
/** The flags for the lines a modified or new file added: skip markers and placeholders. */
|
|
47
|
+
function lineFlags(path, before, after) {
|
|
48
|
+
const flags = [];
|
|
49
|
+
for (const line of addedLines(before ?? "", after)) {
|
|
50
|
+
if (SKIP_MARKER.test(line))
|
|
51
|
+
flags.push(`skip marker added: ${path}: ${clip(line)}`);
|
|
52
|
+
if (PLACEHOLDER_MARKER.test(line) || NOT_IMPLEMENTED.test(line))
|
|
53
|
+
flags.push(`placeholder added: ${path}: ${clip(line)}`);
|
|
54
|
+
}
|
|
55
|
+
return flags;
|
|
56
|
+
}
|
|
57
|
+
function assertionFlags(path, before, after) {
|
|
58
|
+
if (!isTestFile(path))
|
|
59
|
+
return [];
|
|
60
|
+
const was = countAssertions(before);
|
|
61
|
+
const now = countAssertions(after);
|
|
62
|
+
return now < was ? [`fewer assertion lines in ${path}: ${was} before, ${now} after`] : [];
|
|
63
|
+
}
|
|
64
|
+
function flagsForFile(path, before, after) {
|
|
65
|
+
if (after === undefined)
|
|
66
|
+
return before !== undefined && isTestFile(path) ? [`test file deleted: ${path}`] : [];
|
|
67
|
+
const flags = before === undefined && FAKE_NAME.test(baseName(path)) ? [`mock, stub or fake file added: ${path}`] : [];
|
|
68
|
+
flags.push(...lineFlags(path, before, after));
|
|
69
|
+
if (before !== undefined)
|
|
70
|
+
flags.push(...assertionFlags(path, before, after));
|
|
71
|
+
return flags;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Compares the workspace at the start of a turn with the end, and returns one sentence for each pattern the plan names:
|
|
75
|
+
* an added skip marker (`.skip(`, `xit(` or `@pytest.mark.skip`), an added placeholder (`TODO` or `FIXME` in capitals, or
|
|
76
|
+
* "not implemented" in any case), a new file whose name has `mock`, `stub` or `fake` in it, a deleted test file, and a test file with
|
|
77
|
+
* fewer assertion lines than before. Paths come out in sorted order, so the same change gives the same flags.
|
|
78
|
+
*/
|
|
79
|
+
export function flagChanges(before, after) {
|
|
80
|
+
const ignored = (path) => before.skipped.has(path) || after.skipped.has(path);
|
|
81
|
+
const paths = [...new Set([...before.files.keys(), ...after.files.keys()])].sort().filter((path) => !ignored(path));
|
|
82
|
+
const flags = paths.flatMap((path) => flagsForFile(path, before.files.get(path), after.files.get(path)));
|
|
83
|
+
if (flags.length <= MAX_FLAGS)
|
|
84
|
+
return flags;
|
|
85
|
+
return [...flags.slice(0, MAX_FLAGS), `${flags.length - MAX_FLAGS} more flags not listed`];
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=flags.js.map
|