@haystackeditor/cli 0.15.27 → 0.15.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -20
- package/dist/assets/hooks/agent-context/detect.ts +7 -7
- package/dist/assets/hooks/agent-context/parsers/claude.ts +1 -1
- package/dist/assets/hooks/scripts/commit-msg.sh +1 -3
- package/dist/assets/hooks/scripts/post-commit.sh +1 -3
- package/dist/assets/hooks/scripts/pre-push.sh +1 -4
- package/dist/assets/hooks/scripts/prepare-commit-msg.sh +1 -2
- package/dist/assets/telemetry/browser-runtime.js +668 -43
- package/dist/assets/telemetry/runtime.cjs +62 -68
- package/dist/commands/design-verify.d.ts +11 -3
- package/dist/commands/design-verify.js +147 -4
- package/dist/commands/hooks.d.ts +1 -5
- package/dist/commands/hooks.js +10 -112
- package/dist/commands/install-session-hooks.d.ts +2 -2
- package/dist/commands/install-session-hooks.js +71 -21
- package/dist/commands/mcp.js +1 -34
- package/dist/commands/pr.js +2 -7
- package/dist/commands/setup.d.ts +1 -2
- package/dist/commands/setup.js +18 -272
- package/dist/commands/submit.js +1 -2
- package/dist/commands/verify-hosted-mcp.js +12 -2
- package/dist/commands/verify-hosted-reproducibility.d.ts +1 -0
- package/dist/commands/verify-hosted-reproducibility.js +4 -1
- package/dist/commands/verify-hosted.d.ts +20 -0
- package/dist/commands/verify-hosted.js +119 -11
- package/dist/commands/verify-precompute.d.ts +19 -0
- package/dist/commands/verify-precompute.js +328 -0
- package/dist/index.js +45 -44
- package/dist/schema.d.ts +1 -2
- package/dist/schema.js +1 -2
- package/dist/triage/prompts.d.ts +0 -7
- package/dist/triage/prompts.js +0 -145
- package/dist/triage/runner.js +3 -22
- package/dist/triage/types.d.ts +1 -8
- package/dist/types.d.ts +8 -8
- package/dist/types.js +1 -1
- package/dist/utils/design-verifier-api.d.ts +71 -0
- package/dist/utils/design-verifier-api.js +96 -0
- package/dist/utils/design-verifier-result.d.ts +7 -2
- package/dist/utils/design-verifier-result.js +67 -21
- package/dist/utils/git.d.ts +5 -1
- package/dist/utils/git.js +19 -9
- package/dist/utils/github-api.js +11 -14
- package/dist/utils/hooks.d.ts +1 -15
- package/dist/utils/hooks.js +1 -127
- package/dist/utils/telemetry.d.ts +2 -0
- package/dist/utils/telemetry.js +81 -8
- package/package.json +2 -1
- package/schemas/cloud-verifier.v1.json +69 -3
- package/schemas/{pr.v2.json → pr.v3.json} +3 -12
- package/dist/commands/traces.d.ts +0 -48
- package/dist/commands/traces.js +0 -92
- package/dist/triage/traces.d.ts +0 -20
- package/dist/triage/traces.js +0 -311
- package/schemas/traces.v1.json +0 -53
package/dist/index.js
CHANGED
|
@@ -36,7 +36,7 @@ import { cloudVerifierDataStoreDriftCommand } from './commands/cloud-verifier-da
|
|
|
36
36
|
import { cloudVerifierBehaviorsValidateCommand } from './commands/cloud-verifier-behaviors.js';
|
|
37
37
|
import { prepareUniverseReviewCommand } from './commands/prepare-universe-review.js';
|
|
38
38
|
import { scaffoldProvisionalUniverseCommand } from './commands/scaffold-provisional-universe.js';
|
|
39
|
-
import { hooksInstall, hooksStatus
|
|
39
|
+
import { hooksInstall, hooksStatus } from './commands/hooks.js';
|
|
40
40
|
import { submitCommand } from './commands/submit.js';
|
|
41
41
|
import { installSessionHooks, sessionHooksStatus } from './commands/install-session-hooks.js';
|
|
42
42
|
import { listPolicies, addPolicy, removePolicy, initPolicies, addInstruction } from './commands/policy.js';
|
|
@@ -49,7 +49,6 @@ import { prStatusCommand } from './commands/pr-status.js';
|
|
|
49
49
|
import { prReadCommand } from './commands/pr.js';
|
|
50
50
|
import { inboxListCommand } from './commands/inbox.js';
|
|
51
51
|
import { askHaystackCommand } from './commands/ask.js';
|
|
52
|
-
import { tracesGetCommand, tracesListCommand } from './commands/traces.js';
|
|
53
52
|
import { telemetryInstrumentCommand } from './commands/telemetry.js';
|
|
54
53
|
import { setupCommand } from './commands/setup.js';
|
|
55
54
|
import { schemaCommand, listSchemas } from './commands/schema-cmd.js';
|
|
@@ -134,7 +133,6 @@ program
|
|
|
134
133
|
// Defining --no-auto-merge also accepts --auto-merge; default is "ask" unless
|
|
135
134
|
// one is explicitly passed (detected via getOptionValueSource below).
|
|
136
135
|
.option('--no-auto-merge', 'Set auto-merge in the written config (--auto-merge / --no-auto-merge; omit to be asked)')
|
|
137
|
-
.option('--skip-entire', 'Skip the Entire CLI / git-hooks (session tracking) step')
|
|
138
136
|
.option('--answers <file>', 'JSON file of {questionId: value} pre-supplied answers')
|
|
139
137
|
.addHelpText('after', `
|
|
140
138
|
Steps: verify GitHub App → select repos → scan (rules/policies/instructions) →
|
|
@@ -143,7 +141,7 @@ review → write .haystack/pr-rules.yml + review-policy.md + .haystack.json.
|
|
|
143
141
|
Three ways to run:
|
|
144
142
|
• Interactive (default): prompts in your terminal.
|
|
145
143
|
• Autonomous/CI: supply every decision via flags, e.g.
|
|
146
|
-
haystack setup --repo owner/name --yes --no-auto-merge
|
|
144
|
+
haystack setup --repo owner/name --yes --no-auto-merge
|
|
147
145
|
• Coding agent: --json speaks NDJSON on stdout and reads answers on stdin, so an
|
|
148
146
|
agent can auto-answer or relay questions to the user. Pre-supplied answers
|
|
149
147
|
(flags / --answers) skip the matching question.
|
|
@@ -165,7 +163,6 @@ Examples:
|
|
|
165
163
|
repos: options.repo,
|
|
166
164
|
yes: options.yes,
|
|
167
165
|
autoMerge: autoMergeExplicit ? options.autoMerge : undefined,
|
|
168
|
-
skipEntire: options.skipEntire,
|
|
169
166
|
answersFile: options.answers,
|
|
170
167
|
});
|
|
171
168
|
});
|
|
@@ -195,7 +192,7 @@ The local control plane must be running. From the Haystack repository:
|
|
|
195
192
|
pnpm dev
|
|
196
193
|
|
|
197
194
|
Examples:
|
|
198
|
-
haystack verify hosted start owner/repo --base <sha> --head <sha>
|
|
195
|
+
haystack verify hosted start owner/repo --base <sha> --head <sha> --intent-file <path>
|
|
199
196
|
haystack verify posthog-js-478 # start a run, then:
|
|
200
197
|
haystack verify reproducibility posthog-js-478 --runs 3
|
|
201
198
|
haystack verify watch # block until it finishes (exit 1 = tests failed)
|
|
@@ -213,6 +210,18 @@ Examples:
|
|
|
213
210
|
`)
|
|
214
211
|
.action((specimen, options) => runPublicCommand(() => verifyCommand(specimen, options), options.json));
|
|
215
212
|
const verifyRunArg = 'Run id, chapter id (newest run of that chapter), or "latest" (default)';
|
|
213
|
+
verify
|
|
214
|
+
.command('precompute')
|
|
215
|
+
.description('Precompute reusable verification inputs for the current working tree')
|
|
216
|
+
.option('--hook', 'Run silently as a best-effort repository hook')
|
|
217
|
+
.action(async (options) => {
|
|
218
|
+
await runPublicCommand(async () => {
|
|
219
|
+
const { verifyPrecomputeCommand } = await import('./commands/verify-precompute.js');
|
|
220
|
+
await verifyPrecomputeCommand(options);
|
|
221
|
+
});
|
|
222
|
+
if (options.hook)
|
|
223
|
+
process.exitCode = 0;
|
|
224
|
+
});
|
|
216
225
|
const hostedVerify = verify
|
|
217
226
|
.command('hosted')
|
|
218
227
|
.description('Run the production Cloud Verifier against exact GitHub commits');
|
|
@@ -226,6 +235,7 @@ hostedVerify
|
|
|
226
235
|
// validation below accepts the inherited values and still fails closed.
|
|
227
236
|
.option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
|
|
228
237
|
.option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
|
|
238
|
+
.requiredOption('--intent-file <path>', 'Agent-supplied JSON with exactly problem, goal, and intended_outcomes')
|
|
229
239
|
.option('--idempotency-key <key>', 'Stable retry key (default: deterministic from repository and commits)')
|
|
230
240
|
.option('--replay-plan-from <run-id>', 'Reuse the exact validated risk plan from a completed owned run')
|
|
231
241
|
.option('--account <login>', 'Use a specific saved Haystack account')
|
|
@@ -241,9 +251,9 @@ fresh production execution of the same commit pair.
|
|
|
241
251
|
a schema-9 terminal receipt with replay provenance.
|
|
242
252
|
|
|
243
253
|
Examples:
|
|
244
|
-
haystack verify hosted start owner/repo --base <sha> --head <sha>
|
|
245
|
-
haystack verify hosted start owner/repo --base <sha> --head <sha> --replay-plan-from cv_<48 lowercase hex characters>
|
|
246
|
-
haystack verify hosted start owner/repo --base <sha> --head <sha> --no-wait --json
|
|
254
|
+
haystack verify hosted start owner/repo --base <sha> --head <sha> --intent-file <path>
|
|
255
|
+
haystack verify hosted start owner/repo --base <sha> --head <sha> --intent-file <path> --replay-plan-from cv_<48 lowercase hex characters>
|
|
256
|
+
haystack verify hosted start owner/repo --base <sha> --head <sha> --intent-file <path> --no-wait --json
|
|
247
257
|
`)
|
|
248
258
|
.action(async (repository, _options, cmd) => {
|
|
249
259
|
const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
|
|
@@ -274,6 +284,7 @@ hostedVerify
|
|
|
274
284
|
.argument('<repository>', 'Exact GitHub owner/repository name')
|
|
275
285
|
.option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
|
|
276
286
|
.option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
|
|
287
|
+
.requiredOption('--intent-file <path>', 'Agent-supplied JSON reused for every hosted run in the series')
|
|
277
288
|
.option('--runs <n>', 'Number of sequential production runs (default 3; min 2, max 10)')
|
|
278
289
|
.option('--series-key <key>', 'Stable series key for resumable idempotent retries')
|
|
279
290
|
.option('--fresh-plans', 'Run the planner independently for every run instead of replaying the first plan')
|
|
@@ -293,8 +304,8 @@ provenance, and no inconclusive cells. Use this mode to detect planner drift;
|
|
|
293
304
|
use the default mode to isolate executor or assessment drift.
|
|
294
305
|
|
|
295
306
|
Example:
|
|
296
|
-
haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --runs 3
|
|
297
|
-
haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --runs 3 --fresh-plans
|
|
307
|
+
haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --intent-file <path> --runs 3
|
|
308
|
+
haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --intent-file <path> --runs 3 --fresh-plans
|
|
298
309
|
`)
|
|
299
310
|
.action(async (repository, _options, cmd) => {
|
|
300
311
|
const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
|
|
@@ -655,7 +666,7 @@ program
|
|
|
655
666
|
.addHelpText('after', `
|
|
656
667
|
This command is designed for AI coding agents to submit PRs.
|
|
657
668
|
|
|
658
|
-
1. Runs pre-PR triage (code review
|
|
669
|
+
1. Runs pre-PR triage (code review and rules) via sub-agents
|
|
659
670
|
2. Pushes the current branch to origin
|
|
660
671
|
3. Creates a pull request on GitHub
|
|
661
672
|
4. Waits for Haystack analysis results (triggered via GitHub App webhook)
|
|
@@ -664,15 +675,14 @@ Pre-PR Triage:
|
|
|
664
675
|
Before creating the PR, haystack spawns parallel sub-agents to check for:
|
|
665
676
|
• Code review bugs (logic errors, null crashes, security issues)
|
|
666
677
|
• Rule violations (from .haystack/pr-rules.yml)
|
|
667
|
-
• Instruction drift (AI agent deviations from user instructions)
|
|
668
678
|
|
|
669
679
|
Use --force to skip triage entirely.
|
|
670
680
|
Use --no-wait to skip waiting for analysis results.
|
|
671
681
|
|
|
672
682
|
Triage budgets:
|
|
673
683
|
• --max-turns <n> Raise/lower the per-checker tool-use turn cap.
|
|
674
|
-
Defaults: code-review 8, rules-validator 10
|
|
675
|
-
|
|
684
|
+
Defaults: code-review 8, rules-validator 10.
|
|
685
|
+
Applies the same N to both.
|
|
676
686
|
• --triage-timeout <sec> Raise/lower the per-checker wall-clock timeout
|
|
677
687
|
(default: 180s).
|
|
678
688
|
|
|
@@ -790,7 +800,7 @@ inbox
|
|
|
790
800
|
const prProgram = program.command('pr').description('Inspect one pull request');
|
|
791
801
|
prProgram
|
|
792
802
|
.command('get <ref>')
|
|
793
|
-
.description('Get triage
|
|
803
|
+
.description('Get triage and merge blockers')
|
|
794
804
|
.option('--json', 'Output as JSON')
|
|
795
805
|
.action((ref, options) => runPublicCommand(() => prReadCommand(ref, options), options.json));
|
|
796
806
|
program
|
|
@@ -799,19 +809,6 @@ program
|
|
|
799
809
|
.option('--json', 'Output the answer and every consulted customer-facing source as JSON')
|
|
800
810
|
.option('--session <id>', 'Continue a previous Ask Haystack session')
|
|
801
811
|
.action((ref, question, options) => runPublicCommand(() => askHaystackCommand(ref, question, options), options.json));
|
|
802
|
-
const traces = program.command('traces').description('Inspect customer-owned Entire traces');
|
|
803
|
-
traces
|
|
804
|
-
.command('list <ref>')
|
|
805
|
-
.description('List retained checkpoints for a pull request')
|
|
806
|
-
.option('--json', 'Output as JSON')
|
|
807
|
-
.action((ref, options) => runPublicCommand(() => tracesListCommand(ref, options), options.json));
|
|
808
|
-
traces
|
|
809
|
-
.command('get <ref> <checkpoint>')
|
|
810
|
-
.description('Get retained transcript chunks for one checkpoint')
|
|
811
|
-
.option('--cursor <cursor>', 'Zero-based chunk cursor')
|
|
812
|
-
.option('--limit <count>', 'Chunks per page', '20')
|
|
813
|
-
.option('--json', 'Output as JSON')
|
|
814
|
-
.action((ref, checkpoint, options) => runPublicCommand(() => tracesGetCommand(ref, checkpoint, options), options.json));
|
|
815
812
|
const telemetry = program.command('telemetry').description('Add privacy-safe production telemetry without an application SDK');
|
|
816
813
|
telemetry
|
|
817
814
|
.command('instrument <dist>')
|
|
@@ -870,15 +867,27 @@ program
|
|
|
870
867
|
.option('--run <run-id>', 'Resume and wait for an existing exact run id')
|
|
871
868
|
.option('--poll-interval <seconds>', 'Status polling interval', '3')
|
|
872
869
|
.option('--timeout <seconds>', 'Maximum time to wait; the server run continues after timeout', '7200')
|
|
870
|
+
.option('--case <case-id>', 'Select one exact generated case for universe exploration')
|
|
871
|
+
.option('--universe <role>', 'Select control-before, before, or after')
|
|
872
|
+
.option('--replay', 'Replay the registered case command in the selected universe')
|
|
873
|
+
.option('--exec-json <argv>', 'Run a JSON argv array in the selected universe')
|
|
874
|
+
.option('--cwd <path>', 'Repository-relative working directory for --exec-json')
|
|
875
|
+
.option('--command-timeout <seconds>', 'Maximum selected-universe command runtime')
|
|
876
|
+
.option('--action <action-id>', 'Resume one exact universe action')
|
|
873
877
|
.addHelpText('after', `
|
|
874
878
|
Executes the change instead of reading it: infers intent from the diff
|
|
875
|
-
alone, designs pre-registered behavioral test cases, runs
|
|
876
|
-
|
|
877
|
-
|
|
879
|
+
alone, designs pre-registered behavioral test cases, runs control-before,
|
|
880
|
+
before, and after in isolated sandboxes, and reports anomaly candidates
|
|
881
|
+
outside the registered intent.
|
|
878
882
|
|
|
879
883
|
The command waits for a behavioral verdict by default. Use --no-wait to
|
|
880
884
|
return the run id immediately, or --run <run-id> to resume waiting.
|
|
881
885
|
|
|
886
|
+
After a completed run, inspect or replay one retained case universe without
|
|
887
|
+
receiving provider credentials:
|
|
888
|
+
haystack design-verify owner/repo#123 --run RUN --case case-00001 --universe before --replay
|
|
889
|
+
haystack design-verify owner/repo#123 --run RUN --case case-00001 --universe after --exec-json '["rg","TODO"]'
|
|
890
|
+
|
|
882
891
|
PR identifier formats:
|
|
883
892
|
123 PR number (uses current repo)
|
|
884
893
|
owner/repo#123 Fully qualified
|
|
@@ -1188,42 +1197,34 @@ const hooks = program
|
|
|
1188
1197
|
.description('Manage git hooks for AI agent quality checks');
|
|
1189
1198
|
hooks
|
|
1190
1199
|
.command('install')
|
|
1191
|
-
.description('Install Haystack git hooks
|
|
1192
|
-
.option('--version <version>', 'Entire CLI version (default: pinned)')
|
|
1200
|
+
.description('Install Haystack git hooks')
|
|
1193
1201
|
.option('-f, --force', 'Overwrite existing hooks')
|
|
1194
|
-
.option('--skip-entire', 'Skip Entire binary download')
|
|
1195
1202
|
.addHelpText('after', `
|
|
1196
1203
|
This installs:
|
|
1197
1204
|
• Git hooks for AI agent quality checks (pre-commit, commit-msg, etc.)
|
|
1198
1205
|
• Agent context detector (identifies AI agent sessions)
|
|
1199
1206
|
• Truncation checker (prevents code truncation by LLMs)
|
|
1200
|
-
• Entire CLI binary for session tracking (powered by https://entire.dev)
|
|
1201
1207
|
|
|
1202
1208
|
Hooks are installed to <repo>/hooks/ and git is configured to use them.
|
|
1203
1209
|
|
|
1204
1210
|
Examples:
|
|
1205
|
-
haystack hooks install # Install with pinned Entire version
|
|
1206
1211
|
haystack hooks install --force # Overwrite existing hooks
|
|
1207
|
-
haystack hooks install --skip-entire # Only install Haystack hooks
|
|
1208
1212
|
`)
|
|
1209
1213
|
.action(hooksInstall);
|
|
1210
1214
|
hooks
|
|
1211
1215
|
.command('status')
|
|
1212
1216
|
.description('Check hooks installation status')
|
|
1213
1217
|
.action(hooksStatus);
|
|
1214
|
-
hooks
|
|
1215
|
-
.command('update')
|
|
1216
|
-
.description('Update Entire CLI to the latest version')
|
|
1217
|
-
.action(hooksUpdate);
|
|
1218
1218
|
hooks
|
|
1219
1219
|
.command('install-session')
|
|
1220
|
-
.description('Install session-start hooks for coding CLIs')
|
|
1220
|
+
.description('Install session-start and Stop hooks for coding CLIs')
|
|
1221
1221
|
.option('--cli <name>', 'Target CLI: claude, codex, gemini, or all')
|
|
1222
1222
|
.addHelpText('after', `
|
|
1223
1223
|
Configures your coding CLI to run \`haystack triage --hook\` on session start.
|
|
1224
1224
|
This shows pending PR analysis results when you open a new terminal session.
|
|
1225
|
+
Claude Code also runs \`haystack verify precompute --hook\` when a session stops.
|
|
1225
1226
|
|
|
1226
|
-
Claude Code: Native SessionStart
|
|
1227
|
+
Claude Code: Native SessionStart and Stop hooks (.claude/settings.json)
|
|
1227
1228
|
Codex CLI: AGENTS.md instructions
|
|
1228
1229
|
Gemini CLI: GEMINI.md instructions
|
|
1229
1230
|
|
package/dist/schema.d.ts
CHANGED
|
@@ -12,11 +12,10 @@
|
|
|
12
12
|
export declare const SCHEMA_VERSIONS: {
|
|
13
13
|
readonly triage: "2.0.0";
|
|
14
14
|
readonly setup: "1.0.0";
|
|
15
|
-
readonly pr: "
|
|
15
|
+
readonly pr: "3.0.0";
|
|
16
16
|
readonly 'pr-status': "1.0.0";
|
|
17
17
|
readonly inbox: "1.0.0";
|
|
18
18
|
readonly ask: "1.0.0";
|
|
19
|
-
readonly traces: "1.0.0";
|
|
20
19
|
readonly submit: "1.0.0";
|
|
21
20
|
readonly action: "1.0.0";
|
|
22
21
|
readonly 'cloud-verifier': "1.0.0";
|
package/dist/schema.js
CHANGED
|
@@ -12,11 +12,10 @@
|
|
|
12
12
|
export const SCHEMA_VERSIONS = {
|
|
13
13
|
triage: '2.0.0',
|
|
14
14
|
setup: '1.0.0',
|
|
15
|
-
pr: '
|
|
15
|
+
pr: '3.0.0',
|
|
16
16
|
'pr-status': '1.0.0',
|
|
17
17
|
inbox: '1.0.0',
|
|
18
18
|
ask: '1.0.0',
|
|
19
|
-
traces: '1.0.0',
|
|
20
19
|
submit: '1.0.0',
|
|
21
20
|
action: '1.0.0',
|
|
22
21
|
'cloud-verifier': '1.0.0',
|
package/dist/triage/prompts.d.ts
CHANGED
|
@@ -22,10 +22,3 @@ export declare function buildRulesValidatorPrompt(baseBranch: string, rulesYaml:
|
|
|
22
22
|
filename: string;
|
|
23
23
|
content: string;
|
|
24
24
|
}[], precomputedDiff?: string | null): string | null;
|
|
25
|
-
/**
|
|
26
|
-
* Build the intent drift prompt.
|
|
27
|
-
* Only runs if relevant trace files exist.
|
|
28
|
-
*
|
|
29
|
-
* @returns The prompt string, or null if no trace files provided.
|
|
30
|
-
*/
|
|
31
|
-
export declare function buildIntentDriftPrompt(baseBranch: string, traceFiles: string[], outputPath: string, maxTurns: number, timeoutMs: number, precomputedDiff?: string | null): string | null;
|
package/dist/triage/prompts.js
CHANGED
|
@@ -37,21 +37,6 @@ const RULES_VALIDATOR_SCHEMA = `{
|
|
|
37
37
|
"rulesChecked": 3,
|
|
38
38
|
"passed": true
|
|
39
39
|
}`;
|
|
40
|
-
const INTENT_DRIFT_SCHEMA = `{
|
|
41
|
-
"checker": "intent-drift",
|
|
42
|
-
"issues": [
|
|
43
|
-
{
|
|
44
|
-
"file": "relative/path/to/file.ts",
|
|
45
|
-
"line": 42,
|
|
46
|
-
"severity": "error | warning | info",
|
|
47
|
-
"message": "Description of the drift or incomplete fulfillment",
|
|
48
|
-
"pattern": "intent_drift | incomplete_fulfillment | scope_creep | unspecified_decision | ignored_correction | weakened_posture | claimed_but_not_done"
|
|
49
|
-
}
|
|
50
|
-
],
|
|
51
|
-
"summary": "Brief 1-sentence summary",
|
|
52
|
-
"sessionsChecked": 2,
|
|
53
|
-
"passed": true
|
|
54
|
-
}`;
|
|
55
40
|
// ============================================================================
|
|
56
41
|
// Prompt builders
|
|
57
42
|
// ============================================================================
|
|
@@ -235,133 +220,3 @@ ${RULES_VALIDATOR_SCHEMA}
|
|
|
235
220
|
- Set \`passed\` to \`false\` if any violations with severity "error" were found
|
|
236
221
|
- You MUST write the result file even if no violations are found`;
|
|
237
222
|
}
|
|
238
|
-
/**
|
|
239
|
-
* Build the intent drift prompt.
|
|
240
|
-
* Only runs if relevant trace files exist.
|
|
241
|
-
*
|
|
242
|
-
* @returns The prompt string, or null if no trace files provided.
|
|
243
|
-
*/
|
|
244
|
-
export function buildIntentDriftPrompt(baseBranch, traceFiles, outputPath, maxTurns, timeoutMs, precomputedDiff) {
|
|
245
|
-
if (traceFiles.length === 0)
|
|
246
|
-
return null;
|
|
247
|
-
const traceFileList = traceFiles.map(f => `- \`${f}\``).join('\n');
|
|
248
|
-
return `You are an intent drift detector. Your job is to check whether an AI coding agent faithfully implemented what the user asked for.
|
|
249
|
-
|
|
250
|
-
${buildTimeBudgetHeader(maxTurns, timeoutMs)}## Context
|
|
251
|
-
|
|
252
|
-
This PR was created by an AI coding agent. The agent's session transcripts (traces) are stored locally. You will compare what the user asked the agent to do against what was actually implemented in the diff.
|
|
253
|
-
|
|
254
|
-
## Instructions
|
|
255
|
-
|
|
256
|
-
1. Read each of these files (they contain the user's messages extracted from the agent session, one per section):
|
|
257
|
-
${traceFileList}
|
|
258
|
-
|
|
259
|
-
2. From each file, identify:
|
|
260
|
-
- The user's original instruction/prompt (the first message)
|
|
261
|
-
- Any follow-up instructions or corrections from the user
|
|
262
|
-
- The final effective scope after later corrections or reframes
|
|
263
|
-
|
|
264
|
-
3. ${precomputedDiff ? 'Review the diff below' : `Run \`git diff ${baseBranch}...HEAD\``} to see what was actually implemented.
|
|
265
|
-
|
|
266
|
-
4. Compare the user's intent against the actual implementation. Look for:
|
|
267
|
-
|
|
268
|
-
### Intent Drift
|
|
269
|
-
The agent implemented something DIFFERENT from what was asked:
|
|
270
|
-
- User says "delay as long as possible" → agent uses a fixed 30s timeout
|
|
271
|
-
- User says "only when X" → agent does it unconditionally
|
|
272
|
-
- User says "use library A" → agent uses library B
|
|
273
|
-
- User says "lazy load" → agent loads eagerly
|
|
274
|
-
|
|
275
|
-
### Incomplete Fulfillment
|
|
276
|
-
The agent didn't finish everything that was asked:
|
|
277
|
-
- User requested 3 things, agent only did 2
|
|
278
|
-
- User requested a general/systematic guardrail or end-to-end behavior, but the diff only handles one known instance, one special case, or one side of the required wiring
|
|
279
|
-
- User initially mentioned a known example, then clarified "not the specific case" / "the general class"; the diff still implements only the known example or category-specific check
|
|
280
|
-
- Interface fields declared but never populated
|
|
281
|
-
- A new keyed capability, event, route, config value, enum variant, or serialized field is referenced on one side of a boundary but the required registry, producer, consumer, schema, handler, persistence path, or delivery surface is missing
|
|
282
|
-
- A change appears to work through local/dev/test defaults, mocks, or overrides, but the production wiring path needed to deliver the requested behavior was not updated
|
|
283
|
-
- Functions stubbed with TODO/placeholder comments
|
|
284
|
-
- Agent said "I'll skip X for now" for something the user explicitly requested
|
|
285
|
-
- Tests not written when user asked for tests
|
|
286
|
-
|
|
287
|
-
### Scope Creep
|
|
288
|
-
The agent added functionality the user never asked for:
|
|
289
|
-
- User asks to fix a bug → agent also adds a cooldown, retry logic, or caching layer
|
|
290
|
-
- User asks to investigate → agent proactively "fixes" things beyond what was discussed
|
|
291
|
-
- User asks for one feature → agent bundles in extra features "while we're at it"
|
|
292
|
-
- Any new mechanism not traceable to a user instruction
|
|
293
|
-
|
|
294
|
-
### Unspecified Decision
|
|
295
|
-
The user authorized the task's GOAL, but the agent made a specific decision in HOW it carried it out that shapes the program's end output, externally observable behavior, or the data end-users/callers see — and the user never specified or approved that particular decision, and a reasonable user would plausibly want a say in it. Examples (general — do NOT pattern-match on specific keywords):
|
|
296
|
-
- Silently dropping, capping, sampling, or transforming data that flows to the output
|
|
297
|
-
- Picking a default that determines what end-users see
|
|
298
|
-
- Resolving an ambiguous requirement one way when other materially different behaviors were equally valid
|
|
299
|
-
- Choosing a fixed value where the choice changes results
|
|
300
|
-
Do NOT flag internal implementation choices with no observable effect (naming, file layout, helper structure), decisions forced by correctness, or cosmetic defaults a user would not care about. When unsure whether a user would care, do not flag it — reserve this for decisions with real, user-visible ramifications. The difference from scope creep: scope creep is an unrequested *flow*; an unspecified decision is an unrequested, output-shaping choice *inside a requested flow*.
|
|
301
|
-
|
|
302
|
-
### Ignored Correction
|
|
303
|
-
The user gave an explicit correction or redirection and the final code does NOT honor it:
|
|
304
|
-
- User said "don't use a global / use X instead / that's racy, do Y" and the agent shipped the thing it was told not to
|
|
305
|
-
- User said the target is the general class rather than the specific instance, but the agent still shipped only the instance-specific implementation
|
|
306
|
-
- Agent applied the correction, then quietly reverted it in a later step
|
|
307
|
-
- A general later "looks good" does NOT cancel a specific earlier correction
|
|
308
|
-
This is high severity by default: the user actively steered and was overridden.
|
|
309
|
-
|
|
310
|
-
### Weakened Posture
|
|
311
|
-
In service of an authorized goal, the agent relaxed or removed a security, safety, or correctness guard the user never asked to weaken:
|
|
312
|
-
- Loosened an auth/permission/validation check, or broadened access (CORS, scopes)
|
|
313
|
-
- Swallowed or silenced an error, removed an assertion/guard, hardcoded a bypass or credential
|
|
314
|
-
- Disabled a test (.skip / xit), suppressed a type or lint check (any, @ts-ignore, disable comments)
|
|
315
|
-
- Lowered a threshold/timeout that existed as a safeguard
|
|
316
|
-
- Relaxed a CI/pipeline security or quality gate (a workflow check, audit, or validation policy) — especially so that the agent's OWN change would pass that gate
|
|
317
|
-
High severity by default. (If the user explicitly asked to remove the guard, it is not a finding.)
|
|
318
|
-
Tie-breaker for gate relaxations: when the agent's own work was failing a CI gate and the agent changed the gate so it would pass, that is weakened_posture even when the old gate looks buggy or overly strict, and even when a delegation ("shepherd this", "get it merged") covered the goal — whether the relaxation was right is exactly the judgment this flag hands to a human.
|
|
319
|
-
Wording for weakened_posture findings: describe the decision reflectively, lede first, in plain English — sentence 1 says who weakened which protection and why ("To get this change through CI, the agent loosened the check that was blocking it"), sentence 2 gives the concrete change, then hand the call to a human. If a delegation covered the surrounding goal, say so; never claim "no user directive" when one exists. This is not an accusation of violating intent — it is a consequential decision a human should confirm.
|
|
320
|
-
|
|
321
|
-
### Claimed But Not Done
|
|
322
|
-
The agent told the user it completed work that the diff does not actually contain, or contains only in a materially weaker form:
|
|
323
|
-
- "I added error handling for timeouts" but no timeout handling exists in the diff
|
|
324
|
-
- "Added tests for the edge case" but no test assertions exercise it
|
|
325
|
-
- "Made the limit configurable" but the value is still hardcoded
|
|
326
|
-
Compare each concrete claim the agent made to the user against what the diff actually shows. Only flag when the diff clearly contradicts or fails to support the claim; when unsure, do not flag.
|
|
327
|
-
|
|
328
|
-
## Red flags to search for in the diff
|
|
329
|
-
|
|
330
|
-
- Fixed/hardcoded values where dynamic behavior was requested
|
|
331
|
-
- TODO, FIXME, placeholder, stub comments in new code
|
|
332
|
-
- Empty function bodies or early returns
|
|
333
|
-
- Interface fields that are declared but never assigned anywhere
|
|
334
|
-
- Feature-specific or instance-specific checks added after the user asked for a general guardrail against a broader failure mode
|
|
335
|
-
- Earlier narrow examples treated as the whole task even though later user messages broadened or generalized the requested scope
|
|
336
|
-
- New string-keyed names, enum variants, serialized fields, action types, routes, events, or config keys that have no matching entry in the surrounding registry, allow-list, schema, handler, producer, consumer, or template path
|
|
337
|
-
- Comments or neighboring code saying "must also register/list/wire this" where the diff updated only the reference side
|
|
338
|
-
- New mechanisms (cooldowns, retries, caches, rate limits) not requested by the user
|
|
339
|
-
- Decisions that drop, limit, or reshape data flowing to the output, or pick a default that changes what end-users see, with no instruction specifying it
|
|
340
|
-
|
|
341
|
-
## Output
|
|
342
|
-
|
|
343
|
-
Write your results to \`${outputPath}\` as JSON with this exact schema:
|
|
344
|
-
|
|
345
|
-
\`\`\`json
|
|
346
|
-
${INTENT_DRIFT_SCHEMA}
|
|
347
|
-
\`\`\`
|
|
348
|
-
|
|
349
|
-
- Set \`sessionsChecked\` to the number of trace files you read
|
|
350
|
-
- Set \`passed\` to \`true\` only when no intent-drift issues are found
|
|
351
|
-
- Set \`passed\` to \`false\` when any issue is found (any pattern, any severity)
|
|
352
|
-
- For each issue, set \`pattern\` to one of: "intent_drift", "incomplete_fulfillment", "scope_creep", "unspecified_decision", "ignored_correction", "weakened_posture", "claimed_but_not_done"
|
|
353
|
-
- Write each issue's \`message\` in plain English for the PR author, lede first (what concretely happened, then why it matters); never reference this checker's machinery ("per policy", "extracted directives", "session shape")
|
|
354
|
-
- You MUST write the result file even if no issues are found
|
|
355
|
-
|
|
356
|
-
## Severity guidelines
|
|
357
|
-
|
|
358
|
-
- **error**: Core user intent violated — the main thing they asked for is wrong or missing, or large unrequested feature added
|
|
359
|
-
- **warning**: Secondary requirement missed or approximated, or small unrequested mechanism added
|
|
360
|
-
- **info**: Minor simplification that mostly still works as intended${precomputedDiff ? `
|
|
361
|
-
|
|
362
|
-
## Diff (precomputed)
|
|
363
|
-
|
|
364
|
-
\`\`\`diff
|
|
365
|
-
${precomputedDiff}
|
|
366
|
-
\`\`\`` : ''}`;
|
|
367
|
-
}
|
package/dist/triage/runner.js
CHANGED
|
@@ -8,8 +8,7 @@ import { execSync, spawn } from 'child_process';
|
|
|
8
8
|
import { existsSync, readFileSync, mkdirSync, rmSync } from 'fs';
|
|
9
9
|
import { join } from 'path';
|
|
10
10
|
import chalk from 'chalk';
|
|
11
|
-
import { buildCodeReviewPrompt, buildRulesValidatorPrompt
|
|
12
|
-
import { findRelevantTraces } from './traces.js';
|
|
11
|
+
import { buildCodeReviewPrompt, buildRulesValidatorPrompt } from './prompts.js';
|
|
13
12
|
import { resolveDiffBaseRef } from '../utils/git.js';
|
|
14
13
|
import { trackError } from '../utils/telemetry.js';
|
|
15
14
|
// ============================================================================
|
|
@@ -26,7 +25,6 @@ const DEFAULT_TIMEOUT_MS = 180_000; // 3 minutes
|
|
|
26
25
|
const DEFAULT_MAX_TURNS = {
|
|
27
26
|
'code-review': 8,
|
|
28
27
|
'rules-validator': 10,
|
|
29
|
-
'intent-drift': 10,
|
|
30
28
|
};
|
|
31
29
|
// ============================================================================
|
|
32
30
|
// Agent policy file discovery
|
|
@@ -277,8 +275,8 @@ export async function runTriage(gitRoot, baseBranch, options = {}) {
|
|
|
277
275
|
// Resolve the ref to diff against once. Prefers origin/<base> over the bare
|
|
278
276
|
// local <base> ref (which is often stale and balloons the diff with already-
|
|
279
277
|
// merged code). Every git read below — the precomputed diff, the agent diff
|
|
280
|
-
// commands in the prompts
|
|
281
|
-
//
|
|
278
|
+
// commands in the prompts use this single resolved ref so they all see the
|
|
279
|
+
// same fork point.
|
|
282
280
|
const diffBaseRef = resolveDiffBaseRef(baseBranch);
|
|
283
281
|
if (diffBaseRef !== baseBranch) {
|
|
284
282
|
console.log(chalk.dim(` Diff base: ${diffBaseRef}`));
|
|
@@ -324,26 +322,9 @@ export async function runTriage(gitRoot, baseBranch, options = {}) {
|
|
|
324
322
|
});
|
|
325
323
|
}
|
|
326
324
|
}
|
|
327
|
-
// 3. Intent drift (only if relevant trace files exist)
|
|
328
|
-
const traceFiles = findRelevantTraces(gitRoot, diffBaseRef);
|
|
329
|
-
if (traceFiles.length > 0) {
|
|
330
|
-
const intentDriftOutput = join(triageDir, 'intent-drift.json');
|
|
331
|
-
const driftPrompt = buildIntentDriftPrompt(diffBaseRef, traceFiles, intentDriftOutput, maxTurns['intent-drift'], timeoutMs, precomputedDiff);
|
|
332
|
-
if (driftPrompt) {
|
|
333
|
-
checkers.push({
|
|
334
|
-
name: 'intent-drift',
|
|
335
|
-
prompt: driftPrompt,
|
|
336
|
-
outputFile: intentDriftOutput,
|
|
337
|
-
maxTurns: maxTurns['intent-drift'],
|
|
338
|
-
});
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
325
|
// Log what we're running
|
|
342
326
|
const checkerNames = checkers.map(c => c.name);
|
|
343
327
|
console.log(chalk.dim(` Checkers: ${checkerNames.join(', ')}`));
|
|
344
|
-
if (traceFiles.length > 0) {
|
|
345
|
-
console.log(chalk.dim(` Traces: ${traceFiles.length} relevant session(s) found`));
|
|
346
|
-
}
|
|
347
328
|
console.log(chalk.dim(` Running ${checkers.length} checker(s) in parallel...\n`));
|
|
348
329
|
// Spawn all checkers in parallel
|
|
349
330
|
const spawnResults = await Promise.allSettled(checkers.map(async (checker) => {
|
package/dist/triage/types.d.ts
CHANGED
|
@@ -28,14 +28,7 @@ export interface RulesValidatorResult {
|
|
|
28
28
|
passed: boolean;
|
|
29
29
|
rulesChecked: number;
|
|
30
30
|
}
|
|
31
|
-
export
|
|
32
|
-
checker: 'intent-drift';
|
|
33
|
-
issues: TriageIssue[];
|
|
34
|
-
summary: string;
|
|
35
|
-
passed: boolean;
|
|
36
|
-
sessionsChecked: number;
|
|
37
|
-
}
|
|
38
|
-
export type CheckerResult = CodeReviewResult | RulesValidatorResult | IntentDriftResult;
|
|
31
|
+
export type CheckerResult = CodeReviewResult | RulesValidatorResult;
|
|
39
32
|
export interface TriageResult {
|
|
40
33
|
passed: boolean;
|
|
41
34
|
results: CheckerResult[];
|
package/dist/types.d.ts
CHANGED
|
@@ -709,14 +709,14 @@ declare const SecretDeclarationSchema: z.ZodObject<{
|
|
|
709
709
|
}>;
|
|
710
710
|
declare const TriageConfigSchema: z.ZodObject<{
|
|
711
711
|
/** Per-checker max agentic turns. Unspecified checkers use built-in defaults. */
|
|
712
|
-
maxTurns: z.ZodOptional<z.ZodRecord<z.ZodEnum<["code-review", "rules-validator"
|
|
712
|
+
maxTurns: z.ZodOptional<z.ZodRecord<z.ZodEnum<["code-review", "rules-validator"]>, z.ZodNumber>>;
|
|
713
713
|
/** Wall-clock timeout per checker, in milliseconds. */
|
|
714
714
|
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
715
715
|
}, "strip", z.ZodTypeAny, {
|
|
716
|
-
maxTurns?: Partial<Record<"code-review" | "rules-validator"
|
|
716
|
+
maxTurns?: Partial<Record<"code-review" | "rules-validator", number>> | undefined;
|
|
717
717
|
timeoutMs?: number | undefined;
|
|
718
718
|
}, {
|
|
719
|
-
maxTurns?: Partial<Record<"code-review" | "rules-validator"
|
|
719
|
+
maxTurns?: Partial<Record<"code-review" | "rules-validator", number>> | undefined;
|
|
720
720
|
timeoutMs?: number | undefined;
|
|
721
721
|
}>;
|
|
722
722
|
declare const PreferencesSchema: z.ZodObject<{
|
|
@@ -1315,14 +1315,14 @@ export declare const HaystackConfigSchema: z.ZodObject<{
|
|
|
1315
1315
|
/** Pre-PR triage configuration (turn budgets, timeouts) */
|
|
1316
1316
|
triage: z.ZodOptional<z.ZodObject<{
|
|
1317
1317
|
/** Per-checker max agentic turns. Unspecified checkers use built-in defaults. */
|
|
1318
|
-
maxTurns: z.ZodOptional<z.ZodRecord<z.ZodEnum<["code-review", "rules-validator"
|
|
1318
|
+
maxTurns: z.ZodOptional<z.ZodRecord<z.ZodEnum<["code-review", "rules-validator"]>, z.ZodNumber>>;
|
|
1319
1319
|
/** Wall-clock timeout per checker, in milliseconds. */
|
|
1320
1320
|
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
1321
1321
|
}, "strip", z.ZodTypeAny, {
|
|
1322
|
-
maxTurns?: Partial<Record<"code-review" | "rules-validator"
|
|
1322
|
+
maxTurns?: Partial<Record<"code-review" | "rules-validator", number>> | undefined;
|
|
1323
1323
|
timeoutMs?: number | undefined;
|
|
1324
1324
|
}, {
|
|
1325
|
-
maxTurns?: Partial<Record<"code-review" | "rules-validator"
|
|
1325
|
+
maxTurns?: Partial<Record<"code-review" | "rules-validator", number>> | undefined;
|
|
1326
1326
|
timeoutMs?: number | undefined;
|
|
1327
1327
|
}>>;
|
|
1328
1328
|
}, "strip", z.ZodTypeAny, {
|
|
@@ -1455,7 +1455,7 @@ export declare const HaystackConfigSchema: z.ZodObject<{
|
|
|
1455
1455
|
description?: string | undefined;
|
|
1456
1456
|
}> | undefined;
|
|
1457
1457
|
triage?: {
|
|
1458
|
-
maxTurns?: Partial<Record<"code-review" | "rules-validator"
|
|
1458
|
+
maxTurns?: Partial<Record<"code-review" | "rules-validator", number>> | undefined;
|
|
1459
1459
|
timeoutMs?: number | undefined;
|
|
1460
1460
|
} | undefined;
|
|
1461
1461
|
}, {
|
|
@@ -1588,7 +1588,7 @@ export declare const HaystackConfigSchema: z.ZodObject<{
|
|
|
1588
1588
|
description?: string | undefined;
|
|
1589
1589
|
}> | undefined;
|
|
1590
1590
|
triage?: {
|
|
1591
|
-
maxTurns?: Partial<Record<"code-review" | "rules-validator"
|
|
1591
|
+
maxTurns?: Partial<Record<"code-review" | "rules-validator", number>> | undefined;
|
|
1592
1592
|
timeoutMs?: number | undefined;
|
|
1593
1593
|
} | undefined;
|
|
1594
1594
|
}>;
|
package/dist/types.js
CHANGED
|
@@ -272,7 +272,7 @@ const SecretDeclarationSchema = z.object({
|
|
|
272
272
|
// =============================================================================
|
|
273
273
|
// TRIAGE CONFIGURATION
|
|
274
274
|
// =============================================================================
|
|
275
|
-
const TriageCheckerNameSchema = z.enum(['code-review', 'rules-validator'
|
|
275
|
+
const TriageCheckerNameSchema = z.enum(['code-review', 'rules-validator']);
|
|
276
276
|
const TriageConfigSchema = z.object({
|
|
277
277
|
/** Per-checker max agentic turns. Unspecified checkers use built-in defaults. */
|
|
278
278
|
maxTurns: z.record(TriageCheckerNameSchema, z.number().int().positive()).optional(),
|