@haystackeditor/cli 0.15.27 → 0.15.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -20
- package/dist/assets/hooks/agent-context/detect.ts +7 -7
- package/dist/assets/hooks/agent-context/parsers/claude.ts +1 -1
- package/dist/assets/hooks/scripts/commit-msg.sh +1 -3
- package/dist/assets/hooks/scripts/post-commit.sh +1 -3
- package/dist/assets/hooks/scripts/pre-push.sh +1 -4
- package/dist/assets/hooks/scripts/prepare-commit-msg.sh +1 -2
- package/dist/assets/telemetry/browser-runtime.js +668 -43
- package/dist/assets/telemetry/runtime.cjs +62 -68
- package/dist/commands/design-verify.d.ts +11 -3
- package/dist/commands/design-verify.js +147 -4
- package/dist/commands/hooks.d.ts +1 -5
- package/dist/commands/hooks.js +10 -112
- package/dist/commands/install-session-hooks.d.ts +2 -2
- package/dist/commands/install-session-hooks.js +71 -21
- package/dist/commands/mcp.js +1 -34
- package/dist/commands/pr.js +2 -7
- package/dist/commands/setup.d.ts +1 -2
- package/dist/commands/setup.js +18 -272
- package/dist/commands/submit.js +1 -2
- package/dist/commands/verify-hosted-mcp.js +12 -2
- package/dist/commands/verify-hosted-reproducibility.d.ts +1 -0
- package/dist/commands/verify-hosted-reproducibility.js +4 -1
- package/dist/commands/verify-hosted.d.ts +20 -0
- package/dist/commands/verify-hosted.js +119 -11
- package/dist/commands/verify-precompute.d.ts +19 -0
- package/dist/commands/verify-precompute.js +328 -0
- package/dist/index.js +45 -44
- package/dist/schema.d.ts +1 -2
- package/dist/schema.js +1 -2
- package/dist/triage/prompts.d.ts +0 -7
- package/dist/triage/prompts.js +0 -145
- package/dist/triage/runner.js +3 -22
- package/dist/triage/types.d.ts +1 -8
- package/dist/types.d.ts +8 -8
- package/dist/types.js +1 -1
- package/dist/utils/design-verifier-api.d.ts +71 -0
- package/dist/utils/design-verifier-api.js +96 -0
- package/dist/utils/design-verifier-result.d.ts +7 -2
- package/dist/utils/design-verifier-result.js +67 -21
- package/dist/utils/git.d.ts +5 -1
- package/dist/utils/git.js +19 -9
- package/dist/utils/github-api.js +11 -14
- package/dist/utils/hooks.d.ts +1 -15
- package/dist/utils/hooks.js +1 -127
- package/dist/utils/telemetry.d.ts +2 -0
- package/dist/utils/telemetry.js +81 -8
- package/package.json +2 -1
- package/schemas/cloud-verifier.v1.json +69 -3
- package/schemas/{pr.v2.json → pr.v3.json} +3 -12
- package/dist/commands/traces.d.ts +0 -48
- package/dist/commands/traces.js +0 -92
- package/dist/triage/traces.d.ts +0 -20
- package/dist/triage/traces.js +0 -311
- package/schemas/traces.v1.json +0 -53
|
@@ -52,6 +52,48 @@ export interface DesignVerifierExecutionSummary {
|
|
|
52
52
|
traceArtifact: string;
|
|
53
53
|
outputArtifact: string;
|
|
54
54
|
}
|
|
55
|
+
export type DesignVerifierUniverseRole = 'control-before' | 'before' | 'after';
|
|
56
|
+
export interface DesignVerifierUniverseSummary {
|
|
57
|
+
version: 'dv-case-universe-summary-v1';
|
|
58
|
+
universeId: string;
|
|
59
|
+
caseRunId: string;
|
|
60
|
+
caseId: string;
|
|
61
|
+
role: DesignVerifierUniverseRole;
|
|
62
|
+
revision: string;
|
|
63
|
+
capabilities: Array<'inspect' | 'query' | 'replay' | 'exec'>;
|
|
64
|
+
lease: {
|
|
65
|
+
state: 'leased' | 'released';
|
|
66
|
+
expiresAt: string | null;
|
|
67
|
+
available: boolean;
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
export interface DesignVerifierUniverseAction {
|
|
71
|
+
version: 'dv-universe-action-v1';
|
|
72
|
+
actionId: string;
|
|
73
|
+
runId: string;
|
|
74
|
+
caseId: string;
|
|
75
|
+
universe: DesignVerifierUniverseRole;
|
|
76
|
+
universeId: string;
|
|
77
|
+
action: 'replay' | 'exec';
|
|
78
|
+
command: {
|
|
79
|
+
cwd: string;
|
|
80
|
+
argv: string[];
|
|
81
|
+
timeoutMs: number;
|
|
82
|
+
};
|
|
83
|
+
status: 'queued' | 'running' | 'completed' | 'failed';
|
|
84
|
+
createdAt: string;
|
|
85
|
+
startedAt: string | null;
|
|
86
|
+
finishedAt: string | null;
|
|
87
|
+
result: {
|
|
88
|
+
exitCode: number | null;
|
|
89
|
+
stdout: string;
|
|
90
|
+
stderr: string;
|
|
91
|
+
stdoutOmittedBytes: number;
|
|
92
|
+
stderrOmittedBytes: number;
|
|
93
|
+
durationMs: number;
|
|
94
|
+
} | null;
|
|
95
|
+
error: string | null;
|
|
96
|
+
}
|
|
55
97
|
export interface DesignVerifierStageTiming {
|
|
56
98
|
stage: 'prepare' | 'bake' | 'blast-radius' | 'prefetch' | 'case-design' | 'verdict';
|
|
57
99
|
status: string;
|
|
@@ -88,6 +130,7 @@ export interface DesignVerifierStatusResult {
|
|
|
88
130
|
cases: Array<Record<string, any>>;
|
|
89
131
|
scenarios: Array<Record<string, any> & {
|
|
90
132
|
executions?: DesignVerifierExecutionSummary[];
|
|
133
|
+
universes?: DesignVerifierUniverseSummary[];
|
|
91
134
|
}>;
|
|
92
135
|
changedFiles: string[];
|
|
93
136
|
timings?: DesignVerifierTimingSummary;
|
|
@@ -111,6 +154,34 @@ export type DesignVerifierStatusFetchResult = {
|
|
|
111
154
|
status: 'not_found' | 'denied' | 'unavailable' | 'error';
|
|
112
155
|
message: string;
|
|
113
156
|
};
|
|
157
|
+
export type DesignVerifierUniverseActionFetchResult = {
|
|
158
|
+
status: 'ok';
|
|
159
|
+
action: DesignVerifierUniverseAction;
|
|
160
|
+
} | {
|
|
161
|
+
status: 'not_found' | 'denied' | 'unavailable' | 'error';
|
|
162
|
+
message: string;
|
|
163
|
+
};
|
|
164
|
+
export declare function startDesignVerifierUniverseAction(owner: string, repo: string, prNumber: number, runId: string, token: string, request: {
|
|
165
|
+
caseId: string;
|
|
166
|
+
universe: DesignVerifierUniverseRole;
|
|
167
|
+
action: 'replay' | 'exec';
|
|
168
|
+
argv?: string[];
|
|
169
|
+
cwd?: string;
|
|
170
|
+
timeoutMs?: number;
|
|
171
|
+
}): Promise<DesignVerifierUniverseActionFetchResult>;
|
|
172
|
+
export declare function getDesignVerifierUniverseAction(owner: string, repo: string, prNumber: number, runId: string, actionId: string, token: string): Promise<DesignVerifierUniverseActionFetchResult>;
|
|
173
|
+
export declare function waitForDesignVerifierUniverseAction(owner: string, repo: string, prNumber: number, runId: string, actionId: string, token: string, options?: {
|
|
174
|
+
intervalMs?: number;
|
|
175
|
+
timeoutMs?: number;
|
|
176
|
+
}): Promise<{
|
|
177
|
+
status: 'terminal';
|
|
178
|
+
action: DesignVerifierUniverseAction;
|
|
179
|
+
} | {
|
|
180
|
+
status: 'timeout';
|
|
181
|
+
} | {
|
|
182
|
+
status: 'error';
|
|
183
|
+
message: string;
|
|
184
|
+
}>;
|
|
114
185
|
export declare function triggerDesignVerifier(owner: string, repo: string, prNumber: number, token: string): Promise<DesignVerifierTriggerResult>;
|
|
115
186
|
export declare function getDesignVerifierStatus(owner: string, repo: string, prNumber: number, runId: string, token: string): Promise<DesignVerifierStatusFetchResult>;
|
|
116
187
|
export declare function waitForDesignVerifier(owner: string, repo: string, prNumber: number, runId: string, token: string, options?: {
|
|
@@ -54,6 +54,102 @@ function apiHeaders(token) {
|
|
|
54
54
|
Authorization: `Bearer ${token}`,
|
|
55
55
|
};
|
|
56
56
|
}
|
|
57
|
+
function isUniverseAction(value, expected) {
|
|
58
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
59
|
+
return false;
|
|
60
|
+
const action = value;
|
|
61
|
+
return action.version === 'dv-universe-action-v1' && action.runId === expected.runId &&
|
|
62
|
+
(expected.actionId === undefined || action.actionId === expected.actionId) &&
|
|
63
|
+
(expected.caseId === undefined || action.caseId === expected.caseId) &&
|
|
64
|
+
(expected.universe === undefined || action.universe === expected.universe) &&
|
|
65
|
+
/^action-[0-9a-f-]{36}$/.test(action.actionId || '') &&
|
|
66
|
+
/^universe-[0-9a-f]{24}$/.test(action.universeId || '') &&
|
|
67
|
+
['control-before', 'before', 'after'].includes(action.universe) &&
|
|
68
|
+
['replay', 'exec'].includes(action.action) &&
|
|
69
|
+
['queued', 'running', 'completed', 'failed'].includes(action.status) &&
|
|
70
|
+
action.command && Array.isArray(action.command.argv) && typeof action.command.cwd === 'string';
|
|
71
|
+
}
|
|
72
|
+
function universeActionPath(owner, repo, prNumber, runId) {
|
|
73
|
+
return `/api/design-verifier/universe/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/${prNumber}/${encodeURIComponent(runId)}`;
|
|
74
|
+
}
|
|
75
|
+
async function parseUniverseActionResponse(response, expected) {
|
|
76
|
+
let data = {};
|
|
77
|
+
try {
|
|
78
|
+
data = await response.json();
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return { status: 'error', message: `Universe endpoint returned invalid JSON (HTTP ${response.status})` };
|
|
82
|
+
}
|
|
83
|
+
if (response.ok) {
|
|
84
|
+
if (!isUniverseAction(data, expected)) {
|
|
85
|
+
return { status: 'error', message: `Universe endpoint returned an invalid or mismatched result (HTTP ${response.status})` };
|
|
86
|
+
}
|
|
87
|
+
return { status: 'ok', action: data };
|
|
88
|
+
}
|
|
89
|
+
const message = data.message || data.error || `HTTP ${response.status}`;
|
|
90
|
+
if (response.status === 401 || response.status === 403)
|
|
91
|
+
return { status: 'denied', message };
|
|
92
|
+
if (response.status === 404 || response.status === 410)
|
|
93
|
+
return { status: 'not_found', message };
|
|
94
|
+
if (response.status === 502 || response.status === 503)
|
|
95
|
+
return { status: 'unavailable', message };
|
|
96
|
+
return { status: 'error', message };
|
|
97
|
+
}
|
|
98
|
+
export async function startDesignVerifierUniverseAction(owner, repo, prNumber, runId, token, request) {
|
|
99
|
+
let response;
|
|
100
|
+
try {
|
|
101
|
+
response = await fetch(haystackApiUrl(universeActionPath(owner, repo, prNumber, runId)), {
|
|
102
|
+
method: 'POST',
|
|
103
|
+
headers: { ...apiHeaders(token), 'Content-Type': 'application/json' },
|
|
104
|
+
body: JSON.stringify(request),
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
return { status: 'error', message: err.message };
|
|
109
|
+
}
|
|
110
|
+
return parseUniverseActionResponse(response, {
|
|
111
|
+
runId,
|
|
112
|
+
caseId: request.caseId,
|
|
113
|
+
universe: request.universe,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
export async function getDesignVerifierUniverseAction(owner, repo, prNumber, runId, actionId, token) {
|
|
117
|
+
const path = universeActionPath(owner, repo, prNumber, runId);
|
|
118
|
+
let response;
|
|
119
|
+
try {
|
|
120
|
+
response = await fetch(haystackApiUrl(`${path}?actionId=${encodeURIComponent(actionId)}`), {
|
|
121
|
+
method: 'GET',
|
|
122
|
+
headers: apiHeaders(token),
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
catch (err) {
|
|
126
|
+
return { status: 'error', message: err.message };
|
|
127
|
+
}
|
|
128
|
+
return parseUniverseActionResponse(response, { runId, actionId });
|
|
129
|
+
}
|
|
130
|
+
export async function waitForDesignVerifierUniverseAction(owner, repo, prNumber, runId, actionId, token, options = {}) {
|
|
131
|
+
const intervalMs = options.intervalMs ?? 1_000;
|
|
132
|
+
const deadline = Date.now() + (options.timeoutMs ?? 10 * 60_000);
|
|
133
|
+
let consecutiveErrors = 0;
|
|
134
|
+
for (;;) {
|
|
135
|
+
const loaded = await getDesignVerifierUniverseAction(owner, repo, prNumber, runId, actionId, token);
|
|
136
|
+
if (loaded.status === 'ok') {
|
|
137
|
+
consecutiveErrors = 0;
|
|
138
|
+
if (['completed', 'failed'].includes(loaded.action.status)) {
|
|
139
|
+
return { status: 'terminal', action: loaded.action };
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
consecutiveErrors += 1;
|
|
144
|
+
if (loaded.status === 'denied' || loaded.status === 'not_found' || consecutiveErrors >= 3) {
|
|
145
|
+
return { status: 'error', message: loaded.message };
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
if (Date.now() >= deadline)
|
|
149
|
+
return { status: 'timeout' };
|
|
150
|
+
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
151
|
+
}
|
|
152
|
+
}
|
|
57
153
|
export async function triggerDesignVerifier(owner, repo, prNumber, token) {
|
|
58
154
|
const url = haystackApiUrl(`/api/design-verifier/trigger/${owner}/${repo}/${prNumber}`);
|
|
59
155
|
let response;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { DesignVerifierCommandSpec, DesignVerifierExecutionSummary, DesignVerifierStatusResult, DesignVerifierTimingSummary } from './design-verifier-api.js';
|
|
2
|
-
export type DesignVerifierAgentVerdict = 'fix_confirmed' | '
|
|
1
|
+
import type { DesignVerifierCommandSpec, DesignVerifierExecutionSummary, DesignVerifierStatusResult, DesignVerifierTimingSummary, DesignVerifierUniverseSummary } from './design-verifier-api.js';
|
|
2
|
+
export type DesignVerifierAgentVerdict = 'fix_confirmed' | 'anomaly_candidate' | 'no_unexpected_change' | 'inconclusive';
|
|
3
3
|
export interface DesignVerifierAgentCase {
|
|
4
4
|
id: string;
|
|
5
5
|
title: string;
|
|
@@ -7,16 +7,21 @@ export interface DesignVerifierAgentCase {
|
|
|
7
7
|
expectedHead: string;
|
|
8
8
|
verdict: 'confirmed' | 'finding' | 'clean' | 'inconclusive';
|
|
9
9
|
findingKind: string | null;
|
|
10
|
+
findingKinds: string[];
|
|
10
11
|
baseStable: boolean | null;
|
|
11
12
|
observations: Array<{
|
|
12
13
|
path: string;
|
|
13
14
|
base: unknown;
|
|
14
15
|
baseRepeat: unknown;
|
|
15
16
|
head: unknown;
|
|
17
|
+
controlObservedAtSamePath: boolean;
|
|
18
|
+
controlObservedAtAncestorPath: boolean;
|
|
19
|
+
controlObservedAtDescendantPath: boolean;
|
|
16
20
|
}>;
|
|
17
21
|
generatedFiles: string[];
|
|
18
22
|
executedCommand: DesignVerifierCommandSpec | null;
|
|
19
23
|
executions: DesignVerifierExecutionSummary[];
|
|
24
|
+
universes: DesignVerifierUniverseSummary[];
|
|
20
25
|
infrastructure: {
|
|
21
26
|
verified: boolean;
|
|
22
27
|
recipeId: string | null;
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import { isDeepStrictEqual } from 'node:util';
|
|
2
2
|
function findingExplanation(kind) {
|
|
3
3
|
if (kind === 'declared-change-not-observed') {
|
|
4
|
-
return 'The intended behavior change was not observed
|
|
4
|
+
return 'The registered intended behavior change was not observed; investigate the captured outputs before concluding why.';
|
|
5
5
|
}
|
|
6
6
|
if (kind === 'unexpected-change') {
|
|
7
|
-
return 'The proposed revision
|
|
7
|
+
return 'The proposed revision produced a difference outside this case’s registered intent; this is an anomaly candidate for investigation.';
|
|
8
8
|
}
|
|
9
|
-
|
|
9
|
+
if (kind === 'intended-change-control-confounded') {
|
|
10
|
+
return 'The intended difference was observed, but control-before also varied on an intersecting output path; inspect it before claiming causality.';
|
|
11
|
+
}
|
|
12
|
+
return 'Executed behavior produced a suspicious difference against the registered expectation.';
|
|
10
13
|
}
|
|
11
14
|
function sameRecordedValue(left, right) {
|
|
12
15
|
return isDeepStrictEqual(left, right);
|
|
@@ -19,6 +22,14 @@ function executionSummaries(value) {
|
|
|
19
22
|
['completed', 'failed', 'missing'].includes(execution.status) &&
|
|
20
23
|
Array.isArray(execution.diagnostics) && Array.isArray(execution.outputExcerpt));
|
|
21
24
|
}
|
|
25
|
+
function universeSummaries(value, caseId) {
|
|
26
|
+
if (!Array.isArray(value))
|
|
27
|
+
return [];
|
|
28
|
+
return value.filter((universe) => universe?.version === 'dv-case-universe-summary-v1' && universe.caseId === caseId &&
|
|
29
|
+
/^universe-[0-9a-f]{24}$/.test(universe.universeId || '') &&
|
|
30
|
+
['control-before', 'before', 'after'].includes(universe.role) &&
|
|
31
|
+
universe.lease && typeof universe.lease.available === 'boolean');
|
|
32
|
+
}
|
|
22
33
|
export function buildDesignVerifierAgentReport(run) {
|
|
23
34
|
const repository = `${run.owner}/${run.repo}`;
|
|
24
35
|
if (run.status !== 'succeeded' || !run.result) {
|
|
@@ -46,13 +57,19 @@ export function buildDesignVerifierAgentReport(run) {
|
|
|
46
57
|
const summary = verdict.summary || {};
|
|
47
58
|
const findings = Array.isArray(verdict.findings) ? verdict.findings : [];
|
|
48
59
|
const confirmations = Array.isArray(verdict.confirmations) ? verdict.confirmations : [];
|
|
49
|
-
const
|
|
60
|
+
const findingsByCase = new Map();
|
|
61
|
+
for (const finding of findings) {
|
|
62
|
+
const caseFindings = findingsByCase.get(String(finding?.caseId || '')) || [];
|
|
63
|
+
caseFindings.push(finding);
|
|
64
|
+
findingsByCase.set(String(finding?.caseId || ''), caseFindings);
|
|
65
|
+
}
|
|
50
66
|
const confirmedIds = new Set(confirmations.map((confirmation) => confirmation.caseId));
|
|
51
67
|
const scenarioByCase = new Map(run.result.scenarios.map((scenario) => [scenario.caseId, scenario]));
|
|
52
68
|
const probeByCase = new Map(run.result.probes.map((probe) => [probe.caseId, probe]));
|
|
53
69
|
const cases = run.result.cases.map((testCase) => {
|
|
54
70
|
const id = String(testCase.id || '');
|
|
55
|
-
const
|
|
71
|
+
const caseFindings = findingsByCase.get(id) || [];
|
|
72
|
+
const finding = caseFindings[0];
|
|
56
73
|
const scenario = scenarioByCase.get(id);
|
|
57
74
|
const probe = probeByCase.get(id);
|
|
58
75
|
const observations = (Array.isArray(scenario?.divergences) ? scenario.divergences : []).map((item) => ({
|
|
@@ -60,13 +77,18 @@ export function buildDesignVerifierAgentReport(run) {
|
|
|
60
77
|
base: item.base,
|
|
61
78
|
baseRepeat: item.base2,
|
|
62
79
|
head: item.head,
|
|
80
|
+
controlObservedAtSamePath: item.controlObservedAtSamePath === true,
|
|
81
|
+
controlObservedAtAncestorPath: item.controlObservedAtAncestorPath === true,
|
|
82
|
+
controlObservedAtDescendantPath: item.controlObservedAtDescendantPath === true,
|
|
63
83
|
}));
|
|
84
|
+
const hasControlEvidence = typeof scenario?.noisePathCount === 'number' ||
|
|
85
|
+
scenario?.controlScenarioErrorVariation != null;
|
|
64
86
|
return {
|
|
65
87
|
id,
|
|
66
88
|
title: String(testCase.title || scenario?.scenario || id),
|
|
67
89
|
story: String(testCase.story || ''),
|
|
68
90
|
expectedHead: String(testCase.expected_head_observation || ''),
|
|
69
|
-
verdict:
|
|
91
|
+
verdict: caseFindings.length
|
|
70
92
|
? 'finding'
|
|
71
93
|
: confirmedIds.has(id)
|
|
72
94
|
? 'confirmed'
|
|
@@ -74,13 +96,19 @@ export function buildDesignVerifierAgentReport(run) {
|
|
|
74
96
|
? 'clean'
|
|
75
97
|
: 'inconclusive',
|
|
76
98
|
findingKind: finding?.kind || null,
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
99
|
+
findingKinds: caseFindings
|
|
100
|
+
.map((item) => String(item?.kind || ''))
|
|
101
|
+
.filter(Boolean),
|
|
102
|
+
baseStable: hasControlEvidence
|
|
103
|
+
? scenario.noisePathCount === 0 && scenario.controlScenarioErrorVariation == null
|
|
104
|
+
: observations.length
|
|
105
|
+
? observations.every((item) => sameRecordedValue(item.base, item.baseRepeat))
|
|
106
|
+
: scenario?.status === 'clean' ? true : null,
|
|
80
107
|
observations,
|
|
81
108
|
generatedFiles: probe?.files || [],
|
|
82
109
|
executedCommand: probe?.run || null,
|
|
83
110
|
executions: executionSummaries(scenario?.executions),
|
|
111
|
+
universes: universeSummaries(scenario?.universes, id),
|
|
84
112
|
infrastructure: scenario?.infrastructureReceipt &&
|
|
85
113
|
typeof scenario.infrastructureReceipt === 'object'
|
|
86
114
|
? {
|
|
@@ -106,8 +134,8 @@ export function buildDesignVerifierAgentReport(run) {
|
|
|
106
134
|
let explanation;
|
|
107
135
|
let exitCode;
|
|
108
136
|
if (findings.length > 0) {
|
|
109
|
-
agentVerdict = '
|
|
110
|
-
headline = '
|
|
137
|
+
agentVerdict = 'anomaly_candidate';
|
|
138
|
+
headline = 'ANOMALY CANDIDATE';
|
|
111
139
|
explanation = findingExplanation(String(findings[0]?.kind || ''));
|
|
112
140
|
exitCode = 1;
|
|
113
141
|
}
|
|
@@ -120,13 +148,13 @@ export function buildDesignVerifierAgentReport(run) {
|
|
|
120
148
|
else if (confirmations.length > 0) {
|
|
121
149
|
agentVerdict = 'fix_confirmed';
|
|
122
150
|
headline = 'FIX CONFIRMED';
|
|
123
|
-
explanation = 'The intended behavior change was observed on
|
|
151
|
+
explanation = 'The intended behavior change was observed on output paths that did not vary between control-before and before.';
|
|
124
152
|
exitCode = 0;
|
|
125
153
|
}
|
|
126
154
|
else {
|
|
127
155
|
agentVerdict = 'no_unexpected_change';
|
|
128
156
|
headline = 'NO UNEXPECTED CHANGE';
|
|
129
|
-
explanation = '
|
|
157
|
+
explanation = 'The executed cases found no observed difference outside their registered expectations.';
|
|
130
158
|
exitCode = 0;
|
|
131
159
|
}
|
|
132
160
|
return {
|
|
@@ -184,10 +212,10 @@ function commandLabel(command) {
|
|
|
184
212
|
}
|
|
185
213
|
function executionSideLabel(side) {
|
|
186
214
|
if (side === 'base1')
|
|
187
|
-
return '
|
|
215
|
+
return 'before';
|
|
188
216
|
if (side === 'base2')
|
|
189
|
-
return '
|
|
190
|
-
return '
|
|
217
|
+
return 'control-before';
|
|
218
|
+
return 'after';
|
|
191
219
|
}
|
|
192
220
|
function executionNeedsAttention(execution) {
|
|
193
221
|
return execution.status !== 'completed' || execution.diagnostics.length > 0;
|
|
@@ -200,7 +228,7 @@ export function formatDesignVerifierAgentReport(report) {
|
|
|
200
228
|
report.explanation,
|
|
201
229
|
];
|
|
202
230
|
if (report.revisions.base || report.revisions.head) {
|
|
203
|
-
lines.push('', `Revisions:
|
|
231
|
+
lines.push('', `Revisions: before ${report.revisions.base || 'unknown'} (control + comparison)`, ` after ${report.revisions.head || 'unknown'} (comparison)`);
|
|
204
232
|
}
|
|
205
233
|
if (report.timings) {
|
|
206
234
|
lines.push('', 'Timing breakdown:');
|
|
@@ -221,16 +249,30 @@ export function formatDesignVerifierAgentReport(report) {
|
|
|
221
249
|
lines.push(` Behavior: ${testCase.story}`);
|
|
222
250
|
if (testCase.expectedHead)
|
|
223
251
|
lines.push(` Expected on head: ${testCase.expectedHead}`);
|
|
252
|
+
if (testCase.findingKinds.length > 0) {
|
|
253
|
+
lines.push(` Candidate reasons: ${testCase.findingKinds.join(', ')}`);
|
|
254
|
+
}
|
|
224
255
|
if (testCase.observations.length) {
|
|
225
256
|
lines.push(' Observed:');
|
|
226
257
|
for (const observation of testCase.observations) {
|
|
227
258
|
lines.push(` ${observation.path}`);
|
|
228
|
-
lines.push(`
|
|
229
|
-
lines.push(`
|
|
259
|
+
lines.push(` control-before: ${displayValue(observation.baseRepeat)}`);
|
|
260
|
+
lines.push(` before: ${displayValue(observation.base)}`);
|
|
261
|
+
lines.push(` after: ${displayValue(observation.head)}`);
|
|
262
|
+
if (observation.controlObservedAtSamePath || observation.controlObservedAtAncestorPath ||
|
|
263
|
+
observation.controlObservedAtDescendantPath) {
|
|
264
|
+
lines.push(` control evidence: ${observation.controlObservedAtSamePath
|
|
265
|
+
? 'variation observed at this same path'
|
|
266
|
+
: observation.controlObservedAtAncestorPath
|
|
267
|
+
? 'variation observed at an ancestor path'
|
|
268
|
+
: 'variation observed at a descendant path'}`);
|
|
269
|
+
}
|
|
230
270
|
}
|
|
231
271
|
}
|
|
232
272
|
if (testCase.baseStable !== null) {
|
|
233
|
-
lines.push(`
|
|
273
|
+
lines.push(` Control evidence: ${testCase.baseStable
|
|
274
|
+
? 'control-before and before agreed'
|
|
275
|
+
: 'control-before and before varied; inspect alongside the difference'}`);
|
|
234
276
|
}
|
|
235
277
|
if (testCase.infrastructure) {
|
|
236
278
|
lines.push(testCase.infrastructure.verified
|
|
@@ -271,6 +313,10 @@ export function formatDesignVerifierAgentReport(report) {
|
|
|
271
313
|
const command = commandLabel(testCase.executedCommand);
|
|
272
314
|
if (command)
|
|
273
315
|
lines.push(` Executed in the sandbox: ${command}`);
|
|
316
|
+
for (const universe of testCase.universes.filter(item => item.lease.available)) {
|
|
317
|
+
lines.push(` Replay ${universe.role}: haystack design-verify ${report.repository}#${report.pullRequest}` +
|
|
318
|
+
` --run ${report.runId} --case ${testCase.id} --universe ${universe.role} --replay`);
|
|
319
|
+
}
|
|
274
320
|
}
|
|
275
321
|
if (report.changedFiles.length) {
|
|
276
322
|
lines.push('', 'Changed files to inspect:', ...report.changedFiles.map((file) => ` ${file}`));
|
|
@@ -292,7 +338,7 @@ export function designVerifierStageLabel(stage) {
|
|
|
292
338
|
if (stage.startsWith('build:'))
|
|
293
339
|
return 'Building an executable probe';
|
|
294
340
|
if (stage.startsWith('execute:'))
|
|
295
|
-
return 'Running
|
|
341
|
+
return 'Running control-before/before/after comparisons';
|
|
296
342
|
if (stage === 'verdict')
|
|
297
343
|
return 'Comparing observations';
|
|
298
344
|
return stage;
|
package/dist/utils/git.d.ts
CHANGED
|
@@ -93,6 +93,10 @@ export declare function getLastCommitSha(): string;
|
|
|
93
93
|
* Returns the combined log, or empty string if no commits or on error.
|
|
94
94
|
*/
|
|
95
95
|
export declare function getCommitMessagesSinceBase(baseBranch: string): string;
|
|
96
|
+
interface ResolveDiffBaseRefOptions {
|
|
97
|
+
cwd?: string;
|
|
98
|
+
timeoutMs?: number;
|
|
99
|
+
}
|
|
96
100
|
/**
|
|
97
101
|
* Resolve the git ref to diff a PR branch against for review.
|
|
98
102
|
*
|
|
@@ -124,7 +128,7 @@ export declare function getCommitMessagesSinceBase(baseBranch: string): string;
|
|
|
124
128
|
* 4. Fall back to whatever single qualified ref resolves, then the bare
|
|
125
129
|
* <base> name (preserves prior behavior rather than breaking triage).
|
|
126
130
|
*/
|
|
127
|
-
export declare function resolveDiffBaseRef(baseBranch: string): string;
|
|
131
|
+
export declare function resolveDiffBaseRef(baseBranch: string, options?: ResolveDiffBaseRefOptions): string;
|
|
128
132
|
/**
|
|
129
133
|
* Get the number of commits ahead of base branch
|
|
130
134
|
*/
|
package/dist/utils/git.js
CHANGED
|
@@ -376,11 +376,18 @@ export function getCommitMessagesSinceBase(baseBranch) {
|
|
|
376
376
|
}
|
|
377
377
|
}
|
|
378
378
|
}
|
|
379
|
+
function gitCommandOptions(options) {
|
|
380
|
+
return {
|
|
381
|
+
...(options.cwd ? { cwd: options.cwd } : {}),
|
|
382
|
+
...(options.timeoutMs ? { timeout: options.timeoutMs } : {}),
|
|
383
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
384
|
+
};
|
|
385
|
+
}
|
|
379
386
|
/** True if `ref` resolves to a commit in the current repo. */
|
|
380
|
-
function gitRefResolves(ref) {
|
|
387
|
+
function gitRefResolves(ref, options = {}) {
|
|
381
388
|
try {
|
|
382
389
|
execFileSync('git', ['rev-parse', '--verify', '--quiet', '--', `${ref}^{commit}`], {
|
|
383
|
-
|
|
390
|
+
...gitCommandOptions(options),
|
|
384
391
|
});
|
|
385
392
|
return true;
|
|
386
393
|
}
|
|
@@ -389,11 +396,11 @@ function gitRefResolves(ref) {
|
|
|
389
396
|
}
|
|
390
397
|
}
|
|
391
398
|
/** True if `ancestor` is an ancestor of (or equal to) `descendant`. */
|
|
392
|
-
function gitIsAncestor(ancestor, descendant) {
|
|
399
|
+
function gitIsAncestor(ancestor, descendant, options = {}) {
|
|
393
400
|
try {
|
|
394
401
|
// exit 0 => ancestor; exit 1 => not; both non-throwing for us via stdio pipe
|
|
395
402
|
execFileSync('git', ['merge-base', '--is-ancestor', '--', ancestor, descendant], {
|
|
396
|
-
|
|
403
|
+
...gitCommandOptions(options),
|
|
397
404
|
});
|
|
398
405
|
return true;
|
|
399
406
|
}
|
|
@@ -432,7 +439,7 @@ function gitIsAncestor(ancestor, descendant) {
|
|
|
432
439
|
* 4. Fall back to whatever single qualified ref resolves, then the bare
|
|
433
440
|
* <base> name (preserves prior behavior rather than breaking triage).
|
|
434
441
|
*/
|
|
435
|
-
export function resolveDiffBaseRef(baseBranch) {
|
|
442
|
+
export function resolveDiffBaseRef(baseBranch, options = {}) {
|
|
436
443
|
const remoteRef = `refs/remotes/origin/${baseBranch}`;
|
|
437
444
|
const localRef = `refs/heads/${baseBranch}`;
|
|
438
445
|
// Freshen the remote base so the merge-base reflects the true fork point.
|
|
@@ -451,7 +458,10 @@ export function resolveDiffBaseRef(baseBranch) {
|
|
|
451
458
|
let fetched = false;
|
|
452
459
|
try {
|
|
453
460
|
execFileSync('git', ['fetch', 'origin', `+refs/heads/${baseBranch}:${remoteRef}`], {
|
|
454
|
-
|
|
461
|
+
...gitCommandOptions(options),
|
|
462
|
+
env: options.timeoutMs
|
|
463
|
+
? { ...process.env, GIT_TERMINAL_PROMPT: '0' }
|
|
464
|
+
: process.env,
|
|
455
465
|
});
|
|
456
466
|
fetched = true;
|
|
457
467
|
}
|
|
@@ -459,17 +469,17 @@ export function resolveDiffBaseRef(baseBranch) {
|
|
|
459
469
|
// Offline / no creds / base not on remote — fall through to degraded
|
|
460
470
|
// selection below using whatever refs already exist locally.
|
|
461
471
|
}
|
|
462
|
-
const remoteResolves = gitRefResolves(remoteRef);
|
|
472
|
+
const remoteResolves = gitRefResolves(remoteRef, options);
|
|
463
473
|
// Happy path: we just refreshed the remote base, so it's authoritative.
|
|
464
474
|
if (fetched && remoteResolves)
|
|
465
475
|
return remoteRef;
|
|
466
476
|
// Degraded path: fetch failed (or didn't produce the ref). Prefer the more
|
|
467
477
|
// up-to-date of the cached remote ref and the local branch; only a base that
|
|
468
478
|
// lags HEAD's fork point balloons the diff, so pick the one that's ahead.
|
|
469
|
-
const localResolves = gitRefResolves(localRef);
|
|
479
|
+
const localResolves = gitRefResolves(localRef, options);
|
|
470
480
|
if (remoteResolves && localResolves) {
|
|
471
481
|
// If the remote ref is an ancestor of the local ref, local is ahead → use it.
|
|
472
|
-
return gitIsAncestor(remoteRef, localRef) ? localRef : remoteRef;
|
|
482
|
+
return gitIsAncestor(remoteRef, localRef, options) ? localRef : remoteRef;
|
|
473
483
|
}
|
|
474
484
|
if (remoteResolves)
|
|
475
485
|
return remoteRef;
|
package/dist/utils/github-api.js
CHANGED
|
@@ -86,10 +86,7 @@ async function githubApiRequest(method, path, token, body) {
|
|
|
86
86
|
const proxied = await fetch(`${GITHUB_PROXY_BASE}${path}`, init);
|
|
87
87
|
return proxied.status === 403 ? direct : proxied;
|
|
88
88
|
}
|
|
89
|
-
|
|
90
|
-
* Handle API errors consistently
|
|
91
|
-
*/
|
|
92
|
-
async function handleApiError(response, context) {
|
|
89
|
+
async function handleApiError(response, context, operation) {
|
|
93
90
|
if (response.status === 401) {
|
|
94
91
|
throw new Error('GitHub token expired or invalid. Run `haystack login` again.');
|
|
95
92
|
}
|
|
@@ -100,7 +97,7 @@ async function handleApiError(response, context) {
|
|
|
100
97
|
// whether the CLI needs its own token bucket. Routes to PostHog (and on to
|
|
101
98
|
// Slack via the ops-alert pipeline). Fire-and-forget; never blocks.
|
|
102
99
|
trackError('cli_github_rate_limited', {
|
|
103
|
-
|
|
100
|
+
operation,
|
|
104
101
|
status: 403,
|
|
105
102
|
rate_limit_limit: response.headers.get('x-ratelimit-limit'),
|
|
106
103
|
rate_limit_reset: response.headers.get('x-ratelimit-reset'),
|
|
@@ -149,7 +146,7 @@ export async function createPullRequest(options) {
|
|
|
149
146
|
`Push to a new branch name and try again.`);
|
|
150
147
|
}
|
|
151
148
|
}
|
|
152
|
-
await handleApiError(response, `creating PR in ${options.owner}/${options.repo}
|
|
149
|
+
await handleApiError(response, `creating PR in ${options.owner}/${options.repo}`, 'create_pull_request');
|
|
153
150
|
}
|
|
154
151
|
return response.json();
|
|
155
152
|
}
|
|
@@ -190,7 +187,7 @@ export async function findExistingPR(owner, repo, head, token) {
|
|
|
190
187
|
if (!response.ok) {
|
|
191
188
|
if (response.status === 404)
|
|
192
189
|
return null;
|
|
193
|
-
await handleApiError(response,
|
|
190
|
+
await handleApiError(response, 'checking for existing PR', 'find_existing_pull_request');
|
|
194
191
|
}
|
|
195
192
|
const prs = await response.json();
|
|
196
193
|
return prs.length > 0 ? prs[0] : null;
|
|
@@ -205,7 +202,7 @@ export async function addLabelsToIssue(owner, repo, issueNumber, labels, token)
|
|
|
205
202
|
const authToken = await resolveToken(token, { owner, repo });
|
|
206
203
|
const response = await githubApiRequest('POST', `/repos/${owner}/${repo}/issues/${issueNumber}/labels`, authToken, { labels });
|
|
207
204
|
if (!response.ok) {
|
|
208
|
-
await handleApiError(response, `adding labels to PR #${issueNumber}
|
|
205
|
+
await handleApiError(response, `adding labels to PR #${issueNumber}`, 'add_labels');
|
|
209
206
|
}
|
|
210
207
|
}
|
|
211
208
|
/**
|
|
@@ -215,7 +212,7 @@ export async function getLabelsFromIssue(owner, repo, issueNumber, token) {
|
|
|
215
212
|
const authToken = await resolveToken(token, { owner, repo });
|
|
216
213
|
const response = await githubApiRequest('GET', `/repos/${owner}/${repo}/issues/${issueNumber}/labels`, authToken);
|
|
217
214
|
if (!response.ok) {
|
|
218
|
-
await handleApiError(response, `getting labels from PR #${issueNumber}
|
|
215
|
+
await handleApiError(response, `getting labels from PR #${issueNumber}`, 'get_labels');
|
|
219
216
|
}
|
|
220
217
|
const labels = await response.json();
|
|
221
218
|
return labels.map((l) => l.name);
|
|
@@ -228,7 +225,7 @@ export async function removeLabelFromIssue(owner, repo, issueNumber, labelName,
|
|
|
228
225
|
const response = await githubApiRequest('DELETE', `/repos/${owner}/${repo}/issues/${issueNumber}/labels/${encodeURIComponent(labelName)}`, authToken);
|
|
229
226
|
// 404 is fine - label wasn't on the issue
|
|
230
227
|
if (!response.ok && response.status !== 404) {
|
|
231
|
-
await handleApiError(response, `removing label '${labelName}' from PR #${issueNumber}
|
|
228
|
+
await handleApiError(response, `removing label '${labelName}' from PR #${issueNumber}`, 'remove_label');
|
|
232
229
|
}
|
|
233
230
|
}
|
|
234
231
|
/**
|
|
@@ -251,7 +248,7 @@ export async function createLabel(owner, repo, label, token) {
|
|
|
251
248
|
});
|
|
252
249
|
if (!response.ok && response.status !== 422) {
|
|
253
250
|
// 422 means label already exists, which is fine
|
|
254
|
-
await handleApiError(response, `creating label '${label.name}'
|
|
251
|
+
await handleApiError(response, `creating label '${label.name}'`, 'create_label');
|
|
255
252
|
}
|
|
256
253
|
}
|
|
257
254
|
/**
|
|
@@ -278,7 +275,7 @@ export async function requestReviewers(owner, repo, prNumber, reviewers, token)
|
|
|
278
275
|
const authToken = await resolveToken(token, { owner, repo });
|
|
279
276
|
const response = await githubApiRequest('POST', `/repos/${owner}/${repo}/pulls/${prNumber}/requested_reviewers`, authToken, { reviewers });
|
|
280
277
|
if (!response.ok) {
|
|
281
|
-
await handleApiError(response, `requesting reviewers on PR #${prNumber}
|
|
278
|
+
await handleApiError(response, `requesting reviewers on PR #${prNumber}`, 'request_reviewers');
|
|
282
279
|
}
|
|
283
280
|
}
|
|
284
281
|
// ============================================================================
|
|
@@ -291,7 +288,7 @@ export async function getCurrentUser(token) {
|
|
|
291
288
|
const authToken = await resolveToken(token);
|
|
292
289
|
const response = await githubApiRequest('GET', '/user', authToken);
|
|
293
290
|
if (!response.ok) {
|
|
294
|
-
await handleApiError(response, 'getting current user');
|
|
291
|
+
await handleApiError(response, 'getting current user', 'get_current_user');
|
|
295
292
|
}
|
|
296
293
|
return response.json();
|
|
297
294
|
}
|
|
@@ -299,7 +296,7 @@ export async function getRepositoryAccess(owner, repo, token) {
|
|
|
299
296
|
const authToken = await resolveToken(token, { owner, repo });
|
|
300
297
|
const response = await githubApiRequest('GET', `/repos/${owner}/${repo}`, authToken);
|
|
301
298
|
if (!response.ok) {
|
|
302
|
-
await handleApiError(response, `getting repository access for ${owner}/${repo}
|
|
299
|
+
await handleApiError(response, `getting repository access for ${owner}/${repo}`, 'get_repository_access');
|
|
303
300
|
}
|
|
304
301
|
return response.json();
|
|
305
302
|
}
|
package/dist/utils/hooks.d.ts
CHANGED
|
@@ -1,17 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Utilities for haystack hooks install/status
|
|
3
|
-
*
|
|
4
|
-
* Handles Entire CLI binary download, hook file copying, and git configuration.
|
|
2
|
+
* Utilities for haystack hooks install/status commands.
|
|
5
3
|
*/
|
|
6
|
-
export declare const ENTIRE_DEFAULT_VERSION = "0.4.8";
|
|
7
|
-
export declare const ENTIRE_GITHUB_REPO = "entireio/cli";
|
|
8
|
-
export declare const HAYSTACK_BIN_DIR: string;
|
|
9
|
-
export declare const ENTIRE_BIN_PATH: string;
|
|
10
|
-
export declare function getPlatformAssetName(): string;
|
|
11
4
|
export declare function findGitRoot(): string | null;
|
|
12
|
-
export declare function getInstalledEntireVersion(): Promise<string | null>;
|
|
13
|
-
export declare function getLatestEntireVersion(): Promise<string>;
|
|
14
|
-
export declare function downloadEntireBinary(version: string): Promise<void>;
|
|
15
5
|
/** Resolves the path to dist/assets/hooks/ (or src/assets/hooks/ in dev) */
|
|
16
6
|
export declare function getAssetsDir(): string;
|
|
17
7
|
/** Copy hook scripts and TypeScript modules into the target hooks directory */
|
|
@@ -20,7 +10,3 @@ export declare function copyHookFiles(hooksDir: string): Promise<void>;
|
|
|
20
10
|
export declare function installHookDeps(hooksDir: string): Promise<void>;
|
|
21
11
|
/** Copy LLM rules template if none exists */
|
|
22
12
|
export declare function copyLlmRulesTemplate(repoRoot: string): Promise<boolean>;
|
|
23
|
-
/** Create .entire/settings.json and .entire/.gitignore */
|
|
24
|
-
export declare function createEntireConfig(repoRoot: string): Promise<void>;
|
|
25
|
-
/** Add .entire/metadata/ to repo .gitignore if not already present */
|
|
26
|
-
export declare function updateGitignore(repoRoot: string): Promise<boolean>;
|