@wkronmiller/lisa 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +407 -0
- package/bin/lisa-runtime.js +8797 -0
- package/bin/lisa.js +21 -0
- package/completion.ts +58 -0
- package/install.ps1 +51 -0
- package/install.sh +93 -0
- package/lisa.ts +6 -0
- package/package.json +66 -0
- package/skills/README.md +28 -0
- package/skills/claude-code/CLAUDE.md +151 -0
- package/skills/codex/AGENTS.md +151 -0
- package/skills/gemini/GEMINI.md +151 -0
- package/skills/opencode/AGENTS.md +152 -0
- package/src/cli.ts +85 -0
- package/src/harness/base-adapter.ts +47 -0
- package/src/harness/claude-code.ts +106 -0
- package/src/harness/codex.ts +80 -0
- package/src/harness/command.ts +173 -0
- package/src/harness/gemini.ts +74 -0
- package/src/harness/opencode.ts +84 -0
- package/src/harness/registry.ts +29 -0
- package/src/harness/runner.ts +19 -0
- package/src/harness/types.ts +73 -0
- package/src/output-mode.ts +32 -0
- package/src/skill/artifacts.ts +174 -0
- package/src/skill/cli.ts +29 -0
- package/src/skill/install.ts +317 -0
- package/src/spec/agent-guidance.ts +466 -0
- package/src/spec/cli.ts +151 -0
- package/src/spec/commands/check.ts +1 -0
- package/src/spec/commands/config.ts +146 -0
- package/src/spec/commands/diff.ts +1 -0
- package/src/spec/commands/generate.ts +1 -0
- package/src/spec/commands/guide.ts +1 -0
- package/src/spec/commands/harness-list.ts +36 -0
- package/src/spec/commands/implement.ts +1 -0
- package/src/spec/commands/import.ts +1 -0
- package/src/spec/commands/init.ts +1 -0
- package/src/spec/commands/status.ts +87 -0
- package/src/spec/config.ts +63 -0
- package/src/spec/diff.ts +791 -0
- package/src/spec/extensions/benchmark.ts +347 -0
- package/src/spec/extensions/registry.ts +59 -0
- package/src/spec/extensions/types.ts +56 -0
- package/src/spec/grammar/index.ts +14 -0
- package/src/spec/grammar/parser.ts +443 -0
- package/src/spec/grammar/types.ts +70 -0
- package/src/spec/grammar/validator.ts +104 -0
- package/src/spec/loader.ts +174 -0
- package/src/spec/local-config.ts +59 -0
- package/src/spec/parser.ts +226 -0
- package/src/spec/path-utils.ts +73 -0
- package/src/spec/planner.ts +299 -0
- package/src/spec/prompt-renderer.ts +318 -0
- package/src/spec/skill-content.ts +119 -0
- package/src/spec/types.ts +239 -0
- package/src/spec/validator.ts +443 -0
- package/src/spec/workflows/check.ts +1534 -0
- package/src/spec/workflows/diff.ts +209 -0
- package/src/spec/workflows/generate.ts +1270 -0
- package/src/spec/workflows/guide.ts +190 -0
- package/src/spec/workflows/implement.ts +797 -0
- package/src/spec/workflows/import.ts +986 -0
- package/src/spec/workflows/init.ts +548 -0
- package/src/spec/workflows/status.ts +22 -0
- package/src/spec/workspace.ts +541 -0
- package/uninstall.ps1 +21 -0
- package/uninstall.sh +22 -0
|
@@ -0,0 +1,1534 @@
|
|
|
1
|
+
import { createHash } from "crypto";
|
|
2
|
+
import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
|
|
3
|
+
import { tmpdir } from "os";
|
|
4
|
+
import { basename, dirname, join, relative } from "path";
|
|
5
|
+
|
|
6
|
+
import { resolveHarnessCommandOverride } from "../../harness/command";
|
|
7
|
+
import { getHarnessAdapterById } from "../../harness/registry";
|
|
8
|
+
import { runHarnessStage } from "../../harness/runner";
|
|
9
|
+
import type { HarnessInspection, HarnessRequest, HarnessResult } from "../../harness/types";
|
|
10
|
+
import { DEFAULT_SPEC_DIFF_BASE_REF, EXTERNAL_SPEC_DIFF_BASE_REF, collectSpecDiff, readExternalSnapshotCodePaths, readExternalSnapshotRepoRevision, readExternalSpecSnapshot, writeExternalSpecSnapshot } from "../diff";
|
|
11
|
+
import { resolveStageProfile } from "../config";
|
|
12
|
+
import { resolveLocalOverrides } from "../local-config";
|
|
13
|
+
import { getExtensionSidecarsForSpec, verifySpecExtensions } from "../extensions/registry";
|
|
14
|
+
import { loadSpecWorkspace, loadSpecWorkspaceAtRef } from "../loader";
|
|
15
|
+
import { parseSpecEnvironmentConfig } from "../parser";
|
|
16
|
+
import { renderCheckPrompt } from "../prompt-renderer";
|
|
17
|
+
import type {
|
|
18
|
+
LoadedSpecWorkspace,
|
|
19
|
+
ParsedSpecDocument,
|
|
20
|
+
ResolvedStageProfile,
|
|
21
|
+
SpecCheckEvidence,
|
|
22
|
+
SpecCheckMode,
|
|
23
|
+
SpecCheckReport,
|
|
24
|
+
SpecCheckResult,
|
|
25
|
+
SpecCheckStatus,
|
|
26
|
+
SpecDriftWarning,
|
|
27
|
+
ValidationIssue,
|
|
28
|
+
} from "../types";
|
|
29
|
+
import { assertSafeLisaStorageWritePath, listEffectiveWorkspaceFiles, resolveEffectiveProjectSpecPath, resolveWorkspaceLayout, toLogicalPath } from "../workspace";
|
|
30
|
+
import { escapeRegExp, globToRegExp, hasGlobMagic, normalizeRelativePath } from "../path-utils";
|
|
31
|
+
|
|
32
|
+
const SPEC_CHECK_EVIDENCE_TYPES = new Set<SpecCheckEvidence["type"]>(["test", "benchmark", "code-audit", "config"]);
|
|
33
|
+
|
|
34
|
+
interface CheckCommandOptions {
|
|
35
|
+
mode?: SpecCheckMode;
|
|
36
|
+
base?: string;
|
|
37
|
+
head?: string;
|
|
38
|
+
profile?: string;
|
|
39
|
+
harness?: string;
|
|
40
|
+
model?: string;
|
|
41
|
+
help: boolean;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
interface CommandExecutionResult {
|
|
45
|
+
exitCode: number;
|
|
46
|
+
stdout: string;
|
|
47
|
+
stderr: string;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
interface DeterministicCheckResult {
|
|
51
|
+
specId: string;
|
|
52
|
+
path: string;
|
|
53
|
+
codePaths: string[];
|
|
54
|
+
testPaths: string[];
|
|
55
|
+
testCommands: string[];
|
|
56
|
+
matchedCodeFiles: string[];
|
|
57
|
+
matchedTestFiles: string[];
|
|
58
|
+
evidence: SpecCheckEvidence[];
|
|
59
|
+
issues: string[];
|
|
60
|
+
status?: SpecCheckStatus;
|
|
61
|
+
summary: string;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
interface ParsedVerifierAudit {
|
|
65
|
+
status: Exclude<SpecCheckStatus, "SKIPPED">;
|
|
66
|
+
summary: string;
|
|
67
|
+
evidence: SpecCheckEvidence[];
|
|
68
|
+
issues: string[];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
interface CheckTarget {
|
|
72
|
+
specId: string;
|
|
73
|
+
path: string;
|
|
74
|
+
document?: ParsedSpecDocument;
|
|
75
|
+
validationIssues: ValidationIssue[];
|
|
76
|
+
driftWarning?: SpecDriftWarning;
|
|
77
|
+
benchmarkSidecars: ParsedSpecDocument[];
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface CheckCommandIO {
|
|
81
|
+
print(message: string): void;
|
|
82
|
+
error(message: string): void;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface CheckCommandDeps {
|
|
86
|
+
io?: CheckCommandIO;
|
|
87
|
+
runHarness?: (harnessId: string, request: HarnessRequest, cwd: string) => Promise<HarnessResult>;
|
|
88
|
+
inspectHarness?: (harnessId: string, cwd: string, commandOverride?: string) => Promise<HarnessInspection>;
|
|
89
|
+
runCommand?: (command: string, cwd: string) => CommandExecutionResult;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function createConsoleIO(): CheckCommandIO {
|
|
93
|
+
return {
|
|
94
|
+
print(message: string): void {
|
|
95
|
+
console.log(message);
|
|
96
|
+
},
|
|
97
|
+
error(message: string): void {
|
|
98
|
+
console.error(message);
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function printCheckHelp(io: CheckCommandIO): void {
|
|
104
|
+
io.print(`Lisa spec check
|
|
105
|
+
|
|
106
|
+
Usage:
|
|
107
|
+
lisa spec check --changed [options]
|
|
108
|
+
lisa spec check --all [options]
|
|
109
|
+
|
|
110
|
+
Options:
|
|
111
|
+
--changed Verify only changed or drift-impacted specs
|
|
112
|
+
--all Verify every base spec in the workspace
|
|
113
|
+
--base <rev> Diff base revision for --changed (default: HEAD)
|
|
114
|
+
--head <rev> Diff head revision for --changed (default: worktree)
|
|
115
|
+
--profile <name> Override the check stage profile
|
|
116
|
+
--harness <id> Override the configured harness adapter
|
|
117
|
+
--model <name> Override the configured model
|
|
118
|
+
--help, -h Show this help
|
|
119
|
+
`);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function parseCheckArgs(args: string[]): CheckCommandOptions {
|
|
123
|
+
const options: CheckCommandOptions = { help: false };
|
|
124
|
+
|
|
125
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
126
|
+
const arg = args[index];
|
|
127
|
+
if (!arg) {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (arg === "--help" || arg === "-h") {
|
|
132
|
+
options.help = true;
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (arg === "--changed") {
|
|
137
|
+
if (options.mode && options.mode !== "changed") {
|
|
138
|
+
throw new Error("Choose exactly one of `--changed` or `--all`.");
|
|
139
|
+
}
|
|
140
|
+
options.mode = "changed";
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (arg === "--all") {
|
|
145
|
+
if (options.mode && options.mode !== "all") {
|
|
146
|
+
throw new Error("Choose exactly one of `--changed` or `--all`.");
|
|
147
|
+
}
|
|
148
|
+
options.mode = "all";
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (arg === "--base" || arg === "--head" || arg === "--profile" || arg === "--harness" || arg === "--model") {
|
|
153
|
+
const value = args[index + 1];
|
|
154
|
+
if (!value) {
|
|
155
|
+
throw new Error(`${arg} requires a value.`);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
index += 1;
|
|
159
|
+
if (arg === "--base") {
|
|
160
|
+
options.base = value;
|
|
161
|
+
} else if (arg === "--head") {
|
|
162
|
+
options.head = value.toLowerCase() === "worktree" ? undefined : value;
|
|
163
|
+
} else if (arg === "--profile") {
|
|
164
|
+
options.profile = value;
|
|
165
|
+
} else if (arg === "--harness") {
|
|
166
|
+
options.harness = value;
|
|
167
|
+
} else {
|
|
168
|
+
options.model = value;
|
|
169
|
+
}
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
throw new Error(`Unknown lisa spec check option: ${arg}`);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
if (!options.help && !options.mode) {
|
|
177
|
+
throw new Error("Choose exactly one of `--changed` or `--all`.");
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
if (options.mode === "all" && (options.base || args.includes("--base") || args.includes("--head"))) {
|
|
181
|
+
throw new Error("`lisa spec check --all` does not accept `--base` or `--head`. Use `--changed` for diff-scoped checks.");
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
return options;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function resolveProfile(workspace: LoadedSpecWorkspace, options: CheckCommandOptions): ResolvedStageProfile {
|
|
188
|
+
const local = resolveLocalOverrides(workspace.localConfigRoot);
|
|
189
|
+
if (workspace.config) {
|
|
190
|
+
const resolved = resolveStageProfile(workspace.config, "check", {
|
|
191
|
+
profile: options.profile,
|
|
192
|
+
harness: options.harness,
|
|
193
|
+
model: options.model,
|
|
194
|
+
}, local);
|
|
195
|
+
if (!resolved) {
|
|
196
|
+
throw new Error("Unable to resolve a check stage profile from `.specs/config.yaml`.");
|
|
197
|
+
}
|
|
198
|
+
return resolved;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
return {
|
|
202
|
+
stage: "check",
|
|
203
|
+
profileName: options.profile ?? "default-check",
|
|
204
|
+
harness: options.harness ?? local.harness ?? "opencode",
|
|
205
|
+
model: options.model,
|
|
206
|
+
allowEdits: false,
|
|
207
|
+
args: [],
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function decodeOutput(bytes: Uint8Array<ArrayBufferLike>): string {
|
|
212
|
+
return new TextDecoder().decode(bytes);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function runGitCommand(workspaceRoot: string, args: string[], allowFailure = false): CommandExecutionResult {
|
|
216
|
+
const proc = Bun.spawnSync({
|
|
217
|
+
cmd: ["git", ...args],
|
|
218
|
+
cwd: workspaceRoot,
|
|
219
|
+
stdout: "pipe",
|
|
220
|
+
stderr: "pipe",
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
const result = {
|
|
224
|
+
exitCode: proc.exitCode,
|
|
225
|
+
stdout: decodeOutput(proc.stdout),
|
|
226
|
+
stderr: decodeOutput(proc.stderr),
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
if (result.exitCode !== 0 && !allowFailure) {
|
|
230
|
+
throw new Error(result.stderr.trim() || `git ${args.join(" ")} failed with exit code ${result.exitCode}.`);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
return result;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function resolveGitRevision(workspaceRoot: string, revision: string): string {
|
|
237
|
+
return runGitCommand(workspaceRoot, ["rev-parse", revision]).stdout.trim();
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function hasRepoWorktreeDrift(workspaceRoot: string, revision: string): boolean {
|
|
241
|
+
return runGitCommand(workspaceRoot, ["diff", "--quiet", revision, "--", "."], true).exitCode !== 0;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function hasUntrackedFiles(workspaceRoot: string): boolean {
|
|
245
|
+
return runGitCommand(workspaceRoot, ["ls-files", "--others", "--exclude-standard"], true).stdout.trim().length > 0;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function defaultRunCommand(command: string, cwd: string): CommandExecutionResult {
|
|
249
|
+
const proc = Bun.spawnSync({
|
|
250
|
+
cmd: ["sh", "-lc", command],
|
|
251
|
+
cwd,
|
|
252
|
+
stdout: "pipe",
|
|
253
|
+
stderr: "pipe",
|
|
254
|
+
env: process.env,
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
return {
|
|
258
|
+
exitCode: proc.exitCode,
|
|
259
|
+
stdout: decodeOutput(proc.stdout),
|
|
260
|
+
stderr: decodeOutput(proc.stderr),
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function formatCapability(value: boolean): string {
|
|
265
|
+
return value ? "yes" : "no";
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async function defaultInspectHarness(harnessId: string, cwd: string, commandOverride?: string): Promise<HarnessInspection> {
|
|
269
|
+
const adapter = getHarnessAdapterById(harnessId, cwd);
|
|
270
|
+
if (!adapter) {
|
|
271
|
+
throw new Error(`No harness adapter is registered for \`${harnessId}\`.`);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
return {
|
|
275
|
+
adapter,
|
|
276
|
+
availability: commandOverride
|
|
277
|
+
? {
|
|
278
|
+
available: true,
|
|
279
|
+
command: commandOverride,
|
|
280
|
+
reason: "Configured command override supplied by the selected stage profile.",
|
|
281
|
+
}
|
|
282
|
+
: await adapter.detect(),
|
|
283
|
+
capabilities: await adapter.capabilities(),
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function ensureHarnessSupportsCheck(inspection: HarnessInspection, profile: ResolvedStageProfile, options: CheckCommandOptions): void {
|
|
288
|
+
if (!inspection.adapter.supportedStages.includes("check")) {
|
|
289
|
+
throw new Error(`Harness \`${inspection.adapter.id}\` does not support the check stage.`);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
if (profile.allowEdits) {
|
|
293
|
+
throw new Error(`Check stage profile \`${profile.profileName}\` must declare \`allow_edits: false\`.`);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
if (options.model && !inspection.capabilities.supportsModelSelection) {
|
|
297
|
+
throw new Error(`Harness \`${inspection.adapter.id}\` does not support model overrides.`);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
if (!inspection.availability.available) {
|
|
301
|
+
throw new Error(`Harness \`${inspection.adapter.id}\` is unavailable: ${inspection.availability.reason}`);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function printHarnessReport(io: CheckCommandIO, inspection: HarnessInspection, profile: ResolvedStageProfile): void {
|
|
306
|
+
io.print("Harness");
|
|
307
|
+
io.print(`- adapter: ${inspection.adapter.id} (${inspection.adapter.displayName})`);
|
|
308
|
+
io.print(`- profile: ${profile.profileName}`);
|
|
309
|
+
io.print(`- command: ${inspection.availability.command ?? "(auto-detect)"}`);
|
|
310
|
+
io.print(`- availability: ${inspection.availability.reason}`);
|
|
311
|
+
io.print(`- can edit files: ${formatCapability(inspection.capabilities.canEditFiles)}`);
|
|
312
|
+
io.print(`- supports model selection: ${formatCapability(inspection.capabilities.supportsModelSelection)}`);
|
|
313
|
+
io.print(`- supports read-only mode: ${formatCapability(inspection.capabilities.supportsReadOnlyMode)}`);
|
|
314
|
+
io.print(`- workspace isolation: yes (per-spec temporary snapshots)`);
|
|
315
|
+
io.print("");
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function listWorkspaceFiles(workspaceRoot: string): string[] {
|
|
319
|
+
const ignoredDirectories = new Set([".git", "node_modules", ".lisa"]);
|
|
320
|
+
const files: string[] = [];
|
|
321
|
+
|
|
322
|
+
const walk = (currentPath: string): void => {
|
|
323
|
+
for (const entry of readdirSync(currentPath, { withFileTypes: true })) {
|
|
324
|
+
if (entry.isDirectory()) {
|
|
325
|
+
if (ignoredDirectories.has(entry.name)) {
|
|
326
|
+
continue;
|
|
327
|
+
}
|
|
328
|
+
walk(join(currentPath, entry.name));
|
|
329
|
+
continue;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
if (!entry.isFile()) {
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
files.push(normalizeRelativePath(relative(workspaceRoot, join(currentPath, entry.name))));
|
|
337
|
+
}
|
|
338
|
+
};
|
|
339
|
+
|
|
340
|
+
walk(workspaceRoot);
|
|
341
|
+
return files.sort();
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
function resolvePatternMatches(workspaceRoot: string, pattern: string, workspaceFiles: string[]): string[] {
|
|
345
|
+
const normalized = normalizeRelativePath(pattern);
|
|
346
|
+
|
|
347
|
+
if (!hasGlobMagic(normalized)) {
|
|
348
|
+
return existsSync(join(workspaceRoot, normalized)) ? [normalized] : [];
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
const matcher = globToRegExp(normalized);
|
|
352
|
+
return workspaceFiles.filter((file) => matcher.test(file));
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function expandPatterns(
|
|
356
|
+
workspaceRoot: string,
|
|
357
|
+
patterns: string[],
|
|
358
|
+
workspaceFiles: string[],
|
|
359
|
+
): { matches: string[]; missingPatterns: string[] } {
|
|
360
|
+
const matches = new Set<string>();
|
|
361
|
+
const missingPatterns: string[] = [];
|
|
362
|
+
|
|
363
|
+
for (const pattern of patterns) {
|
|
364
|
+
const resolved = resolvePatternMatches(workspaceRoot, pattern, workspaceFiles);
|
|
365
|
+
if (resolved.length === 0) {
|
|
366
|
+
missingPatterns.push(pattern);
|
|
367
|
+
continue;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
for (const match of resolved) {
|
|
371
|
+
matches.add(match);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
return {
|
|
376
|
+
matches: Array.from(matches).sort(),
|
|
377
|
+
missingPatterns,
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function extractStringArray(frontmatter: Record<string, unknown>, key: string): string[] {
|
|
382
|
+
const value = frontmatter[key];
|
|
383
|
+
return Array.isArray(value)
|
|
384
|
+
? value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)
|
|
385
|
+
: [];
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
function truncateOutput(output: string): string {
|
|
389
|
+
const trimmed = output.trim();
|
|
390
|
+
if (trimmed.length <= 240) {
|
|
391
|
+
return trimmed;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
return `${trimmed.slice(0, 237)}...`;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
function formatSpecContent(document: ParsedSpecDocument): string {
|
|
398
|
+
const frontmatter = Bun.YAML.stringify(document.frontmatter).trim();
|
|
399
|
+
if (frontmatter.length === 0) {
|
|
400
|
+
return document.body.trim();
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
return `---\n${frontmatter}\n---\n\n${document.body}`.trim();
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
function deriveSpecIdFromPath(path: string): string {
|
|
407
|
+
const normalized = normalizeRelativePath(path);
|
|
408
|
+
const areaMatch = normalized.match(/\.specs\/(backend|frontend)\//);
|
|
409
|
+
const filename = basename(normalized).replace(/\.md$/i, "");
|
|
410
|
+
return areaMatch ? `${areaMatch[1]}.${filename}` : filename;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function isBaseSpecPath(path: string): boolean {
|
|
414
|
+
const normalized = normalizeRelativePath(path);
|
|
415
|
+
return (normalized.startsWith(".specs/backend/") || normalized.startsWith(".specs/frontend/"))
|
|
416
|
+
&& normalized.endsWith(".md")
|
|
417
|
+
&& !normalized.includes(".bench.");
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
function getWorkspaceIssuesByPath(workspaceRoot: string, issues: ValidationIssue[]): Map<string, ValidationIssue[]> {
|
|
421
|
+
const layout = resolveWorkspaceLayout(workspaceRoot);
|
|
422
|
+
const map = new Map<string, ValidationIssue[]>();
|
|
423
|
+
|
|
424
|
+
for (const issue of issues) {
|
|
425
|
+
const key = normalizeRelativePath(toLogicalPath(layout, issue.path));
|
|
426
|
+
map.set(key, [...(map.get(key) ?? []), issue]);
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
return map;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function getValidationIssuesForTarget(
|
|
433
|
+
workspace: LoadedSpecWorkspace,
|
|
434
|
+
issuesByPath: Map<string, ValidationIssue[]>,
|
|
435
|
+
basePath: string,
|
|
436
|
+
specId: string,
|
|
437
|
+
): ValidationIssue[] {
|
|
438
|
+
const collected = [...(issuesByPath.get(basePath) ?? [])];
|
|
439
|
+
|
|
440
|
+
for (const sidecar of workspace.documents.filter((document) => document.kind !== "base" && document.frontmatter.extends === specId)) {
|
|
441
|
+
const sidecarPath = normalizeRelativePath(toLogicalPath(resolveWorkspaceLayout(workspace.workspacePath), sidecar.path));
|
|
442
|
+
collected.push(...(issuesByPath.get(sidecarPath) ?? []));
|
|
443
|
+
|
|
444
|
+
if (sidecar.extensionKind === "benchmark" && sidecar.frontmatter.required === true && sidecar.status !== "active") {
|
|
445
|
+
collected.push({
|
|
446
|
+
severity: "error",
|
|
447
|
+
path: sidecar.path,
|
|
448
|
+
message: "A required benchmark sidecar must declare `status: active`.",
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
return collected;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
function buildAllTargets(workspace: LoadedSpecWorkspace): CheckTarget[] {
|
|
457
|
+
const issuesByPath = getWorkspaceIssuesByPath(workspace.workspacePath, workspace.issues);
|
|
458
|
+
const targets = new Map<string, CheckTarget>();
|
|
459
|
+
|
|
460
|
+
for (const document of workspace.documents) {
|
|
461
|
+
if (document.kind !== "base") {
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
const relativePath = normalizeRelativePath(toLogicalPath(resolveWorkspaceLayout(workspace.workspacePath), document.path));
|
|
466
|
+
const specId = document.id ?? deriveSpecIdFromPath(relativePath);
|
|
467
|
+
targets.set(relativePath, {
|
|
468
|
+
specId,
|
|
469
|
+
path: document.path,
|
|
470
|
+
document,
|
|
471
|
+
validationIssues: getValidationIssuesForTarget(workspace, issuesByPath, relativePath, specId),
|
|
472
|
+
benchmarkSidecars: getExtensionSidecarsForSpec(workspace, specId),
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
for (const [relativePath, issues] of issuesByPath.entries()) {
|
|
477
|
+
if (!isBaseSpecPath(relativePath) || targets.has(relativePath)) {
|
|
478
|
+
continue;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
targets.set(relativePath, {
|
|
482
|
+
specId: deriveSpecIdFromPath(relativePath),
|
|
483
|
+
path: join(workspace.workspacePath, relativePath),
|
|
484
|
+
validationIssues: issues,
|
|
485
|
+
benchmarkSidecars: [],
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
return Array.from(targets.values()).sort((left, right) => left.specId.localeCompare(right.specId));
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
function listChangedCodePaths(workspaceRoot: string, baseRef: string, headRef?: string): string[] {
|
|
493
|
+
const layout = resolveWorkspaceLayout(workspaceRoot);
|
|
494
|
+
const externalSnapshotCodePaths = layout.storageMode === "external" && baseRef === EXTERNAL_SPEC_DIFF_BASE_REF
|
|
495
|
+
? readExternalSnapshotCodePaths(workspaceRoot)
|
|
496
|
+
: {};
|
|
497
|
+
const effectiveBaseRef = layout.storageMode === "external" && baseRef === EXTERNAL_SPEC_DIFF_BASE_REF
|
|
498
|
+
? readExternalSnapshotRepoRevision(workspaceRoot) ?? DEFAULT_SPEC_DIFF_BASE_REF
|
|
499
|
+
: baseRef;
|
|
500
|
+
const diffArgs = ["diff", "--name-only", ...(headRef ? [effectiveBaseRef, headRef] : [effectiveBaseRef]), "--", "."];
|
|
501
|
+
const diffOutput = runGitCommand(workspaceRoot, diffArgs, true).stdout;
|
|
502
|
+
const untrackedOutput = headRef ? "" : runGitCommand(workspaceRoot, ["ls-files", "--others", "--exclude-standard"], true).stdout;
|
|
503
|
+
const changedPaths = new Set<string>();
|
|
504
|
+
|
|
505
|
+
for (const source of [diffOutput, untrackedOutput]) {
|
|
506
|
+
for (const line of source.split("\n")) {
|
|
507
|
+
const path = normalizeRelativePath(line.trim());
|
|
508
|
+
if (!path || path.startsWith(".specs/") || path.startsWith(".git/") || path.startsWith(".lisa/")) {
|
|
509
|
+
continue;
|
|
510
|
+
}
|
|
511
|
+
const currentHash = existsSync(join(workspaceRoot, path))
|
|
512
|
+
? createHash("sha1").update(readFileSync(join(workspaceRoot, path))).digest("hex")
|
|
513
|
+
: "__missing__";
|
|
514
|
+
const snapshotEntry = externalSnapshotCodePaths[path];
|
|
515
|
+
if (snapshotEntry && (snapshotEntry.verifiedHash === currentHash || snapshotEntry.baselineHash === currentHash)) {
|
|
516
|
+
continue;
|
|
517
|
+
}
|
|
518
|
+
changedPaths.add(path);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
for (const [path, snapshotEntry] of Object.entries(externalSnapshotCodePaths)) {
|
|
523
|
+
if (changedPaths.has(path)) {
|
|
524
|
+
continue;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
const currentHash = existsSync(join(workspaceRoot, path))
|
|
528
|
+
? createHash("sha1").update(readFileSync(join(workspaceRoot, path))).digest("hex")
|
|
529
|
+
: "__missing__";
|
|
530
|
+
if (currentHash !== snapshotEntry.verifiedHash && currentHash !== snapshotEntry.baselineHash) {
|
|
531
|
+
changedPaths.add(path);
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
return Array.from(changedPaths).sort();
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
function listChangedEnvironmentNames(workspaceRoot: string, baseRef: string, headRef?: string): string[] {
|
|
539
|
+
const layout = resolveWorkspaceLayout(workspaceRoot);
|
|
540
|
+
if (layout.storageMode === "external") {
|
|
541
|
+
const previousFiles = readExternalSpecSnapshot(workspaceRoot).files;
|
|
542
|
+
const nextFiles = new Map<string, string>();
|
|
543
|
+
const effective = listEffectiveWorkspaceFiles(layout);
|
|
544
|
+
for (const path of effective.environmentPaths) {
|
|
545
|
+
nextFiles.set(toLogicalPath(layout, path), readFileSync(path, "utf8"));
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
const names = new Set<string>();
|
|
549
|
+
const keys = new Set<string>([
|
|
550
|
+
...Object.keys(previousFiles).filter((key) => key.startsWith(".specs/environments/")),
|
|
551
|
+
...Array.from(nextFiles.keys()),
|
|
552
|
+
]);
|
|
553
|
+
|
|
554
|
+
for (const key of keys) {
|
|
555
|
+
const previous = previousFiles[key];
|
|
556
|
+
const next = nextFiles.get(key);
|
|
557
|
+
if (previous === next) {
|
|
558
|
+
continue;
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
const match = key.match(/^\.specs\/environments\/([^/]+)\.(yaml|yml)$/);
|
|
562
|
+
if (match?.[1]) {
|
|
563
|
+
names.add(match[1]);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
for (const content of [previous, next]) {
|
|
567
|
+
if (!content) {
|
|
568
|
+
continue;
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
try {
|
|
572
|
+
const environment = parseSpecEnvironmentConfig(join(workspaceRoot, key), content);
|
|
573
|
+
if (environment.name) {
|
|
574
|
+
names.add(environment.name);
|
|
575
|
+
}
|
|
576
|
+
} catch {
|
|
577
|
+
// Ignore malformed environment files here; workspace validation handles the concrete issue.
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
return Array.from(names).sort();
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
const diffOutput = runGitCommand(
|
|
586
|
+
workspaceRoot,
|
|
587
|
+
["diff", "--name-only", ...(headRef ? [baseRef, headRef] : [baseRef]), "--", ".specs/environments"],
|
|
588
|
+
true,
|
|
589
|
+
).stdout;
|
|
590
|
+
const untrackedOutput = headRef
|
|
591
|
+
? ""
|
|
592
|
+
: runGitCommand(workspaceRoot, ["ls-files", "--others", "--exclude-standard", "--", ".specs/environments"], true).stdout;
|
|
593
|
+
const names = new Set<string>();
|
|
594
|
+
const readEnvironmentAtRef = (ref: string, path: string): string | undefined => {
|
|
595
|
+
const result = runGitCommand(workspaceRoot, ["show", `${ref}:${path}`], true);
|
|
596
|
+
return result.exitCode === 0 ? result.stdout : undefined;
|
|
597
|
+
};
|
|
598
|
+
|
|
599
|
+
for (const source of [diffOutput, untrackedOutput]) {
|
|
600
|
+
for (const line of source.split("\n")) {
|
|
601
|
+
const path = normalizeRelativePath(line.trim());
|
|
602
|
+
if (!path) {
|
|
603
|
+
continue;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
const match = path.match(/^\.specs\/environments\/([^/]+)\.(yaml|yml)$/);
|
|
607
|
+
if (match?.[1]) {
|
|
608
|
+
names.add(match[1]);
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
const contents = [
|
|
612
|
+
readEnvironmentAtRef(baseRef, path),
|
|
613
|
+
headRef
|
|
614
|
+
? readEnvironmentAtRef(headRef, path)
|
|
615
|
+
: existsSync(join(workspaceRoot, path))
|
|
616
|
+
? readFileSync(join(workspaceRoot, path), "utf8")
|
|
617
|
+
: undefined,
|
|
618
|
+
];
|
|
619
|
+
|
|
620
|
+
for (const content of contents) {
|
|
621
|
+
if (!content) {
|
|
622
|
+
continue;
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
try {
|
|
626
|
+
const environment = parseSpecEnvironmentConfig(join(workspaceRoot, path), content);
|
|
627
|
+
if (environment.name) {
|
|
628
|
+
names.add(environment.name);
|
|
629
|
+
}
|
|
630
|
+
} catch {
|
|
631
|
+
// Ignore malformed environment files here; workspace validation handles the concrete issue.
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
return Array.from(names).sort();
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
function collectDriftWarnings(
|
|
641
|
+
workspace: LoadedSpecWorkspace,
|
|
642
|
+
changedPaths: string[],
|
|
643
|
+
changedSpecIds: Set<string>,
|
|
644
|
+
): SpecDriftWarning[] {
|
|
645
|
+
const warnings: SpecDriftWarning[] = [];
|
|
646
|
+
|
|
647
|
+
for (const document of workspace.documents) {
|
|
648
|
+
if (document.kind !== "base") {
|
|
649
|
+
continue;
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
const specId = document.id ?? deriveSpecIdFromPath(toLogicalPath(resolveWorkspaceLayout(workspace.workspacePath), document.path));
|
|
653
|
+
if (changedSpecIds.has(specId)) {
|
|
654
|
+
continue;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
const codePaths = extractStringArray(document.frontmatter, "code_paths");
|
|
658
|
+
const overlaps = changedPaths.filter((changedPath) => codePaths.some((pattern) => globToRegExp(pattern).test(changedPath)));
|
|
659
|
+
if (overlaps.length === 0) {
|
|
660
|
+
continue;
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
warnings.push({
|
|
664
|
+
specId,
|
|
665
|
+
path: document.path,
|
|
666
|
+
changedPaths: overlaps,
|
|
667
|
+
message: `Changed code overlaps ${specId} without a corresponding spec edit.`,
|
|
668
|
+
});
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
return warnings.sort((left, right) => left.specId.localeCompare(right.specId));
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
function getChangedExtensionIssues(specId: string, diff: ReturnType<typeof collectSpecDiff>): ValidationIssue[] {
|
|
675
|
+
const issues: ValidationIssue[] = [];
|
|
676
|
+
|
|
677
|
+
for (const delta of diff.deltas) {
|
|
678
|
+
if (delta.kind === "base" || delta.extendsSpecId !== specId) {
|
|
679
|
+
const previousExtends = typeof delta.previousDocument?.frontmatter.extends === "string"
|
|
680
|
+
? delta.previousDocument.frontmatter.extends
|
|
681
|
+
: undefined;
|
|
682
|
+
if (previousExtends !== specId) {
|
|
683
|
+
continue;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
const previousDocument = delta.previousDocument;
|
|
688
|
+
const nextDocument = delta.nextDocument;
|
|
689
|
+
const wasRequired = previousDocument?.frontmatter.required === true;
|
|
690
|
+
const stillRequired = nextDocument?.frontmatter.required === true;
|
|
691
|
+
const stillActive = nextDocument?.status === "active";
|
|
692
|
+
const previousExtends = typeof previousDocument?.frontmatter.extends === "string" ? previousDocument.frontmatter.extends : undefined;
|
|
693
|
+
const nextExtends = typeof nextDocument?.frontmatter.extends === "string" ? nextDocument.frontmatter.extends : undefined;
|
|
694
|
+
|
|
695
|
+
if (wasRequired && previousExtends === specId && (delta.fileChange === "deleted" || !stillActive || !stillRequired || nextExtends !== previousExtends)) {
|
|
696
|
+
issues.push({
|
|
697
|
+
severity: "error",
|
|
698
|
+
path: delta.path,
|
|
699
|
+
message: `Required sidecar \`${delta.specId}\` no longer protects \`${specId}\` as an active required sidecar.`,
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
return issues;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
function buildChangedTargets(
|
|
708
|
+
workspace: LoadedSpecWorkspace,
|
|
709
|
+
baseRef: string,
|
|
710
|
+
headRef?: string,
|
|
711
|
+
): { targets: CheckTarget[]; driftWarnings: SpecDriftWarning[]; changedSpecCount: number; effectiveHeadRef: string } {
|
|
712
|
+
const diff = collectSpecDiff(workspace.workspacePath, { baseRef, headRef });
|
|
713
|
+
const issuesByPath = getWorkspaceIssuesByPath(workspace.workspacePath, workspace.issues);
|
|
714
|
+
const changedSpecIds = new Set<string>();
|
|
715
|
+
const changedBasePaths = new Set<string>();
|
|
716
|
+
|
|
717
|
+
for (const delta of diff.deltas) {
|
|
718
|
+
if (delta.kind === "base") {
|
|
719
|
+
changedSpecIds.add(delta.specId);
|
|
720
|
+
changedBasePaths.add(normalizeRelativePath(toLogicalPath(resolveWorkspaceLayout(workspace.workspacePath), delta.path)));
|
|
721
|
+
} else {
|
|
722
|
+
const previousExtends = typeof delta.previousDocument?.frontmatter.extends === "string"
|
|
723
|
+
? delta.previousDocument.frontmatter.extends
|
|
724
|
+
: undefined;
|
|
725
|
+
const nextExtends = typeof delta.nextDocument?.frontmatter.extends === "string"
|
|
726
|
+
? delta.nextDocument.frontmatter.extends
|
|
727
|
+
: undefined;
|
|
728
|
+
if (previousExtends) {
|
|
729
|
+
changedSpecIds.add(previousExtends);
|
|
730
|
+
}
|
|
731
|
+
if (nextExtends) {
|
|
732
|
+
changedSpecIds.add(nextExtends);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
const driftWarnings = collectDriftWarnings(
|
|
738
|
+
workspace,
|
|
739
|
+
listChangedCodePaths(workspace.workspacePath, baseRef, headRef),
|
|
740
|
+
changedSpecIds,
|
|
741
|
+
);
|
|
742
|
+
const changedEnvironmentNames = new Set(listChangedEnvironmentNames(workspace.workspacePath, baseRef, headRef));
|
|
743
|
+
const targets = new Map<string, CheckTarget>();
|
|
744
|
+
|
|
745
|
+
for (const document of workspace.documents) {
|
|
746
|
+
if (document.kind !== "base") {
|
|
747
|
+
continue;
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
const relativePath = normalizeRelativePath(toLogicalPath(resolveWorkspaceLayout(workspace.workspacePath), document.path));
|
|
751
|
+
const specId = document.id ?? deriveSpecIdFromPath(relativePath);
|
|
752
|
+
const sidecars = getExtensionSidecarsForSpec(workspace, specId);
|
|
753
|
+
const changedExtensionIssues = getChangedExtensionIssues(specId, diff);
|
|
754
|
+
const environmentImpacted = sidecars.some((sidecar) => {
|
|
755
|
+
const environmentName = typeof sidecar.frontmatter.environment === "string" ? sidecar.frontmatter.environment : undefined;
|
|
756
|
+
return environmentName !== undefined && changedEnvironmentNames.has(environmentName);
|
|
757
|
+
});
|
|
758
|
+
if (!changedSpecIds.has(specId) && !driftWarnings.some((warning) => warning.specId === specId) && !environmentImpacted) {
|
|
759
|
+
continue;
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
targets.set(relativePath, {
|
|
763
|
+
specId,
|
|
764
|
+
path: document.path,
|
|
765
|
+
document,
|
|
766
|
+
validationIssues: [...getValidationIssuesForTarget(workspace, issuesByPath, relativePath, specId), ...changedExtensionIssues],
|
|
767
|
+
driftWarning: driftWarnings.find((warning) => warning.specId === specId),
|
|
768
|
+
benchmarkSidecars: sidecars,
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
for (const relativePath of changedBasePaths) {
|
|
773
|
+
if (targets.has(relativePath)) {
|
|
774
|
+
continue;
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
targets.set(relativePath, {
|
|
778
|
+
specId: deriveSpecIdFromPath(relativePath),
|
|
779
|
+
path: join(workspace.workspacePath, relativePath),
|
|
780
|
+
validationIssues: issuesByPath.get(relativePath) ?? [],
|
|
781
|
+
benchmarkSidecars: [],
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
return {
|
|
786
|
+
targets: Array.from(targets.values()).sort((left, right) => left.specId.localeCompare(right.specId)),
|
|
787
|
+
driftWarnings,
|
|
788
|
+
changedSpecCount: diff.deltas.length,
|
|
789
|
+
effectiveHeadRef: diff.headRef,
|
|
790
|
+
};
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
async function runDeterministicChecks(
|
|
794
|
+
target: CheckTarget,
|
|
795
|
+
workspace: LoadedSpecWorkspace,
|
|
796
|
+
workspaceRoot: string,
|
|
797
|
+
workspaceFiles: string[],
|
|
798
|
+
testCommandResults: Map<string, CommandExecutionResult>,
|
|
799
|
+
runCommand: (command: string, cwd: string) => CommandExecutionResult,
|
|
800
|
+
): Promise<DeterministicCheckResult> {
|
|
801
|
+
const codePaths = target.document ? extractStringArray(target.document.frontmatter, "code_paths") : [];
|
|
802
|
+
const testPaths = target.document ? extractStringArray(target.document.frontmatter, "test_paths") : [];
|
|
803
|
+
const testCommands = target.document ? extractStringArray(target.document.frontmatter, "test_commands") : [];
|
|
804
|
+
const evidence: SpecCheckEvidence[] = [];
|
|
805
|
+
const issues: string[] = [];
|
|
806
|
+
|
|
807
|
+
for (const issue of target.validationIssues) {
|
|
808
|
+
if (issue.severity === "warning") {
|
|
809
|
+
evidence.push({ type: "config", detail: `Validation warning: ${issue.message}` });
|
|
810
|
+
} else {
|
|
811
|
+
issues.push(issue.message);
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
if (!target.document) {
|
|
816
|
+
issues.push("Spec file could not be loaded for verification.");
|
|
817
|
+
return {
|
|
818
|
+
specId: target.specId,
|
|
819
|
+
path: target.path,
|
|
820
|
+
codePaths,
|
|
821
|
+
testPaths,
|
|
822
|
+
testCommands,
|
|
823
|
+
matchedCodeFiles: [],
|
|
824
|
+
matchedTestFiles: [],
|
|
825
|
+
evidence,
|
|
826
|
+
issues,
|
|
827
|
+
status: "FAIL",
|
|
828
|
+
summary: "Spec could not be parsed or loaded, so conformance could not be checked.",
|
|
829
|
+
};
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
if (target.document.status !== "active") {
|
|
833
|
+
evidence.push({ type: "config", detail: `Skipped ${target.document.status ?? "unknown"} spec.` });
|
|
834
|
+
return {
|
|
835
|
+
specId: target.specId,
|
|
836
|
+
path: target.path,
|
|
837
|
+
codePaths,
|
|
838
|
+
testPaths,
|
|
839
|
+
testCommands,
|
|
840
|
+
matchedCodeFiles: [],
|
|
841
|
+
matchedTestFiles: [],
|
|
842
|
+
evidence,
|
|
843
|
+
issues,
|
|
844
|
+
status: "SKIPPED",
|
|
845
|
+
summary: `Skipped non-active spec with status \`${target.document.status ?? "unknown"}\`.`,
|
|
846
|
+
};
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
const codeMatches = expandPatterns(workspaceRoot, codePaths, workspaceFiles);
|
|
850
|
+
const testMatches = expandPatterns(workspaceRoot, testPaths, workspaceFiles);
|
|
851
|
+
|
|
852
|
+
if (codeMatches.matches.length > 0) {
|
|
853
|
+
evidence.push({ type: "config", detail: `Matched ${codeMatches.matches.length} code path entries.` });
|
|
854
|
+
}
|
|
855
|
+
if (testMatches.matches.length > 0) {
|
|
856
|
+
evidence.push({ type: "test", detail: `Matched ${testMatches.matches.length} test path entries.` });
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
if (codeMatches.missingPatterns.length > 0) {
|
|
860
|
+
issues.push(`Missing declared code paths: ${codeMatches.missingPatterns.join(", ")}`);
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
if (testPaths.length > 0 && testMatches.missingPatterns.length > 0) {
|
|
864
|
+
issues.push(`Missing declared test paths: ${testMatches.missingPatterns.join(", ")}`);
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
if (target.driftWarning) {
|
|
868
|
+
evidence.push({ type: "config", detail: target.driftWarning.message });
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
for (const command of testCommands) {
|
|
872
|
+
const result = testCommandResults.get(command);
|
|
873
|
+
if (!result) {
|
|
874
|
+
issues.push(`Mapped test command was not executed: ${command}`);
|
|
875
|
+
continue;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
if (result.exitCode === 0) {
|
|
879
|
+
evidence.push({ type: "test", detail: `Passed: ${command}` });
|
|
880
|
+
continue;
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
const output = truncateOutput(result.stderr || result.stdout);
|
|
884
|
+
issues.push(`Failed test command: ${command}${output ? ` (${output})` : ""}`);
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
if (testCommands.length === 0 && testMatches.matches.length === 0) {
|
|
888
|
+
issues.push("No mapped tests were found for this active spec.");
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
if (issues.length > 0) {
|
|
892
|
+
return {
|
|
893
|
+
specId: target.specId,
|
|
894
|
+
path: target.path,
|
|
895
|
+
codePaths,
|
|
896
|
+
testPaths,
|
|
897
|
+
testCommands,
|
|
898
|
+
matchedCodeFiles: codeMatches.matches,
|
|
899
|
+
matchedTestFiles: testMatches.matches,
|
|
900
|
+
evidence,
|
|
901
|
+
issues,
|
|
902
|
+
status: "FAIL",
|
|
903
|
+
summary: "Deterministic checks failed.",
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
if (target.document) {
|
|
908
|
+
const extensionResults = await verifySpecExtensions(
|
|
909
|
+
{
|
|
910
|
+
specId: target.specId,
|
|
911
|
+
path: target.path,
|
|
912
|
+
document: target.document,
|
|
913
|
+
sidecars: target.benchmarkSidecars,
|
|
914
|
+
},
|
|
915
|
+
{
|
|
916
|
+
workspaceRoot,
|
|
917
|
+
workspace,
|
|
918
|
+
runCommand,
|
|
919
|
+
},
|
|
920
|
+
);
|
|
921
|
+
|
|
922
|
+
for (const extensionResult of extensionResults) {
|
|
923
|
+
evidence.push(...extensionResult.evidence);
|
|
924
|
+
issues.push(...extensionResult.issues);
|
|
925
|
+
|
|
926
|
+
if (extensionResult.status === "FAIL" || extensionResult.status === "UNSURE") {
|
|
927
|
+
return {
|
|
928
|
+
specId: target.specId,
|
|
929
|
+
path: target.path,
|
|
930
|
+
codePaths,
|
|
931
|
+
testPaths,
|
|
932
|
+
testCommands,
|
|
933
|
+
matchedCodeFiles: codeMatches.matches,
|
|
934
|
+
matchedTestFiles: testMatches.matches,
|
|
935
|
+
evidence,
|
|
936
|
+
issues,
|
|
937
|
+
status: extensionResult.status,
|
|
938
|
+
summary: extensionResult.summary ?? "Extension verification did not produce a conclusive result.",
|
|
939
|
+
};
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
return {
|
|
945
|
+
specId: target.specId,
|
|
946
|
+
path: target.path,
|
|
947
|
+
codePaths,
|
|
948
|
+
testPaths,
|
|
949
|
+
testCommands,
|
|
950
|
+
matchedCodeFiles: codeMatches.matches,
|
|
951
|
+
matchedTestFiles: testMatches.matches,
|
|
952
|
+
evidence,
|
|
953
|
+
issues,
|
|
954
|
+
summary: "Deterministic checks passed; awaiting verifier audit.",
|
|
955
|
+
};
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
function extractJsonPayload(text: string): string | undefined {
|
|
959
|
+
const markerMatch = text.match(/LISA_CHECK_JSON_START\s*([\s\S]*?)\s*LISA_CHECK_JSON_END/);
|
|
960
|
+
if (markerMatch?.[1]) {
|
|
961
|
+
return markerMatch[1].trim();
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
const fencedMatch = text.match(/```json\s*([\s\S]*?)```/i);
|
|
965
|
+
if (fencedMatch?.[1]) {
|
|
966
|
+
return fencedMatch[1].trim();
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
const firstBrace = text.indexOf("{");
|
|
970
|
+
const lastBrace = text.lastIndexOf("}");
|
|
971
|
+
if (firstBrace >= 0 && lastBrace > firstBrace) {
|
|
972
|
+
return text.slice(firstBrace, lastBrace + 1).trim();
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
return undefined;
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
function parseVerifierAudit(result: HarnessResult): ParsedVerifierAudit {
|
|
979
|
+
const raw = typeof result.structuredOutput === "object" && result.structuredOutput !== null
|
|
980
|
+
? result.structuredOutput
|
|
981
|
+
: result.finalText
|
|
982
|
+
? JSON.parse(extractJsonPayload(result.finalText) ?? "null")
|
|
983
|
+
: null;
|
|
984
|
+
|
|
985
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
986
|
+
throw new Error("Verifier did not return a valid JSON audit payload.");
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
const statusValue = typeof (raw as { status?: unknown }).status === "string"
|
|
990
|
+
? (raw as { status: string }).status.toUpperCase()
|
|
991
|
+
: undefined;
|
|
992
|
+
if (statusValue !== "PASS" && statusValue !== "FAIL" && statusValue !== "UNSURE") {
|
|
993
|
+
throw new Error("Verifier audit payload must declare status as PASS, FAIL, or UNSURE.");
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
const evidence = Array.isArray((raw as { evidence?: unknown[] }).evidence)
|
|
997
|
+
? (raw as { evidence: unknown[] }).evidence
|
|
998
|
+
.filter(
|
|
999
|
+
(entry): entry is { type: SpecCheckEvidence["type"]; detail: string } =>
|
|
1000
|
+
typeof entry === "object"
|
|
1001
|
+
&& entry !== null
|
|
1002
|
+
&& typeof (entry as { type?: unknown }).type === "string"
|
|
1003
|
+
&& SPEC_CHECK_EVIDENCE_TYPES.has((entry as { type: SpecCheckEvidence["type"] }).type)
|
|
1004
|
+
&& typeof (entry as { detail?: unknown }).detail === "string",
|
|
1005
|
+
)
|
|
1006
|
+
.map((entry) => ({ type: entry.type, detail: entry.detail.trim() }))
|
|
1007
|
+
.filter((entry) => entry.detail.length > 0)
|
|
1008
|
+
: [];
|
|
1009
|
+
const issues = Array.isArray((raw as { issues?: unknown[] }).issues)
|
|
1010
|
+
? (raw as { issues: unknown[] }).issues.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)
|
|
1011
|
+
: [];
|
|
1012
|
+
|
|
1013
|
+
return {
|
|
1014
|
+
status: statusValue,
|
|
1015
|
+
summary: typeof (raw as { summary?: unknown }).summary === "string"
|
|
1016
|
+
? (raw as { summary: string }).summary.trim()
|
|
1017
|
+
: "Verifier completed without a summary.",
|
|
1018
|
+
evidence,
|
|
1019
|
+
issues,
|
|
1020
|
+
};
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
function ensureParentDirectory(path: string): void {
|
|
1024
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
function copyPathIntoSnapshot(sourcePath: string, snapshotRoot: string, snapshotRelativePath: string): void {
|
|
1028
|
+
if (!existsSync(sourcePath)) {
|
|
1029
|
+
return;
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
const destinationPath = join(snapshotRoot, snapshotRelativePath);
|
|
1033
|
+
ensureParentDirectory(destinationPath);
|
|
1034
|
+
const stats = statSync(sourcePath);
|
|
1035
|
+
if (stats.isDirectory()) {
|
|
1036
|
+
cpSync(sourcePath, destinationPath, { recursive: true });
|
|
1037
|
+
return;
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
cpSync(sourcePath, destinationPath);
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
function copyIntoSnapshot(workspaceRoot: string, snapshotRoot: string, relativePath: string): void {
|
|
1044
|
+
copyPathIntoSnapshot(join(workspaceRoot, relativePath), snapshotRoot, relativePath);
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
function listCommonSnapshotFiles(workspaceRoot: string): string[] {
|
|
1048
|
+
const candidates = ["package.json", "README.md", "bunfig.toml", "bun.lock", "bun.lockb", ".specs/config.yaml"];
|
|
1049
|
+
const commonFiles = candidates.filter((candidate) => existsSync(join(workspaceRoot, candidate)));
|
|
1050
|
+
|
|
1051
|
+
for (const entry of readdirSync(workspaceRoot)) {
|
|
1052
|
+
if (/^tsconfig.*\.json$/i.test(entry)) {
|
|
1053
|
+
commonFiles.push(entry);
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
return Array.from(new Set(commonFiles.map((path) => normalizeRelativePath(path)))).sort();
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
function buildVerifierSnapshot(
|
|
1061
|
+
workspaceRoot: string,
|
|
1062
|
+
deterministic: DeterministicCheckResult,
|
|
1063
|
+
commonFiles: string[],
|
|
1064
|
+
extraContextPaths: string[] = [],
|
|
1065
|
+
): { snapshotRoot: string; contextFiles: string[] } {
|
|
1066
|
+
const layout = resolveWorkspaceLayout(workspaceRoot);
|
|
1067
|
+
const snapshotRoot = mkdtempSync(join(tmpdir(), "lisa-check-"));
|
|
1068
|
+
const contextFiles = new Set<string>();
|
|
1069
|
+
const paths = new Set<string>([
|
|
1070
|
+
...commonFiles,
|
|
1071
|
+
...deterministic.matchedCodeFiles,
|
|
1072
|
+
...deterministic.matchedTestFiles,
|
|
1073
|
+
...extraContextPaths.map((p) => normalizeRelativePath(p)),
|
|
1074
|
+
]);
|
|
1075
|
+
|
|
1076
|
+
const specLogicalPath = normalizeRelativePath(toLogicalPath(layout, deterministic.path));
|
|
1077
|
+
copyPathIntoSnapshot(deterministic.path, snapshotRoot, specLogicalPath);
|
|
1078
|
+
contextFiles.add(join(snapshotRoot, specLogicalPath));
|
|
1079
|
+
|
|
1080
|
+
if (layout.storageMode === "external" && existsSync(layout.configPath)) {
|
|
1081
|
+
copyPathIntoSnapshot(layout.configPath, snapshotRoot, ".specs/config.yaml");
|
|
1082
|
+
contextFiles.add(join(snapshotRoot, ".specs/config.yaml"));
|
|
1083
|
+
}
|
|
1084
|
+
const projectSpecPath = resolveEffectiveProjectSpecPath(layout);
|
|
1085
|
+
if (projectSpecPath && existsSync(projectSpecPath)) {
|
|
1086
|
+
copyPathIntoSnapshot(projectSpecPath, snapshotRoot, ".specs/project.md");
|
|
1087
|
+
contextFiles.add(join(snapshotRoot, ".specs/project.md"));
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
for (const path of paths) {
|
|
1091
|
+
copyIntoSnapshot(workspaceRoot, snapshotRoot, path);
|
|
1092
|
+
const snapshotPath = join(snapshotRoot, path);
|
|
1093
|
+
if (existsSync(snapshotPath)) {
|
|
1094
|
+
contextFiles.add(snapshotPath);
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
return {
|
|
1099
|
+
snapshotRoot,
|
|
1100
|
+
contextFiles: Array.from(contextFiles).sort(),
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
async function mapWithConcurrency<T, U>(
|
|
1105
|
+
items: T[],
|
|
1106
|
+
concurrency: number,
|
|
1107
|
+
mapper: (item: T) => Promise<U>,
|
|
1108
|
+
): Promise<U[]> {
|
|
1109
|
+
const results = new Array<U>(items.length);
|
|
1110
|
+
let cursor = 0;
|
|
1111
|
+
|
|
1112
|
+
const worker = async (): Promise<void> => {
|
|
1113
|
+
while (cursor < items.length) {
|
|
1114
|
+
const index = cursor;
|
|
1115
|
+
cursor += 1;
|
|
1116
|
+
results[index] = await mapper(items[index] as T);
|
|
1117
|
+
}
|
|
1118
|
+
};
|
|
1119
|
+
|
|
1120
|
+
const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, () => worker());
|
|
1121
|
+
await Promise.all(workers);
|
|
1122
|
+
return results;
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
async function runVerifierAudit(
|
|
1126
|
+
target: CheckTarget,
|
|
1127
|
+
deterministic: DeterministicCheckResult,
|
|
1128
|
+
workspaceRoot: string,
|
|
1129
|
+
profile: ResolvedStageProfile,
|
|
1130
|
+
runHarness: (harnessId: string, request: HarnessRequest, cwd: string) => Promise<HarnessResult>,
|
|
1131
|
+
commonSnapshotFiles: string[],
|
|
1132
|
+
): Promise<ParsedVerifierAudit> {
|
|
1133
|
+
const specContextPaths = Array.isArray(target.document?.frontmatter.context_paths)
|
|
1134
|
+
? target.document.frontmatter.context_paths.filter((entry): entry is string => typeof entry === "string")
|
|
1135
|
+
: [];
|
|
1136
|
+
const { snapshotRoot, contextFiles } = buildVerifierSnapshot(workspaceRoot, deterministic, commonSnapshotFiles, specContextPaths);
|
|
1137
|
+
|
|
1138
|
+
try {
|
|
1139
|
+
const specContent = existsSync(target.path)
|
|
1140
|
+
? readFileSync(target.path, "utf8")
|
|
1141
|
+
: target.document
|
|
1142
|
+
? formatSpecContent(target.document)
|
|
1143
|
+
: "";
|
|
1144
|
+
const prompt = renderCheckPrompt({
|
|
1145
|
+
workspacePath: workspaceRoot,
|
|
1146
|
+
spec: target.document as ParsedSpecDocument,
|
|
1147
|
+
specContent,
|
|
1148
|
+
codePaths: deterministic.codePaths,
|
|
1149
|
+
testPaths: deterministic.testPaths,
|
|
1150
|
+
testCommands: deterministic.testCommands,
|
|
1151
|
+
matchedCodeFiles: deterministic.matchedCodeFiles,
|
|
1152
|
+
matchedTestFiles: deterministic.matchedTestFiles,
|
|
1153
|
+
deterministicEvidence: deterministic.evidence,
|
|
1154
|
+
deterministicIssues: deterministic.issues,
|
|
1155
|
+
driftWarning: target.driftWarning?.message,
|
|
1156
|
+
});
|
|
1157
|
+
|
|
1158
|
+
const result = await runHarness(
|
|
1159
|
+
profile.harness,
|
|
1160
|
+
{
|
|
1161
|
+
stage: "check",
|
|
1162
|
+
prompt,
|
|
1163
|
+
cwd: snapshotRoot,
|
|
1164
|
+
allowEdits: false,
|
|
1165
|
+
contextFiles,
|
|
1166
|
+
env: profile.command ? { LISA_HARNESS_COMMAND: resolveHarnessCommandOverride(profile.command, dirname(resolveWorkspaceLayout(workspaceRoot).configPath)) } : undefined,
|
|
1167
|
+
model: profile.model,
|
|
1168
|
+
profile: profile.profileName,
|
|
1169
|
+
extraArgs: profile.args,
|
|
1170
|
+
limits: { maxTurns: 4 },
|
|
1171
|
+
},
|
|
1172
|
+
snapshotRoot,
|
|
1173
|
+
);
|
|
1174
|
+
|
|
1175
|
+
if (result.status === "blocked") {
|
|
1176
|
+
return {
|
|
1177
|
+
status: "UNSURE",
|
|
1178
|
+
summary: "Verifier harness reported a blocked run.",
|
|
1179
|
+
evidence: [],
|
|
1180
|
+
issues: [result.abortReason?.trim() || result.finalText?.trim() || "Verifier harness blocked without a diagnostic."],
|
|
1181
|
+
};
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
if (result.status !== "success") {
|
|
1185
|
+
return {
|
|
1186
|
+
status: "UNSURE",
|
|
1187
|
+
summary: "Verifier harness did not complete successfully.",
|
|
1188
|
+
evidence: [],
|
|
1189
|
+
issues: [result.finalText?.trim() || "Verifier harness failed without a diagnostic."],
|
|
1190
|
+
};
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
try {
|
|
1194
|
+
return parseVerifierAudit(result);
|
|
1195
|
+
} catch (error) {
|
|
1196
|
+
return {
|
|
1197
|
+
status: "UNSURE",
|
|
1198
|
+
summary: "Verifier output could not be parsed.",
|
|
1199
|
+
evidence: [],
|
|
1200
|
+
issues: [error instanceof Error ? error.message : String(error)],
|
|
1201
|
+
};
|
|
1202
|
+
}
|
|
1203
|
+
} finally {
|
|
1204
|
+
rmSync(snapshotRoot, { recursive: true, force: true });
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
function mergeVerificationResult(
|
|
1209
|
+
deterministic: DeterministicCheckResult,
|
|
1210
|
+
audit?: ParsedVerifierAudit,
|
|
1211
|
+
): SpecCheckResult {
|
|
1212
|
+
if (deterministic.status === "SKIPPED") {
|
|
1213
|
+
return {
|
|
1214
|
+
specId: deterministic.specId,
|
|
1215
|
+
path: deterministic.path,
|
|
1216
|
+
status: "SKIPPED",
|
|
1217
|
+
summary: deterministic.summary,
|
|
1218
|
+
evidence: deterministic.evidence,
|
|
1219
|
+
issues: deterministic.issues,
|
|
1220
|
+
};
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
if (deterministic.status === "FAIL") {
|
|
1224
|
+
return {
|
|
1225
|
+
specId: deterministic.specId,
|
|
1226
|
+
path: deterministic.path,
|
|
1227
|
+
status: "FAIL",
|
|
1228
|
+
summary: deterministic.summary,
|
|
1229
|
+
evidence: deterministic.evidence,
|
|
1230
|
+
issues: deterministic.issues,
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
if (deterministic.status === "UNSURE") {
|
|
1235
|
+
return {
|
|
1236
|
+
specId: deterministic.specId,
|
|
1237
|
+
path: deterministic.path,
|
|
1238
|
+
status: "UNSURE",
|
|
1239
|
+
summary: deterministic.summary,
|
|
1240
|
+
evidence: deterministic.evidence,
|
|
1241
|
+
issues: deterministic.issues,
|
|
1242
|
+
};
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
if (!audit) {
|
|
1246
|
+
return {
|
|
1247
|
+
specId: deterministic.specId,
|
|
1248
|
+
path: deterministic.path,
|
|
1249
|
+
status: "UNSURE",
|
|
1250
|
+
summary: "Deterministic checks passed, but no verifier audit result was produced.",
|
|
1251
|
+
evidence: deterministic.evidence,
|
|
1252
|
+
issues: [...deterministic.issues, "Missing verifier audit result."],
|
|
1253
|
+
};
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
return {
|
|
1257
|
+
specId: deterministic.specId,
|
|
1258
|
+
path: deterministic.path,
|
|
1259
|
+
status: audit.status,
|
|
1260
|
+
summary: audit.summary,
|
|
1261
|
+
evidence: [...deterministic.evidence, ...audit.evidence],
|
|
1262
|
+
issues: [...deterministic.issues, ...audit.issues],
|
|
1263
|
+
};
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
function summarizeResults(results: SpecCheckResult[]): SpecCheckReport["summary"] {
|
|
1267
|
+
return results.reduce(
|
|
1268
|
+
(summary, result) => {
|
|
1269
|
+
if (result.status === "PASS") {
|
|
1270
|
+
summary.pass += 1;
|
|
1271
|
+
} else if (result.status === "FAIL") {
|
|
1272
|
+
summary.fail += 1;
|
|
1273
|
+
} else if (result.status === "UNSURE") {
|
|
1274
|
+
summary.unsure += 1;
|
|
1275
|
+
} else {
|
|
1276
|
+
summary.skipped += 1;
|
|
1277
|
+
}
|
|
1278
|
+
return summary;
|
|
1279
|
+
},
|
|
1280
|
+
{ pass: 0, fail: 0, unsure: 0, skipped: 0 },
|
|
1281
|
+
);
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
function renderMarkdownReport(report: SpecCheckReport): string {
|
|
1285
|
+
const lines = [
|
|
1286
|
+
"# Lisa Spec Report",
|
|
1287
|
+
"",
|
|
1288
|
+
`- Generated: ${report.generatedAt}`,
|
|
1289
|
+
`- Mode: ${report.mode}`,
|
|
1290
|
+
report.baseRef ? `- Range: ${report.baseRef} -> ${report.headRef ?? "WORKTREE"}` : undefined,
|
|
1291
|
+
`- Profile: ${report.profile}`,
|
|
1292
|
+
`- Harness: ${report.harness}`,
|
|
1293
|
+
"",
|
|
1294
|
+
"## Summary",
|
|
1295
|
+
`- PASS: ${report.summary.pass}`,
|
|
1296
|
+
`- FAIL: ${report.summary.fail}`,
|
|
1297
|
+
`- UNSURE: ${report.summary.unsure}`,
|
|
1298
|
+
`- SKIPPED: ${report.summary.skipped}`,
|
|
1299
|
+
"",
|
|
1300
|
+
].filter((line): line is string => line !== undefined);
|
|
1301
|
+
|
|
1302
|
+
if (report.driftWarnings.length > 0) {
|
|
1303
|
+
lines.push("## Drift Warnings", "");
|
|
1304
|
+
for (const warning of report.driftWarnings) {
|
|
1305
|
+
lines.push(`- ${warning.specId}: ${warning.message}`);
|
|
1306
|
+
lines.push(` Changed paths: ${warning.changedPaths.join(", ")}`);
|
|
1307
|
+
}
|
|
1308
|
+
lines.push("");
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
lines.push("## Results", "");
|
|
1312
|
+
for (const result of report.results) {
|
|
1313
|
+
lines.push(`### ${result.specId} - ${result.status}`);
|
|
1314
|
+
lines.push("");
|
|
1315
|
+
lines.push(`- Path: ${normalizeRelativePath(toLogicalPath(resolveWorkspaceLayout(report.workspacePath), result.path))}`);
|
|
1316
|
+
lines.push(`- Summary: ${result.summary}`);
|
|
1317
|
+
if (result.evidence.length > 0) {
|
|
1318
|
+
for (const entry of result.evidence) {
|
|
1319
|
+
lines.push(`- Evidence (${entry.type}): ${entry.detail}`);
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
if (result.issues.length > 0) {
|
|
1323
|
+
for (const issue of result.issues) {
|
|
1324
|
+
lines.push(`- Issue: ${issue}`);
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
lines.push("");
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1330
|
+
return lines.join("\n").trimEnd() + "\n";
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
function writeArtifacts(workspaceRoot: string, report: SpecCheckReport): { jsonPath: string; markdownPath: string; checkDir: string } {
|
|
1334
|
+
const layout = resolveWorkspaceLayout(workspaceRoot);
|
|
1335
|
+
const runtimeDir = layout.runtimeRoot;
|
|
1336
|
+
const checkDir = join(runtimeDir, "check");
|
|
1337
|
+
if (layout.storageMode === "external") {
|
|
1338
|
+
assertSafeLisaStorageWritePath(runtimeDir, checkDir, "Lisa check artifacts");
|
|
1339
|
+
}
|
|
1340
|
+
mkdirSync(checkDir, { recursive: true });
|
|
1341
|
+
|
|
1342
|
+
const jsonPath = join(runtimeDir, "spec-report.json");
|
|
1343
|
+
const markdownPath = join(runtimeDir, "spec-report.md");
|
|
1344
|
+
if (layout.storageMode === "external") {
|
|
1345
|
+
assertSafeLisaStorageWritePath(runtimeDir, jsonPath, "Lisa check artifacts");
|
|
1346
|
+
assertSafeLisaStorageWritePath(runtimeDir, markdownPath, "Lisa check artifacts");
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
writeFileSync(jsonPath, JSON.stringify(report, null, 2) + "\n");
|
|
1350
|
+
writeFileSync(markdownPath, renderMarkdownReport(report));
|
|
1351
|
+
|
|
1352
|
+
for (const result of report.results) {
|
|
1353
|
+
const fileName = `${result.specId.replace(/[^a-zA-Z0-9._-]+/g, "_")}.json`;
|
|
1354
|
+
writeFileSync(join(checkDir, fileName), JSON.stringify(result, null, 2) + "\n");
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
return { jsonPath, markdownPath, checkDir };
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
function printResults(io: CheckCommandIO, results: SpecCheckResult[]): void {
|
|
1361
|
+
io.print("Results");
|
|
1362
|
+
if (results.length === 0) {
|
|
1363
|
+
io.print("- no specs selected");
|
|
1364
|
+
io.print("");
|
|
1365
|
+
return;
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
for (const result of results) {
|
|
1369
|
+
io.print(`- ${result.specId}: ${result.status} - ${result.summary}`);
|
|
1370
|
+
}
|
|
1371
|
+
io.print("");
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
export async function runSpecCheckCommand(
|
|
1375
|
+
args: string[],
|
|
1376
|
+
cwd = process.cwd(),
|
|
1377
|
+
deps: CheckCommandDeps = {},
|
|
1378
|
+
): Promise<number> {
|
|
1379
|
+
const io = deps.io ?? createConsoleIO();
|
|
1380
|
+
|
|
1381
|
+
try {
|
|
1382
|
+
const options = parseCheckArgs(args);
|
|
1383
|
+
if (options.help) {
|
|
1384
|
+
printCheckHelp(io);
|
|
1385
|
+
return 0;
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
const layout = resolveWorkspaceLayout(cwd);
|
|
1389
|
+
const workspaceRoot = layout.workspacePath;
|
|
1390
|
+
if (layout.storageMode === "external" && (args.includes("--base") || args.includes("--head"))) {
|
|
1391
|
+
io.error("External Lisa workspaces do not support `--base` or `--head` for spec history yet.");
|
|
1392
|
+
return 1;
|
|
1393
|
+
}
|
|
1394
|
+
if (options.head) {
|
|
1395
|
+
const requestedHead = resolveGitRevision(workspaceRoot, options.head);
|
|
1396
|
+
const currentHead = resolveGitRevision(workspaceRoot, "HEAD");
|
|
1397
|
+
if (requestedHead !== currentHead) {
|
|
1398
|
+
io.error("`spec check --head <rev>` can only target the currently checked out commit. Check out that revision first or omit `--head`.");
|
|
1399
|
+
return 1;
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1402
|
+
if (hasRepoWorktreeDrift(workspaceRoot, options.head) || hasUntrackedFiles(workspaceRoot)) {
|
|
1403
|
+
io.error("`spec check --head <rev>` requires the full workspace to match that revision. Commit or stash local changes first.");
|
|
1404
|
+
return 1;
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
const workspace = options.head ? loadSpecWorkspaceAtRef(options.head, workspaceRoot) : loadSpecWorkspace(workspaceRoot);
|
|
1409
|
+
const profile = resolveProfile(workspace, options);
|
|
1410
|
+
const inspectHarness = deps.inspectHarness ?? defaultInspectHarness;
|
|
1411
|
+
const inspection = await inspectHarness(profile.harness, workspaceRoot, profile.command);
|
|
1412
|
+
ensureHarnessSupportsCheck(inspection, profile, options);
|
|
1413
|
+
printHarnessReport(io, inspection, profile);
|
|
1414
|
+
|
|
1415
|
+
if (workspace.issues.length > 0) {
|
|
1416
|
+
io.print("Workspace Validation");
|
|
1417
|
+
for (const issue of workspace.issues) {
|
|
1418
|
+
io.print(`- ${issue.severity}: ${normalizeRelativePath(toLogicalPath(layout, issue.path))} - ${issue.message}`);
|
|
1419
|
+
}
|
|
1420
|
+
io.print("");
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
let targets: CheckTarget[];
|
|
1424
|
+
let driftWarnings: SpecDriftWarning[] = [];
|
|
1425
|
+
let baseRef: string | undefined;
|
|
1426
|
+
let headRef: string | undefined;
|
|
1427
|
+
|
|
1428
|
+
if (options.mode === "changed") {
|
|
1429
|
+
baseRef = options.base ?? (layout.storageMode === "external" ? EXTERNAL_SPEC_DIFF_BASE_REF : DEFAULT_SPEC_DIFF_BASE_REF);
|
|
1430
|
+
const changedSelection = buildChangedTargets(workspace, baseRef, options.head);
|
|
1431
|
+
targets = changedSelection.targets;
|
|
1432
|
+
driftWarnings = changedSelection.driftWarnings;
|
|
1433
|
+
headRef = changedSelection.effectiveHeadRef;
|
|
1434
|
+
|
|
1435
|
+
io.print("Selection");
|
|
1436
|
+
io.print(`- mode: changed`);
|
|
1437
|
+
io.print(`- base: ${baseRef}`);
|
|
1438
|
+
io.print(`- head: ${headRef}`);
|
|
1439
|
+
io.print(`- selected specs: ${targets.length}`);
|
|
1440
|
+
io.print(`- changed spec deltas: ${changedSelection.changedSpecCount}`);
|
|
1441
|
+
io.print("");
|
|
1442
|
+
} else {
|
|
1443
|
+
targets = buildAllTargets(workspace);
|
|
1444
|
+
io.print("Selection");
|
|
1445
|
+
io.print(`- mode: all`);
|
|
1446
|
+
io.print(`- selected specs: ${targets.length}`);
|
|
1447
|
+
io.print("");
|
|
1448
|
+
}
|
|
1449
|
+
|
|
1450
|
+
if (driftWarnings.length > 0) {
|
|
1451
|
+
io.print("Drift Warnings");
|
|
1452
|
+
for (const warning of driftWarnings) {
|
|
1453
|
+
io.print(`- ${warning.specId}: ${warning.changedPaths.join(", ")}`);
|
|
1454
|
+
}
|
|
1455
|
+
io.print("");
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
const workspaceFiles = listWorkspaceFiles(workspaceRoot);
|
|
1459
|
+
const activeTargets = targets.filter((target) => target.document?.status === "active");
|
|
1460
|
+
const testCommands = Array.from(
|
|
1461
|
+
new Set(activeTargets.flatMap((target) => extractStringArray(target.document?.frontmatter ?? {}, "test_commands"))),
|
|
1462
|
+
).sort();
|
|
1463
|
+
const runCommand = deps.runCommand ?? defaultRunCommand;
|
|
1464
|
+
const testCommandResults = new Map<string, CommandExecutionResult>(
|
|
1465
|
+
testCommands.map((command) => [command, runCommand(command, workspaceRoot)]),
|
|
1466
|
+
);
|
|
1467
|
+
|
|
1468
|
+
if (testCommands.length > 0) {
|
|
1469
|
+
io.print("Deterministic Tests");
|
|
1470
|
+
for (const command of testCommandResults.keys()) {
|
|
1471
|
+
const result = testCommandResults.get(command) as CommandExecutionResult;
|
|
1472
|
+
io.print(`- ${command} (${result.exitCode === 0 ? "pass" : "fail"})`);
|
|
1473
|
+
}
|
|
1474
|
+
io.print("");
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
const deterministicResults = await Promise.all(
|
|
1478
|
+
targets.map((target) => runDeterministicChecks(target, workspace, workspaceRoot, workspaceFiles, testCommandResults, runCommand)),
|
|
1479
|
+
);
|
|
1480
|
+
const commonSnapshotFiles = listCommonSnapshotFiles(workspaceRoot);
|
|
1481
|
+
const runHarness = deps.runHarness ?? runHarnessStage;
|
|
1482
|
+
const auditable = deterministicResults.filter((result) => result.status === undefined);
|
|
1483
|
+
const targetBySpecId = new Map(targets.map((target) => [target.specId, target]));
|
|
1484
|
+
const audits = new Map<string, ParsedVerifierAudit>();
|
|
1485
|
+
|
|
1486
|
+
for (const audit of await mapWithConcurrency(auditable, 4, async (deterministic) => {
|
|
1487
|
+
const target = targetBySpecId.get(deterministic.specId) as CheckTarget;
|
|
1488
|
+
return {
|
|
1489
|
+
specId: deterministic.specId,
|
|
1490
|
+
audit: await runVerifierAudit(target, deterministic, workspaceRoot, profile, runHarness, commonSnapshotFiles),
|
|
1491
|
+
};
|
|
1492
|
+
})) {
|
|
1493
|
+
audits.set(audit.specId, audit.audit);
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
const results = deterministicResults.map((deterministic) => mergeVerificationResult(deterministic, audits.get(deterministic.specId)));
|
|
1497
|
+
const report: SpecCheckReport = {
|
|
1498
|
+
workspacePath: workspaceRoot,
|
|
1499
|
+
mode: options.mode as SpecCheckMode,
|
|
1500
|
+
baseRef,
|
|
1501
|
+
headRef,
|
|
1502
|
+
generatedAt: new Date().toISOString(),
|
|
1503
|
+
profile: profile.profileName,
|
|
1504
|
+
harness: profile.harness,
|
|
1505
|
+
results,
|
|
1506
|
+
driftWarnings,
|
|
1507
|
+
summary: summarizeResults(results),
|
|
1508
|
+
};
|
|
1509
|
+
const artifacts = writeArtifacts(workspaceRoot, report);
|
|
1510
|
+
if (
|
|
1511
|
+
layout.storageMode === "external"
|
|
1512
|
+
&& report.summary.fail === 0
|
|
1513
|
+
&& report.summary.unsure === 0
|
|
1514
|
+
&& results.some((result) => result.status === "PASS")
|
|
1515
|
+
&& (
|
|
1516
|
+
options.mode === "all"
|
|
1517
|
+
|| (options.mode === "changed" && driftWarnings.length === 0)
|
|
1518
|
+
)
|
|
1519
|
+
) {
|
|
1520
|
+
writeExternalSpecSnapshot(workspaceRoot);
|
|
1521
|
+
}
|
|
1522
|
+
|
|
1523
|
+
printResults(io, results);
|
|
1524
|
+
io.print("Artifacts");
|
|
1525
|
+
io.print(`- json: ${layout.storageMode === "external" ? artifacts.jsonPath : normalizeRelativePath(relative(workspaceRoot, artifacts.jsonPath))}`);
|
|
1526
|
+
io.print(`- markdown: ${layout.storageMode === "external" ? artifacts.markdownPath : normalizeRelativePath(relative(workspaceRoot, artifacts.markdownPath))}`);
|
|
1527
|
+
io.print(`- per-spec: ${layout.storageMode === "external" ? artifacts.checkDir : normalizeRelativePath(relative(workspaceRoot, artifacts.checkDir))}`);
|
|
1528
|
+
|
|
1529
|
+
return report.summary.fail > 0 || report.summary.unsure > 0 ? 1 : 0;
|
|
1530
|
+
} catch (error) {
|
|
1531
|
+
io.error(error instanceof Error ? error.message : String(error));
|
|
1532
|
+
return 1;
|
|
1533
|
+
}
|
|
1534
|
+
}
|