ai-spec-dev 0.30.1 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +5 -1
- package/README.md +29 -1
- package/RELEASE_LOG.md +188 -0
- package/cli/commands/config.ts +93 -0
- package/cli/commands/export.ts +66 -0
- package/cli/commands/init.ts +153 -0
- package/cli/commands/learn.ts +30 -0
- package/cli/commands/logs.ts +106 -0
- package/cli/commands/model.ts +156 -0
- package/cli/commands/restore.ts +22 -0
- package/cli/commands/review.ts +63 -0
- package/cli/commands/trend.ts +36 -0
- package/cli/commands/update.ts +178 -0
- package/cli/commands/workspace.ts +219 -0
- package/cli/index.ts +301 -1
- package/cli/utils.ts +83 -0
- package/core/dsl-feedback.ts +255 -0
- package/core/prompt-hasher.ts +42 -0
- package/core/run-logger.ts +21 -0
- package/core/run-trend.ts +241 -0
- package/core/self-evaluator.ts +276 -0
- package/dist/cli/index.js +1089 -445
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/index.mjs +1089 -445
- package/dist/cli/index.mjs.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs.map +1 -1
- package/package.json +6 -3
- package/purpose.md +189 -2
- package/tests/dsl-extractor.test.ts +264 -0
- package/tests/dsl-feedback.test.ts +266 -0
- package/tests/dsl-validator.test.ts +283 -0
- package/tests/error-feedback.test.ts +292 -0
- package/tests/provider-utils.test.ts +173 -0
- package/tests/run-trend.test.ts +186 -0
- package/tests/self-evaluator.test.ts +339 -0
- package/tests/spec-assessor.test.ts +142 -0
- package/tests/task-generator.test.ts +230 -0
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import * as path from "path";
|
|
3
|
+
import * as fs from "fs-extra";
|
|
4
|
+
import chalk from "chalk";
|
|
5
|
+
import { input, select, confirm } from "@inquirer/prompts";
|
|
6
|
+
import {
|
|
7
|
+
WorkspaceLoader,
|
|
8
|
+
WorkspaceConfig,
|
|
9
|
+
RepoConfig,
|
|
10
|
+
WORKSPACE_CONFIG_FILE,
|
|
11
|
+
detectRepoType,
|
|
12
|
+
} from "../../core/workspace-loader";
|
|
13
|
+
|
|
14
|
+
export function registerWorkspace(program: Command): void {
|
|
15
|
+
const workspaceCmd = program
|
|
16
|
+
.command("workspace")
|
|
17
|
+
.description("Manage multi-repo workspace configuration");
|
|
18
|
+
|
|
19
|
+
// ── workspace init ──────────────────────────────────────────────────────────
|
|
20
|
+
workspaceCmd
|
|
21
|
+
.command("init")
|
|
22
|
+
.description(`Interactive workspace setup — creates ${WORKSPACE_CONFIG_FILE}`)
|
|
23
|
+
.action(async () => {
|
|
24
|
+
const currentDir = process.cwd();
|
|
25
|
+
const configPath = path.join(currentDir, WORKSPACE_CONFIG_FILE);
|
|
26
|
+
|
|
27
|
+
if (await fs.pathExists(configPath)) {
|
|
28
|
+
const overwrite = await confirm({
|
|
29
|
+
message: `${WORKSPACE_CONFIG_FILE} already exists. Overwrite?`,
|
|
30
|
+
default: false,
|
|
31
|
+
});
|
|
32
|
+
if (!overwrite) {
|
|
33
|
+
console.log(chalk.gray(" Cancelled."));
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
console.log(chalk.blue("\n─── Workspace Setup ────────────────────────────"));
|
|
39
|
+
|
|
40
|
+
const workspaceName = await input({
|
|
41
|
+
message: "Workspace name:",
|
|
42
|
+
validate: (v) => v.trim().length > 0 || "Name cannot be empty",
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
const repos: RepoConfig[] = [];
|
|
46
|
+
|
|
47
|
+
const useAutoScan = await confirm({
|
|
48
|
+
message: "Auto-scan sibling directories for repos?",
|
|
49
|
+
default: true,
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
if (useAutoScan) {
|
|
53
|
+
const workspaceLoader = new WorkspaceLoader(currentDir);
|
|
54
|
+
const detected = await workspaceLoader.autoDetect();
|
|
55
|
+
|
|
56
|
+
if (detected.length === 0) {
|
|
57
|
+
console.log(chalk.yellow(" No recognizable repos found in sibling directories."));
|
|
58
|
+
} else {
|
|
59
|
+
console.log(chalk.cyan("\n Detected repos:"));
|
|
60
|
+
for (const r of detected) {
|
|
61
|
+
console.log(chalk.gray(` - ${r.name}: ${r.role} (${r.type}) at ${r.path}`));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const keepAll = await confirm({
|
|
65
|
+
message: `Include all ${detected.length} detected repo(s)?`,
|
|
66
|
+
default: true,
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
if (keepAll) {
|
|
70
|
+
repos.push(...detected);
|
|
71
|
+
} else {
|
|
72
|
+
for (const r of detected) {
|
|
73
|
+
const keep = await confirm({
|
|
74
|
+
message: `Include "${r.name}" (${r.role}, ${r.type})?`,
|
|
75
|
+
default: true,
|
|
76
|
+
});
|
|
77
|
+
if (keep) repos.push(r);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
console.log(chalk.green(` ✔ ${repos.length} repo(s) added from auto-scan.`));
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const repoTypeChoices = [
|
|
85
|
+
{ name: "node-express (Node.js/Express backend)", value: "node-express" },
|
|
86
|
+
{ name: "node-koa (Node.js/Koa backend)", value: "node-koa" },
|
|
87
|
+
{ name: "go (Go backend)", value: "go" },
|
|
88
|
+
{ name: "python (Python backend)", value: "python" },
|
|
89
|
+
{ name: "java (Java/Spring backend)", value: "java" },
|
|
90
|
+
{ name: "rust (Rust backend)", value: "rust" },
|
|
91
|
+
{ name: "php (PHP/Lumen/Laravel backend)", value: "php" },
|
|
92
|
+
{ name: "react (React frontend)", value: "react" },
|
|
93
|
+
{ name: "next (Next.js)", value: "next" },
|
|
94
|
+
{ name: "vue (Vue frontend)", value: "vue" },
|
|
95
|
+
{ name: "react-native (React Native mobile)", value: "react-native" },
|
|
96
|
+
{ name: "unknown", value: "unknown" },
|
|
97
|
+
];
|
|
98
|
+
|
|
99
|
+
let addMore = await confirm({
|
|
100
|
+
message: repos.length > 0 ? "Manually add more repos?" : "Add repos manually?",
|
|
101
|
+
default: repos.length === 0,
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
while (addMore) {
|
|
105
|
+
console.log(chalk.cyan(`\n Adding repo #${repos.length + 1}`));
|
|
106
|
+
|
|
107
|
+
const repoName = await input({
|
|
108
|
+
message: "Repo name (e.g. api, web, app):",
|
|
109
|
+
validate: (v) => {
|
|
110
|
+
if (!v.trim()) return "Name cannot be empty";
|
|
111
|
+
if (repos.some((r) => r.name === v.trim())) return "Name already used";
|
|
112
|
+
return true;
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
const repoPath = await input({
|
|
117
|
+
message: `Relative path to "${repoName}" from here (default: ./${repoName}):`,
|
|
118
|
+
default: `./${repoName}`,
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
const absPath = path.resolve(currentDir, repoPath);
|
|
122
|
+
let detectedType = "unknown";
|
|
123
|
+
let detectedRole = "shared";
|
|
124
|
+
|
|
125
|
+
if (await fs.pathExists(absPath)) {
|
|
126
|
+
const { type, role } = await detectRepoType(absPath);
|
|
127
|
+
detectedType = type;
|
|
128
|
+
detectedRole = role;
|
|
129
|
+
console.log(chalk.gray(` Auto-detected: type=${type}, role=${role}`));
|
|
130
|
+
} else {
|
|
131
|
+
console.log(chalk.yellow(` Path "${absPath}" not found — type/role will be manual.`));
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const repoType = await select({
|
|
135
|
+
message: `Repo type for "${repoName}":`,
|
|
136
|
+
choices: repoTypeChoices,
|
|
137
|
+
default: detectedType,
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
const repoRole = await select({
|
|
141
|
+
message: `Repo role for "${repoName}":`,
|
|
142
|
+
choices: [
|
|
143
|
+
{ name: "backend", value: "backend" },
|
|
144
|
+
{ name: "frontend", value: "frontend" },
|
|
145
|
+
{ name: "mobile", value: "mobile" },
|
|
146
|
+
{ name: "shared", value: "shared" },
|
|
147
|
+
],
|
|
148
|
+
default: detectedRole,
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
repos.push({
|
|
152
|
+
name: repoName,
|
|
153
|
+
path: repoPath,
|
|
154
|
+
type: repoType as RepoConfig["type"],
|
|
155
|
+
role: repoRole as RepoConfig["role"],
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
console.log(chalk.green(` ✔ Added: ${repoName} (${repoRole}, ${repoType})`));
|
|
159
|
+
|
|
160
|
+
addMore = await confirm({
|
|
161
|
+
message: "Add another repo?",
|
|
162
|
+
default: false,
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const workspaceConfig: WorkspaceConfig = { name: workspaceName, repos };
|
|
167
|
+
|
|
168
|
+
console.log(chalk.cyan("\n Workspace summary:"));
|
|
169
|
+
console.log(chalk.gray(` Name: ${workspaceName}`));
|
|
170
|
+
for (const r of repos) {
|
|
171
|
+
console.log(chalk.gray(` - ${r.name}: ${r.role} (${r.type}) at ${r.path}`));
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const ok = await confirm({ message: `Save to ${WORKSPACE_CONFIG_FILE}?`, default: true });
|
|
175
|
+
if (!ok) {
|
|
176
|
+
console.log(chalk.gray(" Cancelled."));
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const loader = new WorkspaceLoader(currentDir);
|
|
181
|
+
const saved = await loader.save(workspaceConfig);
|
|
182
|
+
console.log(chalk.green(`\n ✔ Workspace saved: ${saved}`));
|
|
183
|
+
console.log(chalk.gray(` Run \`ai-spec create "your feature"\` — workspace mode will activate automatically.`));
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
// ── workspace status ────────────────────────────────────────────────────────
|
|
187
|
+
workspaceCmd
|
|
188
|
+
.command("status")
|
|
189
|
+
.description("Show current workspace configuration")
|
|
190
|
+
.action(async () => {
|
|
191
|
+
const currentDir = process.cwd();
|
|
192
|
+
const loader = new WorkspaceLoader(currentDir);
|
|
193
|
+
const config = await loader.load();
|
|
194
|
+
|
|
195
|
+
if (!config) {
|
|
196
|
+
console.log(chalk.yellow(`No ${WORKSPACE_CONFIG_FILE} found in ${currentDir}`));
|
|
197
|
+
console.log(chalk.gray(" Run `ai-spec workspace init` to create one."));
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
console.log(chalk.bold(`\nWorkspace: ${config.name}`));
|
|
202
|
+
console.log(chalk.gray(` Config: ${path.join(currentDir, WORKSPACE_CONFIG_FILE)}`));
|
|
203
|
+
console.log(chalk.gray(` Repos (${config.repos.length}):\n`));
|
|
204
|
+
|
|
205
|
+
for (const repo of config.repos) {
|
|
206
|
+
const absPath = loader.resolveAbsPath(repo);
|
|
207
|
+
const exists = await fs.pathExists(absPath);
|
|
208
|
+
const status = exists ? chalk.green("found") : chalk.red("not found");
|
|
209
|
+
|
|
210
|
+
console.log(
|
|
211
|
+
` ${chalk.bold(repo.name.padEnd(12))} ${repo.role.padEnd(10)} ${repo.type.padEnd(16)} ${status}`
|
|
212
|
+
);
|
|
213
|
+
console.log(chalk.gray(` path: ${absPath}`));
|
|
214
|
+
if (repo.constitution) {
|
|
215
|
+
console.log(chalk.green(` constitution: found`));
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
});
|
|
219
|
+
}
|
package/cli/index.ts
CHANGED
|
@@ -73,6 +73,17 @@ import { SpecUpdater } from "../core/spec-updater";
|
|
|
73
73
|
import { exportOpenApi } from "../core/openapi-exporter";
|
|
74
74
|
import { generateRunId, RunLogger, setActiveLogger } from "../core/run-logger";
|
|
75
75
|
import { RunSnapshot, setActiveSnapshot } from "../core/run-snapshot";
|
|
76
|
+
import { computePromptHash } from "../core/prompt-hasher";
|
|
77
|
+
import { runSelfEval, printSelfEval } from "../core/self-evaluator";
|
|
78
|
+
import { loadRunLogs, buildTrendReport, printTrendReport } from "../core/run-trend";
|
|
79
|
+
import {
|
|
80
|
+
assessDslRichness,
|
|
81
|
+
buildDslGapRefinementPrompt,
|
|
82
|
+
extractStructuralFindings,
|
|
83
|
+
buildStructuralAmendmentPrompt,
|
|
84
|
+
printDslGaps,
|
|
85
|
+
printStructuralFindings,
|
|
86
|
+
} from "../core/dsl-feedback";
|
|
76
87
|
|
|
77
88
|
// ─── Config File ──────────────────────────────────────────────────────────────
|
|
78
89
|
|
|
@@ -305,6 +316,11 @@ program
|
|
|
305
316
|
});
|
|
306
317
|
setActiveLogger(runLogger);
|
|
307
318
|
|
|
319
|
+
// Record prompt hash immediately — links this RunLog to the prompt version
|
|
320
|
+
// in use, enabling cross-run harnessScore comparisons (Harness Engineering).
|
|
321
|
+
const promptHash = computePromptHash();
|
|
322
|
+
runLogger.setPromptHash(promptHash);
|
|
323
|
+
|
|
308
324
|
// ── Step 1: Context ───────────────────────────────────────────────────────
|
|
309
325
|
console.log(chalk.blue("[1/6] Loading project context..."));
|
|
310
326
|
runLogger.stageStart("context_load");
|
|
@@ -517,6 +533,60 @@ program
|
|
|
517
533
|
}
|
|
518
534
|
}
|
|
519
535
|
|
|
536
|
+
// ── Loop 1: DSL Gap Feedback ──────────────────────────────────────────────
|
|
537
|
+
// Runs only in interactive mode (not --auto / --fast / --skip-dsl).
|
|
538
|
+
// Checks for common completeness gaps in the freshly-extracted DSL and
|
|
539
|
+
// offers a targeted spec refinement AI call to fill them before codegen.
|
|
540
|
+
// Zero extra AI calls unless the user explicitly opts in.
|
|
541
|
+
if (extractedDsl && !opts.auto && !opts.fast && !opts.skipDsl) {
|
|
542
|
+
const dslGaps = assessDslRichness(extractedDsl);
|
|
543
|
+
|
|
544
|
+
if (dslGaps.length > 0) {
|
|
545
|
+
printDslGaps(dslGaps);
|
|
546
|
+
runLogger.stageStart("dsl_gap_feedback", { gapCount: dslGaps.length, gaps: dslGaps.map((g) => g.code) });
|
|
547
|
+
|
|
548
|
+
const refineChoice = await select({
|
|
549
|
+
message: "How would you like to proceed?",
|
|
550
|
+
choices: [
|
|
551
|
+
{ name: "🔧 Refine spec (AI fills the gaps, then re-extract DSL)", value: "refine" },
|
|
552
|
+
{ name: "⏭ Skip — proceed with the current DSL", value: "skip" },
|
|
553
|
+
],
|
|
554
|
+
});
|
|
555
|
+
|
|
556
|
+
if (refineChoice === "refine") {
|
|
557
|
+
console.log(chalk.blue(" Refining spec to fill DSL gaps..."));
|
|
558
|
+
try {
|
|
559
|
+
const refinedSpec = await specProvider.generate(
|
|
560
|
+
buildDslGapRefinementPrompt(finalSpec, dslGaps),
|
|
561
|
+
"You are a Senior Tech Lead doing a targeted spec revision. Output only the complete revised Markdown spec."
|
|
562
|
+
);
|
|
563
|
+
finalSpec = refinedSpec;
|
|
564
|
+
console.log(chalk.green(" ✔ Spec refined."));
|
|
565
|
+
|
|
566
|
+
// Re-extract DSL from the improved spec
|
|
567
|
+
console.log(chalk.blue(" Re-extracting DSL from refined spec..."));
|
|
568
|
+
const isFrontend2 = isFrontendDeps(context.dependencies);
|
|
569
|
+
const reExtractor = new DslExtractor(specProvider);
|
|
570
|
+
const reExtractedDsl = await reExtractor.extract(finalSpec, { auto: true, isFrontend: isFrontend2 });
|
|
571
|
+
if (reExtractedDsl) {
|
|
572
|
+
extractedDsl = reExtractedDsl;
|
|
573
|
+
console.log(chalk.green(` ✔ DSL re-extracted: ${extractedDsl.endpoints.length} endpoint(s), ${extractedDsl.models.length} model(s).`));
|
|
574
|
+
runLogger.stageEnd("dsl_gap_feedback", { action: "refined", endpoints: extractedDsl.endpoints.length, models: extractedDsl.models.length });
|
|
575
|
+
} else {
|
|
576
|
+
console.log(chalk.yellow(" ⚠ Re-extraction failed — keeping original DSL."));
|
|
577
|
+
runLogger.stageEnd("dsl_gap_feedback", { action: "refined_but_reextract_failed" });
|
|
578
|
+
}
|
|
579
|
+
} catch (err) {
|
|
580
|
+
console.log(chalk.yellow(` ⚠ Spec refinement failed: ${(err as Error).message} — keeping original DSL.`));
|
|
581
|
+
runLogger.stageEnd("dsl_gap_feedback", { action: "refinement_error", error: (err as Error).message });
|
|
582
|
+
}
|
|
583
|
+
} else {
|
|
584
|
+
runLogger.stageEnd("dsl_gap_feedback", { action: "skipped" });
|
|
585
|
+
console.log(chalk.gray(" Continuing with current DSL."));
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
520
590
|
// ── Step 4: Git Worktree ──────────────────────────────────────────────────
|
|
521
591
|
// Frontend projects (React / Vue / Next / React-Native) work directly on a
|
|
522
592
|
// feature branch — no worktree needed. node_modules is not copied into
|
|
@@ -625,14 +695,16 @@ program
|
|
|
625
695
|
// ── Step 8: Error Feedback Loop ───────────────────────────────────────────
|
|
626
696
|
// In TDD mode, the error feedback loop is the primary driver:
|
|
627
697
|
// it runs tests, collects failures, and fixes implementation until tests pass.
|
|
698
|
+
let compilePassed = false;
|
|
628
699
|
if (opts.skipErrorFeedback) {
|
|
629
700
|
console.log(chalk.gray("[8/9] Skipping error feedback (--skip-error-feedback)."));
|
|
701
|
+
compilePassed = true; // treat skip as neutral pass for self-eval
|
|
630
702
|
} else {
|
|
631
703
|
if (opts.tdd) {
|
|
632
704
|
console.log(chalk.cyan("[8/9] TDD mode — error feedback loop driving implementation to pass tests..."));
|
|
633
705
|
}
|
|
634
706
|
runLogger.stageStart("error_feedback");
|
|
635
|
-
await runErrorFeedback(codegenProvider, workingDir, extractedDsl, {
|
|
707
|
+
compilePassed = await runErrorFeedback(codegenProvider, workingDir, extractedDsl, {
|
|
636
708
|
maxCycles: opts.tdd ? 3 : 2, // TDD gets one extra cycle
|
|
637
709
|
});
|
|
638
710
|
runLogger.stageEnd("error_feedback");
|
|
@@ -665,6 +737,96 @@ program
|
|
|
665
737
|
await accumulateReviewKnowledge(specProvider, currentDir, reviewResult);
|
|
666
738
|
}
|
|
667
739
|
|
|
740
|
+
// ── Loop 2: Review → DSL Structural Feedback ─────────────────────────────
|
|
741
|
+
// Runs only in interactive mode when a review was produced.
|
|
742
|
+
// Classifies Pass 1 (architecture) findings as design-level vs implementation-
|
|
743
|
+
// level issues. Design issues belong in the Spec/DSL — not just in §9.
|
|
744
|
+
// If found: offer to amend the spec → re-extract DSL → overwrite saved DSL file.
|
|
745
|
+
// The user can then run `ai-spec update --codegen` to regenerate affected files.
|
|
746
|
+
if (reviewResult && !opts.skipReview && !opts.auto && extractedDsl && savedDslFile) {
|
|
747
|
+
const structuralFindings = extractStructuralFindings(reviewResult);
|
|
748
|
+
|
|
749
|
+
if (structuralFindings.length > 0) {
|
|
750
|
+
printStructuralFindings(structuralFindings);
|
|
751
|
+
runLogger.stageStart("review_dsl_feedback", { findingCount: structuralFindings.length, categories: structuralFindings.map((f) => f.category) });
|
|
752
|
+
|
|
753
|
+
const savedSpecContent = await fs.readFile(specFile, "utf-8");
|
|
754
|
+
|
|
755
|
+
const patchChoice = await select({
|
|
756
|
+
message: "These are design issues in the Spec/DSL. How would you like to handle them?",
|
|
757
|
+
choices: [
|
|
758
|
+
{ name: "🔧 Amend spec + update DSL (AI fixes the design issues, no regen yet)", value: "amend" },
|
|
759
|
+
{ name: "📝 Note in §9 only (already done — no DSL change)", value: "note" },
|
|
760
|
+
{ name: "⏭ Skip", value: "skip" },
|
|
761
|
+
],
|
|
762
|
+
});
|
|
763
|
+
|
|
764
|
+
if (patchChoice === "amend") {
|
|
765
|
+
console.log(chalk.blue(" Amending spec to address structural findings..."));
|
|
766
|
+
try {
|
|
767
|
+
const amendedSpec = await specProvider.generate(
|
|
768
|
+
buildStructuralAmendmentPrompt(savedSpecContent, structuralFindings),
|
|
769
|
+
"You are a Senior Tech Lead doing a targeted spec correction. Output only the complete revised Markdown spec."
|
|
770
|
+
);
|
|
771
|
+
|
|
772
|
+
// Snapshot spec + DSL before overwriting so `ai-spec restore <runId>` works
|
|
773
|
+
await runSnapshot.snapshotFile(specFile);
|
|
774
|
+
if (savedDslFile) await runSnapshot.snapshotFile(savedDslFile);
|
|
775
|
+
|
|
776
|
+
// Overwrite the saved spec file with the amendment
|
|
777
|
+
await fs.writeFile(specFile, amendedSpec, "utf-8");
|
|
778
|
+
console.log(chalk.green(` ✔ Spec updated: ${specFile}`));
|
|
779
|
+
|
|
780
|
+
// Re-extract DSL from the amended spec
|
|
781
|
+
console.log(chalk.blue(" Re-extracting DSL from amended spec..."));
|
|
782
|
+
const isFrontend3 = isFrontendDeps(context.dependencies);
|
|
783
|
+
const amendExtractor = new DslExtractor(specProvider);
|
|
784
|
+
const amendedDsl = await amendExtractor.extract(amendedSpec, { auto: true, isFrontend: isFrontend3 });
|
|
785
|
+
if (amendedDsl) {
|
|
786
|
+
// Overwrite saved DSL file
|
|
787
|
+
const dslWriter = new DslExtractor(specProvider);
|
|
788
|
+
const newDslPath = await dslWriter.saveDsl(amendedDsl, specFile);
|
|
789
|
+
extractedDsl = amendedDsl;
|
|
790
|
+
console.log(chalk.green(` ✔ DSL updated: ${newDslPath}`));
|
|
791
|
+
console.log(chalk.cyan(
|
|
792
|
+
`\n Next step: run ${chalk.white("ai-spec update --codegen")} to regenerate files affected by the DSL change.`
|
|
793
|
+
));
|
|
794
|
+
runLogger.stageEnd("review_dsl_feedback", {
|
|
795
|
+
action: "amended",
|
|
796
|
+
endpoints: amendedDsl.endpoints.length,
|
|
797
|
+
models: amendedDsl.models.length,
|
|
798
|
+
});
|
|
799
|
+
} else {
|
|
800
|
+
console.log(chalk.yellow(" ⚠ DSL re-extraction failed — spec was updated but DSL file unchanged."));
|
|
801
|
+
runLogger.stageEnd("review_dsl_feedback", { action: "amended_spec_only" });
|
|
802
|
+
}
|
|
803
|
+
} catch (err) {
|
|
804
|
+
console.log(chalk.yellow(` ⚠ Spec amendment failed: ${(err as Error).message}`));
|
|
805
|
+
runLogger.stageEnd("review_dsl_feedback", { action: "amendment_error", error: (err as Error).message });
|
|
806
|
+
}
|
|
807
|
+
} else {
|
|
808
|
+
runLogger.stageEnd("review_dsl_feedback", { action: patchChoice });
|
|
809
|
+
if (patchChoice === "note") {
|
|
810
|
+
console.log(chalk.gray(" Structural findings retained in §9. DSL unchanged."));
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
// ── Step 10: Harness Self-Evaluation ──────────────────────────────────────
|
|
817
|
+
// Zero AI calls — deterministic scoring from file-system state + review text.
|
|
818
|
+
// Records harnessScore + promptHash in RunLog for cross-run trend analysis.
|
|
819
|
+
runLogger.stageStart("self_eval");
|
|
820
|
+
const selfEvalResult = runSelfEval({
|
|
821
|
+
dsl: extractedDsl,
|
|
822
|
+
generatedFiles,
|
|
823
|
+
compilePassed,
|
|
824
|
+
reviewText: reviewResult,
|
|
825
|
+
promptHash,
|
|
826
|
+
logger: runLogger,
|
|
827
|
+
});
|
|
828
|
+
printSelfEval(selfEvalResult);
|
|
829
|
+
|
|
668
830
|
// ── Done ──────────────────────────────────────────────────────────────────
|
|
669
831
|
runLogger.finish();
|
|
670
832
|
console.log(chalk.bold.green("\n✔ All done!"));
|
|
@@ -2230,4 +2392,142 @@ program
|
|
|
2230
2392
|
}
|
|
2231
2393
|
});
|
|
2232
2394
|
|
|
2395
|
+
// ═══════════════════════════════════════════════════════════════════════════════
|
|
2396
|
+
// Command: trend
|
|
2397
|
+
// ═══════════════════════════════════════════════════════════════════════════════
|
|
2398
|
+
|
|
2399
|
+
program
|
|
2400
|
+
.command("trend")
|
|
2401
|
+
.description("Show harness score trend across past create runs")
|
|
2402
|
+
.option("--last <n>", "Number of recent scored runs to show (default: 15)", "15")
|
|
2403
|
+
.option("--prompt <hash>", "Filter to a specific prompt hash (prefix match)")
|
|
2404
|
+
.option("--json", "Output raw JSON instead of formatted table")
|
|
2405
|
+
.action(async (opts: { last: string; prompt?: string; json?: boolean }) => {
|
|
2406
|
+
const currentDir = process.cwd();
|
|
2407
|
+
const last = parseInt(opts.last, 10) || 15;
|
|
2408
|
+
|
|
2409
|
+
const logs = await loadRunLogs(currentDir);
|
|
2410
|
+
if (logs.length === 0) {
|
|
2411
|
+
console.log(chalk.yellow(
|
|
2412
|
+
"\n No run logs found. Run `ai-spec create` at least once to start tracking.\n"
|
|
2413
|
+
));
|
|
2414
|
+
return;
|
|
2415
|
+
}
|
|
2416
|
+
|
|
2417
|
+
const report = buildTrendReport(logs, {
|
|
2418
|
+
last,
|
|
2419
|
+
promptFilter: opts.prompt,
|
|
2420
|
+
});
|
|
2421
|
+
|
|
2422
|
+
if (opts.json) {
|
|
2423
|
+
console.log(JSON.stringify(report, null, 2));
|
|
2424
|
+
return;
|
|
2425
|
+
}
|
|
2426
|
+
|
|
2427
|
+
printTrendReport(report, currentDir);
|
|
2428
|
+
});
|
|
2429
|
+
|
|
2430
|
+
// ═══════════════════════════════════════════════════════════════════════════════
|
|
2431
|
+
// Command: logs
|
|
2432
|
+
// ═══════════════════════════════════════════════════════════════════════════════
|
|
2433
|
+
|
|
2434
|
+
program
|
|
2435
|
+
.command("logs")
|
|
2436
|
+
.description("List recent run logs with stage timing")
|
|
2437
|
+
.argument("[runId]", "Show detailed stage breakdown for a specific run ID")
|
|
2438
|
+
.option("--last <n>", "Number of runs to list (default: 10)", "10")
|
|
2439
|
+
.action(async (runId: string | undefined, opts: { last: string }) => {
|
|
2440
|
+
const currentDir = process.cwd();
|
|
2441
|
+
const logDir = path.join(currentDir, ".ai-spec-logs");
|
|
2442
|
+
|
|
2443
|
+
if (!(await fs.pathExists(logDir))) {
|
|
2444
|
+
console.log(chalk.yellow("\n No run logs found (.ai-spec-logs/ does not exist).\n"));
|
|
2445
|
+
return;
|
|
2446
|
+
}
|
|
2447
|
+
|
|
2448
|
+
if (runId) {
|
|
2449
|
+
// ── Detail view for a single run ──────────────────────────────────────
|
|
2450
|
+
const logPath = path.join(logDir, `${runId}.json`);
|
|
2451
|
+
if (!(await fs.pathExists(logPath))) {
|
|
2452
|
+
console.log(chalk.red(`\n Run not found: ${runId}\n`));
|
|
2453
|
+
return;
|
|
2454
|
+
}
|
|
2455
|
+
const log = await fs.readJson(logPath);
|
|
2456
|
+
|
|
2457
|
+
console.log(chalk.cyan(`\n─── Run: ${log.runId} ─────────────────────────────────`));
|
|
2458
|
+
console.log(chalk.gray(` Started : ${log.startedAt}`));
|
|
2459
|
+
if (log.endedAt) console.log(chalk.gray(` Ended : ${log.endedAt}`));
|
|
2460
|
+
if (log.totalDurationMs !== undefined)
|
|
2461
|
+
console.log(chalk.gray(` Duration: ${(log.totalDurationMs / 1000).toFixed(1)}s`));
|
|
2462
|
+
if (log.provider) console.log(chalk.gray(` Provider: ${log.provider} / ${log.model ?? "?"}`));
|
|
2463
|
+
if (log.promptHash) console.log(chalk.gray(` Prompt : ${log.promptHash}`));
|
|
2464
|
+
if (log.harnessScore !== undefined)
|
|
2465
|
+
console.log(chalk.white(` Score : ${log.harnessScore}/10`));
|
|
2466
|
+
if (log.filesWritten?.length)
|
|
2467
|
+
console.log(chalk.gray(` Files : ${log.filesWritten.length} written`));
|
|
2468
|
+
if (log.errors?.length)
|
|
2469
|
+
console.log(chalk.yellow(` Errors : ${log.errors.length}`));
|
|
2470
|
+
|
|
2471
|
+
if (log.entries?.length) {
|
|
2472
|
+
console.log(chalk.bold("\n Stages:\n"));
|
|
2473
|
+
const doneEvents = (log.entries as Array<{ event: string; data?: Record<string, unknown>; ts: string }>)
|
|
2474
|
+
.filter((e) => e.event.endsWith(":done") || e.event.endsWith(":failed"));
|
|
2475
|
+
|
|
2476
|
+
for (const entry of doneEvents) {
|
|
2477
|
+
const isOk = entry.event.endsWith(":done");
|
|
2478
|
+
const stage = entry.event.replace(/:done$|:failed$/, "");
|
|
2479
|
+
const dur = entry.data?.durationMs
|
|
2480
|
+
? chalk.gray(` ${(Number(entry.data.durationMs) / 1000).toFixed(1)}s`)
|
|
2481
|
+
: "";
|
|
2482
|
+
const mark = isOk ? chalk.green("✔") : chalk.red("✘");
|
|
2483
|
+
console.log(` ${mark} ${stage.padEnd(20)}${dur}`);
|
|
2484
|
+
}
|
|
2485
|
+
}
|
|
2486
|
+
console.log(chalk.cyan("─".repeat(52)));
|
|
2487
|
+
return;
|
|
2488
|
+
}
|
|
2489
|
+
|
|
2490
|
+
// ── List view ─────────────────────────────────────────────────────────────
|
|
2491
|
+
const logs = await loadRunLogs(currentDir);
|
|
2492
|
+
const last = parseInt(opts.last, 10) || 10;
|
|
2493
|
+
const shown = logs.slice(0, last);
|
|
2494
|
+
|
|
2495
|
+
if (shown.length === 0) {
|
|
2496
|
+
console.log(chalk.yellow("\n No run logs found.\n"));
|
|
2497
|
+
return;
|
|
2498
|
+
}
|
|
2499
|
+
|
|
2500
|
+
console.log(chalk.cyan("\n─── Run Logs ────────────────────────────────────────────────"));
|
|
2501
|
+
console.log(chalk.gray(
|
|
2502
|
+
"\n " +
|
|
2503
|
+
"Run ID ".padEnd(26) +
|
|
2504
|
+
"Date " +
|
|
2505
|
+
"Score ".padStart(6) +
|
|
2506
|
+
" Files Dur\n"
|
|
2507
|
+
));
|
|
2508
|
+
|
|
2509
|
+
for (const log of shown) {
|
|
2510
|
+
const date = log.startedAt.slice(0, 10);
|
|
2511
|
+
const score = log.harnessScore !== undefined
|
|
2512
|
+
? (log.harnessScore >= 8 ? chalk.green : log.harnessScore >= 6 ? chalk.yellow : chalk.red)(
|
|
2513
|
+
log.harnessScore.toFixed(1).padStart(5)
|
|
2514
|
+
)
|
|
2515
|
+
: chalk.gray(" —");
|
|
2516
|
+
const files = String(log.filesWritten?.length ?? 0).padStart(5);
|
|
2517
|
+
const dur = log.totalDurationMs !== undefined
|
|
2518
|
+
? chalk.gray((log.totalDurationMs / 1000).toFixed(0) + "s")
|
|
2519
|
+
: chalk.gray("—");
|
|
2520
|
+
const errMark = (log.errors?.length ?? 0) > 0
|
|
2521
|
+
? chalk.yellow(` ⚠${log.errors.length}`)
|
|
2522
|
+
: "";
|
|
2523
|
+
|
|
2524
|
+
console.log(` ${chalk.white(log.runId.padEnd(25))} ${chalk.gray(date)} ${score} ${chalk.gray(files)} ${dur}${errMark}`);
|
|
2525
|
+
}
|
|
2526
|
+
|
|
2527
|
+
console.log(chalk.gray(`\n Showing ${shown.length} of ${logs.length} run(s) · logs: .ai-spec-logs/`));
|
|
2528
|
+
console.log(chalk.cyan("─".repeat(63)));
|
|
2529
|
+
console.log(chalk.gray(` Tip: ai-spec logs <runId> to see stage breakdown`));
|
|
2530
|
+
console.log(chalk.gray(` ai-spec trend to see score trend by prompt version\n`));
|
|
2531
|
+
});
|
|
2532
|
+
|
|
2233
2533
|
program.parse();
|
package/cli/utils.ts
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import * as path from "path";
|
|
2
|
+
import * as fs from "fs-extra";
|
|
3
|
+
import chalk from "chalk";
|
|
4
|
+
import { input, select } from "@inquirer/prompts";
|
|
5
|
+
import { CodeGenMode } from "../core/code-generator";
|
|
6
|
+
import { ENV_KEY_MAP } from "../core/spec-generator";
|
|
7
|
+
import { getSavedKey, saveKey, KEY_STORE_FILE } from "../core/key-store";
|
|
8
|
+
|
|
9
|
+
// ─── Config ───────────────────────────────────────────────────────────────────
|
|
10
|
+
|
|
11
|
+
export interface AiSpecConfig {
|
|
12
|
+
provider?: string;
|
|
13
|
+
model?: string;
|
|
14
|
+
codegen?: CodeGenMode;
|
|
15
|
+
codegenProvider?: string;
|
|
16
|
+
codegenModel?: string;
|
|
17
|
+
/** Minimum overall spec score (1-10) required to pass Approval Gate. 0 = disabled (default). */
|
|
18
|
+
minSpecScore?: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export const CONFIG_FILE = ".ai-spec.json";
|
|
22
|
+
|
|
23
|
+
export async function loadConfig(dir: string): Promise<AiSpecConfig> {
|
|
24
|
+
const p = path.join(dir, CONFIG_FILE);
|
|
25
|
+
if (await fs.pathExists(p)) {
|
|
26
|
+
return fs.readJson(p);
|
|
27
|
+
}
|
|
28
|
+
return {};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// ─── API Key Resolution ───────────────────────────────────────────────────────
|
|
32
|
+
|
|
33
|
+
export async function resolveApiKey(
|
|
34
|
+
providerName: string,
|
|
35
|
+
cliKey?: string
|
|
36
|
+
): Promise<string> {
|
|
37
|
+
if (cliKey) return cliKey;
|
|
38
|
+
|
|
39
|
+
const envVar = ENV_KEY_MAP[providerName];
|
|
40
|
+
if (envVar && process.env[envVar]) return process.env[envVar]!;
|
|
41
|
+
|
|
42
|
+
const savedKey = await getSavedKey(providerName);
|
|
43
|
+
if (savedKey) {
|
|
44
|
+
const masked = savedKey.slice(0, 6) + "..." + savedKey.slice(-4);
|
|
45
|
+
const choice = await select({
|
|
46
|
+
message: `${providerName} API key (saved: ${masked}):`,
|
|
47
|
+
choices: [
|
|
48
|
+
{ name: "Use saved key", value: "reuse" },
|
|
49
|
+
{ name: "Enter a new key", value: "new" },
|
|
50
|
+
],
|
|
51
|
+
});
|
|
52
|
+
if (choice === "reuse") return savedKey;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const newKey = await input({
|
|
56
|
+
message: `Enter your ${providerName} API key${envVar ? ` (or set ${envVar} env var)` : ""}:`,
|
|
57
|
+
validate: (v) => v.trim().length > 0 || "API key cannot be empty",
|
|
58
|
+
});
|
|
59
|
+
await saveKey(providerName, newKey.trim());
|
|
60
|
+
console.log(chalk.gray(` Key saved to ${KEY_STORE_FILE}`));
|
|
61
|
+
return newKey.trim();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// ─── Banner ───────────────────────────────────────────────────────────────────
|
|
65
|
+
|
|
66
|
+
export function printBanner(opts: {
|
|
67
|
+
specProvider: string;
|
|
68
|
+
specModel: string;
|
|
69
|
+
codegenMode: string;
|
|
70
|
+
codegenProvider: string;
|
|
71
|
+
codegenModel: string;
|
|
72
|
+
}) {
|
|
73
|
+
console.log(chalk.blue("\n" + "─".repeat(52)));
|
|
74
|
+
console.log(chalk.bold(" ai-spec — AI-driven Development Orchestrator"));
|
|
75
|
+
console.log(chalk.blue("─".repeat(52)));
|
|
76
|
+
console.log(chalk.gray(` Spec : ${opts.specProvider} / ${opts.specModel}`));
|
|
77
|
+
console.log(
|
|
78
|
+
chalk.gray(
|
|
79
|
+
` Codegen : ${opts.codegenMode} (${opts.codegenProvider} / ${opts.codegenModel})`
|
|
80
|
+
)
|
|
81
|
+
);
|
|
82
|
+
console.log(chalk.blue("─".repeat(52) + "\n"));
|
|
83
|
+
}
|