@codewithjuber/forgekit 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +12 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.codex-plugin/plugin.json +29 -0
- package/.mcp.json +8 -0
- package/ARCHITECTURE.md +314 -0
- package/CHANGELOG.md +467 -0
- package/LICENSE +21 -0
- package/ONBOARDING.md +180 -0
- package/README.md +286 -0
- package/bin/claude-init.sh +90 -0
- package/bin/claude-taste.sh +33 -0
- package/bin/learn-consolidate.sh +51 -0
- package/brand.json +15 -0
- package/global/CLAUDE.md +31 -0
- package/global/crew/frontend-verifier.md +36 -0
- package/global/crew/independent-reviewer.md +29 -0
- package/global/crew/scout.md +24 -0
- package/global/crew/verifier.md +28 -0
- package/global/guards/_guardlib.sh +36 -0
- package/global/guards/cortex.sh +14 -0
- package/global/guards/cost-budget.sh +41 -0
- package/global/guards/doom-loop.sh +25 -0
- package/global/guards/format-on-edit.sh +32 -0
- package/global/guards/lean-guard.sh +20 -0
- package/global/guards/protect-paths.sh +45 -0
- package/global/guards/recall-load.sh +22 -0
- package/global/guards/secret-redact.sh +18 -0
- package/global/guards/session-learner.sh +72 -0
- package/global/recall/MEMORY.md +7 -0
- package/global/rules/self-correction.md +17 -0
- package/global/rules/stack-notes.md +24 -0
- package/global/rules/tech-currency.md +19 -0
- package/global/settings.template.json +183 -0
- package/global/statusline.sh +51 -0
- package/global/taste/brutalist.json +9 -0
- package/global/taste/brutalist.md +19 -0
- package/global/taste/corporate.json +9 -0
- package/global/taste/corporate.md +19 -0
- package/global/taste/editorial.json +9 -0
- package/global/taste/editorial.md +19 -0
- package/global/taste/minimalist.json +9 -0
- package/global/taste/minimalist.md +20 -0
- package/global/taste/playful.json +9 -0
- package/global/taste/playful.md +19 -0
- package/global/tools/atlas/SKILL.md +27 -0
- package/global/tools/code-modernization/SKILL.md +275 -0
- package/global/tools/code-modernization/references/cost-impact-preflight.md +54 -0
- package/global/tools/code-modernization/references/design-patterns-cheatsheet.md +24 -0
- package/global/tools/code-modernization/references/research-protocol.md +42 -0
- package/global/tools/code-modernization/scripts/preflight_scan.py +190 -0
- package/global/tools/cognitive-substrate/SKILL.md +56 -0
- package/global/tools/cognitive-substrate/references/capability-map.md +17 -0
- package/global/tools/cost-guard/SKILL.md +50 -0
- package/global/tools/design-md/SKILL.md +54 -0
- package/global/tools/dev-radar/SKILL.md +56 -0
- package/global/tools/explore-plan-code/SKILL.md +24 -0
- package/global/tools/lean/SKILL.md +41 -0
- package/global/tools/recall/SKILL.md +31 -0
- package/global/tools/reuse-first/SKILL.md +64 -0
- package/global/tools/self-improve/SKILL.md +44 -0
- package/global/tools/taste/SKILL.md +26 -0
- package/global/tools/tech-selector/SKILL.md +35 -0
- package/global/tools/ui-workflow/SKILL.md +44 -0
- package/hooks/hooks.json +107 -0
- package/install.sh +88 -0
- package/package.json +93 -0
- package/public/index.html +45 -0
- package/scripts/build-pages.mjs +180 -0
- package/scripts/bump.mjs +322 -0
- package/skills/cognitive-substrate/SKILL.md +56 -0
- package/skills/cognitive-substrate/references/capability-map.md +17 -0
- package/source/mcp.json +10 -0
- package/source/rules.json +106 -0
- package/source/substrate.json +41 -0
- package/src/adjudicate.js +84 -0
- package/src/anchor.js +210 -0
- package/src/atlas.js +487 -0
- package/src/brain.js +84 -0
- package/src/brand.js +25 -0
- package/src/cli.js +1509 -0
- package/src/context.js +273 -0
- package/src/cortex.js +251 -0
- package/src/cortex_distill.js +55 -0
- package/src/cortex_features.js +81 -0
- package/src/cortex_hook.js +197 -0
- package/src/cortex_hook_main.js +139 -0
- package/src/cortex_mcp.js +352 -0
- package/src/cost_report.js +271 -0
- package/src/dash.html +396 -0
- package/src/dash.js +220 -0
- package/src/diagnose.js +0 -0
- package/src/doctor.js +315 -0
- package/src/embed.js +244 -0
- package/src/emit/_shared.js +39 -0
- package/src/emit/aider.js +22 -0
- package/src/emit/claude.js +44 -0
- package/src/emit/codex.js +17 -0
- package/src/emit/continue.js +28 -0
- package/src/emit/copilot.js +12 -0
- package/src/emit/cursor.js +23 -0
- package/src/emit/gemini.js +40 -0
- package/src/emit/mcp.js +94 -0
- package/src/emit/windsurf.js +22 -0
- package/src/emit/zed.js +34 -0
- package/src/eval.js +47 -0
- package/src/extract.js +82 -0
- package/src/harden.js +44 -0
- package/src/imagine.js +301 -0
- package/src/init.js +178 -0
- package/src/lean.js +149 -0
- package/src/ledger.js +475 -0
- package/src/ledger_bridge.js +279 -0
- package/src/ledger_read.js +152 -0
- package/src/ledger_store.js +360 -0
- package/src/lessons.js +185 -0
- package/src/lessons_store.js +137 -0
- package/src/metrics.js +54 -0
- package/src/model_tiers.js +17 -0
- package/src/model_tiers.json +39 -0
- package/src/predictor.js +143 -0
- package/src/preflight.js +410 -0
- package/src/providers.js +320 -0
- package/src/recall.js +103 -0
- package/src/reuse.js +0 -0
- package/src/route.js +323 -0
- package/src/scope.js +122 -0
- package/src/skillgate.js +89 -0
- package/src/speclock.js +64 -0
- package/src/substrate.js +492 -0
- package/src/sync.js +132 -0
- package/src/taste.js +55 -0
- package/src/uicheck.js +96 -0
- package/src/uifingerprint.js +861 -0
- package/src/uivisual.js +334 -0
- package/src/util.js +71 -0
- package/src/verify.js +117 -0
- package/templates/project-layer/.claude/settings.json +22 -0
- package/templates/project-layer/.claude/skills/hostlelo-deploy/SKILL.md +38 -0
- package/templates/project-layer/AGENTS.md +28 -0
- package/templates/project-layer/CLAUDE.md +40 -0
package/src/metrics.js
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// forge metrics — the measurement backbone of the cost model
|
|
2
|
+
// (docs/plans/substrate-v2/05-cost-model.md): every substrate stage appends one
|
|
3
|
+
// JSONL line, and savings are later ARITHMETIC on these lines, never an estimate
|
|
4
|
+
// asserted after the fact. Append-only, corrupt-line tolerant, best-effort — a
|
|
5
|
+
// metrics failure must never break the stage it measures.
|
|
6
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
|
|
9
|
+
export const metricsPath = (root = process.cwd()) => join(root, ".forge", "metrics.jsonl");
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Record one stage event: { t?, stage, outcome?, tokensIn?, tokensOut?, savedEstimate?,
|
|
13
|
+
* tier?, ref?, ... }. `t` defaults to wall-clock ms (metrics are telemetry, not
|
|
14
|
+
* content-addressed protocol state — clock use is fine here, unlike the ledger).
|
|
15
|
+
*/
|
|
16
|
+
export function record(root, entry) {
|
|
17
|
+
try {
|
|
18
|
+
const dir = join(root, ".forge");
|
|
19
|
+
mkdirSync(dir, { recursive: true });
|
|
20
|
+
appendFileSync(metricsPath(root), `${JSON.stringify({ t: Date.now(), ...entry })}\n`);
|
|
21
|
+
return { ok: true };
|
|
22
|
+
} catch {
|
|
23
|
+
return { ok: false };
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** All events, oldest first (corrupt lines skipped). Optional stage filter.
|
|
28
|
+
* @param {string} root
|
|
29
|
+
* @param {{stage?: string}} [opts] */
|
|
30
|
+
export function read(root, { stage } = {}) {
|
|
31
|
+
const path = metricsPath(root);
|
|
32
|
+
if (!existsSync(path)) return [];
|
|
33
|
+
const out = [];
|
|
34
|
+
for (const line of readFileSync(path, "utf8").split("\n")) {
|
|
35
|
+
if (!line.trim()) continue;
|
|
36
|
+
try {
|
|
37
|
+
const e = JSON.parse(line);
|
|
38
|
+
if (!stage || e.stage === stage) out.push(e);
|
|
39
|
+
} catch {}
|
|
40
|
+
}
|
|
41
|
+
return out;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Counts by stage → outcome, plus summed token-saving estimates per stage. */
|
|
45
|
+
export function summarize(root) {
|
|
46
|
+
const stages = {};
|
|
47
|
+
for (const e of read(root)) {
|
|
48
|
+
const s = (stages[e.stage] ??= { events: 0, byOutcome: {}, savedEstimate: 0 });
|
|
49
|
+
s.events++;
|
|
50
|
+
if (e.outcome) s.byOutcome[e.outcome] = (s.byOutcome[e.outcome] ?? 0) + 1;
|
|
51
|
+
if (Number.isFinite(e.savedEstimate)) s.savedEstimate += e.savedEstimate;
|
|
52
|
+
}
|
|
53
|
+
return stages;
|
|
54
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// forge model tiers — the routing target table. Cheapest capable model per complexity tier.
|
|
2
|
+
// Costs are per-million tokens (input/output). The premise: a prime-number finder does not
|
|
3
|
+
// need Fable 5. Size the model to the task. Data lives in model_tiers.json so rotation is
|
|
4
|
+
// a config change, not a code change.
|
|
5
|
+
import { readFileSync } from "node:fs";
|
|
6
|
+
|
|
7
|
+
const data = JSON.parse(readFileSync(new URL("./model_tiers.json", import.meta.url), "utf8"));
|
|
8
|
+
|
|
9
|
+
/** Currency for every inCost/outCost below. */
|
|
10
|
+
export const PRICING_CURRENCY = data.pricingCurrency;
|
|
11
|
+
/** Date the prices were last checked. `forge doctor` warns when this goes stale (re-verify via dev-radar). */
|
|
12
|
+
export const PRICING_VERIFIED = data.pricingVerified;
|
|
13
|
+
|
|
14
|
+
export const MODELS = data.models;
|
|
15
|
+
|
|
16
|
+
/** Cheap → expensive. */
|
|
17
|
+
export const TIER_ORDER = data.tierOrder;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{
|
|
2
|
+
"pricingCurrency": "USD",
|
|
3
|
+
"pricingVerified": "2026-07-05",
|
|
4
|
+
"models": {
|
|
5
|
+
"haiku": {
|
|
6
|
+
"id": "claude-haiku-4-5-20251001",
|
|
7
|
+
"name": "Haiku 4.5",
|
|
8
|
+
"tier": "simple",
|
|
9
|
+
"inCost": 1,
|
|
10
|
+
"outCost": 5,
|
|
11
|
+
"use": "lint, formatting, docs, stubs, trivial well-defined edits"
|
|
12
|
+
},
|
|
13
|
+
"sonnet": {
|
|
14
|
+
"id": "claude-sonnet-5",
|
|
15
|
+
"name": "Sonnet 5",
|
|
16
|
+
"tier": "medium",
|
|
17
|
+
"inCost": 3,
|
|
18
|
+
"outCost": 15,
|
|
19
|
+
"use": "refactoring, feature work, tests, code review (the default)"
|
|
20
|
+
},
|
|
21
|
+
"opus": {
|
|
22
|
+
"id": "claude-opus-4-8",
|
|
23
|
+
"name": "Opus 4.8",
|
|
24
|
+
"tier": "complex",
|
|
25
|
+
"inCost": 5,
|
|
26
|
+
"outCost": 25,
|
|
27
|
+
"use": "architecture, cross-module refactor, novel algorithms, multi-layer debugging"
|
|
28
|
+
},
|
|
29
|
+
"fable": {
|
|
30
|
+
"id": "claude-fable-5",
|
|
31
|
+
"name": "Fable 5",
|
|
32
|
+
"tier": "extreme",
|
|
33
|
+
"inCost": 10,
|
|
34
|
+
"outCost": 50,
|
|
35
|
+
"use": "only the hardest research-grade reasoning — rarely worth it"
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
"tierOrder": ["haiku", "sonnet", "opus", "fable"]
|
|
39
|
+
}
|
package/src/predictor.js
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
// forge predictor — the moonshot, made honest. Predicts "does this edit resemble a past
|
|
2
|
+
// mistake / look risky?" from cheap structured features. A hand-weighted HEURISTIC ships
|
|
3
|
+
// day one (and is the permanent fallback); a tiny logistic model is trained locally on the
|
|
4
|
+
// repo's own correction history and only takes over if it MEASURABLY beats the heuristic on
|
|
5
|
+
// future-held-out data. Zero runtime deps: the "model" is a JSON weight vector; inference
|
|
6
|
+
// is a dot product. Everything here is pure so the kill-criteria can't be hand-waved.
|
|
7
|
+
|
|
8
|
+
/** Heuristic weights (hand-set priors). bias is the intercept. */
|
|
9
|
+
export const DEFAULT_WEIGHTS = {
|
|
10
|
+
bias: -1.5,
|
|
11
|
+
caller_fanout: 0.9, // editing a high-fanout symbol's signature risks breaking callers
|
|
12
|
+
lesson_match: 1.4, // this edit matches an active lesson's trigger
|
|
13
|
+
churn: 0.5, // fragile, frequently-changed area
|
|
14
|
+
test_coverage_gap: 0.7, // edited code has no covering test
|
|
15
|
+
signature_change: 0.6, // alters an API surface, not a body
|
|
16
|
+
no_caller_update: 1.1, // signature changed but no caller touched in the same diff
|
|
17
|
+
past_mistake_here: 1.0, // this exact spot bit us before
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
export const FEATURE_KEYS = Object.keys(DEFAULT_WEIGHTS).filter((k) => k !== "bias");
|
|
21
|
+
|
|
22
|
+
const sigmoid = (z) => 1 / (1 + Math.exp(-z));
|
|
23
|
+
|
|
24
|
+
/** Heuristic risk in [0,1]. Advisory only — never blocks. */
|
|
25
|
+
export function heuristicRisk(features, weights = DEFAULT_WEIGHTS) {
|
|
26
|
+
let z = weights.bias ?? 0;
|
|
27
|
+
for (const k of FEATURE_KEYS) z += (weights[k] ?? 0) * (features[k] ?? 0);
|
|
28
|
+
return sigmoid(z);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Coarse band — we never claim false-precision probabilities at these sample sizes. */
|
|
32
|
+
export function band(risk) {
|
|
33
|
+
return risk >= 0.66 ? "high" : risk >= 0.4 ? "med" : "low";
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Train logistic regression by batch gradient descent (pure, zero-dep). */
|
|
37
|
+
export function trainLogistic(samples, keys = FEATURE_KEYS, { epochs = 300, lr = 0.3 } = {}) {
|
|
38
|
+
const w = { bias: 0 };
|
|
39
|
+
for (const k of keys) w[k] = 0;
|
|
40
|
+
for (let e = 0; e < epochs; e++) {
|
|
41
|
+
for (const s of samples) {
|
|
42
|
+
let z = w.bias;
|
|
43
|
+
for (const k of keys) z += w[k] * (s.features[k] ?? 0);
|
|
44
|
+
const err = sigmoid(z) - s.label;
|
|
45
|
+
w.bias -= lr * err;
|
|
46
|
+
for (const k of keys) w[k] -= lr * err * (s.features[k] ?? 0);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return w;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function predictLogistic(w, features, keys = FEATURE_KEYS) {
|
|
53
|
+
let z = w.bias ?? 0;
|
|
54
|
+
for (const k of keys) z += (w[k] ?? 0) * (features[k] ?? 0);
|
|
55
|
+
return sigmoid(z);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Average precision (area under precision-recall). PR, not ROC: mistakes are the rare
|
|
60
|
+
* positive class, and ROC-AUC flatters a model under imbalance.
|
|
61
|
+
* @param {{score:number, label:number}[]} scored
|
|
62
|
+
*/
|
|
63
|
+
export function aucPr(scored) {
|
|
64
|
+
const positives = scored.filter((s) => s.label === 1).length;
|
|
65
|
+
if (!positives) return 0;
|
|
66
|
+
const sorted = [...scored].sort((a, b) => b.score - a.score);
|
|
67
|
+
let tp = 0;
|
|
68
|
+
let fp = 0;
|
|
69
|
+
let sumPrecision = 0;
|
|
70
|
+
for (const s of sorted) {
|
|
71
|
+
if (s.label === 1) {
|
|
72
|
+
tp += 1;
|
|
73
|
+
sumPrecision += tp / (tp + fp);
|
|
74
|
+
} else {
|
|
75
|
+
fp += 1;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return sumPrecision / positives;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Prequential (train-on-past / test-on-future) evaluation + the KILL CRITERIA that decide
|
|
83
|
+
* whether the learned model is allowed to take over. This is the anti-vaporware gate.
|
|
84
|
+
* @param {{features:object, label:number}[]} samples - time-ordered.
|
|
85
|
+
* @returns {{mode:"heuristic"|"learned"|"disabled", reason:string, heuristicAucPr?:number, learnedAucPr?:number, weights?:object, n:number}}
|
|
86
|
+
*/
|
|
87
|
+
export function evaluate(samples, keys = FEATURE_KEYS, opts = {}) {
|
|
88
|
+
const { minSamples = 20, floor = 0.6, margin = 0.05 } = opts;
|
|
89
|
+
if (samples.length < minSamples) {
|
|
90
|
+
return {
|
|
91
|
+
mode: "heuristic",
|
|
92
|
+
reason: "cold-start — not enough labels yet",
|
|
93
|
+
n: samples.length,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
const cut = Math.floor(samples.length * 0.8);
|
|
97
|
+
const train = samples.slice(0, cut);
|
|
98
|
+
const test = samples.slice(cut);
|
|
99
|
+
|
|
100
|
+
const heuristicAucPr = aucPr(
|
|
101
|
+
test.map((s) => ({ score: heuristicRisk(s.features), label: s.label })),
|
|
102
|
+
);
|
|
103
|
+
// If even the heuristic can't separate mistakes here, the features carry no signal for
|
|
104
|
+
// this repo — disable prediction entirely rather than nag on noise.
|
|
105
|
+
if (heuristicAucPr < floor) {
|
|
106
|
+
return {
|
|
107
|
+
mode: "disabled",
|
|
108
|
+
reason: "features carry no signal in this repo",
|
|
109
|
+
heuristicAucPr,
|
|
110
|
+
n: samples.length,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const w = trainLogistic(train, keys);
|
|
115
|
+
const learnedAucPr = aucPr(
|
|
116
|
+
test.map((s) => ({
|
|
117
|
+
score: predictLogistic(w, s.features, keys),
|
|
118
|
+
label: s.label,
|
|
119
|
+
})),
|
|
120
|
+
);
|
|
121
|
+
const beats = learnedAucPr >= heuristicAucPr + margin;
|
|
122
|
+
return {
|
|
123
|
+
mode: beats ? "learned" : "heuristic",
|
|
124
|
+
reason: beats
|
|
125
|
+
? `learned beats heuristic by ≥${margin} AUC-PR`
|
|
126
|
+
: "heuristic retained — learned did not beat it by the margin",
|
|
127
|
+
heuristicAucPr: Number(heuristicAucPr.toFixed(3)),
|
|
128
|
+
learnedAucPr: Number(learnedAucPr.toFixed(3)),
|
|
129
|
+
weights: beats ? w : undefined,
|
|
130
|
+
n: samples.length,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** The live predictor: use the learned weights only if evaluate() blessed them. */
|
|
135
|
+
export function riskFor(features, evaluation) {
|
|
136
|
+
if (evaluation?.mode === "disabled") return { risk: 0, band: "low", path: "disabled" };
|
|
137
|
+
if (evaluation?.mode === "learned" && evaluation.weights) {
|
|
138
|
+
const risk = predictLogistic(evaluation.weights, features);
|
|
139
|
+
return { risk, band: band(risk), path: "learned" };
|
|
140
|
+
}
|
|
141
|
+
const risk = heuristicRisk(features);
|
|
142
|
+
return { risk, band: band(risk), path: "heuristic" };
|
|
143
|
+
}
|
package/src/preflight.js
ADDED
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
// forge preflight — the assumption detector. Before the agent spends a token, scan the task
|
|
2
|
+
// for code identifiers/files it NAMES but the repo doesn't DEFINE — those are the things it
|
|
3
|
+
// will silently ASSUME. The richer assumption gate also scores specification completeness.
|
|
4
|
+
import { existsSync } from "node:fs";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { adjudicate, asText, asUnit, buildRunner, llmEnabled } from "./adjudicate.js";
|
|
7
|
+
import { build as buildAtlas, has, load as loadAtlas } from "./atlas.js";
|
|
8
|
+
import { CODE_EXT } from "./util.js";
|
|
9
|
+
|
|
10
|
+
const STOP = new Set([
|
|
11
|
+
"the",
|
|
12
|
+
"this",
|
|
13
|
+
"that",
|
|
14
|
+
"add",
|
|
15
|
+
"fix",
|
|
16
|
+
"make",
|
|
17
|
+
"use",
|
|
18
|
+
"code",
|
|
19
|
+
"file",
|
|
20
|
+
"test",
|
|
21
|
+
"error",
|
|
22
|
+
"errors",
|
|
23
|
+
"function",
|
|
24
|
+
"class",
|
|
25
|
+
"value",
|
|
26
|
+
"data",
|
|
27
|
+
"type",
|
|
28
|
+
"name",
|
|
29
|
+
"todo",
|
|
30
|
+
"note",
|
|
31
|
+
]);
|
|
32
|
+
|
|
33
|
+
const rx = (pattern) => new RegExp(pattern, "i");
|
|
34
|
+
|
|
35
|
+
const DIMENSIONS = [
|
|
36
|
+
{
|
|
37
|
+
key: "inputs_outputs",
|
|
38
|
+
description: "exact input/output behavior",
|
|
39
|
+
question: "What exact inputs, outputs, examples, or return values should this satisfy?",
|
|
40
|
+
applies: rx(
|
|
41
|
+
"\\b(function|api|endpoint|parse|convert|transform|read|write|generate|compute|return|implement)\\b",
|
|
42
|
+
),
|
|
43
|
+
cues: rx(
|
|
44
|
+
"(->|=>|input|output|returns?|given|example|e\\.g\\.|for example|format|json|csv|schema|signature|expected|```)",
|
|
45
|
+
),
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
key: "target_scope",
|
|
49
|
+
description: "target file/module/component scope",
|
|
50
|
+
question: "Which specific file, module, component, or symbol should this change touch?",
|
|
51
|
+
applies: rx(
|
|
52
|
+
"\\b(fix|change|edit|update|modify|refactor|add to|remove from|integrate|wire|bug|issue|error)\\b",
|
|
53
|
+
),
|
|
54
|
+
cues: rx(
|
|
55
|
+
"\\b(file|module|class|function|component|path|directory|service|layer|endpoint)\\b|`[\\w./-]+`|\\w+\\.\\w+",
|
|
56
|
+
),
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
key: "success_criteria",
|
|
60
|
+
description: "external success criteria",
|
|
61
|
+
question:
|
|
62
|
+
"How will we verify it: tests, acceptance criteria, benchmark, or reference behavior?",
|
|
63
|
+
applies: rx(
|
|
64
|
+
"\\b(fix|optimi[sz]e|make it (faster|work|better)|improve|ensure|feature|behavior)\\b",
|
|
65
|
+
),
|
|
66
|
+
cues: rx(
|
|
67
|
+
"(->|=>|test|passes|acceptance|criteria|expected|should return|should equal|should match|verify|assert|benchmark|correct when|e\\.g\\.|example|```)",
|
|
68
|
+
),
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
key: "constraints",
|
|
72
|
+
description: "hard constraints",
|
|
73
|
+
question:
|
|
74
|
+
"What constraints must be respected: performance, dependencies, style, or compatibility?",
|
|
75
|
+
applies: rx("\\b(design|architect|production|scal|migrate|distributed|concurren|refactor)\\b"),
|
|
76
|
+
cues: rx(
|
|
77
|
+
"\\b(must|should|constraint|limit|no new dependenc|only use|standard library|without|performance|latency|O\\(|backward|compatib|convention|style)\\b",
|
|
78
|
+
),
|
|
79
|
+
},
|
|
80
|
+
];
|
|
81
|
+
|
|
82
|
+
const VAGUE = rx(
|
|
83
|
+
"\\b(some|somehow|etc|and so on|things?|stuff|appropriate(ly)?|as needed|handle (it|everything)|make it (work|better|nice|good)|clean it up|cleaner|the usual|standard way|properly|correctly|the way we (discussed|talked)|like before|as before)\\b",
|
|
84
|
+
);
|
|
85
|
+
const ANCHORS = [
|
|
86
|
+
/```/,
|
|
87
|
+
/->|=>/,
|
|
88
|
+
/\b\w+\([^)]*\)/,
|
|
89
|
+
/'[^']+'|"[^"]+"/,
|
|
90
|
+
/\b\w+\.\w{1,5}\b/,
|
|
91
|
+
/\b\d+\b.*(->|=>|=|:)|\(\d/,
|
|
92
|
+
/\b(e\.g\.|for example|such as|example:)\b/i,
|
|
93
|
+
];
|
|
94
|
+
const SPECIFIC =
|
|
95
|
+
/\b(python|javascript|typescript|java|rust|golang|react|django|flask|node|redis|sql|postgres|asyncio|dijkstra|lru|adjacency|owasp|regex)\b|token-?bucket|binary heap|condition[- ]variable|recursive-?descent|in-?order|standard library|o\(\s*\d|o\(n|o\(1/gi;
|
|
96
|
+
|
|
97
|
+
const isCodeIdent = (p, backticked) => {
|
|
98
|
+
if (!p || p.length < 2) return false;
|
|
99
|
+
if (STOP.has(p.toLowerCase())) return false;
|
|
100
|
+
if (backticked) return /^[A-Za-z_$][\w$]*$/.test(p);
|
|
101
|
+
return (
|
|
102
|
+
/[a-z][A-Z]/.test(p) || (p.includes("_") && /[a-z]/i.test(p)) || /^[A-Z][a-z]+[A-Z]/.test(p)
|
|
103
|
+
);
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
export function referencedEntities(text) {
|
|
107
|
+
const s = String(text);
|
|
108
|
+
const symbols = new Set();
|
|
109
|
+
const files = new Set();
|
|
110
|
+
const consider = (raw, backticked) => {
|
|
111
|
+
const tok = raw
|
|
112
|
+
.trim()
|
|
113
|
+
.replace(/\(\)$/, "")
|
|
114
|
+
.replace(/[.,;:]+$/, "");
|
|
115
|
+
if (!tok) return;
|
|
116
|
+
if (tok.includes("/") || CODE_EXT.test(tok)) {
|
|
117
|
+
files.add(tok);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
for (const part of tok.split(".").filter(Boolean)) {
|
|
121
|
+
if (isCodeIdent(part, backticked)) symbols.add(part);
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
for (const m of s.matchAll(/`([^`]+)`/g)) for (const t of m[1].split(/\s+/)) consider(t, true);
|
|
125
|
+
for (const m of s.matchAll(/[A-Za-z_$][\w$./-]*/g)) {
|
|
126
|
+
const t = m[0];
|
|
127
|
+
if (t.includes("/") || CODE_EXT.test(t) || /[a-z][A-Z]/.test(t) || t.includes("_"))
|
|
128
|
+
consider(t, false);
|
|
129
|
+
}
|
|
130
|
+
return { symbols: [...symbols], files: [...files] };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const AMBIGUITY = [
|
|
134
|
+
/\bsome(how|thing)?\b/i,
|
|
135
|
+
/\betc\.?\b/i,
|
|
136
|
+
/\band so on\b/i,
|
|
137
|
+
/\bas needed\b/i,
|
|
138
|
+
/\bappropriate(ly)?\b/i,
|
|
139
|
+
/\bproper(ly)?\b/i,
|
|
140
|
+
/\ba few\b/i,
|
|
141
|
+
/\bseveral\b/i,
|
|
142
|
+
/\bmake it work\b/i,
|
|
143
|
+
/\bhandle (the )?errors?\b/i,
|
|
144
|
+
/\bvarious\b/i,
|
|
145
|
+
/\band more\b/i,
|
|
146
|
+
];
|
|
147
|
+
|
|
148
|
+
export function ambiguityMarkers(text) {
|
|
149
|
+
const out = [];
|
|
150
|
+
for (const re of AMBIGUITY) {
|
|
151
|
+
const m = String(text).match(re);
|
|
152
|
+
if (m) out.push(m[0].trim().toLowerCase());
|
|
153
|
+
}
|
|
154
|
+
return [...new Set(out)];
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function assessTask(text, { askThreshold = 0.6 } = {}) {
|
|
158
|
+
const task = String(text || "");
|
|
159
|
+
const words = task.trim().split(/\s+/).filter(Boolean).length;
|
|
160
|
+
const concreteness = ANCHORS.filter((a) => a.test(task)).length;
|
|
161
|
+
const specifics = [...new Set([...task.matchAll(SPECIFIC)].map((m) => m[0].toLowerCase()))];
|
|
162
|
+
const vagueHits = [
|
|
163
|
+
...new Set([...task.matchAll(new RegExp(VAGUE.source, "gi"))].map((m) => m[0].toLowerCase())),
|
|
164
|
+
];
|
|
165
|
+
const reasons = [];
|
|
166
|
+
let score =
|
|
167
|
+
0.45 +
|
|
168
|
+
Math.min(0.45, 0.18 * concreteness) +
|
|
169
|
+
Math.min(0.2, 0.06 * specifics.length) -
|
|
170
|
+
0.22 * vagueHits.length;
|
|
171
|
+
if (words <= 7) {
|
|
172
|
+
score -= 0.22;
|
|
173
|
+
reasons.push("very short request");
|
|
174
|
+
}
|
|
175
|
+
if (words >= 30 && specifics.length >= 2) {
|
|
176
|
+
score += 0.2;
|
|
177
|
+
reasons.push(`long detailed spec (${words} words)`);
|
|
178
|
+
} else if (words >= 22 && specifics.length >= 1) {
|
|
179
|
+
score += 0.1;
|
|
180
|
+
reasons.push(`detailed spec (${words} words)`);
|
|
181
|
+
}
|
|
182
|
+
if (concreteness) reasons.push(`${concreteness} concrete anchor(s)`);
|
|
183
|
+
if (specifics.length) reasons.push(`${specifics.length} named technical specific(s)`);
|
|
184
|
+
if (vagueHits.length) reasons.push(`${vagueHits.length} vague filler(s)`);
|
|
185
|
+
|
|
186
|
+
const missing = [];
|
|
187
|
+
const questions = [];
|
|
188
|
+
for (const d of DIMENSIONS) {
|
|
189
|
+
if (d.applies.test(task) && !d.cues.test(task)) {
|
|
190
|
+
missing.push({ key: d.key, description: d.description });
|
|
191
|
+
questions.push(d.question);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
const completeness = Math.max(0, Math.min(1, score));
|
|
195
|
+
const hardUnderspecified =
|
|
196
|
+
concreteness === 0 && (words <= 10 || (vagueHits.length >= 1 && specifics.length === 0));
|
|
197
|
+
const shouldAsk = completeness < askThreshold || hardUnderspecified;
|
|
198
|
+
if (hardUnderspecified && !reasons.includes("very short request"))
|
|
199
|
+
reasons.push("no concrete anchor to act on");
|
|
200
|
+
const risk = completeness < 0.45 ? "high" : completeness < 0.7 ? "medium" : "low";
|
|
201
|
+
return {
|
|
202
|
+
completeness,
|
|
203
|
+
risk,
|
|
204
|
+
shouldAsk,
|
|
205
|
+
hardUnderspecified,
|
|
206
|
+
missing,
|
|
207
|
+
questions: (shouldAsk && !questions.length
|
|
208
|
+
? ["What exactly should this produce, and how will we know it is correct?"]
|
|
209
|
+
: questions
|
|
210
|
+
).slice(0, 3),
|
|
211
|
+
reasons,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// M2 assumption gate — LLM proposer. Scores the paper's s(x) completeness functional over the
|
|
216
|
+
// same four dimensions the rubric names, and may add clarifying questions. PROPOSER ONLY:
|
|
217
|
+
// reconcileAssumption() bounds it against the deterministic score and grounds every question.
|
|
218
|
+
const DIM_KEYS = DIMENSIONS.map((d) => d.key);
|
|
219
|
+
|
|
220
|
+
export function buildAssumptionPrompt(task) {
|
|
221
|
+
return `A coding agent received this task. Judge how completely it is specified BEFORE any code is written.
|
|
222
|
+
Task: """${String(task).slice(0, 1200)}"""
|
|
223
|
+
Score specification completeness in [0,1] over these dimensions: inputs_outputs, target_scope,
|
|
224
|
+
success_criteria, constraints. List the concrete missing-information questions a careful engineer
|
|
225
|
+
would ask first. Respond with STRICT JSON and nothing else:
|
|
226
|
+
{"completeness":<0..1>,"missing":["<dimension name>"...],"questions":["<question>"...]}
|
|
227
|
+
Do not echo credentials or personal data. No text outside the JSON object.`;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export function parseAssumptionProposal(obj) {
|
|
231
|
+
const completeness = asUnit(obj.completeness);
|
|
232
|
+
if (completeness == null) return null;
|
|
233
|
+
const missing = Array.isArray(obj.missing)
|
|
234
|
+
? obj.missing.map((m) => asText(m, 40)).filter((m) => DIM_KEYS.includes(m))
|
|
235
|
+
: [];
|
|
236
|
+
const questions = Array.isArray(obj.questions)
|
|
237
|
+
? [...new Set(obj.questions.map((q) => asText(q, 200)).filter(Boolean))].slice(0, 5)
|
|
238
|
+
: [];
|
|
239
|
+
return { completeness, missing, questions };
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** Ask the model for an assumption reading (proposer). Returns null when off/unavailable. */
|
|
243
|
+
export function assessTaskLLM(task, { run = buildRunner() } = {}) {
|
|
244
|
+
return adjudicate({ prompt: buildAssumptionPrompt(task), parse: parseAssumptionProposal, run });
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Verify-don't-trust reconcile for M2. The model may only move completeness within ±band of the
|
|
249
|
+
* deterministic score, so a clearly-specified or clearly-vague task can never be flipped — only a
|
|
250
|
+
* borderline reading shifts. In `bidirectional` mode (default) the ask is recomputed purely from
|
|
251
|
+
* that bounded completeness, so a verified reading can also CLEAR a false ask — but two hard
|
|
252
|
+
* floors the model can never override still force the ask: a task with no concrete anchor
|
|
253
|
+
* (`hardUnderspecified`), or one naming symbols/files the repo doesn't define (`hasUnresolved`).
|
|
254
|
+
* With `bidirectional:false` the gate only ever tightens (the conservative pre-bidirectional
|
|
255
|
+
* behaviour). Extra questions survive only if they map to a rubric-flagged dimension or (via
|
|
256
|
+
* `grounded`) reference a real repo entity.
|
|
257
|
+
* @param {object} det - assessTask() result
|
|
258
|
+
* @param {{completeness:number, missing:string[], questions:string[]}|null} proposal
|
|
259
|
+
* @param {object} [opts]
|
|
260
|
+
* @param {number} [opts.askThreshold]
|
|
261
|
+
* @param {number} [opts.band]
|
|
262
|
+
* @param {(q:string)=>boolean} [opts.grounded]
|
|
263
|
+
* @param {boolean} [opts.bidirectional]
|
|
264
|
+
* @param {boolean} [opts.hasUnresolved]
|
|
265
|
+
*/
|
|
266
|
+
export function reconcileAssumption(
|
|
267
|
+
det,
|
|
268
|
+
proposal,
|
|
269
|
+
{
|
|
270
|
+
askThreshold = 0.6,
|
|
271
|
+
band = 0.25,
|
|
272
|
+
grounded = () => false,
|
|
273
|
+
bidirectional = true,
|
|
274
|
+
hasUnresolved = false,
|
|
275
|
+
} = {},
|
|
276
|
+
) {
|
|
277
|
+
if (!proposal) return { ...det, provenance: { path: "deterministic" } };
|
|
278
|
+
const bounded = Math.max(
|
|
279
|
+
det.completeness - band,
|
|
280
|
+
Math.min(det.completeness + band, proposal.completeness),
|
|
281
|
+
);
|
|
282
|
+
const completeness = Math.max(0, Math.min(1, bounded));
|
|
283
|
+
const flaggedDims = new Set(det.missing.map((m) => m.key));
|
|
284
|
+
const extraQuestions = proposal.questions.filter(
|
|
285
|
+
(q) => proposal.missing.some((m) => flaggedDims.has(m)) || grounded(q),
|
|
286
|
+
);
|
|
287
|
+
const questions = [...new Set([...det.questions, ...extraQuestions])].slice(0, 3);
|
|
288
|
+
// Bidirectional (default): the ask follows the bounded completeness, guarded by two floors the
|
|
289
|
+
// model can't override. Tighten-only: the rubric's ask always stands, the model can only add one.
|
|
290
|
+
const shouldAsk = bidirectional
|
|
291
|
+
? det.hardUnderspecified || hasUnresolved || completeness < askThreshold
|
|
292
|
+
: det.shouldAsk || det.hardUnderspecified || completeness < askThreshold;
|
|
293
|
+
const risk = completeness < 0.45 ? "high" : completeness < 0.7 ? "medium" : "low";
|
|
294
|
+
const moved =
|
|
295
|
+
Math.abs(completeness - det.completeness) > 1e-9 || questions.length !== det.questions.length;
|
|
296
|
+
let path;
|
|
297
|
+
if (shouldAsk && !det.shouldAsk) path = "llm-tightened";
|
|
298
|
+
else if (!shouldAsk && det.shouldAsk) path = "llm-cleared";
|
|
299
|
+
else path = moved ? "llm-verified" : "llm-agreed";
|
|
300
|
+
return {
|
|
301
|
+
...det,
|
|
302
|
+
completeness,
|
|
303
|
+
risk,
|
|
304
|
+
shouldAsk,
|
|
305
|
+
questions:
|
|
306
|
+
shouldAsk && !questions.length
|
|
307
|
+
? ["What exactly should this produce, and how will we know it is correct?"]
|
|
308
|
+
: questions,
|
|
309
|
+
provenance: { path, detCompleteness: det.completeness },
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
export function informationGap(text, deps = {}) {
|
|
314
|
+
const { hasSymbol = () => false, fileExists = () => false } = deps;
|
|
315
|
+
const { symbols, files } = referencedEntities(text);
|
|
316
|
+
const ambiguous = ambiguityMarkers(text);
|
|
317
|
+
const unresolvedSymbols = symbols.filter((s) => !hasSymbol(s));
|
|
318
|
+
const unresolvedFiles = files.filter((f) => !fileExists(f));
|
|
319
|
+
const problems = unresolvedSymbols.length + unresolvedFiles.length + ambiguous.length;
|
|
320
|
+
const denom = symbols.length + files.length + ambiguous.length;
|
|
321
|
+
const gap = denom === 0 ? 0 : Math.min(1, problems / denom);
|
|
322
|
+
return {
|
|
323
|
+
gap,
|
|
324
|
+
unresolved: { symbols: unresolvedSymbols, files: unresolvedFiles },
|
|
325
|
+
ambiguous,
|
|
326
|
+
entities: { symbols, files },
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
export function clarifyBlock(result, { threshold = 0.5 } = {}) {
|
|
331
|
+
const nRef = result.unresolved.symbols.length + result.unresolved.files.length;
|
|
332
|
+
const assumption = result.assumption;
|
|
333
|
+
if (nRef === 0 && result.gap < threshold && !assumption?.shouldAsk) return "";
|
|
334
|
+
const lines = [
|
|
335
|
+
"## Before starting — clarify (Forge Preflight)",
|
|
336
|
+
"This task has unknowns that would otherwise become assumptions:",
|
|
337
|
+
"",
|
|
338
|
+
];
|
|
339
|
+
for (const s of result.unresolved.symbols)
|
|
340
|
+
lines.push(`- \`${s}\` — not found in the code. Different name, or should it be created?`);
|
|
341
|
+
for (const f of result.unresolved.files)
|
|
342
|
+
lines.push(`- \`${f}\` — file not found. Confirm the path, or that it is new.`);
|
|
343
|
+
if (result.ambiguous.length)
|
|
344
|
+
lines.push(
|
|
345
|
+
`- Ambiguous: ${result.ambiguous.map((a) => `"${a}"`).join(", ")} — state concrete acceptance criteria.`,
|
|
346
|
+
);
|
|
347
|
+
if (assumption?.shouldAsk) for (const q of assumption.questions) lines.push(`- ${q}`);
|
|
348
|
+
lines.push("", "_Advisory: ask rather than assume._");
|
|
349
|
+
return lines.join("\n");
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* @param {string} root
|
|
354
|
+
* @param {string} text
|
|
355
|
+
* @param {object} [opts]
|
|
356
|
+
* @param {boolean} [opts.allowBuild]
|
|
357
|
+
* @param {number} [opts.askThreshold]
|
|
358
|
+
* @param {boolean} [opts.llm]
|
|
359
|
+
* @param {string} [opts.model]
|
|
360
|
+
* @param {number} [opts.timeoutMs]
|
|
361
|
+
* @param {(p:string)=>string} [opts.run]
|
|
362
|
+
* @param {boolean} [opts.bidirectional]
|
|
363
|
+
* @param {number} [opts.band]
|
|
364
|
+
*/
|
|
365
|
+
export function preflightRepo(
|
|
366
|
+
root,
|
|
367
|
+
text,
|
|
368
|
+
{
|
|
369
|
+
allowBuild = true,
|
|
370
|
+
askThreshold = 0.6,
|
|
371
|
+
llm,
|
|
372
|
+
model,
|
|
373
|
+
timeoutMs,
|
|
374
|
+
run,
|
|
375
|
+
bidirectional = true,
|
|
376
|
+
band,
|
|
377
|
+
} = {},
|
|
378
|
+
) {
|
|
379
|
+
const atlas = loadAtlas(root) || (allowBuild ? buildAtlas({ root }) : null);
|
|
380
|
+
// When the graph is capped (files were dropped) we can't be sure a symbol is truly absent, so
|
|
381
|
+
// treat everything as resolvable rather than raising false "not found in the code" clarifications.
|
|
382
|
+
const hasSymbol = atlas && !atlas.capped ? (name) => has(atlas, name) : () => true;
|
|
383
|
+
const gap = informationGap(text, {
|
|
384
|
+
hasSymbol,
|
|
385
|
+
fileExists: (f) => existsSync(join(root, f)),
|
|
386
|
+
});
|
|
387
|
+
const det = assessTask(text, { askThreshold });
|
|
388
|
+
// M2 proposer: only when opted in. The rubric is the external judge; the model refines it
|
|
389
|
+
// within bounds and can add grounded questions. Fail-safe: null proposal keeps `det`.
|
|
390
|
+
if (!llmEnabled({ llm }))
|
|
391
|
+
return { ...gap, assumption: { ...det, provenance: { path: "deterministic" } } };
|
|
392
|
+
const proposal = assessTaskLLM(text, { run: run || buildRunner({ model, timeoutMs }) });
|
|
393
|
+
const grounded = (q) => {
|
|
394
|
+
const { symbols, files } = referencedEntities(q);
|
|
395
|
+
return symbols.some(hasSymbol) || files.some((f) => existsSync(join(root, f)));
|
|
396
|
+
};
|
|
397
|
+
// Repo grounding is a hard floor on clearing: if the task names entities the repo lacks, the
|
|
398
|
+
// model can never wave the gate through no matter how "complete" it judges the prose.
|
|
399
|
+
const hasUnresolved = gap.unresolved.symbols.length + gap.unresolved.files.length > 0;
|
|
400
|
+
return {
|
|
401
|
+
...gap,
|
|
402
|
+
assumption: reconcileAssumption(det, proposal, {
|
|
403
|
+
askThreshold,
|
|
404
|
+
grounded,
|
|
405
|
+
bidirectional,
|
|
406
|
+
hasUnresolved,
|
|
407
|
+
...(typeof band === "number" ? { band } : {}),
|
|
408
|
+
}),
|
|
409
|
+
};
|
|
410
|
+
}
|