@agentsfleet/orly 0.10.5 → 0.10.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -5
- package/README.md +26 -8
- package/audits/agents-md.md +7 -3
- package/audits/spec-template.sh +30 -113
- package/audits/spec-template.ts +193 -0
- package/core/operating-model.md +5 -5
- package/dispatch/lifecycle.md +98 -42
- package/dispatch/verify.md +30 -48
- package/dispatch/write_spec.md +56 -82
- package/docs/ORLY_ARCHITECTURE.md +13 -3
- package/docs/RULE_ENFORCEMENT.md +2 -2
- package/docs/TEMPLATE.md +82 -77
- package/docs/VERIFY_TIERS.md +20 -26
- package/llms.txt +17 -12
- package/package.json +1 -1
- package/registry.json +4 -0
- package/skills/orly-spec-new/SKILL.md +3 -1
- package/skills/orly-write-integration-test/SKILL.md +1 -1
- package/skills/orly-write-unit-test/SKILL.md +1 -1
- package/src/cli.ts +14 -16
- package/src/criteria.test.ts +47 -6
- package/src/criteria.ts +30 -49
- package/src/criteria_spec.ts +53 -6
- package/src/criteria_support.ts +5 -4
- package/src/gates.test.ts +2 -2
- package/src/gates_spec.test.ts +40 -1
- package/src/gates_test_support.ts +18 -4
- package/src/install.ts +16 -81
- package/src/render.test.ts +5 -7
- package/src/setup.test.ts +116 -0
- package/src/spec_baseline.test.ts +96 -0
- package/src/spec_template.test.ts +109 -0
- package/src/spec_template_io.test.ts +97 -0
- package/src/validation.ts +9 -0
package/src/cli.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { isString, OrlyError, readJsonObject, RulesModel } from "./model";
|
|
|
9
9
|
import { Renderer } from "./render";
|
|
10
10
|
import { beginTelemetry, installedOrlyVersion, isTelemetrySkill, recordTelemetry } from "./telemetry";
|
|
11
11
|
import { verifyRenders, writeEvidence } from "./verify";
|
|
12
|
+
import { commandSetupErrors } from "./validation";
|
|
12
13
|
|
|
13
14
|
type CliResult = { exitCode: number; gate?: string; failedCriterion?: string };
|
|
14
15
|
|
|
@@ -152,15 +153,18 @@ async function materialise(model: RulesModel, args: string[], isInit: boolean):
|
|
|
152
153
|
installHooks: !args.includes(NO_HOOKS_FLAG),
|
|
153
154
|
orlyVersion: await packageVersion(model),
|
|
154
155
|
});
|
|
155
|
-
|
|
156
|
-
|
|
156
|
+
const setupErrors = result.ok ? commandSetupErrors((await readConfig(targetRoot))!.commands) : [];
|
|
157
|
+
if (args.includes(JSON_FLAG)) console.log(JSON.stringify({ ...result, setupErrors }, undefined, JSON_INDENT));
|
|
158
|
+
else printInstall(result, setupErrors);
|
|
157
159
|
return result.ok ? 0 : 1;
|
|
158
160
|
}
|
|
159
161
|
|
|
160
|
-
function printInstall(result: InstallResult): void {
|
|
162
|
+
function printInstall(result: InstallResult, setupErrors: string[]): void {
|
|
161
163
|
for (const error of result.errors) console.log(`${FAIL_GLYPH} ${error.path}: ${error.message} — ${error.suggestion}`);
|
|
162
164
|
if (!result.ok) return;
|
|
163
165
|
console.log(`${PASS_GLYPH} ${result.written.length} written, ${result.skipped.length} already current (${result.packs.length} packs)`);
|
|
166
|
+
for (const error of setupErrors) console.log(`🟠 setup incomplete: ${error}`);
|
|
167
|
+
if (setupErrors.length > 0) console.log("Then run orly doctor to check the completed setup.");
|
|
164
168
|
}
|
|
165
169
|
|
|
166
170
|
function printGate(report: GateReport): void {
|
|
@@ -175,9 +179,6 @@ function projectRoot(): string {
|
|
|
175
179
|
}
|
|
176
180
|
|
|
177
181
|
async function doctorGlobal(model: RulesModel): Promise<number> {
|
|
178
|
-
// Every repository is the same case now: doctor reports on the ruleset
|
|
179
|
-
// installed where the caller is standing. There is no machine-level carrier
|
|
180
|
-
// to check — the rules ride in each repository's own commit.
|
|
181
182
|
const errors = await doctorInstall(model);
|
|
182
183
|
if (errors.length > 0) {
|
|
183
184
|
for (const error of errors) console.log(`${FAIL_GLYPH} ${error}`);
|
|
@@ -188,7 +189,10 @@ async function doctorGlobal(model: RulesModel): Promise<number> {
|
|
|
188
189
|
console.log(`${FAIL_GLYPH} no ${CONFIG_PATH} here — run \`orly init\` first`);
|
|
189
190
|
return 1;
|
|
190
191
|
}
|
|
191
|
-
|
|
192
|
+
const setupErrors = commandSetupErrors(config.commands);
|
|
193
|
+
for (const error of setupErrors) console.log(`${FAIL_GLYPH} setup incomplete: ${error}`);
|
|
194
|
+
if (setupErrors.length > 0) return 1;
|
|
195
|
+
console.log(`${PASS_GLYPH} installed rules match ${CONFIG_PATH}; work and verification commands are declared`);
|
|
192
196
|
return 0;
|
|
193
197
|
}
|
|
194
198
|
|
|
@@ -266,12 +270,6 @@ function requireNoArguments(args: string[], usage: string): void {
|
|
|
266
270
|
if (args.length > 0) throw new OrlyError(usage);
|
|
267
271
|
}
|
|
268
272
|
|
|
269
|
-
function optionValue(args: string[], name: string): string {
|
|
270
|
-
const value = optionalValue(args, name);
|
|
271
|
-
if (!value) throw new OrlyError(`${name} is required`);
|
|
272
|
-
return value;
|
|
273
|
-
}
|
|
274
|
-
|
|
275
273
|
function optionalValue(args: string[], name: string): string | undefined {
|
|
276
274
|
const index = args.indexOf(name);
|
|
277
275
|
return index < 0 ? undefined : args[index + 1];
|
|
@@ -301,10 +299,10 @@ Gates (read-only; no PR without every criterion green or a recorded override):
|
|
|
301
299
|
orly gate <work|verify|pr> run one gate
|
|
302
300
|
work does this commit conform? the declared conform command; no git
|
|
303
301
|
state, so a commit hook's own dirty tree never blocks it
|
|
304
|
-
verify does the work hold up?
|
|
305
|
-
|
|
302
|
+
verify does the work hold up? configuration, docs language, and the
|
|
303
|
+
non-test verify.* set; unfinished Sections may be pushed
|
|
306
304
|
pr can this ship? branch, tree, pushed, every spec criterion, and
|
|
307
|
-
|
|
305
|
+
every declared verify.* command
|
|
308
306
|
orly override <CRITERION> --reason <REASON>
|
|
309
307
|
empty commit with an Orly-Override trailer
|
|
310
308
|
|
package/src/criteria.test.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { afterEach, describe, expect, test } from "bun:test";
|
|
2
2
|
import { rmSync } from "node:fs";
|
|
3
|
-
import { join } from "node:path";
|
|
3
|
+
import { join, relative } from "node:path";
|
|
4
4
|
|
|
5
5
|
import { criteriaFor, runCommand } from "./criteria";
|
|
6
|
-
import {
|
|
6
|
+
import { activeSpecPath, closedSpecPath } from "./gates";
|
|
7
|
+
import { cleanupTemporaryDirectories, closedSpecRepository, git, modelFor, newRepository, newSpecRepository } from "./gates_test_support";
|
|
7
8
|
|
|
8
9
|
const WORK = "work";
|
|
9
10
|
const VERIFY = "verify";
|
|
@@ -32,12 +33,12 @@ describe("criteriaFor", () => {
|
|
|
32
33
|
expect(named(WORK, await contextFor(project))).toEqual(["cmd.conform", REPO_CONFIG]);
|
|
33
34
|
});
|
|
34
35
|
|
|
35
|
-
test("verify
|
|
36
|
+
test("verify permits unfinished Sections and excludes boundary test suites", async () => {
|
|
36
37
|
const project = newSpecRepository();
|
|
37
38
|
const names = named(VERIFY, await contextFor(project));
|
|
38
39
|
|
|
39
|
-
expect(names).toContain("spec.dimensions");
|
|
40
|
-
expect(names).toContain("cmd.verify.unit");
|
|
40
|
+
expect(names).not.toContain("spec.dimensions");
|
|
41
|
+
expect(names).not.toContain("cmd.verify.unit");
|
|
41
42
|
// conform is the work gate's tier now, so the chain runs it once.
|
|
42
43
|
expect(names).not.toContain("cmd.conform");
|
|
43
44
|
// The slow tier belongs to the pr gate; leaking it here would make every
|
|
@@ -63,6 +64,20 @@ describe("criteriaFor", () => {
|
|
|
63
64
|
expect(names).toContain("git.tree");
|
|
64
65
|
});
|
|
65
66
|
|
|
67
|
+
test("final verification runs test lanes even when push verification never ran", async () => {
|
|
68
|
+
const root = newRepository();
|
|
69
|
+
const commands = Object.fromEntries(["unit", "integration", "memory", "lint"].map((lane) => [
|
|
70
|
+
`verify.${lane}`, [["sh", "-c", `echo ${lane} >> ran.txt`]],
|
|
71
|
+
]));
|
|
72
|
+
const context = {
|
|
73
|
+
root, model: await modelFor(root, undefined, commands), acceptDirty: false,
|
|
74
|
+
surfaces: { changed: ["app.rs"], code: ["app.rs"], docs: [], userSurface: [] },
|
|
75
|
+
};
|
|
76
|
+
const checks = criteriaFor(PR, context).filter((entry) => entry.name.startsWith("cmd."));
|
|
77
|
+
expect(checks.map((entry) => entry.evaluate(context).ok)).toEqual([true, true, true, true]);
|
|
78
|
+
expect((await Bun.file(join(root, "ran.txt")).text()).trim().split("\n").sort()).toEqual(["integration", "lint", "memory", "unit"]);
|
|
79
|
+
});
|
|
80
|
+
|
|
66
81
|
test("an unknown gate name yields no criteria rather than throwing", async () => {
|
|
67
82
|
const project = newSpecRepository();
|
|
68
83
|
expect(criteriaFor("nonsense", await contextFor(project))).toEqual([]);
|
|
@@ -108,14 +123,19 @@ describe("runCommand", () => {
|
|
|
108
123
|
expect(runCommand(process.cwd(), ["true"])).toEqual({ ok: true, detail: "exit 0" });
|
|
109
124
|
});
|
|
110
125
|
|
|
111
|
-
test("a non-zero exit
|
|
126
|
+
test("a non-zero exit retains all output, including earlier findings", () => {
|
|
112
127
|
const result = runCommand(process.cwd(), ["sh", "-c", "echo first; echo decisive >&2; exit 3"]);
|
|
113
128
|
|
|
114
129
|
expect(result.ok).toBeFalse();
|
|
115
130
|
expect(result.detail).toContain("exit 3");
|
|
131
|
+
expect(result.detail).toContain("first");
|
|
116
132
|
expect(result.detail).toEndWith("decisive");
|
|
117
133
|
});
|
|
118
134
|
|
|
135
|
+
test("a successful command retains its measured counts", () => {
|
|
136
|
+
expect(runCommand(process.cwd(), ["echo", "12 passed, 0 failed"]).detail).toContain("12 passed, 0 failed");
|
|
137
|
+
});
|
|
138
|
+
|
|
119
139
|
test("a silent failure still reports a detail rather than an empty string", () => {
|
|
120
140
|
const result = runCommand(process.cwd(), ["false"]);
|
|
121
141
|
|
|
@@ -146,3 +166,24 @@ describe("repo.config", () => {
|
|
|
146
166
|
expect(verdict?.detail).toContain("orly init");
|
|
147
167
|
});
|
|
148
168
|
});
|
|
169
|
+
|
|
170
|
+
describe("the final tree check", () => {
|
|
171
|
+
for (const stage of ["active", "done"]) {
|
|
172
|
+
test(`requires a ${stage} spec edit to be committed`, async () => {
|
|
173
|
+
const root = stage === "active" ? newSpecRepository() : closedSpecRepository("feat/spec");
|
|
174
|
+
const model = await modelFor(root);
|
|
175
|
+
const specPath = relative(root, activeSpecPath(root) ?? closedSpecPath(root)!);
|
|
176
|
+
const context = { root, model, specPath, acceptDirty: false };
|
|
177
|
+
const check = criteriaFor(PR, context).find((entry) => entry.name === "git.tree")!;
|
|
178
|
+
expect(check.evaluate(context).ok).toBe(true);
|
|
179
|
+
|
|
180
|
+
const path = join(root, specPath);
|
|
181
|
+
await Bun.write(path, `${await Bun.file(path).text()}\nUpdated evidence.\n`);
|
|
182
|
+
expect(check.evaluate(context).ok).toBe(false);
|
|
183
|
+
git(root, "add", specPath);
|
|
184
|
+
expect(check.evaluate(context).ok).toBe(false);
|
|
185
|
+
git(root, "commit", "-qm", "docs: record evidence");
|
|
186
|
+
expect(check.evaluate(context).ok).toBe(true);
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
});
|
package/src/criteria.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { documentationSurfaces, scanSurfaces } from "./doc_rules";
|
|
|
12
12
|
import { isObject, JsonObject, objectValue, OrlyError, RulesModel } from "./model";
|
|
13
13
|
import { readConfigSync, RepoConfig } from "./config";
|
|
14
14
|
import { classifyBranch, SurfaceReport } from "./surfaces";
|
|
15
|
+
import { commandSetupErrors } from "./validation";
|
|
15
16
|
|
|
16
17
|
export type { Criterion, CriterionContext, CriterionResult, Verdict };
|
|
17
18
|
export { runCommand };
|
|
@@ -22,6 +23,9 @@ const VERIFY_PREFIX = "verify.";
|
|
|
22
23
|
const REPOSITORIES_LABEL = "repositories";
|
|
23
24
|
const UNINSTALLED = "no .oracle/orly.json here — run `orly init` first";
|
|
24
25
|
const REV_PARSE = "rev-parse";
|
|
26
|
+
const HEAD = "HEAD";
|
|
27
|
+
const UPSTREAM = "@{upstream}";
|
|
28
|
+
const NEWLINE = "\n";
|
|
25
29
|
const ABBREV_REF = "--abbrev-ref";
|
|
26
30
|
const WORKTREE_LIST = ["worktree", "list", "--porcelain"];
|
|
27
31
|
const WORKTREE_PREFIX = "worktree ";
|
|
@@ -47,37 +51,20 @@ const SLOW_TIER = "slow";
|
|
|
47
51
|
// The slow tier is a fixed name set, not a prefix rule: lint and version
|
|
48
52
|
// checks are verify.* too, and demoting them to skip-on-prose would be wrong.
|
|
49
53
|
const SLOW_COMMANDS = ["verify.integration", "verify.memory"];
|
|
54
|
+
const UNIT_COMMAND = "verify.unit";
|
|
55
|
+
const ALL_TIER = "all";
|
|
50
56
|
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
// One gate per cadence, and each question asked at the only moment it can be
|
|
56
|
-
// answered honestly:
|
|
57
|
-
//
|
|
58
|
-
// work — does this commit conform? Config plus the declared `conform`
|
|
59
|
-
// command. Nothing about branch or tree: inside a commit hook the
|
|
60
|
-
// tree is dirty BY CONSTRUCTION (that is what is being committed),
|
|
61
|
-
// and the operating model itself says a new spec is committed on the
|
|
62
|
-
// default branch. Judging either here made the generated pre-commit
|
|
63
|
-
// hook refuse every commit — including the one installing orly —
|
|
64
|
-
// with `--no-verify`, which Hard Safety forbids, as the only way out.
|
|
65
|
-
// verify — does the work hold up? Spec dimensions, documentation language,
|
|
66
|
-
// and the fast `verify.*` set. Runs before the work leaves the
|
|
67
|
-
// machine, not after every edit.
|
|
68
|
-
// pr — can this ship? The whole-branch facts (branch shape, clean tree,
|
|
69
|
-
// pushed) plus every spec criterion and the slow suites. A tree is
|
|
70
|
-
// only required clean where a reviewer is about to read it, and a
|
|
71
|
-
// branch is only wrong for being `main` where a PR would open from
|
|
72
|
-
// it.
|
|
57
|
+
// Commit checks read the index. Push checks permit in-flight Sections and omit
|
|
58
|
+
// the boundary test suites. The final gate runs all declared verification
|
|
59
|
+
// itself, including checks a custom repository hook may not have invoked.
|
|
73
60
|
export function criteriaFor(gate: string, context: CriterionContext): Criterion[] {
|
|
74
61
|
if (gate === "work") return [repositoryConfig(), ...commandCriteria(context, CONFORM_TIER)];
|
|
75
|
-
if (gate === "verify") return [
|
|
62
|
+
if (gate === "verify") return [repositoryConfig(), docsLanguage(), ...commandCriteria(context, FAST_TIER)];
|
|
76
63
|
if (gate === "pr") {
|
|
77
64
|
return [
|
|
78
65
|
gitBranch(), gitTree(), gitPushed(), specGate(), openQuestions(), productClarity(), specDimensions(),
|
|
79
66
|
specMoved(), specBaseline(), specOrdering(), specDeferrals(),
|
|
80
|
-
docsUpdated(), ...commandCriteria(context,
|
|
67
|
+
repositoryConfig(), docsLanguage(), docsUpdated(), ...commandCriteria(context, ALL_TIER),
|
|
81
68
|
];
|
|
82
69
|
}
|
|
83
70
|
return [];
|
|
@@ -93,14 +80,10 @@ function gitBranch(): Criterion {
|
|
|
93
80
|
|
|
94
81
|
function gitTree(): Criterion {
|
|
95
82
|
return criterion(GIT_TREE, (context) => {
|
|
96
|
-
// The active spec is excluded while work is in flight: Dimensions get
|
|
97
|
-
// marked DONE as the agent goes, and the tree check must not block on that
|
|
98
|
-
// bookkeeping. Committing the spec stays a CHORE(close) obligation.
|
|
99
83
|
const dirty = gitOutput(context.root, ["status", "--porcelain=v1", "-uall"])
|
|
100
84
|
.split(/\r?\n/)
|
|
101
|
-
.filter(Boolean)
|
|
102
|
-
|
|
103
|
-
if (dirty.length === 0) return { ok: true, detail: "clean (active spec excluded)" };
|
|
85
|
+
.filter(Boolean);
|
|
86
|
+
if (dirty.length === 0) return { ok: true, detail: "clean" };
|
|
104
87
|
if (context.acceptDirty) return { ok: true, detail: `${dirty.length} dirty path(s) accepted` };
|
|
105
88
|
return { ok: false, detail: `${dirty.length} uncommitted path(s), first: ${dirty[0] ?? ""}` };
|
|
106
89
|
});
|
|
@@ -108,11 +91,12 @@ function gitTree(): Criterion {
|
|
|
108
91
|
|
|
109
92
|
function gitPushed(): Criterion {
|
|
110
93
|
return criterion(GIT_PUSHED, (context) => {
|
|
111
|
-
const upstream = gitOutput(context.root, [REV_PARSE, ABBREV_REF, "--symbolic-full-name",
|
|
94
|
+
const upstream = gitOutput(context.root, [REV_PARSE, ABBREV_REF, "--symbolic-full-name", UPSTREAM]);
|
|
112
95
|
if (upstream.length === 0) return { ok: false, detail: "branch has no upstream; push it first" };
|
|
113
|
-
const
|
|
114
|
-
const
|
|
115
|
-
|
|
96
|
+
const head = gitOutput(context.root, [REV_PARSE, HEAD]);
|
|
97
|
+
const remote = gitOutput(context.root, [REV_PARSE, UPSTREAM]);
|
|
98
|
+
const ok = head.length > 0 && head === remote;
|
|
99
|
+
return { ok, detail: ok ? `in sync with ${upstream}` : `HEAD differs from ${upstream}; push the reviewed revision` };
|
|
116
100
|
});
|
|
117
101
|
}
|
|
118
102
|
|
|
@@ -121,27 +105,23 @@ function repositoryConfig(): Criterion {
|
|
|
121
105
|
try {
|
|
122
106
|
const config = readConfigSync(context.root);
|
|
123
107
|
if (!config) return { ok: false, detail: UNINSTALLED };
|
|
124
|
-
const
|
|
125
|
-
return { ok:
|
|
108
|
+
const errors = commandSetupErrors(config.commands);
|
|
109
|
+
return { ok: errors.length === 0, detail: errors.length > 0 ? errors.join("; ") : `${Object.keys(config.commands).length} command(s) declared` };
|
|
126
110
|
} catch (error) {
|
|
127
111
|
return { ok: false, detail: error instanceof Error ? error.message : String(error) };
|
|
128
112
|
}
|
|
129
113
|
});
|
|
130
114
|
}
|
|
131
115
|
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
// `conform` command alone. Fast tier = every verify.* that is not slow. Slow
|
|
135
|
-
// tier = verify.integration and verify.memory; those auto-pass with a printed
|
|
136
|
-
// skip when the branch carries no code, so a prose-only branch never pays for
|
|
137
|
-
// the slow suites.
|
|
116
|
+
// Commands and source paths are repository-owned. Only integration and memory
|
|
117
|
+
// checks are conditional on code; every other declared lane always runs.
|
|
138
118
|
function commandCriteria(context: CriterionContext, tier: string): Criterion[] {
|
|
139
119
|
const config = resolvedConfig(context);
|
|
140
|
-
if (!config) return
|
|
120
|
+
if (!config) return [];
|
|
141
121
|
const commands = config.commands;
|
|
142
|
-
const selected = Object.keys(commands).filter((key) => tierOf(key) === tier).sort();
|
|
122
|
+
const selected = Object.keys(commands).filter((key) => tier === ALL_TIER ? key.startsWith(VERIFY_PREFIX) : tierOf(key) === tier).sort();
|
|
143
123
|
return selected.map((key) => criterion(`cmd.${key}`, (inner) => {
|
|
144
|
-
if (
|
|
124
|
+
if (SLOW_COMMANDS.includes(key) && report(inner, config.surfaces).code.length === 0) {
|
|
145
125
|
return { ok: true, detail: "skipped — no code files on this branch" };
|
|
146
126
|
}
|
|
147
127
|
return runInvocations(inner.root, commands[key]);
|
|
@@ -150,7 +130,7 @@ function commandCriteria(context: CriterionContext, tier: string): Criterion[] {
|
|
|
150
130
|
|
|
151
131
|
function tierOf(key: string): string {
|
|
152
132
|
if (key === CONFORM_COMMAND) return CONFORM_TIER;
|
|
153
|
-
if (SLOW_COMMANDS.includes(key)) return SLOW_TIER;
|
|
133
|
+
if (key === UNIT_COMMAND || SLOW_COMMANDS.includes(key)) return SLOW_TIER;
|
|
154
134
|
if (key.startsWith(VERIFY_PREFIX)) return FAST_TIER;
|
|
155
135
|
return "";
|
|
156
136
|
}
|
|
@@ -213,12 +193,13 @@ function resolvedConfig(context: CriterionContext): RepoConfig | undefined {
|
|
|
213
193
|
|
|
214
194
|
function runInvocations(root: string, invocations: unknown): Verdict {
|
|
215
195
|
if (!Array.isArray(invocations) || invocations.length === 0) return { ok: false, detail: "command group is empty" };
|
|
196
|
+
const evidence: string[] = [];
|
|
216
197
|
for (const invocation of invocations) {
|
|
217
198
|
if (!Array.isArray(invocation) || invocation.length === 0) return { ok: false, detail: "command invocation is empty" };
|
|
218
199
|
const argv = invocation.map((argument) => String(argument));
|
|
219
200
|
const result = runCommand(root, argv);
|
|
220
|
-
|
|
201
|
+
evidence.push(`${argv.join(" ")} -> ${result.detail}`);
|
|
202
|
+
if (!result.ok) return { ok: false, detail: evidence.join(NEWLINE) };
|
|
221
203
|
}
|
|
222
|
-
return { ok: true, detail: `${invocations.length} invocation(s) exit 0` };
|
|
204
|
+
return { ok: true, detail: `${invocations.length} invocation(s) exit 0\n${evidence.join(NEWLINE)}` };
|
|
223
205
|
}
|
|
224
|
-
|
package/src/criteria_spec.ts
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { readConfigSync } from "./config";
|
|
1
4
|
import { Criterion, CriterionContext, criterion, gitOutput, runCommand, Verdict } from "./criteria_support";
|
|
2
|
-
import { branchDiff, defaultMergeBase } from "./surfaces";
|
|
5
|
+
import { branchDiff, classifyBranch, defaultMergeBase } from "./surfaces";
|
|
6
|
+
|
|
7
|
+
const BASELINE_HEADER = "Test Baseline";
|
|
8
|
+
const REVISION_HEADER = "Baseline revision";
|
|
9
|
+
const EVIDENCE_HEADER = "Baseline evidence";
|
|
10
|
+
const BASELINE_LANES = ["unit", "integration"];
|
|
11
|
+
const FULL_REVISION = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/;
|
|
12
|
+
const NOT_APPLICABLE = /^n\/a\s+[—-]\s+\S/i;
|
|
13
|
+
|
|
3
14
|
|
|
4
15
|
const OPEN_QUESTION = "[?]";
|
|
5
16
|
const PRODUCT_CLARITY_HEADING = "## Product Clarity";
|
|
@@ -17,7 +28,6 @@ const SPEC_BASELINE = "spec.baseline";
|
|
|
17
28
|
const SPEC_ORDERING = "spec.ordering";
|
|
18
29
|
const SPEC_DEFERRALS = "spec.deferrals";
|
|
19
30
|
const STATUS_DONE = "Status: DONE";
|
|
20
|
-
const BASELINE_HEADER = "Test Baseline:";
|
|
21
31
|
const INDY_ACK = "> Indy (";
|
|
22
32
|
// deferred/deferral(s) only — never Zig's defer/errdefer keywords.
|
|
23
33
|
const DEFERRAL_CLAIM = /\bdeferr(ed|al|als)\b/i;
|
|
@@ -78,10 +88,29 @@ export function specMoved(): Criterion {
|
|
|
78
88
|
|
|
79
89
|
export function specBaseline(): Criterion {
|
|
80
90
|
return specCriterion(SPEC_BASELINE, (context) => {
|
|
81
|
-
const
|
|
82
|
-
return
|
|
83
|
-
|
|
84
|
-
|
|
91
|
+
const baseline = header(context.specText ?? "", BASELINE_HEADER);
|
|
92
|
+
if (!baseline) return { ok: false, detail: "no Test Baseline header — declare it at opening" };
|
|
93
|
+
let config;
|
|
94
|
+
try {
|
|
95
|
+
config = readConfigSync(context.root);
|
|
96
|
+
} catch (error) {
|
|
97
|
+
return { ok: false, detail: `cannot read baseline configuration: ${error instanceof Error ? error.message : String(error)}` };
|
|
98
|
+
}
|
|
99
|
+
if (!config) return { ok: false, detail: "baseline needs the repository's declared commands" };
|
|
100
|
+
const lanes = BASELINE_LANES.filter((lane) => `verify.${lane}` in config.commands);
|
|
101
|
+
const surfaces = context.surfaces ?? classifyBranch(context.root, config.surfaces);
|
|
102
|
+
if (NOT_APPLICABLE.test(baseline)) {
|
|
103
|
+
const knownBase = defaultMergeBase(context.root).length > 0;
|
|
104
|
+
const allowed = lanes.length === 0 || (knownBase && surfaces.code.length === 0);
|
|
105
|
+
return { ok: allowed, detail: allowed ? "baseline not applicable: no code or no baseline lanes" : "n/a cannot replace a code branch's declared baseline lanes" };
|
|
106
|
+
}
|
|
107
|
+
for (const lane of lanes) {
|
|
108
|
+
if (!new RegExp(`(?:^|\\s)${lane}=\\d+(?=\\s|$)`).test(baseline)) {
|
|
109
|
+
return { ok: false, detail: `Test Baseline carries no count for ${lane} — measure before the Pull Request` };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (lanes.length === 0) return { ok: false, detail: "no baseline lanes declared — record n/a with the reason" };
|
|
113
|
+
return baselineEvidence(context);
|
|
85
114
|
});
|
|
86
115
|
}
|
|
87
116
|
|
|
@@ -125,3 +154,21 @@ function specLines(context: CriterionContext): string[] {
|
|
|
125
154
|
function dimensionLabels(lines: string[]): string {
|
|
126
155
|
return lines.map((line) => line.trim().slice(DIMENSION_PREFIX.length).split("*")[0]?.trim() ?? "?").join(", ");
|
|
127
156
|
}
|
|
157
|
+
|
|
158
|
+
function baselineEvidence(context: CriterionContext): Verdict {
|
|
159
|
+
const text = context.specText ?? "";
|
|
160
|
+
const revision = header(text, REVISION_HEADER);
|
|
161
|
+
if (!FULL_REVISION.test(revision)) return { ok: false, detail: "Baseline revision must name a full comparison commit" };
|
|
162
|
+
const ancestor = gitOutput(context.root, ["merge-base", revision, "HEAD"]);
|
|
163
|
+
if (ancestor !== revision) return { ok: false, detail: "Baseline revision is missing or is not an ancestor of HEAD" };
|
|
164
|
+
const evidence = header(text, EVIDENCE_HEADER);
|
|
165
|
+
if (!evidence || (!/^https?:\/\/\S+$/.test(evidence) && !existsSync(resolve(context.root, evidence)))) {
|
|
166
|
+
return { ok: false, detail: "Baseline evidence must name an existing report or a run URL" };
|
|
167
|
+
}
|
|
168
|
+
return { ok: true, detail: "baseline counts, comparison revision, and evidence reference recorded; report contents require review" };
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function header(text: string, name: string): string {
|
|
172
|
+
const prefix = `**${name}:**`;
|
|
173
|
+
return (text.split(/\r?\n/).find((line) => line.startsWith(prefix))?.slice(prefix.length).trim() ?? "").replaceAll("`", "");
|
|
174
|
+
}
|
package/src/criteria_support.ts
CHANGED
|
@@ -41,10 +41,11 @@ export function runCommand(root: string, command: string[]): Verdict {
|
|
|
41
41
|
} catch (error) {
|
|
42
42
|
return { ok: false, detail: `${command[0] ?? "command"} could not be run: ${error instanceof Error ? error.message : String(error)}` };
|
|
43
43
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
const output = [result.stdout.toString(), result.stderr.toString()].filter(Boolean).join("\n").trim();
|
|
45
|
+
if (output) return { ok: result.exitCode === 0, detail: `exit ${result.exitCode}:\n${output}` };
|
|
46
|
+
return result.exitCode === 0
|
|
47
|
+
? { ok: true, detail: "exit 0" }
|
|
48
|
+
: { ok: false, detail: `exit ${result.exitCode}: ${NO_OUTPUT}` };
|
|
48
49
|
}
|
|
49
50
|
|
|
50
51
|
export function gitOutput(root: string, command: string[]): string {
|
package/src/gates.test.ts
CHANGED
|
@@ -35,9 +35,9 @@ describe("gate groups", () => {
|
|
|
35
35
|
const model = await modelFor(project);
|
|
36
36
|
|
|
37
37
|
expect(names(runGate(model, project, "work"))).toEqual(["cmd.conform", "repo.config"]);
|
|
38
|
-
expect(names(runGate(model, project, "verify"))).toEqual(["
|
|
38
|
+
expect(names(runGate(model, project, "verify"))).toEqual(["docs.language", "repo.config"]);
|
|
39
39
|
expect(names(runGate(model, project, "pr"))).toEqual([
|
|
40
|
-
"docs.updated", "git.branch", "git.pushed", "git.tree", "spec.baseline", "spec.deferrals", "spec.dimensions",
|
|
40
|
+
"cmd.verify.unit", "docs.language", "docs.updated", "git.branch", "git.pushed", "git.tree", "repo.config", "spec.baseline", "spec.deferrals", "spec.dimensions",
|
|
41
41
|
"spec.gate", "spec.moved", "spec.open-questions", "spec.ordering", "spec.product-clarity",
|
|
42
42
|
]);
|
|
43
43
|
});
|
package/src/gates_spec.test.ts
CHANGED
|
@@ -8,6 +8,8 @@ import {
|
|
|
8
8
|
newRepository, newSpecRepository, orly, SPEC_RELATIVE, specFixture,
|
|
9
9
|
} from "./gates_test_support";
|
|
10
10
|
|
|
11
|
+
const BASELINE_FIXTURE = "**Test Baseline:** unit=0 integration=0";
|
|
12
|
+
|
|
11
13
|
afterEach(cleanupTemporaryDirectories);
|
|
12
14
|
|
|
13
15
|
describe("spec discovery", () => {
|
|
@@ -16,7 +18,7 @@ describe("spec discovery", () => {
|
|
|
16
18
|
const model = await modelFor(project);
|
|
17
19
|
git(project, "checkout", "-q", "-b", "fix/adhoc");
|
|
18
20
|
|
|
19
|
-
const verify = runGate(model, project, "
|
|
21
|
+
const verify = runGate(model, project, "pr");
|
|
20
22
|
expect(verify.results.find((result) => result.name === "spec.dimensions")?.detail).toContain("no active spec");
|
|
21
23
|
expect(runGate(model, project, "work").results.find((result) => result.name === "cmd.conform")?.ok).toBeTrue();
|
|
22
24
|
});
|
|
@@ -122,6 +124,43 @@ describe("closed-spec follow-through", () => {
|
|
|
122
124
|
expect(pr.results.find((result) => result.name === "spec.baseline")?.ok).toBeTrue();
|
|
123
125
|
});
|
|
124
126
|
|
|
127
|
+
// CHORE(open) declares the header and the boundary fills it. The gate that
|
|
128
|
+
// grades the boundary is `pr`, so `pending` surviving to here is the
|
|
129
|
+
// measurement skipped rather than the measurement not yet due.
|
|
130
|
+
test("a Test Baseline still reading `pending` is red at the Pull Request gate", async () => {
|
|
131
|
+
const project = newRepository();
|
|
132
|
+
const model = await modelFor(project);
|
|
133
|
+
git(project, "checkout", "-q", "-b", "feat/pending-baseline");
|
|
134
|
+
mkdirSync(join(project, "docs/v1/active"), { recursive: true });
|
|
135
|
+
await Bun.write(
|
|
136
|
+
join(project, SPEC_RELATIVE),
|
|
137
|
+
specFixture("IN_PROGRESS", "feat/pending-baseline").replace(BASELINE_FIXTURE, "**Test Baseline:** pending — measured before the Pull Request"),
|
|
138
|
+
);
|
|
139
|
+
git(project, "add", ".");
|
|
140
|
+
git(project, "commit", "-q", "-m", "chore(open): the spec declares its baseline");
|
|
141
|
+
|
|
142
|
+
const baseline = runGate(model, project, "pr").results.find((result) => result.name === "spec.baseline");
|
|
143
|
+
expect(baseline?.ok).toBeFalse();
|
|
144
|
+
expect(baseline?.detail).toContain("carries no count");
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
test("a branch carrying no code records n/a and spec.baseline is green", async () => {
|
|
148
|
+
const project = newRepository();
|
|
149
|
+
const model = await modelFor(project);
|
|
150
|
+
git(project, "checkout", "-q", "-b", "feat/no-code-baseline");
|
|
151
|
+
mkdirSync(join(project, "docs/v1/active"), { recursive: true });
|
|
152
|
+
await Bun.write(
|
|
153
|
+
join(project, SPEC_RELATIVE),
|
|
154
|
+
specFixture("IN_PROGRESS", "feat/no-code-baseline").replace(BASELINE_FIXTURE, "**Test Baseline:** n/a — no code on this branch"),
|
|
155
|
+
);
|
|
156
|
+
git(project, "add", ".");
|
|
157
|
+
git(project, "commit", "-q", "-m", "chore(open): a docs-only stream measures nothing");
|
|
158
|
+
|
|
159
|
+
const baseline = runGate(model, project, "pr").results.find((result) => result.name === "spec.baseline");
|
|
160
|
+
expect(baseline?.ok).toBeTrue();
|
|
161
|
+
expect(baseline?.detail).toContain("not applicable");
|
|
162
|
+
});
|
|
163
|
+
|
|
125
164
|
test("Status: DONE while the spec still lives under active/ is red on spec.moved", async () => {
|
|
126
165
|
const project = newRepository();
|
|
127
166
|
const model = await modelFor(project);
|
|
@@ -10,6 +10,10 @@ import { CONFIG_PATH } from "./config";
|
|
|
10
10
|
import { RulesModel } from "./model";
|
|
11
11
|
|
|
12
12
|
const GIT = "git";
|
|
13
|
+
const HEAD = "HEAD";
|
|
14
|
+
const REV_PARSE = "rev-parse";
|
|
15
|
+
const IN_PROGRESS = "IN_PROGRESS";
|
|
16
|
+
const BASELINE_EVIDENCE = "**Baseline evidence:** README.md";
|
|
13
17
|
const QUIET = "-q";
|
|
14
18
|
const COMMIT = "commit";
|
|
15
19
|
const CONFIG = "config";
|
|
@@ -21,6 +25,7 @@ const ACCEPTANCE_RUBRIC_HEADING = "Acceptance Rubric";
|
|
|
21
25
|
const FIXTURE = "fixture";
|
|
22
26
|
const AUDITS_DIR = "audits";
|
|
23
27
|
const SPEC_GATE_SCRIPT = `${AUDITS_DIR}/spec-template.sh`;
|
|
28
|
+
const SPEC_GATE_CONTENT = `${AUDITS_DIR}/spec-template.ts`;
|
|
24
29
|
|
|
25
30
|
export const ROOT = resolve(import.meta.dir, "..");
|
|
26
31
|
export const SPEC_RELATIVE = "docs/v1/active/M99_001_P2_CLI_FIXTURE.md";
|
|
@@ -105,7 +110,10 @@ export function closedSpecRepository(branch: string, extraLines: string[] = []):
|
|
|
105
110
|
mkdirSync(join(project, "docs/v1/done"), { recursive: true });
|
|
106
111
|
mkdirSync(join(project, AUDITS_DIR), { recursive: true });
|
|
107
112
|
copyFileSync(join(ROOT, SPEC_GATE_SCRIPT), join(project, SPEC_GATE_SCRIPT));
|
|
108
|
-
|
|
113
|
+
copyFileSync(join(ROOT, SPEC_GATE_CONTENT), join(project, SPEC_GATE_CONTENT));
|
|
114
|
+
Bun.write(join(project, "docs/v1/done/M99_001_P2_CLI_FIXTURE.md"), specFixture("DONE", branch, [
|
|
115
|
+
`**Baseline revision:** ${gitOutput(project, REV_PARSE, HEAD)}`, BASELINE_EVIDENCE, ...extraLines,
|
|
116
|
+
]));
|
|
109
117
|
git(project, ADD, ".");
|
|
110
118
|
git(project, COMMIT, QUIET, MESSAGE, "chore: close the fixture spec");
|
|
111
119
|
return project;
|
|
@@ -116,7 +124,10 @@ export function newSpecRepository(): string {
|
|
|
116
124
|
mkdirSync(join(project, "docs/v1/active"), { recursive: true });
|
|
117
125
|
mkdirSync(join(project, AUDITS_DIR), { recursive: true });
|
|
118
126
|
copyFileSync(join(ROOT, SPEC_GATE_SCRIPT), join(project, SPEC_GATE_SCRIPT));
|
|
119
|
-
|
|
127
|
+
copyFileSync(join(ROOT, SPEC_GATE_CONTENT), join(project, SPEC_GATE_CONTENT));
|
|
128
|
+
Bun.write(join(project, SPEC_RELATIVE), specFixture(IN_PROGRESS, undefined, [
|
|
129
|
+
`**Baseline revision:** ${gitOutput(project, REV_PARSE, HEAD)}`, BASELINE_EVIDENCE,
|
|
130
|
+
]));
|
|
120
131
|
git(project, ADD, ".");
|
|
121
132
|
git(project, COMMIT, QUIET, MESSAGE, "test: add fixture spec");
|
|
122
133
|
return project;
|
|
@@ -124,7 +135,7 @@ export function newSpecRepository(): string {
|
|
|
124
135
|
|
|
125
136
|
// A spec carrying every section audits/spec-template.sh requires, with all
|
|
126
137
|
// Dimensions already DONE so the gates' own criteria are what is under test.
|
|
127
|
-
export function specFixture(status =
|
|
138
|
+
export function specFixture(status = IN_PROGRESS, branch?: string, extraLines: string[] = []): string {
|
|
128
139
|
const sections = [
|
|
129
140
|
"PR Intent & comprehension handshake",
|
|
130
141
|
"Overview",
|
|
@@ -139,6 +150,7 @@ export function specFixture(status = "IN_PROGRESS", branch?: string, extraLines:
|
|
|
139
150
|
"Metrics & Observability",
|
|
140
151
|
"Test Specification (tiered)",
|
|
141
152
|
ACCEPTANCE_RUBRIC_HEADING,
|
|
153
|
+
"Dead Code Sweep",
|
|
142
154
|
"Out of Scope",
|
|
143
155
|
"Product Clarity (authoring record)",
|
|
144
156
|
"Decomposition & alternatives",
|
|
@@ -149,7 +161,9 @@ export function specFixture(status = "IN_PROGRESS", branch?: string, extraLines:
|
|
|
149
161
|
// Family 2b) — the fixture models a compliant spec, so its rubric carries
|
|
150
162
|
// them the way a real spec's S-rows do.
|
|
151
163
|
heading === ACCEPTANCE_RUBRIC_HEADING
|
|
152
|
-
? `## ${heading}\n\n|
|
|
164
|
+
? `## ${heading}\n\n| # | Criterion | Verify | Expected | Priority | Graded |\n|---|---|---|---|---|---|\n| S1 | Declared checks | \`${TRUE_COMMAND}\` | exit 0 | P0 | |\n`
|
|
165
|
+
: heading.startsWith("Implementing agent") ? `## ${heading}\n\n1. \`README.md\` — fixture reference.\n`
|
|
166
|
+
: heading.startsWith("Test Specification") ? `## ${heading}\n\n| Dimension | Tier | Test | Asserts |\n|---|---|---|---|\n| 1.1 | unit | \`fixture_test\` | Valid input is accepted. |\n`
|
|
153
167
|
: `## ${heading}\n\nFixture content for ${heading}.\n`,
|
|
154
168
|
);
|
|
155
169
|
return [
|