@agentsfleet/orly 0.10.5 → 0.10.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -6
- package/README.md +26 -8
- package/audits/agents-md.md +7 -3
- package/audits/doc-read.sh +97 -16
- package/audits/spec-template.sh +30 -113
- package/audits/spec-template.ts +193 -0
- package/core/operating-model.md +6 -6
- package/dispatch/lifecycle.md +98 -42
- package/dispatch/verify.md +30 -48
- package/dispatch/write_spec.md +56 -82
- package/docs/ORLY_ARCHITECTURE.md +13 -3
- package/docs/RULE_ENFORCEMENT.md +2 -2
- package/docs/TEMPLATE.md +82 -77
- package/docs/VERIFY_TIERS.md +20 -26
- package/llms.txt +17 -12
- package/package.json +1 -1
- package/registry.json +4 -0
- package/skills/orly-spec-new/SKILL.md +3 -1
- package/skills/orly-write-integration-test/SKILL.md +1 -1
- package/skills/orly-write-unit-test/SKILL.md +1 -1
- package/src/cli.ts +14 -16
- package/src/criteria.test.ts +47 -6
- package/src/criteria.ts +30 -49
- package/src/criteria_spec.ts +53 -6
- package/src/criteria_support.ts +5 -4
- package/src/gates.test.ts +2 -2
- package/src/gates_spec.test.ts +40 -1
- package/src/gates_test_support.ts +18 -4
- package/src/install.ts +16 -81
- package/src/render.test.ts +5 -7
- package/src/setup.test.ts +116 -0
- package/src/spec_baseline.test.ts +96 -0
- package/src/spec_template.test.ts +109 -0
- package/src/spec_template_io.test.ts +97 -0
- package/src/validation.ts +9 -0
package/llms.txt
CHANGED
|
@@ -36,17 +36,19 @@ Run from the repository root. Execute these in order and stop at the first
|
|
|
36
36
|
non-zero exit.
|
|
37
37
|
|
|
38
38
|
```bash
|
|
39
|
-
# 1.
|
|
40
|
-
|
|
39
|
+
# 1. install the executable used by Git hooks
|
|
40
|
+
bun add -g @agentsfleet/orly
|
|
41
41
|
|
|
42
|
-
# 2.
|
|
43
|
-
|
|
42
|
+
# 2. preview, then initialize
|
|
43
|
+
orly init --dry-run
|
|
44
|
+
orly init
|
|
44
45
|
|
|
45
|
-
# 3.
|
|
46
|
-
|
|
46
|
+
# 3. complete any reported setup items, then check configuration and files
|
|
47
|
+
orly doctor
|
|
47
48
|
```
|
|
48
49
|
|
|
49
50
|
Then commit the result. Teammates receive the rules on clone.
|
|
51
|
+
Keep `orly` on `PATH`; hooks use the installed executable.
|
|
50
52
|
|
|
51
53
|
IMPORTANT: git does not clone hooks. Each checkout must run `orly init` once.
|
|
52
54
|
If a human reports that gates are not firing, this is the first thing to check.
|
|
@@ -67,8 +69,11 @@ If a human reports that gates are not firing, this is the first thing to check.
|
|
|
67
69
|
| `.oracle/orly.json` | packs, gate commands, install record | shared — edit the commands |
|
|
68
70
|
|
|
69
71
|
`orly init` seeds `.oracle/orly.json` with gate commands found in `Makefile` or
|
|
70
|
-
`package.json`.
|
|
71
|
-
|
|
72
|
+
`package.json`. Read the repository's build files to complete missing commands;
|
|
73
|
+
ask only when the repository does not define a check. Do not invent commands.
|
|
74
|
+
Declare `conform` and at least one named `verify.*` command. Documentation
|
|
75
|
+
repositories can use `verify.docs` without application unit or integration
|
|
76
|
+
suites. `init` reports missing configuration; `doctor` rejects it.
|
|
72
77
|
|
|
73
78
|
---
|
|
74
79
|
|
|
@@ -91,7 +96,7 @@ move it. `orly gate pr` reads where it ended up.
|
|
|
91
96
|
```
|
|
92
97
|
docs/v1/pending/ spec written, committed on main
|
|
93
98
|
│
|
|
94
|
-
│ CHORE(open) branch cut,
|
|
99
|
+
│ CHORE(open) branch cut, comparison revision recorded, baseline measurement pending
|
|
95
100
|
▼
|
|
96
101
|
docs/v1/active/ PLAN → EXECUTE → CONFORM → VERIFY → REVIEW → DOCUMENT → COMMIT
|
|
97
102
|
│
|
|
@@ -117,14 +122,14 @@ Rules for you:
|
|
|
117
122
|
| same gate fails twice on the same cause | stop, report the gate name and output to the user |
|
|
118
123
|
| `orly init` refuses a file | orly did not write that file. Report it. Do **not** pass `--force` on your own initiative |
|
|
119
124
|
| hooks not firing | run `git config core.hooksPath` and confirm `orly init` was run in this checkout |
|
|
120
|
-
| gate command missing from `.oracle/orly.json` |
|
|
125
|
+
| gate command missing from `.oracle/orly.json` | read the build files and declare the existing check; ask only if none exists |
|
|
121
126
|
| `orly gate pr` red and the failure is genuinely inapplicable | ask the user; only they authorise `orly override` |
|
|
122
127
|
| gstack `/review` unavailable | continue; orly records the skipped review in the PR notes |
|
|
123
128
|
|
|
124
129
|
Overrides are recorded as an empty commit that rides into the Pull Request:
|
|
125
130
|
|
|
126
131
|
```bash
|
|
127
|
-
|
|
132
|
+
orly override <criterion> --reason "<why>"
|
|
128
133
|
```
|
|
129
134
|
|
|
130
135
|
Never record an override without an explicit instruction from the user, and
|
|
@@ -155,7 +160,7 @@ never with a placeholder reason.
|
|
|
155
160
|
| `orly gate` | run declared checks in order, stopping at the first failure |
|
|
156
161
|
| `orly gate pr` | the Pull Request criteria check |
|
|
157
162
|
| `orly override <criterion> --reason <why>` | record a gate exception (user-authorised only) |
|
|
158
|
-
| `orly doctor` |
|
|
163
|
+
| `orly doctor` | check installed files and required command configuration |
|
|
159
164
|
|
|
160
165
|
---
|
|
161
166
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agentsfleet/orly",
|
|
3
|
-
"version": "0.10.
|
|
3
|
+
"version": "0.10.7",
|
|
4
4
|
"description": "Repository-scoped engineering harness: renders agent rules, materialises the gates that enforce them, and proves the PR boundary \u2014 for any coding agent.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/registry.json
CHANGED
|
@@ -74,6 +74,8 @@ This is the step that prevents greptile findings — the spec becomes a pre-comm
|
|
|
74
74
|
- **Every Failure Mode → a negative test.** **Every Invariant → enforced by code** (compiler, lint, comptime assertion, runtime check) — never by review discipline.
|
|
75
75
|
- **Every Metrics row → event/test proof.** User-facing or operator-facing specs declare product/operator signals, privacy guards, and analytics/funnel playbook updates; internal-only cleanup explicitly says no signal changed.
|
|
76
76
|
- **Rubric = 5–12 outcome rows** — one per Section outcome, failure class, or hygiene gate (never per Dimension); each with a copy-paste Verify command and a mechanically checkable Expected (exit code / literal substring / match count). Standard rows come pre-filled in the template — prune to the touched surface.
|
|
77
|
+
- **Stage-owned values** — keep branch and baseline measurement pending at authoring. Opening records the full comparison commit; unit and integration measurement is due before the Pull Request. The lifecycle dispatch owns command timing and hydration.
|
|
78
|
+
- **Decision ownership** — identify agent choices, preparation outputs, and required human decisions. Human acceptance uses tier `manual` with a procedure and durable evidence; never invent the person's sign-off.
|
|
77
79
|
- **Reporting spine** — **Discovery (consult log)** carries consults, skill-chain outcomes, and Indy-acked deferral quotes; the **Acceptance Rubric's Graded column** carries the VERIFY verdicts (✅/❌ + one decisive output line each). Both empty at creation, populated as work proceeds.
|
|
78
80
|
|
|
79
81
|
→ Fills **Sections (+ Dimensions)**, **Metrics & Observability**, **Failure Modes**, **Invariants**, **Test Specification (tiered)**, **Acceptance Rubric**, **Discovery**.
|
|
@@ -90,7 +92,7 @@ cp docs/TEMPLATE.md docs/v{N}/pending/M{N}_{WS}_P{P}_{CATEGORIES}_{NAME}.md
|
|
|
90
92
|
```
|
|
91
93
|
|
|
92
94
|
|
|
93
|
-
**Fill grammar** — the template body is ordered for the *executing* agent (execution read order), not for authoring; fill sections in the order of Steps 1–3 above, not top-to-bottom. Replace every `{
|
|
95
|
+
**Fill grammar** — the template body is ordered for the *executing* agent (execution read order), not for authoring; fill sections in the order of Steps 1–3 above, not top-to-bottom. Replace every `{{fill:description}}` authoring slot with instance content, then **delete every `<!-- tpl: … -->` guidance comment** — the SPEC TEMPLATE GATE BLOCKs any survivor, unfilled slot sentinels, and missing required sections. The SPEC AUTHORING RULES banner is the one comment that stays.
|
|
94
96
|
|
|
95
97
|
**Inputs** — Milestone `M{N}` (next free, sortable) · Workstream `{WS}` zero-padded (`001`…) · Priority (**P0** blocking · **P1** customer/operator-facing · **P2** tooling · **P3** deferrable) · Category set alphabetised (`API` Zig/Go · `CLI` agentsfleet/Node · `UI` Next.js · `OBS` Grafana · `SKILL` SKILL.md · `INFRA` Terraform) · Name UPPER_SNAKE_CASE ≤6 words describing the outcome (`BUN_VENDOR_UTILITIES`, not `BUMP_BUN_DEPS`) · Prototype tag (`v1.0.0`, `v2.0.0` — drives `docs/v1/` vs `docs/v2/`).
|
|
96
98
|
|
|
@@ -69,7 +69,7 @@ Spec asserts "503 on Redis down", code returns 200 → test the spec, flag the c
|
|
|
69
69
|
1. **Read the spec** — Failure Modes, Error Contracts, Concurrency Contracts, Resource Limits, Streaming Contracts tables are authoritative.
|
|
70
70
|
2. **Read `docs/greptile-learnings/RULES.md`** — every rule maps to a regression test.
|
|
71
71
|
3. **Bring up real deps** — `make up` (or stack equivalent). Verify health before writing tests.
|
|
72
|
-
4. **
|
|
72
|
+
4. **Follow lifecycle timing** — record the comparison revision at opening; measure full unit and integration baselines before the Pull Request. Run focused reproduction and Section proofs while implementing; first application hydration occurs at Section verification. `dispatch/lifecycle.md` owns the sequence.
|
|
73
73
|
5. **Map the ordered request path** — list every dependency interaction in execution order, including repeated acquisitions of the same resource, plus every `catch`/`orelse`/`except`/`Err` in the chain. That list seeds T4 and its partial-completion matrix.
|
|
74
74
|
|
|
75
75
|
## Three execution modes
|
|
@@ -51,7 +51,7 @@ Spec contradicts runtime → surface the conflict before testing. Code contradic
|
|
|
51
51
|
1. **Read the spec, then the code** — spec defines what to test; code is what gets tested. Don't let code shape expectations.
|
|
52
52
|
2. **Read `docs/greptile-learnings/RULES.md`** — every rule is a regression test.
|
|
53
53
|
3. **Detect the stack** from project files (`pyproject.toml`, `Cargo.toml`, `build.zig`, `package.json` + `react`/`bin`, `*.sh`, `openapi.yaml`).
|
|
54
|
-
4. **
|
|
54
|
+
4. **Follow lifecycle timing** — record the comparison revision at opening; measure full unit and integration baselines before the Pull Request. Run focused reproduction and Section proofs while implementing; first application hydration occurs at Section verification. `dispatch/lifecycle.md` owns the sequence.
|
|
55
55
|
5. **Follow existing naming and fixture conventions.**
|
|
56
56
|
|
|
57
57
|
## Step 0 — the diff is the spec: build the required-test ledger first
|
package/src/cli.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { isString, OrlyError, readJsonObject, RulesModel } from "./model";
|
|
|
9
9
|
import { Renderer } from "./render";
|
|
10
10
|
import { beginTelemetry, installedOrlyVersion, isTelemetrySkill, recordTelemetry } from "./telemetry";
|
|
11
11
|
import { verifyRenders, writeEvidence } from "./verify";
|
|
12
|
+
import { commandSetupErrors } from "./validation";
|
|
12
13
|
|
|
13
14
|
type CliResult = { exitCode: number; gate?: string; failedCriterion?: string };
|
|
14
15
|
|
|
@@ -152,15 +153,18 @@ async function materialise(model: RulesModel, args: string[], isInit: boolean):
|
|
|
152
153
|
installHooks: !args.includes(NO_HOOKS_FLAG),
|
|
153
154
|
orlyVersion: await packageVersion(model),
|
|
154
155
|
});
|
|
155
|
-
|
|
156
|
-
|
|
156
|
+
const setupErrors = result.ok ? commandSetupErrors((await readConfig(targetRoot))!.commands) : [];
|
|
157
|
+
if (args.includes(JSON_FLAG)) console.log(JSON.stringify({ ...result, setupErrors }, undefined, JSON_INDENT));
|
|
158
|
+
else printInstall(result, setupErrors);
|
|
157
159
|
return result.ok ? 0 : 1;
|
|
158
160
|
}
|
|
159
161
|
|
|
160
|
-
function printInstall(result: InstallResult): void {
|
|
162
|
+
function printInstall(result: InstallResult, setupErrors: string[]): void {
|
|
161
163
|
for (const error of result.errors) console.log(`${FAIL_GLYPH} ${error.path}: ${error.message} — ${error.suggestion}`);
|
|
162
164
|
if (!result.ok) return;
|
|
163
165
|
console.log(`${PASS_GLYPH} ${result.written.length} written, ${result.skipped.length} already current (${result.packs.length} packs)`);
|
|
166
|
+
for (const error of setupErrors) console.log(`🟠 setup incomplete: ${error}`);
|
|
167
|
+
if (setupErrors.length > 0) console.log("Then run orly doctor to check the completed setup.");
|
|
164
168
|
}
|
|
165
169
|
|
|
166
170
|
function printGate(report: GateReport): void {
|
|
@@ -175,9 +179,6 @@ function projectRoot(): string {
|
|
|
175
179
|
}
|
|
176
180
|
|
|
177
181
|
async function doctorGlobal(model: RulesModel): Promise<number> {
|
|
178
|
-
// Every repository is the same case now: doctor reports on the ruleset
|
|
179
|
-
// installed where the caller is standing. There is no machine-level carrier
|
|
180
|
-
// to check — the rules ride in each repository's own commit.
|
|
181
182
|
const errors = await doctorInstall(model);
|
|
182
183
|
if (errors.length > 0) {
|
|
183
184
|
for (const error of errors) console.log(`${FAIL_GLYPH} ${error}`);
|
|
@@ -188,7 +189,10 @@ async function doctorGlobal(model: RulesModel): Promise<number> {
|
|
|
188
189
|
console.log(`${FAIL_GLYPH} no ${CONFIG_PATH} here — run \`orly init\` first`);
|
|
189
190
|
return 1;
|
|
190
191
|
}
|
|
191
|
-
|
|
192
|
+
const setupErrors = commandSetupErrors(config.commands);
|
|
193
|
+
for (const error of setupErrors) console.log(`${FAIL_GLYPH} setup incomplete: ${error}`);
|
|
194
|
+
if (setupErrors.length > 0) return 1;
|
|
195
|
+
console.log(`${PASS_GLYPH} installed rules match ${CONFIG_PATH}; work and verification commands are declared`);
|
|
192
196
|
return 0;
|
|
193
197
|
}
|
|
194
198
|
|
|
@@ -266,12 +270,6 @@ function requireNoArguments(args: string[], usage: string): void {
|
|
|
266
270
|
if (args.length > 0) throw new OrlyError(usage);
|
|
267
271
|
}
|
|
268
272
|
|
|
269
|
-
function optionValue(args: string[], name: string): string {
|
|
270
|
-
const value = optionalValue(args, name);
|
|
271
|
-
if (!value) throw new OrlyError(`${name} is required`);
|
|
272
|
-
return value;
|
|
273
|
-
}
|
|
274
|
-
|
|
275
273
|
function optionalValue(args: string[], name: string): string | undefined {
|
|
276
274
|
const index = args.indexOf(name);
|
|
277
275
|
return index < 0 ? undefined : args[index + 1];
|
|
@@ -301,10 +299,10 @@ Gates (read-only; no PR without every criterion green or a recorded override):
|
|
|
301
299
|
orly gate <work|verify|pr> run one gate
|
|
302
300
|
work does this commit conform? the declared conform command; no git
|
|
303
301
|
state, so a commit hook's own dirty tree never blocks it
|
|
304
|
-
verify does the work hold up?
|
|
305
|
-
|
|
302
|
+
verify does the work hold up? configuration, docs language, and the
|
|
303
|
+
non-test verify.* set; unfinished Sections may be pushed
|
|
306
304
|
pr can this ship? branch, tree, pushed, every spec criterion, and
|
|
307
|
-
|
|
305
|
+
every declared verify.* command
|
|
308
306
|
orly override <CRITERION> --reason <REASON>
|
|
309
307
|
empty commit with an Orly-Override trailer
|
|
310
308
|
|
package/src/criteria.test.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { afterEach, describe, expect, test } from "bun:test";
|
|
2
2
|
import { rmSync } from "node:fs";
|
|
3
|
-
import { join } from "node:path";
|
|
3
|
+
import { join, relative } from "node:path";
|
|
4
4
|
|
|
5
5
|
import { criteriaFor, runCommand } from "./criteria";
|
|
6
|
-
import {
|
|
6
|
+
import { activeSpecPath, closedSpecPath } from "./gates";
|
|
7
|
+
import { cleanupTemporaryDirectories, closedSpecRepository, git, modelFor, newRepository, newSpecRepository } from "./gates_test_support";
|
|
7
8
|
|
|
8
9
|
const WORK = "work";
|
|
9
10
|
const VERIFY = "verify";
|
|
@@ -32,12 +33,12 @@ describe("criteriaFor", () => {
|
|
|
32
33
|
expect(named(WORK, await contextFor(project))).toEqual(["cmd.conform", REPO_CONFIG]);
|
|
33
34
|
});
|
|
34
35
|
|
|
35
|
-
test("verify
|
|
36
|
+
test("verify permits unfinished Sections and excludes boundary test suites", async () => {
|
|
36
37
|
const project = newSpecRepository();
|
|
37
38
|
const names = named(VERIFY, await contextFor(project));
|
|
38
39
|
|
|
39
|
-
expect(names).toContain("spec.dimensions");
|
|
40
|
-
expect(names).toContain("cmd.verify.unit");
|
|
40
|
+
expect(names).not.toContain("spec.dimensions");
|
|
41
|
+
expect(names).not.toContain("cmd.verify.unit");
|
|
41
42
|
// conform is the work gate's tier now, so the chain runs it once.
|
|
42
43
|
expect(names).not.toContain("cmd.conform");
|
|
43
44
|
// The slow tier belongs to the pr gate; leaking it here would make every
|
|
@@ -63,6 +64,20 @@ describe("criteriaFor", () => {
|
|
|
63
64
|
expect(names).toContain("git.tree");
|
|
64
65
|
});
|
|
65
66
|
|
|
67
|
+
test("final verification runs test lanes even when push verification never ran", async () => {
|
|
68
|
+
const root = newRepository();
|
|
69
|
+
const commands = Object.fromEntries(["unit", "integration", "memory", "lint"].map((lane) => [
|
|
70
|
+
`verify.${lane}`, [["sh", "-c", `echo ${lane} >> ran.txt`]],
|
|
71
|
+
]));
|
|
72
|
+
const context = {
|
|
73
|
+
root, model: await modelFor(root, undefined, commands), acceptDirty: false,
|
|
74
|
+
surfaces: { changed: ["app.rs"], code: ["app.rs"], docs: [], userSurface: [] },
|
|
75
|
+
};
|
|
76
|
+
const checks = criteriaFor(PR, context).filter((entry) => entry.name.startsWith("cmd."));
|
|
77
|
+
expect(checks.map((entry) => entry.evaluate(context).ok)).toEqual([true, true, true, true]);
|
|
78
|
+
expect((await Bun.file(join(root, "ran.txt")).text()).trim().split("\n").sort()).toEqual(["integration", "lint", "memory", "unit"]);
|
|
79
|
+
});
|
|
80
|
+
|
|
66
81
|
test("an unknown gate name yields no criteria rather than throwing", async () => {
|
|
67
82
|
const project = newSpecRepository();
|
|
68
83
|
expect(criteriaFor("nonsense", await contextFor(project))).toEqual([]);
|
|
@@ -108,14 +123,19 @@ describe("runCommand", () => {
|
|
|
108
123
|
expect(runCommand(process.cwd(), ["true"])).toEqual({ ok: true, detail: "exit 0" });
|
|
109
124
|
});
|
|
110
125
|
|
|
111
|
-
test("a non-zero exit
|
|
126
|
+
test("a non-zero exit retains all output, including earlier findings", () => {
|
|
112
127
|
const result = runCommand(process.cwd(), ["sh", "-c", "echo first; echo decisive >&2; exit 3"]);
|
|
113
128
|
|
|
114
129
|
expect(result.ok).toBeFalse();
|
|
115
130
|
expect(result.detail).toContain("exit 3");
|
|
131
|
+
expect(result.detail).toContain("first");
|
|
116
132
|
expect(result.detail).toEndWith("decisive");
|
|
117
133
|
});
|
|
118
134
|
|
|
135
|
+
test("a successful command retains its measured counts", () => {
|
|
136
|
+
expect(runCommand(process.cwd(), ["echo", "12 passed, 0 failed"]).detail).toContain("12 passed, 0 failed");
|
|
137
|
+
});
|
|
138
|
+
|
|
119
139
|
test("a silent failure still reports a detail rather than an empty string", () => {
|
|
120
140
|
const result = runCommand(process.cwd(), ["false"]);
|
|
121
141
|
|
|
@@ -146,3 +166,24 @@ describe("repo.config", () => {
|
|
|
146
166
|
expect(verdict?.detail).toContain("orly init");
|
|
147
167
|
});
|
|
148
168
|
});
|
|
169
|
+
|
|
170
|
+
describe("the final tree check", () => {
|
|
171
|
+
for (const stage of ["active", "done"]) {
|
|
172
|
+
test(`requires a ${stage} spec edit to be committed`, async () => {
|
|
173
|
+
const root = stage === "active" ? newSpecRepository() : closedSpecRepository("feat/spec");
|
|
174
|
+
const model = await modelFor(root);
|
|
175
|
+
const specPath = relative(root, activeSpecPath(root) ?? closedSpecPath(root)!);
|
|
176
|
+
const context = { root, model, specPath, acceptDirty: false };
|
|
177
|
+
const check = criteriaFor(PR, context).find((entry) => entry.name === "git.tree")!;
|
|
178
|
+
expect(check.evaluate(context).ok).toBe(true);
|
|
179
|
+
|
|
180
|
+
const path = join(root, specPath);
|
|
181
|
+
await Bun.write(path, `${await Bun.file(path).text()}\nUpdated evidence.\n`);
|
|
182
|
+
expect(check.evaluate(context).ok).toBe(false);
|
|
183
|
+
git(root, "add", specPath);
|
|
184
|
+
expect(check.evaluate(context).ok).toBe(false);
|
|
185
|
+
git(root, "commit", "-qm", "docs: record evidence");
|
|
186
|
+
expect(check.evaluate(context).ok).toBe(true);
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
});
|
package/src/criteria.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { documentationSurfaces, scanSurfaces } from "./doc_rules";
|
|
|
12
12
|
import { isObject, JsonObject, objectValue, OrlyError, RulesModel } from "./model";
|
|
13
13
|
import { readConfigSync, RepoConfig } from "./config";
|
|
14
14
|
import { classifyBranch, SurfaceReport } from "./surfaces";
|
|
15
|
+
import { commandSetupErrors } from "./validation";
|
|
15
16
|
|
|
16
17
|
export type { Criterion, CriterionContext, CriterionResult, Verdict };
|
|
17
18
|
export { runCommand };
|
|
@@ -22,6 +23,9 @@ const VERIFY_PREFIX = "verify.";
|
|
|
22
23
|
const REPOSITORIES_LABEL = "repositories";
|
|
23
24
|
const UNINSTALLED = "no .oracle/orly.json here — run `orly init` first";
|
|
24
25
|
const REV_PARSE = "rev-parse";
|
|
26
|
+
const HEAD = "HEAD";
|
|
27
|
+
const UPSTREAM = "@{upstream}";
|
|
28
|
+
const NEWLINE = "\n";
|
|
25
29
|
const ABBREV_REF = "--abbrev-ref";
|
|
26
30
|
const WORKTREE_LIST = ["worktree", "list", "--porcelain"];
|
|
27
31
|
const WORKTREE_PREFIX = "worktree ";
|
|
@@ -47,37 +51,20 @@ const SLOW_TIER = "slow";
|
|
|
47
51
|
// The slow tier is a fixed name set, not a prefix rule: lint and version
|
|
48
52
|
// checks are verify.* too, and demoting them to skip-on-prose would be wrong.
|
|
49
53
|
const SLOW_COMMANDS = ["verify.integration", "verify.memory"];
|
|
54
|
+
const UNIT_COMMAND = "verify.unit";
|
|
55
|
+
const ALL_TIER = "all";
|
|
50
56
|
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
// One gate per cadence, and each question asked at the only moment it can be
|
|
56
|
-
// answered honestly:
|
|
57
|
-
//
|
|
58
|
-
// work — does this commit conform? Config plus the declared `conform`
|
|
59
|
-
// command. Nothing about branch or tree: inside a commit hook the
|
|
60
|
-
// tree is dirty BY CONSTRUCTION (that is what is being committed),
|
|
61
|
-
// and the operating model itself says a new spec is committed on the
|
|
62
|
-
// default branch. Judging either here made the generated pre-commit
|
|
63
|
-
// hook refuse every commit — including the one installing orly —
|
|
64
|
-
// with `--no-verify`, which Hard Safety forbids, as the only way out.
|
|
65
|
-
// verify — does the work hold up? Spec dimensions, documentation language,
|
|
66
|
-
// and the fast `verify.*` set. Runs before the work leaves the
|
|
67
|
-
// machine, not after every edit.
|
|
68
|
-
// pr — can this ship? The whole-branch facts (branch shape, clean tree,
|
|
69
|
-
// pushed) plus every spec criterion and the slow suites. A tree is
|
|
70
|
-
// only required clean where a reviewer is about to read it, and a
|
|
71
|
-
// branch is only wrong for being `main` where a PR would open from
|
|
72
|
-
// it.
|
|
57
|
+
// Commit checks read the index. Push checks permit in-flight Sections and omit
|
|
58
|
+
// the boundary test suites. The final gate runs all declared verification
|
|
59
|
+
// itself, including checks a custom repository hook may not have invoked.
|
|
73
60
|
export function criteriaFor(gate: string, context: CriterionContext): Criterion[] {
|
|
74
61
|
if (gate === "work") return [repositoryConfig(), ...commandCriteria(context, CONFORM_TIER)];
|
|
75
|
-
if (gate === "verify") return [
|
|
62
|
+
if (gate === "verify") return [repositoryConfig(), docsLanguage(), ...commandCriteria(context, FAST_TIER)];
|
|
76
63
|
if (gate === "pr") {
|
|
77
64
|
return [
|
|
78
65
|
gitBranch(), gitTree(), gitPushed(), specGate(), openQuestions(), productClarity(), specDimensions(),
|
|
79
66
|
specMoved(), specBaseline(), specOrdering(), specDeferrals(),
|
|
80
|
-
docsUpdated(), ...commandCriteria(context,
|
|
67
|
+
repositoryConfig(), docsLanguage(), docsUpdated(), ...commandCriteria(context, ALL_TIER),
|
|
81
68
|
];
|
|
82
69
|
}
|
|
83
70
|
return [];
|
|
@@ -93,14 +80,10 @@ function gitBranch(): Criterion {
|
|
|
93
80
|
|
|
94
81
|
function gitTree(): Criterion {
|
|
95
82
|
return criterion(GIT_TREE, (context) => {
|
|
96
|
-
// The active spec is excluded while work is in flight: Dimensions get
|
|
97
|
-
// marked DONE as the agent goes, and the tree check must not block on that
|
|
98
|
-
// bookkeeping. Committing the spec stays a CHORE(close) obligation.
|
|
99
83
|
const dirty = gitOutput(context.root, ["status", "--porcelain=v1", "-uall"])
|
|
100
84
|
.split(/\r?\n/)
|
|
101
|
-
.filter(Boolean)
|
|
102
|
-
|
|
103
|
-
if (dirty.length === 0) return { ok: true, detail: "clean (active spec excluded)" };
|
|
85
|
+
.filter(Boolean);
|
|
86
|
+
if (dirty.length === 0) return { ok: true, detail: "clean" };
|
|
104
87
|
if (context.acceptDirty) return { ok: true, detail: `${dirty.length} dirty path(s) accepted` };
|
|
105
88
|
return { ok: false, detail: `${dirty.length} uncommitted path(s), first: ${dirty[0] ?? ""}` };
|
|
106
89
|
});
|
|
@@ -108,11 +91,12 @@ function gitTree(): Criterion {
|
|
|
108
91
|
|
|
109
92
|
function gitPushed(): Criterion {
|
|
110
93
|
return criterion(GIT_PUSHED, (context) => {
|
|
111
|
-
const upstream = gitOutput(context.root, [REV_PARSE, ABBREV_REF, "--symbolic-full-name",
|
|
94
|
+
const upstream = gitOutput(context.root, [REV_PARSE, ABBREV_REF, "--symbolic-full-name", UPSTREAM]);
|
|
112
95
|
if (upstream.length === 0) return { ok: false, detail: "branch has no upstream; push it first" };
|
|
113
|
-
const
|
|
114
|
-
const
|
|
115
|
-
|
|
96
|
+
const head = gitOutput(context.root, [REV_PARSE, HEAD]);
|
|
97
|
+
const remote = gitOutput(context.root, [REV_PARSE, UPSTREAM]);
|
|
98
|
+
const ok = head.length > 0 && head === remote;
|
|
99
|
+
return { ok, detail: ok ? `in sync with ${upstream}` : `HEAD differs from ${upstream}; push the reviewed revision` };
|
|
116
100
|
});
|
|
117
101
|
}
|
|
118
102
|
|
|
@@ -121,27 +105,23 @@ function repositoryConfig(): Criterion {
|
|
|
121
105
|
try {
|
|
122
106
|
const config = readConfigSync(context.root);
|
|
123
107
|
if (!config) return { ok: false, detail: UNINSTALLED };
|
|
124
|
-
const
|
|
125
|
-
return { ok:
|
|
108
|
+
const errors = commandSetupErrors(config.commands);
|
|
109
|
+
return { ok: errors.length === 0, detail: errors.length > 0 ? errors.join("; ") : `${Object.keys(config.commands).length} command(s) declared` };
|
|
126
110
|
} catch (error) {
|
|
127
111
|
return { ok: false, detail: error instanceof Error ? error.message : String(error) };
|
|
128
112
|
}
|
|
129
113
|
});
|
|
130
114
|
}
|
|
131
115
|
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
// `conform` command alone. Fast tier = every verify.* that is not slow. Slow
|
|
135
|
-
// tier = verify.integration and verify.memory; those auto-pass with a printed
|
|
136
|
-
// skip when the branch carries no code, so a prose-only branch never pays for
|
|
137
|
-
// the slow suites.
|
|
116
|
+
// Commands and source paths are repository-owned. Only integration and memory
|
|
117
|
+
// checks are conditional on code; every other declared lane always runs.
|
|
138
118
|
function commandCriteria(context: CriterionContext, tier: string): Criterion[] {
|
|
139
119
|
const config = resolvedConfig(context);
|
|
140
|
-
if (!config) return
|
|
120
|
+
if (!config) return [];
|
|
141
121
|
const commands = config.commands;
|
|
142
|
-
const selected = Object.keys(commands).filter((key) => tierOf(key) === tier).sort();
|
|
122
|
+
const selected = Object.keys(commands).filter((key) => tier === ALL_TIER ? key.startsWith(VERIFY_PREFIX) : tierOf(key) === tier).sort();
|
|
143
123
|
return selected.map((key) => criterion(`cmd.${key}`, (inner) => {
|
|
144
|
-
if (
|
|
124
|
+
if (SLOW_COMMANDS.includes(key) && report(inner, config.surfaces).code.length === 0) {
|
|
145
125
|
return { ok: true, detail: "skipped — no code files on this branch" };
|
|
146
126
|
}
|
|
147
127
|
return runInvocations(inner.root, commands[key]);
|
|
@@ -150,7 +130,7 @@ function commandCriteria(context: CriterionContext, tier: string): Criterion[] {
|
|
|
150
130
|
|
|
151
131
|
function tierOf(key: string): string {
|
|
152
132
|
if (key === CONFORM_COMMAND) return CONFORM_TIER;
|
|
153
|
-
if (SLOW_COMMANDS.includes(key)) return SLOW_TIER;
|
|
133
|
+
if (key === UNIT_COMMAND || SLOW_COMMANDS.includes(key)) return SLOW_TIER;
|
|
154
134
|
if (key.startsWith(VERIFY_PREFIX)) return FAST_TIER;
|
|
155
135
|
return "";
|
|
156
136
|
}
|
|
@@ -213,12 +193,13 @@ function resolvedConfig(context: CriterionContext): RepoConfig | undefined {
|
|
|
213
193
|
|
|
214
194
|
function runInvocations(root: string, invocations: unknown): Verdict {
|
|
215
195
|
if (!Array.isArray(invocations) || invocations.length === 0) return { ok: false, detail: "command group is empty" };
|
|
196
|
+
const evidence: string[] = [];
|
|
216
197
|
for (const invocation of invocations) {
|
|
217
198
|
if (!Array.isArray(invocation) || invocation.length === 0) return { ok: false, detail: "command invocation is empty" };
|
|
218
199
|
const argv = invocation.map((argument) => String(argument));
|
|
219
200
|
const result = runCommand(root, argv);
|
|
220
|
-
|
|
201
|
+
evidence.push(`${argv.join(" ")} -> ${result.detail}`);
|
|
202
|
+
if (!result.ok) return { ok: false, detail: evidence.join(NEWLINE) };
|
|
221
203
|
}
|
|
222
|
-
return { ok: true, detail: `${invocations.length} invocation(s) exit 0` };
|
|
204
|
+
return { ok: true, detail: `${invocations.length} invocation(s) exit 0\n${evidence.join(NEWLINE)}` };
|
|
223
205
|
}
|
|
224
|
-
|
package/src/criteria_spec.ts
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { readConfigSync } from "./config";
|
|
1
4
|
import { Criterion, CriterionContext, criterion, gitOutput, runCommand, Verdict } from "./criteria_support";
|
|
2
|
-
import { branchDiff, defaultMergeBase } from "./surfaces";
|
|
5
|
+
import { branchDiff, classifyBranch, defaultMergeBase } from "./surfaces";
|
|
6
|
+
|
|
7
|
+
const BASELINE_HEADER = "Test Baseline";
|
|
8
|
+
const REVISION_HEADER = "Baseline revision";
|
|
9
|
+
const EVIDENCE_HEADER = "Baseline evidence";
|
|
10
|
+
const BASELINE_LANES = ["unit", "integration"];
|
|
11
|
+
const FULL_REVISION = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/;
|
|
12
|
+
const NOT_APPLICABLE = /^n\/a\s+[—-]\s+\S/i;
|
|
13
|
+
|
|
3
14
|
|
|
4
15
|
const OPEN_QUESTION = "[?]";
|
|
5
16
|
const PRODUCT_CLARITY_HEADING = "## Product Clarity";
|
|
@@ -17,7 +28,6 @@ const SPEC_BASELINE = "spec.baseline";
|
|
|
17
28
|
const SPEC_ORDERING = "spec.ordering";
|
|
18
29
|
const SPEC_DEFERRALS = "spec.deferrals";
|
|
19
30
|
const STATUS_DONE = "Status: DONE";
|
|
20
|
-
const BASELINE_HEADER = "Test Baseline:";
|
|
21
31
|
const INDY_ACK = "> Indy (";
|
|
22
32
|
// deferred/deferral(s) only — never Zig's defer/errdefer keywords.
|
|
23
33
|
const DEFERRAL_CLAIM = /\bdeferr(ed|al|als)\b/i;
|
|
@@ -78,10 +88,29 @@ export function specMoved(): Criterion {
|
|
|
78
88
|
|
|
79
89
|
export function specBaseline(): Criterion {
|
|
80
90
|
return specCriterion(SPEC_BASELINE, (context) => {
|
|
81
|
-
const
|
|
82
|
-
return
|
|
83
|
-
|
|
84
|
-
|
|
91
|
+
const baseline = header(context.specText ?? "", BASELINE_HEADER);
|
|
92
|
+
if (!baseline) return { ok: false, detail: "no Test Baseline header — declare it at opening" };
|
|
93
|
+
let config;
|
|
94
|
+
try {
|
|
95
|
+
config = readConfigSync(context.root);
|
|
96
|
+
} catch (error) {
|
|
97
|
+
return { ok: false, detail: `cannot read baseline configuration: ${error instanceof Error ? error.message : String(error)}` };
|
|
98
|
+
}
|
|
99
|
+
if (!config) return { ok: false, detail: "baseline needs the repository's declared commands" };
|
|
100
|
+
const lanes = BASELINE_LANES.filter((lane) => `verify.${lane}` in config.commands);
|
|
101
|
+
const surfaces = context.surfaces ?? classifyBranch(context.root, config.surfaces);
|
|
102
|
+
if (NOT_APPLICABLE.test(baseline)) {
|
|
103
|
+
const knownBase = defaultMergeBase(context.root).length > 0;
|
|
104
|
+
const allowed = lanes.length === 0 || (knownBase && surfaces.code.length === 0);
|
|
105
|
+
return { ok: allowed, detail: allowed ? "baseline not applicable: no code or no baseline lanes" : "n/a cannot replace a code branch's declared baseline lanes" };
|
|
106
|
+
}
|
|
107
|
+
for (const lane of lanes) {
|
|
108
|
+
if (!new RegExp(`(?:^|\\s)${lane}=\\d+(?=\\s|$)`).test(baseline)) {
|
|
109
|
+
return { ok: false, detail: `Test Baseline carries no count for ${lane} — measure before the Pull Request` };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (lanes.length === 0) return { ok: false, detail: "no baseline lanes declared — record n/a with the reason" };
|
|
113
|
+
return baselineEvidence(context);
|
|
85
114
|
});
|
|
86
115
|
}
|
|
87
116
|
|
|
@@ -125,3 +154,21 @@ function specLines(context: CriterionContext): string[] {
|
|
|
125
154
|
function dimensionLabels(lines: string[]): string {
|
|
126
155
|
return lines.map((line) => line.trim().slice(DIMENSION_PREFIX.length).split("*")[0]?.trim() ?? "?").join(", ");
|
|
127
156
|
}
|
|
157
|
+
|
|
158
|
+
function baselineEvidence(context: CriterionContext): Verdict {
|
|
159
|
+
const text = context.specText ?? "";
|
|
160
|
+
const revision = header(text, REVISION_HEADER);
|
|
161
|
+
if (!FULL_REVISION.test(revision)) return { ok: false, detail: "Baseline revision must name a full comparison commit" };
|
|
162
|
+
const ancestor = gitOutput(context.root, ["merge-base", revision, "HEAD"]);
|
|
163
|
+
if (ancestor !== revision) return { ok: false, detail: "Baseline revision is missing or is not an ancestor of HEAD" };
|
|
164
|
+
const evidence = header(text, EVIDENCE_HEADER);
|
|
165
|
+
if (!evidence || (!/^https?:\/\/\S+$/.test(evidence) && !existsSync(resolve(context.root, evidence)))) {
|
|
166
|
+
return { ok: false, detail: "Baseline evidence must name an existing report or a run URL" };
|
|
167
|
+
}
|
|
168
|
+
return { ok: true, detail: "baseline counts, comparison revision, and evidence reference recorded; report contents require review" };
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function header(text: string, name: string): string {
|
|
172
|
+
const prefix = `**${name}:**`;
|
|
173
|
+
return (text.split(/\r?\n/).find((line) => line.startsWith(prefix))?.slice(prefix.length).trim() ?? "").replaceAll("`", "");
|
|
174
|
+
}
|
package/src/criteria_support.ts
CHANGED
|
@@ -41,10 +41,11 @@ export function runCommand(root: string, command: string[]): Verdict {
|
|
|
41
41
|
} catch (error) {
|
|
42
42
|
return { ok: false, detail: `${command[0] ?? "command"} could not be run: ${error instanceof Error ? error.message : String(error)}` };
|
|
43
43
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
const output = [result.stdout.toString(), result.stderr.toString()].filter(Boolean).join("\n").trim();
|
|
45
|
+
if (output) return { ok: result.exitCode === 0, detail: `exit ${result.exitCode}:\n${output}` };
|
|
46
|
+
return result.exitCode === 0
|
|
47
|
+
? { ok: true, detail: "exit 0" }
|
|
48
|
+
: { ok: false, detail: `exit ${result.exitCode}: ${NO_OUTPUT}` };
|
|
48
49
|
}
|
|
49
50
|
|
|
50
51
|
export function gitOutput(root: string, command: string[]): string {
|
package/src/gates.test.ts
CHANGED
|
@@ -35,9 +35,9 @@ describe("gate groups", () => {
|
|
|
35
35
|
const model = await modelFor(project);
|
|
36
36
|
|
|
37
37
|
expect(names(runGate(model, project, "work"))).toEqual(["cmd.conform", "repo.config"]);
|
|
38
|
-
expect(names(runGate(model, project, "verify"))).toEqual(["
|
|
38
|
+
expect(names(runGate(model, project, "verify"))).toEqual(["docs.language", "repo.config"]);
|
|
39
39
|
expect(names(runGate(model, project, "pr"))).toEqual([
|
|
40
|
-
"docs.updated", "git.branch", "git.pushed", "git.tree", "spec.baseline", "spec.deferrals", "spec.dimensions",
|
|
40
|
+
"cmd.verify.unit", "docs.language", "docs.updated", "git.branch", "git.pushed", "git.tree", "repo.config", "spec.baseline", "spec.deferrals", "spec.dimensions",
|
|
41
41
|
"spec.gate", "spec.moved", "spec.open-questions", "spec.ordering", "spec.product-clarity",
|
|
42
42
|
]);
|
|
43
43
|
});
|