@tech-leads-club/harness-toolkit 0.3.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -8
- package/bin/tlc-cli.ts +40 -1
- package/bin/tlc-exec.d.mts +1 -0
- package/bin/tlc-exec.mjs +47 -2
- package/capabilities/catalog.json +47 -30
- package/dist/compact-before.mjs +84 -78
- package/dist/doctor.mjs +87 -81
- package/dist/help-topic.mjs +6 -5
- package/dist/init-project.mjs +147 -9
- package/dist/install-runtime.mjs +85 -79
- package/dist/lessons-cli.mjs +87 -81
- package/dist/obs-cli.mjs +83 -77
- package/dist/price-lookup.mjs +2 -2
- package/dist/prompt-submit.mjs +84 -78
- package/dist/refresh-model-prices.mjs +85 -79
- package/dist/response-after.mjs +84 -78
- package/dist/run.mjs +84 -78
- package/dist/session-end.mjs +90 -84
- package/dist/session-start.mjs +92 -86
- package/dist/shim.mjs +82 -76
- package/dist/stop.mjs +90 -84
- package/dist/subagent-start.mjs +84 -78
- package/dist/subagent-stop.mjs +85 -79
- package/dist/support.mjs +88 -82
- package/dist/tlc-cli.mjs +104 -98
- package/dist/tool-after.mjs +84 -78
- package/dist/tool-before.mjs +84 -78
- package/dist/tool-failure.mjs +84 -78
- package/dist/uninstall-runtime.mjs +4 -4
- package/docs/architecture.md +1 -0
- package/docs/concepts.md +86 -0
- package/docs/diagnose.md +20 -0
- package/docs/init.md +10 -2
- package/docs/lessons.md +12 -0
- package/docs/log.md +5 -0
- package/package.json +1 -1
- package/skills/harness-init/references/capabilities.md +54 -0
- package/src/core/core.facade.ts +54 -0
- package/src/core/floor/floor.paths.ts +2 -2
- package/src/core/floor/floor.policy-surface.ts +6 -1
- package/src/core/lesson/lesson.select.ts +30 -7
- package/src/core/policy/policy.defaults.ts +3 -0
- package/src/core/policy/policy.integrity.ts +2 -2
- package/src/core/policy/policy.loader.ts +14 -3
- package/src/core/policy/policy.shadow.ts +97 -0
- package/src/core/policy/policy.types.ts +8 -0
- package/src/core/release/release.decisions.ts +3 -13
- package/src/core/rules/rules.decide.ts +123 -0
- package/src/core/rules/rules.observe.ts +76 -0
- package/src/core/rules/rules.parse.ts +142 -0
- package/src/core/rules/rules.proof.ts +130 -0
- package/src/core/rules/rules.service.ts +141 -0
- package/src/core/rules/rules.store.ts +77 -0
- package/src/core/rules/rules.trigger.ts +101 -0
- package/src/core/rules/rules.types.ts +64 -0
- package/src/entrypoints/shim.ts +9 -1
- package/src/entrypoints/stop.ts +75 -1
- package/src/entrypoints/subagent-stop.ts +9 -1
- package/src/entrypoints/support.ts +32 -0
- package/src/entrypoints/tool-after.ts +7 -2
- package/src/entrypoints/tool-before.ts +44 -3
- package/src/platform/frontmatter.ts +142 -0
- package/src/platform/links.ts +32 -0
- package/src/platform/paths.ts +58 -4
- package/src/platform/pricing.ts +3 -3
- package/src/platform/screen.ts +62 -3
- package/tools/doctor.ts +162 -2
- package/tools/help-topic.ts +39 -23
- package/tools/init-project.ts +51 -6
- package/tools/install-runtime.ts +23 -2
- package/tools/lessons-cli.ts +4 -1
- package/tools/refresh-model-prices.ts +2 -2
- package/tools/uninstall-runtime.ts +11 -3
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import { flagsDir,
|
|
3
|
+
import { flagsDir, machineConfigPath, projectConfigPath } from "../../platform/paths.ts";
|
|
4
4
|
import { lessonsSyncMode, resolveSyncMode, type SyncModeResolution } from "../lesson/lesson.sync.ts";
|
|
5
5
|
import { DEFAULTS } from "./policy.defaults.ts";
|
|
6
6
|
import { type PostureResolution, resolvePosture } from "./policy.posture.ts";
|
|
@@ -55,7 +55,7 @@ type ConfigPair = { fromUser: PartialPolicy; fromProject: PartialPolicy };
|
|
|
55
55
|
|
|
56
56
|
function readConfigPair(root: string): ConfigPair {
|
|
57
57
|
return {
|
|
58
|
-
fromUser: readJsonFile<PartialPolicy>(
|
|
58
|
+
fromUser: readJsonFile<PartialPolicy>(machineConfigPath()) ?? {},
|
|
59
59
|
fromProject: readJsonFile<PartialPolicy>(projectConfigPath(root)) ?? {},
|
|
60
60
|
};
|
|
61
61
|
}
|
|
@@ -89,7 +89,7 @@ export function resolveProjectSyncMode(root: string): SyncModeResolution {
|
|
|
89
89
|
if (resolution.coercedFrom === undefined) {
|
|
90
90
|
return resolution;
|
|
91
91
|
}
|
|
92
|
-
const path = fromProject === undefined ?
|
|
92
|
+
const path = fromProject === undefined ? machineConfigPath() : projectConfigPath(root);
|
|
93
93
|
return { ...resolution, coercedIn: path };
|
|
94
94
|
}
|
|
95
95
|
|
|
@@ -112,6 +112,17 @@ export function loadPolicy(root: string): Policy {
|
|
|
112
112
|
return merged;
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
+
/**
|
|
116
|
+
* The policy as it would be with this project's config absent — `DEFAULTS` merged with the user tier.
|
|
117
|
+
*
|
|
118
|
+
* why here: the merge lives in this module, and asking "what would this resolve to without the project file" with
|
|
119
|
+
* a second copy of the merge is how the two answers drift ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
120
|
+
*/
|
|
121
|
+
export function resolvedWithoutProjectTier(): Record<string, unknown> {
|
|
122
|
+
const fromUser = readJsonFile<PartialPolicy>(machineConfigPath()) ?? {};
|
|
123
|
+
return deepMerge(DEFAULTS, fromUser) as unknown as Record<string, unknown>;
|
|
124
|
+
}
|
|
125
|
+
|
|
115
126
|
export function isUnderCodePaths(relativePath: string, codePaths: string[]): boolean {
|
|
116
127
|
const normalized = relativePath.replace(/\\/g, "/");
|
|
117
128
|
return codePaths.some((prefix) => normalized === prefix || normalized.startsWith(`${prefix}/`));
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which keys a project config restates rather than decides.
|
|
3
|
+
*
|
|
4
|
+
* why this is worth reporting: the layers are `DEFAULTS < user < project`, so a project key naming the value the
|
|
5
|
+
* lower tiers already resolve to changes nothing today and shadows them for ever. The moment the operator edits
|
|
6
|
+
* the machine-wide config, every project that restated the old value keeps it — and nothing said so. `init` writes
|
|
7
|
+
* the whole default policy when there is no config yet, and the wizard writes every knob it collected, so this is
|
|
8
|
+
* the common case rather than the odd one ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
9
|
+
*
|
|
10
|
+
* invariant: pure, and it reports rather than decides. A key that restates a default is not a fault — it is a key
|
|
11
|
+
* that has stopped tracking the tier below it, which is a thing an operator may want and must be able to see.
|
|
12
|
+
*/
|
|
13
|
+
/** A leaf the project config names, and the value it would have had without naming it. */
|
|
14
|
+
export type ShadowedKey = { path: string; value: unknown };
|
|
15
|
+
|
|
16
|
+
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
|
17
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* why a JSON comparison rather than a deep walk of its own: the values are config leaves — scalars and small
|
|
22
|
+
* arrays — and `codePaths: ["src"]` has to compare equal to `codePaths: ["src"]`. A second structural comparator
|
|
23
|
+
* here would be the copy that disagrees with the merge later.
|
|
24
|
+
*/
|
|
25
|
+
function sameValue(a: unknown, b: unknown): boolean {
|
|
26
|
+
return JSON.stringify(a) === JSON.stringify(b);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Every leaf in `project` whose value the lower tiers already resolve to.
|
|
31
|
+
*
|
|
32
|
+
* `resolved` is the policy as it would be with the project config absent — `DEFAULTS` merged with the user tier.
|
|
33
|
+
*/
|
|
34
|
+
export function shadowedKeys(
|
|
35
|
+
project: Record<string, unknown>,
|
|
36
|
+
resolved: Record<string, unknown>,
|
|
37
|
+
prefix = "",
|
|
38
|
+
): ShadowedKey[] {
|
|
39
|
+
const found: ShadowedKey[] = [];
|
|
40
|
+
for (const [key, value] of Object.entries(project)) {
|
|
41
|
+
// why skipped: `version` is the config's own shape marker rather than a setting, so naming it is required
|
|
42
|
+
// rather than redundant.
|
|
43
|
+
if (prefix === "" && key === "version") {
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
const path = prefix === "" ? key : `${prefix}.${key}`;
|
|
47
|
+
const below = resolved[key];
|
|
48
|
+
if (isPlainObject(value) && isPlainObject(below)) {
|
|
49
|
+
found.push(...shadowedKeys(value, below, path));
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (sameValue(value, below)) {
|
|
53
|
+
found.push({ path, value });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return found;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The project config with every restatement removed.
|
|
61
|
+
*
|
|
62
|
+
* why this and not just a report: `init` wrote the whole default policy when a project had no config yet, and the
|
|
63
|
+
* wizard wrote every knob it collected — so a fresh project shadowed the machine tier in dozens of places before
|
|
64
|
+
* anyone had chosen anything. Reporting it after the fact leaves the operator to undo it by hand; not writing it
|
|
65
|
+
* is the fix ([/decisions/ad-101.md](/decisions/ad-101.md)).
|
|
66
|
+
*
|
|
67
|
+
* invariant: pruning cannot change the effective policy. A leaf is dropped only when the tiers below already
|
|
68
|
+
* resolve to it, so the merge produces the same value with the key absent.
|
|
69
|
+
*
|
|
70
|
+
* why empty objects go too: `{ shipGate: {} }` is a block that decides nothing, and leaving it behind would make
|
|
71
|
+
* a pruned config read as though it had opinions.
|
|
72
|
+
*/
|
|
73
|
+
export function pruneShadowed(
|
|
74
|
+
project: Record<string, unknown>,
|
|
75
|
+
resolved: Record<string, unknown>,
|
|
76
|
+
prefix = "",
|
|
77
|
+
): Record<string, unknown> {
|
|
78
|
+
const kept: Record<string, unknown> = {};
|
|
79
|
+
for (const [key, value] of Object.entries(project)) {
|
|
80
|
+
if (prefix === "" && key === "version") {
|
|
81
|
+
kept[key] = value;
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
const below = resolved[key];
|
|
85
|
+
if (isPlainObject(value) && isPlainObject(below)) {
|
|
86
|
+
const inner = pruneShadowed(value, below, `${prefix}${key}.`);
|
|
87
|
+
if (Object.keys(inner).length > 0) {
|
|
88
|
+
kept[key] = inner;
|
|
89
|
+
}
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
if (!sameValue(value, below)) {
|
|
93
|
+
kept[key] = value;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return kept;
|
|
97
|
+
}
|
|
@@ -110,6 +110,13 @@ export type Policy = {
|
|
|
110
110
|
enabled: boolean;
|
|
111
111
|
windowMinutes: number;
|
|
112
112
|
};
|
|
113
|
+
/**
|
|
114
|
+
* Operator rules: the operator declares the trigger and the proof, and the harness enforces it. Off by default,
|
|
115
|
+
* and with no rule files the whole mechanism is inert ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
116
|
+
*/
|
|
117
|
+
rules: {
|
|
118
|
+
enabled: boolean;
|
|
119
|
+
};
|
|
113
120
|
shell: {
|
|
114
121
|
catastrophicAsk: boolean;
|
|
115
122
|
stallDetection: boolean;
|
|
@@ -139,6 +146,7 @@ export type PartialPolicy = Partial<Policy> & {
|
|
|
139
146
|
obs?: Partial<Policy["obs"]>;
|
|
140
147
|
untrustedContent?: Partial<Policy["untrustedContent"]>;
|
|
141
148
|
planGate?: Partial<Policy["planGate"]>;
|
|
149
|
+
rules?: Partial<Policy["rules"]>;
|
|
142
150
|
shell?: Partial<Policy["shell"]>;
|
|
143
151
|
intelligence?: Partial<Policy["intelligence"]> & {
|
|
144
152
|
lessons?: Partial<LessonsPolicyConfig>;
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
+
// invariant: one frontmatter reader, in platform. This file used to carry a private single-field copy
|
|
4
|
+
// ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
5
|
+
import { frontmatterField } from "../../platform/frontmatter.ts";
|
|
3
6
|
|
|
4
7
|
/**
|
|
5
8
|
* why: the substance of a changelog already exists in this repository as thirty decision records, each carrying why,
|
|
@@ -22,19 +25,6 @@ export type DecisionSummary = {
|
|
|
22
25
|
path: string;
|
|
23
26
|
};
|
|
24
27
|
|
|
25
|
-
function frontmatterField(text: string, field: string): string | undefined {
|
|
26
|
-
// why: line-scoped, matching how `check-docs-bundle` reads the same files. The values here are single-line
|
|
27
|
-
// quoted strings by convention, and the bundle check is what enforces that.
|
|
28
|
-
const match = new RegExp(`^${field}:\\s*"?(.+?)"?\\s*$`, "m").exec(text);
|
|
29
|
-
const value = match?.[1]?.trim();
|
|
30
|
-
if (value === undefined || value === "") {
|
|
31
|
-
return undefined;
|
|
32
|
-
}
|
|
33
|
-
// hazard: an escaped quote inside the value survived the outer-quote strip and reached the operator as a literal
|
|
34
|
-
// `\"` in their terminal. Seen in a real update run ([/decisions/ad-034.md](/decisions/ad-034.md)).
|
|
35
|
-
return value.replace(/\\(["'\\])/g, "$1");
|
|
36
|
-
}
|
|
37
|
-
|
|
38
28
|
export function decisionsDir(repoRoot: string): string {
|
|
39
29
|
return join(repoRoot, "docs", "decisions");
|
|
40
30
|
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What the harness answers when a rule fires and its proof is missing.
|
|
3
|
+
*
|
|
4
|
+
* invariant: pure, and posture only reaches `ask`. `deny`, `follow-up` and `warn` are verification, and the
|
|
5
|
+
* evidence bar is identical at all three postures — a posture that switched a check off is the defect the posture
|
|
6
|
+
* feature exists to remove ([/decisions/ad-025.md](/decisions/ad-025.md) item 4).
|
|
7
|
+
*/
|
|
8
|
+
import type { Decision } from "../../contracts/decision.ts";
|
|
9
|
+
import type { OperatorMode } from "../policy/policy.types.ts";
|
|
10
|
+
import { missingProofs, type Observation, type ProofContext, proofLabel } from "./rules.proof.ts";
|
|
11
|
+
import type { Rule, RuleVerdict } from "./rules.types.ts";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* `ask` is an interruption, which is the one thing posture governs. `paired` promises a check-in before a sizable
|
|
15
|
+
* move, so it asks. `solo` and `focus` name what reaches the operator — a destructive action, a dead end, and for
|
|
16
|
+
* `solo` a real ambiguity — and a missing proof is none of those, so the harness settles it itself.
|
|
17
|
+
*
|
|
18
|
+
* invariant: it hardens rather than softens. Softening would let a posture clear a verification, which is exactly
|
|
19
|
+
* what posture must never do — and it is the same direction `ask` already degrades in on a host that cannot ask.
|
|
20
|
+
*/
|
|
21
|
+
export function effectiveVerdict(declared: RuleVerdict, mode: OperatorMode): RuleVerdict {
|
|
22
|
+
return declared === "ask" && mode !== "paired" ? "deny" : declared;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* why the body verbatim: the rule's instruction is the operator's, with their project's context and their
|
|
27
|
+
* attachments. The harness adds the rule's name and what is missing, and changes nothing else.
|
|
28
|
+
*/
|
|
29
|
+
export function ruleMessage(rule: Rule, missing: readonly ReturnType<typeof proofLabel>[]): string {
|
|
30
|
+
const head = `rule ${rule.name} (${rule.tier}): missing ${missing.join(", ")}`;
|
|
31
|
+
return rule.body.trim() === "" ? head : `${head}\n\n${rule.body.trim()}`;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export type RuleOutcome = {
|
|
35
|
+
rule: Rule;
|
|
36
|
+
verdict: RuleVerdict;
|
|
37
|
+
missing: string[];
|
|
38
|
+
message: string;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* invariant: a rule whose proof holds produces nothing at all. Silence on the healthy path is what keeps this
|
|
43
|
+
* from being a wall an operator learns to ignore ([/decisions/ad-034.md](/decisions/ad-034.md)).
|
|
44
|
+
*/
|
|
45
|
+
export function evaluateRules(
|
|
46
|
+
rules: readonly Rule[],
|
|
47
|
+
observations: readonly Observation[],
|
|
48
|
+
context: ProofContext & { mode: OperatorMode },
|
|
49
|
+
): RuleOutcome[] {
|
|
50
|
+
const outcomes: RuleOutcome[] = [];
|
|
51
|
+
for (const rule of rules) {
|
|
52
|
+
const missing = missingProofs(rule, observations, context);
|
|
53
|
+
if (missing.length === 0) {
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
const labels = missing.map(proofLabel);
|
|
57
|
+
outcomes.push({
|
|
58
|
+
rule,
|
|
59
|
+
verdict: effectiveVerdict(rule.otherwise, context.mode),
|
|
60
|
+
missing: labels,
|
|
61
|
+
message: ruleMessage(rule, labels),
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
return outcomes;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const SEVERITY: Record<RuleVerdict, number> = { warn: 0, "follow-up": 1, ask: 2, deny: 3 };
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* invariant: the strictest outcome decides, which is how every host resolves two hooks answering one event. A
|
|
71
|
+
* `warn` beside a `deny` must not soften the `deny`.
|
|
72
|
+
*/
|
|
73
|
+
export function strictest(outcomes: readonly RuleOutcome[]): RuleOutcome | null {
|
|
74
|
+
return outcomes.reduce<RuleOutcome | null>(
|
|
75
|
+
(best, outcome) => (best === null || SEVERITY[outcome.verdict] > SEVERITY[best.verdict] ? outcome : best),
|
|
76
|
+
null,
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The decision for an action-time trigger. `follow-up` and `warn` never block an action — they are answers to the
|
|
82
|
+
* end of a turn, and `stopDecision` is where they are answered.
|
|
83
|
+
*/
|
|
84
|
+
export function actionDecision(outcome: RuleOutcome): Decision {
|
|
85
|
+
const rule = `rule:${outcome.rule.name}`;
|
|
86
|
+
if (outcome.verdict === "deny") {
|
|
87
|
+
return { kind: "deny", reason: outcome.message, rule };
|
|
88
|
+
}
|
|
89
|
+
if (outcome.verdict === "ask") {
|
|
90
|
+
return { kind: "ask", reason: outcome.message, userNote: outcome.message, rule };
|
|
91
|
+
}
|
|
92
|
+
return { kind: "abstain" };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The same four verdicts at the other moment. A stop can only be allowed or continued, so the matrix is:
|
|
97
|
+
*
|
|
98
|
+
* | verdict | at an action | at the stop |
|
|
99
|
+
* |-------------|-------------------------------|--------------------------------------------------|
|
|
100
|
+
* | `deny` | refuses the action | refuses the stop |
|
|
101
|
+
* | `ask` | asks (`paired`), else refuses | refuses the stop — no host offers an ask here |
|
|
102
|
+
* | `follow-up` | allows | refuses the stop, framed as the next action |
|
|
103
|
+
* | `warn` | allows | advisory text, and the stop is allowed |
|
|
104
|
+
*
|
|
105
|
+
* why `ask` refuses rather than passes: it already hardens to `deny` under `solo` and `focus`, and a verdict that
|
|
106
|
+
* quietly became "allow" at the one moment its channel is missing would be a bar that vanishes
|
|
107
|
+
* ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
108
|
+
*
|
|
109
|
+
* hazard: `follow-up` and `warn` were declared, parsed and evaluated, and then discarded — the caller read only
|
|
110
|
+
* `.outcomes.length`. `on: stop` was the same: `firingRules` handled it while nothing ever called this with a stop
|
|
111
|
+
* event. Three members of a closed vocabulary that a rule could name and `doctor` would list as active.
|
|
112
|
+
*/
|
|
113
|
+
export function stopDecision(outcome: RuleOutcome): Decision {
|
|
114
|
+
switch (outcome.verdict) {
|
|
115
|
+
case "deny":
|
|
116
|
+
case "ask":
|
|
117
|
+
return { kind: "continue", text: `BLOCKED: ${outcome.message}` };
|
|
118
|
+
case "follow-up":
|
|
119
|
+
return { kind: "continue", text: `NEED: ${outcome.message}` };
|
|
120
|
+
case "warn":
|
|
121
|
+
return { kind: "context", text: `ADVISORY: ${outcome.message}` };
|
|
122
|
+
}
|
|
123
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What an event proves, if anything.
|
|
3
|
+
*
|
|
4
|
+
* invariant: pure. The caller supplies the sha and the clock; this decides whether the event is worth recording
|
|
5
|
+
* and as what.
|
|
6
|
+
*
|
|
7
|
+
* why `tool.after` and not an exit code: measured on 3,755 real `shell.after` records from this machine, the
|
|
8
|
+
* payload carries `tool_response: {stdout, stderr, interrupted, …}` and **no exit code**, in any of the three
|
|
9
|
+
* shapes the two hosts send. But a tool that fails arrives as `tool.failure` — a different event, 64 of them in
|
|
10
|
+
* the same file, one of which is a `Bash` that exited non-zero. So observing at `tool.after` already means the
|
|
11
|
+
* command ran and did not fail, and no exit code is needed to say it
|
|
12
|
+
* ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
13
|
+
*
|
|
14
|
+
* hazard: 65 of those `shell.after` records carry a non-empty stderr. Treating stderr as failure would discard
|
|
15
|
+
* proof from every command that warns.
|
|
16
|
+
*/
|
|
17
|
+
import type { Observation } from "./rules.proof.ts";
|
|
18
|
+
|
|
19
|
+
export type ObservableEvent = {
|
|
20
|
+
event: string;
|
|
21
|
+
toolName?: string;
|
|
22
|
+
command?: string;
|
|
23
|
+
filePath?: string;
|
|
24
|
+
/**
|
|
25
|
+
* hazard: this was `subagentType`, which is the field a *spawn* carries. On `subagent.stop` both hosts put the
|
|
26
|
+
* type in `spawnSubagentType` — measured by running a real payload of each shape through the providers. So the
|
|
27
|
+
* producer read `undefined` and recorded nothing, and the unit test passed because its fixture invented the
|
|
28
|
+
* field name it was asserting about ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
29
|
+
*/
|
|
30
|
+
spawnSubagentType?: string;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export type ObserveContext = { sha: string | null; sessionKey: string; at: string };
|
|
34
|
+
|
|
35
|
+
/** What the event says happened, with no `when` attached to it yet. */
|
|
36
|
+
export type ObservedFact = { kind: Observation["kind"]; value: string };
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* invariant: one event, at most one observation. A `tool.after` that carries both a command and a path is a
|
|
40
|
+
* command — the path is its argument, not a file the turn wrote.
|
|
41
|
+
*
|
|
42
|
+
* why this is separate from `observationFrom`: the sha is a process spawn and this answers whether anything is
|
|
43
|
+
* worth spawning for. One place still decides what an event means
|
|
44
|
+
* ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
45
|
+
*/
|
|
46
|
+
export function observedFact(event: ObservableEvent): ObservedFact | null {
|
|
47
|
+
if (event.event === "subagent.stop") {
|
|
48
|
+
// why the type and not the id: a rule says "the jury reviewed", not "agent 7f3a reviewed".
|
|
49
|
+
return event.spawnSubagentType === undefined
|
|
50
|
+
? null
|
|
51
|
+
: { kind: "subagent", value: event.spawnSubagentType };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
if (event.event === "tool.after" || event.event === "shell.after") {
|
|
55
|
+
return event.command === undefined ? null : { kind: "command", value: event.command };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (event.event === "edit.after") {
|
|
59
|
+
return event.filePath === undefined ? null : { kind: "file", value: event.filePath };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function observationFrom(event: ObservableEvent, context: ObserveContext): Observation | null {
|
|
66
|
+
const fact = observedFact(event);
|
|
67
|
+
return fact === null ? null : { ...fact, sha: context.sha, sessionKey: context.sessionKey, at: context.at };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* A gate outcome is not an event the host sends — the harness decides it. So it is recorded where it is decided,
|
|
72
|
+
* with the gate's own name.
|
|
73
|
+
*/
|
|
74
|
+
export function gateObservation(gate: string, context: ObserveContext): Observation {
|
|
75
|
+
return { kind: "gate", value: gate, sha: context.sha, sessionKey: context.sessionKey, at: context.at };
|
|
76
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading an operator rule, and merging the two tiers.
|
|
3
|
+
*
|
|
4
|
+
* invariant: pure. The caller reads the directories and hands over `{ name, tier, text }`; this decides what they
|
|
5
|
+
* mean. That is what makes the whole vocabulary testable without a filesystem
|
|
6
|
+
* ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
7
|
+
*/
|
|
8
|
+
import { asList, parseFrontmatterDoc } from "../../platform/frontmatter.ts";
|
|
9
|
+
import {
|
|
10
|
+
PROOF_KINDS,
|
|
11
|
+
type ProofWindow,
|
|
12
|
+
RULE_VERDICTS,
|
|
13
|
+
type Rule,
|
|
14
|
+
type RuleError,
|
|
15
|
+
type RuleProof,
|
|
16
|
+
type RuleSet,
|
|
17
|
+
type RuleTier,
|
|
18
|
+
type RuleTrigger,
|
|
19
|
+
type RuleVerdict,
|
|
20
|
+
} from "./rules.types.ts";
|
|
21
|
+
|
|
22
|
+
export type RuleSource = { name: string; tier: RuleTier; text: string };
|
|
23
|
+
|
|
24
|
+
const BARE_TRIGGERS = new Set(["pr-open", "commit", "push", "stop"]);
|
|
25
|
+
|
|
26
|
+
/** why: `tool(Write)` and `command(gh pr create)` carry an argument; the other four do not. */
|
|
27
|
+
function parseTrigger(raw: string): RuleTrigger | string {
|
|
28
|
+
const bare = raw.trim();
|
|
29
|
+
if (BARE_TRIGGERS.has(bare)) {
|
|
30
|
+
return { kind: bare } as RuleTrigger;
|
|
31
|
+
}
|
|
32
|
+
const call = /^(tool|command)\(([^)]*)\)$/.exec(bare);
|
|
33
|
+
const verb = call?.[1];
|
|
34
|
+
const argument = call?.[2]?.trim();
|
|
35
|
+
if (verb === undefined || argument === undefined || argument === "") {
|
|
36
|
+
return `unknown trigger "${bare}" — use one of pr-open, commit, push, stop, tool(<name>), command(<pattern>)`;
|
|
37
|
+
}
|
|
38
|
+
return verb === "tool" ? { kind: "tool", name: argument } : { kind: "command", pattern: argument };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* `subagent(the-jury) since HEAD` → one proof.
|
|
43
|
+
*
|
|
44
|
+
* invariant: an unknown kind is an error, never a proof that silently never holds. A rule that cannot be
|
|
45
|
+
* evaluated has to say so at read time, or it reads as protection and is not.
|
|
46
|
+
*/
|
|
47
|
+
function parseProof(raw: string): RuleProof | string {
|
|
48
|
+
const [head, ...tail] = raw.trim().split(/\s+since\s+/i);
|
|
49
|
+
const call = /^([a-z]+)\(([^)]*)\)$/.exec((head ?? "").trim());
|
|
50
|
+
const kind = call?.[1];
|
|
51
|
+
const value = call?.[2]?.trim();
|
|
52
|
+
if (kind === undefined || !PROOF_KINDS.has(kind) || value === undefined || value === "") {
|
|
53
|
+
return `unknown proof "${raw.trim()}" — use subagent(<type>), command(<pattern>), gate(<name>) or file(<glob>)`;
|
|
54
|
+
}
|
|
55
|
+
const windowRaw = (tail[0] ?? "head").trim().toLowerCase();
|
|
56
|
+
if (windowRaw !== "head" && windowRaw !== "session") {
|
|
57
|
+
return `unknown window "since ${windowRaw}" in "${raw.trim()}" — use since HEAD or since session`;
|
|
58
|
+
}
|
|
59
|
+
return { kind, value, since: windowRaw as ProofWindow } as RuleProof;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function parseRule(source: RuleSource): { rule: Rule } | { error: RuleError } {
|
|
63
|
+
const fail = (error: string) => ({ error: { name: source.name, tier: source.tier, error } });
|
|
64
|
+
const { doc, error } = parseFrontmatterDoc(source.text);
|
|
65
|
+
if (doc === null) {
|
|
66
|
+
return fail(error ?? "unreadable");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const enabledRaw = doc.fields.enabled;
|
|
70
|
+
const enabled = enabledRaw === undefined ? true : String(enabledRaw).trim() !== "false";
|
|
71
|
+
|
|
72
|
+
const onRaw = doc.fields.on;
|
|
73
|
+
if (typeof onRaw !== "string" || onRaw.trim() === "") {
|
|
74
|
+
return fail("no `on:` trigger");
|
|
75
|
+
}
|
|
76
|
+
const trigger = parseTrigger(onRaw);
|
|
77
|
+
if (typeof trigger === "string") {
|
|
78
|
+
return fail(trigger);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const verdictRaw = doc.fields.otherwise;
|
|
82
|
+
if (typeof verdictRaw !== "string" || !RULE_VERDICTS.has(verdictRaw.trim())) {
|
|
83
|
+
return fail("`otherwise:` must be one of deny, ask, follow-up, warn");
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const proofs: RuleProof[] = [];
|
|
87
|
+
for (const entry of asList(doc.fields.require)) {
|
|
88
|
+
const proof = parseProof(entry);
|
|
89
|
+
if (typeof proof === "string") {
|
|
90
|
+
return fail(proof);
|
|
91
|
+
}
|
|
92
|
+
proofs.push(proof);
|
|
93
|
+
}
|
|
94
|
+
// why: a disabled rule is allowed to declare nothing — it exists to switch a global off and to say why.
|
|
95
|
+
if (proofs.length === 0 && enabled) {
|
|
96
|
+
return fail("no `require:` proof, so nothing could ever satisfy this rule");
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return {
|
|
100
|
+
rule: {
|
|
101
|
+
name: source.name,
|
|
102
|
+
tier: source.tier,
|
|
103
|
+
enabled,
|
|
104
|
+
on: trigger,
|
|
105
|
+
require: proofs,
|
|
106
|
+
otherwise: verdictRaw.trim() as RuleVerdict,
|
|
107
|
+
body: doc.body,
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Both tiers apply. A project rule of the same name replaces the global one, which is how the lesson tiers
|
|
114
|
+
* already behave — union, deduplicated by id, nearer tier winning
|
|
115
|
+
* ([/decisions/ad-040.md](/decisions/ad-040.md)). Writing the same rule once per repository is the friction this
|
|
116
|
+
* removes.
|
|
117
|
+
*
|
|
118
|
+
* invariant: a rule switched off is kept, in `disabled`, so `doctor` can say which global a project turned off
|
|
119
|
+
* rather than leaving the operator to guess why nothing fired.
|
|
120
|
+
*/
|
|
121
|
+
export function buildRuleSet(sources: readonly RuleSource[]): RuleSet {
|
|
122
|
+
const byName = new Map<string, Rule>();
|
|
123
|
+
const errors: RuleError[] = [];
|
|
124
|
+
|
|
125
|
+
for (const tier of ["global", "project"] as const) {
|
|
126
|
+
for (const source of sources.filter((candidate) => candidate.tier === tier)) {
|
|
127
|
+
const parsed = parseRule(source);
|
|
128
|
+
if ("error" in parsed) {
|
|
129
|
+
errors.push(parsed.error);
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
byName.set(parsed.rule.name, parsed.rule);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const all = [...byName.values()];
|
|
137
|
+
return {
|
|
138
|
+
rules: all.filter((rule) => rule.enabled),
|
|
139
|
+
disabled: all.filter((rule) => !rule.enabled),
|
|
140
|
+
errors,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether the proof a rule demands exists.
|
|
3
|
+
*
|
|
4
|
+
* invariant: a proof is satisfied only by something the harness itself observed. Never by the agent asserting it,
|
|
5
|
+
* and never by a model's verdict — the same change gets different verdicts across runs, so a probabilistic
|
|
6
|
+
* reviewer cannot be an enforcement mechanism ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
7
|
+
*
|
|
8
|
+
* invariant: pure. The store hands over the observations; this decides what they mean.
|
|
9
|
+
*/
|
|
10
|
+
import { matchesPhrase } from "./rules.trigger.ts";
|
|
11
|
+
import type { Rule, RuleProof } from "./rules.types.ts";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* One thing the harness saw.
|
|
15
|
+
*
|
|
16
|
+
* why `sha`: freshness is part of the proof. A review of the code as it was two commits ago reviewed something
|
|
17
|
+
* else, so an observation carries the HEAD it was made against and `since HEAD` compares them.
|
|
18
|
+
*/
|
|
19
|
+
export type Observation = {
|
|
20
|
+
kind: RuleProof["kind"];
|
|
21
|
+
/** The subagent type, the command as words, the gate name, or the path. */
|
|
22
|
+
value: string;
|
|
23
|
+
/** HEAD when it was observed. `null` when the project is not a git checkout. */
|
|
24
|
+
sha: string | null;
|
|
25
|
+
sessionKey: string;
|
|
26
|
+
at: string;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export type ProofContext = { sha: string | null; sessionKey: string };
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* why a suffix and a directory instead of a glob engine: nothing else in this repository needs globs, and an
|
|
33
|
+
* engine written for one caller is generality nobody asked for. Three shapes, each stated: an exact path,
|
|
34
|
+
* `*.<ext>`, and a `dir/` prefix. A rule that needs more than that is a decision with a reason, not a silent
|
|
35
|
+
* extension of this function.
|
|
36
|
+
*/
|
|
37
|
+
function pathMatches(pattern: string, path: string): boolean {
|
|
38
|
+
if (pattern === path) {
|
|
39
|
+
return true;
|
|
40
|
+
}
|
|
41
|
+
if (pattern.startsWith("*")) {
|
|
42
|
+
return path.endsWith(pattern.slice(1));
|
|
43
|
+
}
|
|
44
|
+
if (pattern.endsWith("/")) {
|
|
45
|
+
return path.startsWith(pattern);
|
|
46
|
+
}
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function valueMatches(proof: RuleProof, observation: Observation): boolean {
|
|
51
|
+
if (proof.kind !== observation.kind) {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
switch (proof.kind) {
|
|
55
|
+
case "command":
|
|
56
|
+
// why the same phrase rule as the trigger: the operator wrote words in an order, in both places.
|
|
57
|
+
return matchesPhrase(observation.value.split(/\s+/), proof.value);
|
|
58
|
+
case "file":
|
|
59
|
+
return pathMatches(proof.value, observation.value);
|
|
60
|
+
default:
|
|
61
|
+
return observation.value === proof.value;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* invariant: `since HEAD` needs a sha on both sides. When the project is not a git checkout there is no HEAD to
|
|
67
|
+
* compare, so a `since HEAD` proof cannot be satisfied — and saying so is the honest answer, rather than treating
|
|
68
|
+
* "no sha" as "any sha".
|
|
69
|
+
*/
|
|
70
|
+
function windowMatches(proof: RuleProof, observation: Observation, context: ProofContext): boolean {
|
|
71
|
+
if (proof.since === "session") {
|
|
72
|
+
return observation.sessionKey === context.sessionKey;
|
|
73
|
+
}
|
|
74
|
+
return context.sha !== null && observation.sha === context.sha;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function proofSatisfied(
|
|
78
|
+
proof: RuleProof,
|
|
79
|
+
observations: readonly Observation[],
|
|
80
|
+
context: ProofContext,
|
|
81
|
+
): boolean {
|
|
82
|
+
return observations.some(
|
|
83
|
+
(observation) => valueMatches(proof, observation) && windowMatches(proof, observation, context),
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** invariant: every proof must hold. The list is a conjunction, which is why there is no boolean algebra. */
|
|
88
|
+
export function missingProofs(
|
|
89
|
+
rule: Rule,
|
|
90
|
+
observations: readonly Observation[],
|
|
91
|
+
context: ProofContext,
|
|
92
|
+
): RuleProof[] {
|
|
93
|
+
return rule.require.filter((proof) => !proofSatisfied(proof, observations, context));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function proofLabel(proof: RuleProof): string {
|
|
97
|
+
return `${proof.kind}(${proof.value}) since ${proof.since === "head" ? "HEAD" : "session"}`;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Whether recording this kind could ever matter here.
|
|
102
|
+
*
|
|
103
|
+
* why: the observing rails fire on every tool call, and a fact nothing reads is a write and a process spawn for
|
|
104
|
+
* nothing. An operator whose only rule wants `subagent(the-jury)` pays no git on any command
|
|
105
|
+
* ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
106
|
+
*/
|
|
107
|
+
export function kindIsRequired(rules: readonly Rule[], kind: RuleProof["kind"]): boolean {
|
|
108
|
+
return rules.some((rule) => rule.require.some((proof) => proof.kind === kind));
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* What `doctor` needs to tell an operator that a rule can never be satisfied here.
|
|
113
|
+
*
|
|
114
|
+
* why this rather than a capability flag: both hosts report subagent types today, so a flag for it would be a flag
|
|
115
|
+
* that is never false — the shape of a rail that reads as protection and measures nothing
|
|
116
|
+
* ([/decisions/ad-034.md](/decisions/ad-034.md)). What is worth saying is factual: this rule wants a kind of
|
|
117
|
+
* observation that has never been recorded in this project.
|
|
118
|
+
*/
|
|
119
|
+
export function unobservedKinds(
|
|
120
|
+
rules: readonly Rule[],
|
|
121
|
+
observations: readonly Observation[],
|
|
122
|
+
): Array<{ rule: string; kinds: RuleProof["kind"][] }> {
|
|
123
|
+
const seen = new Set(observations.map((observation) => observation.kind));
|
|
124
|
+
return rules
|
|
125
|
+
.map((rule) => ({
|
|
126
|
+
rule: rule.name,
|
|
127
|
+
kinds: [...new Set(rule.require.map((proof) => proof.kind))].filter((kind) => !seen.has(kind)),
|
|
128
|
+
}))
|
|
129
|
+
.filter((entry) => entry.kinds.length > 0);
|
|
130
|
+
}
|