@tech-leads-club/harness-toolkit 0.3.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -8
- package/bin/tlc-cli.ts +40 -1
- package/bin/tlc-exec.d.mts +1 -0
- package/bin/tlc-exec.mjs +47 -2
- package/capabilities/catalog.json +47 -30
- package/dist/compact-before.mjs +84 -78
- package/dist/doctor.mjs +87 -81
- package/dist/help-topic.mjs +6 -5
- package/dist/init-project.mjs +147 -9
- package/dist/install-runtime.mjs +85 -79
- package/dist/lessons-cli.mjs +87 -81
- package/dist/obs-cli.mjs +83 -77
- package/dist/price-lookup.mjs +2 -2
- package/dist/prompt-submit.mjs +84 -78
- package/dist/refresh-model-prices.mjs +85 -79
- package/dist/response-after.mjs +84 -78
- package/dist/run.mjs +84 -78
- package/dist/session-end.mjs +90 -84
- package/dist/session-start.mjs +92 -86
- package/dist/shim.mjs +82 -76
- package/dist/stop.mjs +90 -84
- package/dist/subagent-start.mjs +84 -78
- package/dist/subagent-stop.mjs +85 -79
- package/dist/support.mjs +88 -82
- package/dist/tlc-cli.mjs +104 -98
- package/dist/tool-after.mjs +84 -78
- package/dist/tool-before.mjs +84 -78
- package/dist/tool-failure.mjs +84 -78
- package/dist/uninstall-runtime.mjs +4 -4
- package/docs/architecture.md +1 -0
- package/docs/concepts.md +86 -0
- package/docs/diagnose.md +20 -0
- package/docs/init.md +10 -2
- package/docs/lessons.md +12 -0
- package/docs/log.md +5 -0
- package/package.json +1 -1
- package/skills/harness-init/references/capabilities.md +54 -0
- package/src/core/core.facade.ts +54 -0
- package/src/core/floor/floor.paths.ts +2 -2
- package/src/core/floor/floor.policy-surface.ts +6 -1
- package/src/core/lesson/lesson.select.ts +30 -7
- package/src/core/policy/policy.defaults.ts +3 -0
- package/src/core/policy/policy.integrity.ts +2 -2
- package/src/core/policy/policy.loader.ts +14 -3
- package/src/core/policy/policy.shadow.ts +97 -0
- package/src/core/policy/policy.types.ts +8 -0
- package/src/core/release/release.decisions.ts +3 -13
- package/src/core/rules/rules.decide.ts +123 -0
- package/src/core/rules/rules.observe.ts +76 -0
- package/src/core/rules/rules.parse.ts +142 -0
- package/src/core/rules/rules.proof.ts +130 -0
- package/src/core/rules/rules.service.ts +141 -0
- package/src/core/rules/rules.store.ts +77 -0
- package/src/core/rules/rules.trigger.ts +101 -0
- package/src/core/rules/rules.types.ts +64 -0
- package/src/entrypoints/shim.ts +9 -1
- package/src/entrypoints/stop.ts +75 -1
- package/src/entrypoints/subagent-stop.ts +9 -1
- package/src/entrypoints/support.ts +32 -0
- package/src/entrypoints/tool-after.ts +7 -2
- package/src/entrypoints/tool-before.ts +44 -3
- package/src/platform/frontmatter.ts +142 -0
- package/src/platform/links.ts +32 -0
- package/src/platform/paths.ts +58 -4
- package/src/platform/pricing.ts +3 -3
- package/src/platform/screen.ts +62 -3
- package/tools/doctor.ts +162 -2
- package/tools/help-topic.ts +39 -23
- package/tools/init-project.ts +51 -6
- package/tools/install-runtime.ts +23 -2
- package/tools/lessons-cli.ts +4 -1
- package/tools/refresh-model-prices.ts +2 -2
- package/tools/uninstall-runtime.ts +11 -3
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The composition the entrypoints call: read the rules, see what fired, decide.
|
|
3
|
+
*
|
|
4
|
+
* why here and not in the entrypoint: the entrypoints are adapters. Which rules apply, what proves them and what
|
|
5
|
+
* the verdict is are all decisions, and decisions live in core ([/decisions/ad-016.md](/decisions/ad-016.md)).
|
|
6
|
+
*
|
|
7
|
+
* invariant: with the capability off, or with no rule files, this reads two directory entries and returns nothing.
|
|
8
|
+
* That is what makes the feature inert until an operator declares something (AC1).
|
|
9
|
+
*/
|
|
10
|
+
import type { Decision } from "../../contracts/decision.ts";
|
|
11
|
+
import type { OperatorMode } from "../policy/policy.types.ts";
|
|
12
|
+
import { actionDecision, evaluateRules, type RuleOutcome, stopDecision, strictest } from "./rules.decide.ts";
|
|
13
|
+
import {
|
|
14
|
+
gateObservation,
|
|
15
|
+
type ObservableEvent,
|
|
16
|
+
type ObserveContext,
|
|
17
|
+
observationFrom,
|
|
18
|
+
observedFact,
|
|
19
|
+
} from "./rules.observe.ts";
|
|
20
|
+
import { buildRuleSet } from "./rules.parse.ts";
|
|
21
|
+
import { kindIsRequired } from "./rules.proof.ts";
|
|
22
|
+
import { readObservations, readRuleSources, recordObservation } from "./rules.store.ts";
|
|
23
|
+
import { firingRules, type TriggerContext } from "./rules.trigger.ts";
|
|
24
|
+
import type { RuleError, RuleSet } from "./rules.types.ts";
|
|
25
|
+
|
|
26
|
+
export type RulesConfig = { enabled: boolean };
|
|
27
|
+
|
|
28
|
+
export function loadRules(root: string, config: RulesConfig): RuleSet {
|
|
29
|
+
if (!config.enabled) {
|
|
30
|
+
return { rules: [], disabled: [], errors: [] };
|
|
31
|
+
}
|
|
32
|
+
return buildRuleSet(readRuleSources(root));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Whether this event is worth a sha.
|
|
37
|
+
*
|
|
38
|
+
* hazard: nothing called `observe` at all in the first cut of this feature. The store was never written, so no
|
|
39
|
+
* proof could exist, so every rule that parsed denied for ever — and `require:` is mandatory, so that was every
|
|
40
|
+
* rule. The end-to-end run that appeared to show the loop working was a script calling `observe` by hand, which
|
|
41
|
+
* supplied the missing half and hid it ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
42
|
+
*
|
|
43
|
+
* why the question is asked before the answer is fetched: the observing rails fire on every tool call and the sha
|
|
44
|
+
* is a process spawn. An operator whose only rule wants `subagent(the-jury)` pays two directory reads per command
|
|
45
|
+
* and no git at all.
|
|
46
|
+
*/
|
|
47
|
+
export function wantsObservation(root: string, config: RulesConfig, event: ObservableEvent): boolean {
|
|
48
|
+
const fact = observedFact(event);
|
|
49
|
+
return fact !== null && kindIsRequired(loadRules(root, config).rules, fact.kind);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** invariant: with the capability off nothing is written, so a machine that never opted in carries no new file. */
|
|
53
|
+
export function observe(
|
|
54
|
+
root: string,
|
|
55
|
+
config: RulesConfig,
|
|
56
|
+
event: ObservableEvent,
|
|
57
|
+
context: ObserveContext,
|
|
58
|
+
): void {
|
|
59
|
+
if (!config.enabled) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
const observation = observationFrom(event, context);
|
|
63
|
+
if (observation !== null) {
|
|
64
|
+
recordObservation(root, observation);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* A gate is the one proof the harness decides rather than witnesses, so it is recorded where it is decided.
|
|
70
|
+
*
|
|
71
|
+
* invariant: only a gate that passed. Recording a failure as an observation would make "the gate ran" satisfy a
|
|
72
|
+
* rule that asked for "the gate passed".
|
|
73
|
+
*/
|
|
74
|
+
export function wantsGateObservation(root: string, config: RulesConfig): boolean {
|
|
75
|
+
return kindIsRequired(loadRules(root, config).rules, "gate");
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function observeGate(root: string, config: RulesConfig, gate: string, context: ObserveContext): void {
|
|
79
|
+
if (!config.enabled) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
recordObservation(root, gateObservation(gate, context));
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export type RulesVerdict = {
|
|
86
|
+
decision: Decision;
|
|
87
|
+
/** Everything that fired, so a `follow-up` or a `warn` can be reported even when the action is allowed. */
|
|
88
|
+
outcomes: RuleOutcome[];
|
|
89
|
+
errors: RuleError[];
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const NOTHING: RulesVerdict = { decision: { kind: "abstain" }, outcomes: [], errors: [] };
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* The action-time answer. `deny` and `ask` block here; `follow-up` and `warn` are answers to the end of a turn and
|
|
96
|
+
* to the record, so they abstain and are returned for the caller to report.
|
|
97
|
+
*/
|
|
98
|
+
export function decideAction(
|
|
99
|
+
root: string,
|
|
100
|
+
config: RulesConfig,
|
|
101
|
+
trigger: TriggerContext,
|
|
102
|
+
context: RuleContext,
|
|
103
|
+
): RulesVerdict {
|
|
104
|
+
return decide(root, config, trigger, context, actionDecision);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The end-of-turn answer, and the only caller that can see an `on: stop` rule.
|
|
109
|
+
*
|
|
110
|
+
* why a second entry rather than a flag: the trigger is fixed and the mapping differs, so a boolean would make one
|
|
111
|
+
* function answer two questions ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
112
|
+
*/
|
|
113
|
+
export function decideStop(root: string, config: RulesConfig, context: RuleContext): RulesVerdict {
|
|
114
|
+
return decide(root, config, { event: "stop" }, context, stopDecision);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export type RuleContext = { sha: string | null; sessionKey: string; mode: OperatorMode };
|
|
118
|
+
|
|
119
|
+
function decide(
|
|
120
|
+
root: string,
|
|
121
|
+
config: RulesConfig,
|
|
122
|
+
trigger: TriggerContext,
|
|
123
|
+
context: RuleContext,
|
|
124
|
+
map: (outcome: RuleOutcome) => Decision,
|
|
125
|
+
): RulesVerdict {
|
|
126
|
+
const set = loadRules(root, config);
|
|
127
|
+
if (set.rules.length === 0 && set.errors.length === 0) {
|
|
128
|
+
return NOTHING;
|
|
129
|
+
}
|
|
130
|
+
const firing = firingRules(set.rules, trigger);
|
|
131
|
+
if (firing.length === 0) {
|
|
132
|
+
return { ...NOTHING, errors: set.errors };
|
|
133
|
+
}
|
|
134
|
+
const outcomes = evaluateRules(firing, readObservations(root), context);
|
|
135
|
+
const worst = strictest(outcomes);
|
|
136
|
+
return {
|
|
137
|
+
decision: worst === null ? { kind: "abstain" } : map(worst),
|
|
138
|
+
outcomes,
|
|
139
|
+
errors: set.errors,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where the harness records what it observed, and where it reads the rules from.
|
|
3
|
+
*
|
|
4
|
+
* invariant: the agent cannot write here. This lives under the project state directory, which the floor's
|
|
5
|
+
* `policy-surface-write` refuses to an agent through a shell redirect, an interpreter or a write tool — and the
|
|
6
|
+
* mutating `tlc harness` subcommands are refused from inside a session. So the only writer is the harness
|
|
7
|
+
* observing a host event, which is what makes a proof unforgeable rather than conventional
|
|
8
|
+
* ([/decisions/ad-100.md](/decisions/ad-100.md), [/decisions/ad-022.md](/decisions/ad-022.md)).
|
|
9
|
+
*
|
|
10
|
+
* why append-only jsonl and not a merged document: two sessions observe at the same time, and an append is the one
|
|
11
|
+
* write that needs no lock. The reader takes the tail, because a proof is about now.
|
|
12
|
+
*/
|
|
13
|
+
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
14
|
+
import { basename, join } from "node:path";
|
|
15
|
+
import { appendRecord, readTail } from "../../platform/fs-jsonl.ts";
|
|
16
|
+
import { machineHome, projectStateDir } from "../../platform/paths.ts";
|
|
17
|
+
import type { RuleSource } from "./rules.parse.ts";
|
|
18
|
+
import type { Observation } from "./rules.proof.ts";
|
|
19
|
+
import type { RuleTier } from "./rules.types.ts";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* why a bound: an observation older than this window cannot satisfy `since HEAD` anyway, and a file that grows
|
|
23
|
+
* without limit is a file nobody prunes. The tail is generous enough that a long session keeps its own proofs.
|
|
24
|
+
*/
|
|
25
|
+
const OBSERVATION_TAIL = 500;
|
|
26
|
+
|
|
27
|
+
export function observationsPath(root: string): string {
|
|
28
|
+
return join(projectStateDir(root), "rule-observations.jsonl");
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** The project's rules, versioned with it. */
|
|
32
|
+
export function projectRulesDir(root: string): string {
|
|
33
|
+
return join(projectStateDir(root), "..", "rules");
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** This machine's rules, every repository — the tier that follows the operator across products. */
|
|
37
|
+
export function globalRulesDir(): string {
|
|
38
|
+
return join(machineHome(), "rules");
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function readDir(dir: string, tier: RuleTier): RuleSource[] {
|
|
42
|
+
if (!existsSync(dir)) {
|
|
43
|
+
return [];
|
|
44
|
+
}
|
|
45
|
+
return readdirSync(dir, { withFileTypes: true })
|
|
46
|
+
.filter((entry) => entry.isFile() && entry.name.endsWith(".md"))
|
|
47
|
+
.map((entry) => {
|
|
48
|
+
const path = join(dir, entry.name);
|
|
49
|
+
return { name: basename(entry.name, ".md"), tier, text: readFileSync(path, "utf8") };
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* invariant: both tiers are read, global first, so `buildRuleSet` can let the project win by name. Absent
|
|
55
|
+
* directories are absent rules, not an error — no rules means no behaviour change
|
|
56
|
+
* ([/decisions/ad-040.md](/decisions/ad-040.md)).
|
|
57
|
+
*/
|
|
58
|
+
export function readRuleSources(root: string): RuleSource[] {
|
|
59
|
+
return [...readDir(globalRulesDir(), "global"), ...readDir(projectRulesDir(root), "project")];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function recordObservation(root: string, observation: Observation): void {
|
|
63
|
+
try {
|
|
64
|
+
appendRecord(observationsPath(root), observation);
|
|
65
|
+
} catch {
|
|
66
|
+
// why swallowed: an unwritable state directory must not fail the turn that was being observed. The proof will
|
|
67
|
+
// be missing, which the gate reports as missing rather than as an error nobody can act on.
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function readObservations(root: string): Observation[] {
|
|
72
|
+
try {
|
|
73
|
+
return readTail<Observation>(observationsPath(root), OBSERVATION_TAIL);
|
|
74
|
+
} catch {
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether a rule's trigger fires on this event.
|
|
3
|
+
*
|
|
4
|
+
* invariant: pure, and it never reads a host payload. It takes the published event shape, so a rule written once
|
|
5
|
+
* fires the same way on every provider ([/decisions/ad-004.md](/decisions/ad-004.md)).
|
|
6
|
+
*
|
|
7
|
+
* hazard: a shell trigger cannot be a substring test against the whole command. `x && gh pr create` is a pull
|
|
8
|
+
* request being opened, and a heredoc body containing the words `gh pr create` is a document. `tokenizeShell`
|
|
9
|
+
* separates both and is the only splitter in this repository — a second regex here would be the duplication that
|
|
10
|
+
* makes one of them wrong later ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
11
|
+
*/
|
|
12
|
+
import { tokenizeShell } from "../floor/floor.tokenize.ts";
|
|
13
|
+
import type { Rule, RuleTrigger } from "./rules.types.ts";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* What the harness reads to decide whether a trigger fires. A subset of the event, named so the vocabulary is
|
|
17
|
+
* visible: adding a trigger that needs a new field has to widen this deliberately.
|
|
18
|
+
*/
|
|
19
|
+
export type TriggerContext = {
|
|
20
|
+
event: string;
|
|
21
|
+
toolName?: string;
|
|
22
|
+
command?: string;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* why a set per trigger rather than one pattern the operator writes: `pr-open` has to mean the same thing in
|
|
27
|
+
* every repository, or a rule copied between them silently stops firing. An operator who wants their own shape
|
|
28
|
+
* writes `command(<pattern>)`.
|
|
29
|
+
*/
|
|
30
|
+
const SHELL_SHAPES: Record<"pr-open" | "commit" | "push", readonly string[][]> = {
|
|
31
|
+
"pr-open": [
|
|
32
|
+
["gh", "pr", "create"],
|
|
33
|
+
["gh", "pr", "ready"],
|
|
34
|
+
],
|
|
35
|
+
commit: [["git", "commit"]],
|
|
36
|
+
push: [["git", "push"]],
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* invariant: `tokenizeShell` already declines to emit segments from a heredoc body, so a body is never mistaken
|
|
41
|
+
* for a command and a command after one is still seen. Measured both ways on
|
|
42
|
+
* `cat <<EOF > runbook.md\ngh pr create --fill\nEOF` and on the same with a real command after the terminator:
|
|
43
|
+
* identical output.
|
|
44
|
+
*
|
|
45
|
+
* hazard: the first version of this called `splitHeredocs` first as well. It changed nothing — the mutation that
|
|
46
|
+
* removed it survived, which is what exposed it as dead rather than as untested
|
|
47
|
+
* ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
48
|
+
*/
|
|
49
|
+
function subCommands(command: string): string[][] {
|
|
50
|
+
return tokenizeShell(command)
|
|
51
|
+
.map((segment) => segment.words.map((word) => word.text))
|
|
52
|
+
.filter((words) => words.length > 0);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** why prefix rather than equality: `gh pr create --fill --base main` is the same act as `gh pr create`. */
|
|
56
|
+
function startsWithShape(words: readonly string[], shape: readonly string[]): boolean {
|
|
57
|
+
return shape.every((token, index) => words[index] === token);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* why a phrase and not a word: an operator writes `command(gh pr review)`, meaning those words in that order.
|
|
62
|
+
* Matching the raw string against the whole command would let a heredoc or an unrelated argument satisfy it.
|
|
63
|
+
*/
|
|
64
|
+
export function matchesPhrase(words: readonly string[], pattern: string): boolean {
|
|
65
|
+
const phrase = pattern.trim().split(/\s+/);
|
|
66
|
+
if (phrase.length === 0) {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
return words.some((_, start) => phrase.every((token, index) => words[start + index] === token));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function triggerMatches(trigger: RuleTrigger, context: TriggerContext): boolean {
|
|
73
|
+
switch (trigger.kind) {
|
|
74
|
+
case "stop":
|
|
75
|
+
return context.event === "stop";
|
|
76
|
+
case "tool":
|
|
77
|
+
return context.toolName === trigger.name;
|
|
78
|
+
case "pr-open":
|
|
79
|
+
case "commit":
|
|
80
|
+
case "push": {
|
|
81
|
+
if (context.command === undefined) {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
const shapes = SHELL_SHAPES[trigger.kind];
|
|
85
|
+
return subCommands(context.command).some((words) =>
|
|
86
|
+
shapes.some((shape) => startsWithShape(words, shape)),
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
default: {
|
|
90
|
+
if (context.command === undefined) {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
return subCommands(context.command).some((words) => matchesPhrase(words, trigger.pattern));
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** invariant: a disabled rule never fires. It exists to switch a global off and to record why. */
|
|
99
|
+
export function firingRules(rules: readonly Rule[], context: TriggerContext): Rule[] {
|
|
100
|
+
return rules.filter((rule) => rule.enabled && triggerMatches(rule.on, context));
|
|
101
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What an operator rule is.
|
|
3
|
+
*
|
|
4
|
+
* why two closed vocabularies: a gate may only rest on something the harness observed, and a trigger may only be
|
|
5
|
+
* something it can recognise. An open expression language would let an operator declare a rule the harness cannot
|
|
6
|
+
* evaluate, and the honest answer to that is a parse error rather than a rule that never fires
|
|
7
|
+
* ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/** Where the rule came from. `project` replaces a `global` of the same name, and both apply otherwise. */
|
|
11
|
+
export type RuleTier = "global" | "project";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* What happens when the proof is missing.
|
|
15
|
+
*
|
|
16
|
+
* invariant: `deny`, `follow-up` and `warn` are verification and are identical at every posture. `ask` is an
|
|
17
|
+
* interruption, which is the one thing posture governs ([/decisions/ad-025.md](/decisions/ad-025.md)).
|
|
18
|
+
*/
|
|
19
|
+
export type RuleVerdict = "deny" | "ask" | "follow-up" | "warn";
|
|
20
|
+
|
|
21
|
+
export const RULE_VERDICTS: ReadonlySet<string> = new Set<RuleVerdict>(["deny", "ask", "follow-up", "warn"]);
|
|
22
|
+
|
|
23
|
+
/** When the rule is evaluated. */
|
|
24
|
+
export type RuleTrigger =
|
|
25
|
+
| { kind: "pr-open" }
|
|
26
|
+
| { kind: "commit" }
|
|
27
|
+
| { kind: "push" }
|
|
28
|
+
| { kind: "stop" }
|
|
29
|
+
| { kind: "tool"; name: string }
|
|
30
|
+
| { kind: "command"; pattern: string };
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* What counts as proof, and how fresh it must be.
|
|
34
|
+
*
|
|
35
|
+
* why `head` by default: a review of the code as it was two commits ago is not a review of what the pull request
|
|
36
|
+
* carries.
|
|
37
|
+
*/
|
|
38
|
+
export type ProofWindow = "head" | "session";
|
|
39
|
+
|
|
40
|
+
export type RuleProof =
|
|
41
|
+
| { kind: "subagent"; value: string; since: ProofWindow }
|
|
42
|
+
| { kind: "command"; value: string; since: ProofWindow }
|
|
43
|
+
| { kind: "gate"; value: string; since: ProofWindow }
|
|
44
|
+
| { kind: "file"; value: string; since: ProofWindow };
|
|
45
|
+
|
|
46
|
+
export const PROOF_KINDS: ReadonlySet<string> = new Set(["subagent", "command", "gate", "file"]);
|
|
47
|
+
|
|
48
|
+
export type Rule = {
|
|
49
|
+
/** The file name without its extension. This is the id the tiers dedupe on. */
|
|
50
|
+
name: string;
|
|
51
|
+
tier: RuleTier;
|
|
52
|
+
enabled: boolean;
|
|
53
|
+
on: RuleTrigger;
|
|
54
|
+
/** invariant: every proof must hold. There is no boolean algebra here on purpose. */
|
|
55
|
+
require: RuleProof[];
|
|
56
|
+
otherwise: RuleVerdict;
|
|
57
|
+
/** The operator's own text, injected verbatim when the rule fires. */
|
|
58
|
+
body: string;
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
/** A rule that could not be read. Named, so `doctor` can report it instead of the harness ignoring it silently. */
|
|
62
|
+
export type RuleError = { name: string; tier: RuleTier; error: string };
|
|
63
|
+
|
|
64
|
+
export type RuleSet = { rules: Rule[]; errors: RuleError[]; disabled: Rule[] };
|
package/src/entrypoints/shim.ts
CHANGED
|
@@ -79,6 +79,14 @@ if (existsSync(execBin)) {
|
|
|
79
79
|
} else if (existsSync(srcHandler)) {
|
|
80
80
|
run(process.env.BUN_BIN || "bun", ["run", srcHandler]);
|
|
81
81
|
} else {
|
|
82
|
+
/**
|
|
83
|
+
* hazard: this exited 127 with nothing on stdout. The project shim answers a host hook, so a shim that cannot
|
|
84
|
+
* find its handler was standing between the agent and its tools with no verdict at all — the same defect the
|
|
85
|
+
* launcher had one layer down ([/decisions/ad-101.md](/decisions/ad-101.md)).
|
|
86
|
+
*
|
|
87
|
+
* invariant: the diagnosis still reaches stderr. Failing open is not failing silently.
|
|
88
|
+
*/
|
|
82
89
|
console.error(`tlc shim: handler not found: ${handler}`);
|
|
83
|
-
process.
|
|
90
|
+
process.stdout.write("{}");
|
|
91
|
+
process.exit(0);
|
|
84
92
|
}
|
package/src/entrypoints/stop.ts
CHANGED
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
import { flagsDir } from "../platform/paths.ts";
|
|
14
14
|
import type { Handler, HandlerContext } from "./run.ts";
|
|
15
15
|
import { main } from "./run.ts";
|
|
16
|
-
import { formatLessonsBlock, obsConfigFor, sessionIdFromKey } from "./support.ts";
|
|
16
|
+
import { currentGitSha, formatLessonsBlock, obsConfigFor, sessionIdFromKey } from "./support.ts";
|
|
17
17
|
|
|
18
18
|
const STAGNATION_FOLLOWUP = [
|
|
19
19
|
"BLOCKED: identical validation fingerprint repeated — no progress between attempts.",
|
|
@@ -21,6 +21,31 @@ const STAGNATION_FOLLOWUP = [
|
|
|
21
21
|
"NEED: change approach. Do not repeat the same fix. Inspect root cause, try a different path, or escalate with BLOCKED/TRIED/NEED.",
|
|
22
22
|
].join("\n");
|
|
23
23
|
|
|
24
|
+
/**
|
|
25
|
+
* A gate is the one proof the harness decides rather than witnesses, so it is recorded where every gate already
|
|
26
|
+
* funnels through — the same argument `recordGateOutcome` makes for itself: a gate added later cannot be
|
|
27
|
+
* forgotten ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
28
|
+
*
|
|
29
|
+
* invariant: only a gate that passed. Recording a failure would let "the gate ran" satisfy a rule that asked for
|
|
30
|
+
* "the gate passed", which is the whole point of asking.
|
|
31
|
+
*/
|
|
32
|
+
async function observeGateForRules(args: {
|
|
33
|
+
root: string;
|
|
34
|
+
sessionKey: string;
|
|
35
|
+
policy: Policy;
|
|
36
|
+
gate: string;
|
|
37
|
+
passed: boolean;
|
|
38
|
+
}): Promise<void> {
|
|
39
|
+
if (!args.passed || !coreFacade.rules.wantsGate(args.root, args.policy.rules)) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
coreFacade.rules.observeGate(args.root, args.policy.rules, args.gate, {
|
|
43
|
+
sha: await currentGitSha(args.root),
|
|
44
|
+
sessionKey: args.sessionKey,
|
|
45
|
+
at: new Date().toISOString(),
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
24
49
|
/**
|
|
25
50
|
* hazard: `gate.outcome` was consumed in two places — the rollup counter and the session report's
|
|
26
51
|
* "Gates pass/fail" line — and emitted by nothing. Both read structurally zero, so the report printed a
|
|
@@ -200,6 +225,7 @@ async function runLockedGate(args: {
|
|
|
200
225
|
|
|
201
226
|
// invariant: recorded outside the lock. A measurement must not widen the window in which one gate blocks another.
|
|
202
227
|
recordGateOutcome({ ...args, artifact, reused: cached !== null });
|
|
228
|
+
await observeGateForRules({ ...args, gate: args.gate, passed: artifact.passed });
|
|
203
229
|
await creditPendingLessons({
|
|
204
230
|
root: args.root,
|
|
205
231
|
provider: args.provider,
|
|
@@ -387,6 +413,28 @@ async function failGate(args: {
|
|
|
387
413
|
return { kind: "continue", text: parts.join("\n") };
|
|
388
414
|
}
|
|
389
415
|
|
|
416
|
+
/**
|
|
417
|
+
* why the sha is fetched twice-or-never: `git rev-parse` is a process spawn and this runs on every stop. The first
|
|
418
|
+
* pass answers whether any `on: stop` rule fired at all, which costs two directory reads; only then is a sha worth
|
|
419
|
+
* a process, and the second pass is the one whose verdict counts. Same shape as the action-time rail
|
|
420
|
+
* ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
421
|
+
*/
|
|
422
|
+
async function decideStopRules(
|
|
423
|
+
root: string,
|
|
424
|
+
policy: Policy,
|
|
425
|
+
sessionKey: string,
|
|
426
|
+
): Promise<ReturnType<typeof coreFacade.rules.decideStop>> {
|
|
427
|
+
const context = { sessionKey, mode: policy.mode };
|
|
428
|
+
const dryRun = coreFacade.rules.decideStop(root, policy.rules, { ...context, sha: null });
|
|
429
|
+
if (dryRun.outcomes.length === 0) {
|
|
430
|
+
return dryRun;
|
|
431
|
+
}
|
|
432
|
+
return coreFacade.rules.decideStop(root, policy.rules, {
|
|
433
|
+
...context,
|
|
434
|
+
sha: await currentGitSha(root),
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
|
|
390
438
|
export const stopHandler: Handler = async (event: HarnessEvent, ctx: HandlerContext): Promise<Decision> => {
|
|
391
439
|
const { policy, capabilities } = ctx;
|
|
392
440
|
const root = event.projectDir;
|
|
@@ -829,6 +877,32 @@ export const stopHandler: Handler = async (event: HarnessEvent, ctx: HandlerCont
|
|
|
829
877
|
});
|
|
830
878
|
}
|
|
831
879
|
|
|
880
|
+
/**
|
|
881
|
+
* The operator's own bar, last among the blockers so every gate the harness owns has already spoken — and after
|
|
882
|
+
* the gates specifically, because a rule asking for `gate(lint) since HEAD` can only be satisfied once lint has
|
|
883
|
+
* run this turn ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
884
|
+
*
|
|
885
|
+
* invariant: `warn` returns `context`, which does not block. Everything else refuses the stop, so a rule the
|
|
886
|
+
* operator wrote cannot be ended past.
|
|
887
|
+
*/
|
|
888
|
+
const stopRules = await decideStopRules(root, policy, sessionKey);
|
|
889
|
+
if (stopRules.decision.kind !== "abstain") {
|
|
890
|
+
if (stopRules.decision.kind !== "context") {
|
|
891
|
+
const worst = coreFacade.rules.strictest(stopRules.outcomes);
|
|
892
|
+
await coreFacade.handoff.patchHandoff(root, provider, {
|
|
893
|
+
slice: {
|
|
894
|
+
last_gate_result: "fail",
|
|
895
|
+
last_failure_category: "policy",
|
|
896
|
+
blockers: `Rule ${worst?.rule.name ?? "unknown"} is not satisfied for this HEAD.`,
|
|
897
|
+
next_action: worst?.missing.length
|
|
898
|
+
? `Produce ${worst.missing.join(", ")}, then stop again.`
|
|
899
|
+
: "Satisfy the rule named in the follow-up, then stop again.",
|
|
900
|
+
},
|
|
901
|
+
});
|
|
902
|
+
}
|
|
903
|
+
return stopRules.decision;
|
|
904
|
+
}
|
|
905
|
+
|
|
832
906
|
// why: the pairing of a failure with what resolved it is captured here, immediately before the record that
|
|
833
907
|
// holds the failure identity is cleared. This is the one moment both halves exist
|
|
834
908
|
// ([/decisions/ad-028.md](/decisions/ad-028.md)).
|
|
@@ -2,10 +2,18 @@ import type { Decision, HarnessEvent } from "../contracts/index.ts";
|
|
|
2
2
|
import { coreFacade } from "../core/index.ts";
|
|
3
3
|
import type { Handler, HandlerContext } from "./run.ts";
|
|
4
4
|
import { main } from "./run.ts";
|
|
5
|
+
import { observeForRules } from "./support.ts";
|
|
5
6
|
|
|
6
7
|
// why: no legacy predecessor covers subagent.stop verification — this reuses the same unfinished-work
|
|
7
8
|
// signal (blockers/pending/in_progress/previous_gaps) already carried on the handoff slice.
|
|
8
|
-
export const subagentStopHandler: Handler = (
|
|
9
|
+
export const subagentStopHandler: Handler = async (
|
|
10
|
+
event: HarnessEvent,
|
|
11
|
+
ctx: HandlerContext,
|
|
12
|
+
): Promise<Decision> => {
|
|
13
|
+
// why before the verdict: this records that a subagent of this type finished, which is the proof an operator
|
|
14
|
+
// rule asks for. It cannot change the decision below ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
15
|
+
await observeForRules(event, ctx);
|
|
16
|
+
|
|
9
17
|
const handoff = coreFacade.handoff.readHandoff(event.projectDir, event.provider);
|
|
10
18
|
const unfinishedWork =
|
|
11
19
|
Boolean(handoff.blockers) ||
|
|
@@ -180,3 +180,35 @@ export function formatLessonsBlock(lessons: HarnessLesson[], title: string, omit
|
|
|
180
180
|
}
|
|
181
181
|
return lines.join("\n");
|
|
182
182
|
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* The producer half of the feature: what the harness witnessed, written where only the harness can write it.
|
|
186
|
+
*
|
|
187
|
+
* hazard: this did not exist in the first cut. `observe` had no caller, so the store was never written, no proof
|
|
188
|
+
* could ever be satisfied, and every rule that parsed denied for ever — `require:` is mandatory, so that was
|
|
189
|
+
* every rule ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
190
|
+
*
|
|
191
|
+
* why `wants` first: this runs on every tool call and the sha is a process spawn. Nothing is asked of git unless
|
|
192
|
+
* a declared rule requires this kind of proof, so an operator whose only rule wants a subagent pays no git on any
|
|
193
|
+
* command.
|
|
194
|
+
*
|
|
195
|
+
* invariant: after the event, never able to change it. A rail that records what happened must not become a rail
|
|
196
|
+
* that decides whether it may.
|
|
197
|
+
*/
|
|
198
|
+
export async function observeForRules(
|
|
199
|
+
event: HarnessEvent,
|
|
200
|
+
// why the shape and not `HandlerContext`: `run.ts` already imports this module, so naming its type here would
|
|
201
|
+
// close an import cycle. Only the one field is needed.
|
|
202
|
+
ctx: { policy: { rules: Policy["rules"] } },
|
|
203
|
+
): Promise<void> {
|
|
204
|
+
const config = ctx.policy.rules;
|
|
205
|
+
if (!coreFacade.rules.wants(event.projectDir, config, event)) {
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
const sha = await currentGitSha(event.projectDir);
|
|
209
|
+
coreFacade.rules.observe(event.projectDir, config, event, {
|
|
210
|
+
sha,
|
|
211
|
+
sessionKey: event.sessionKey,
|
|
212
|
+
at: new Date().toISOString(),
|
|
213
|
+
});
|
|
214
|
+
}
|
|
@@ -4,7 +4,7 @@ import { estimateCostUsd, mapPoolToNeutral } from "../platform/pricing.ts";
|
|
|
4
4
|
import { readClaudeUsage } from "../providers/index.ts";
|
|
5
5
|
import type { Handler, HandlerContext } from "./run.ts";
|
|
6
6
|
import { main } from "./run.ts";
|
|
7
|
-
import { OBS_CONFIG_AUDIT, obsConfigFor } from "./support.ts";
|
|
7
|
+
import { OBS_CONFIG_AUDIT, obsConfigFor, observeForRules } from "./support.ts";
|
|
8
8
|
|
|
9
9
|
const OBS_KIND_BY_EVENT: Partial<Record<HarnessEventKind, ObsKind>> = {
|
|
10
10
|
"tool.after": "tool.end",
|
|
@@ -48,7 +48,12 @@ function usageGenAi(event: HarnessEvent, ctx: HandlerContext): Record<string, un
|
|
|
48
48
|
};
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
export const toolAfterHandler: Handler = (event: HarnessEvent, ctx: HandlerContext) => {
|
|
51
|
+
export const toolAfterHandler: Handler = async (event: HarnessEvent, ctx: HandlerContext) => {
|
|
52
|
+
// why here and not at `*.before`: arriving on an after-event is what says the tool ran and did not fail. A
|
|
53
|
+
// failure comes as `tool.failure`, a different event this rail never sees, and the payload carries no exit code
|
|
54
|
+
// in any of the three shapes the two hosts send ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
55
|
+
await observeForRules(event, ctx);
|
|
56
|
+
|
|
52
57
|
coreFacade.observability.recordAudit(event.projectDir, event.event, event.raw, ctx.policy.obs.globalSpool);
|
|
53
58
|
|
|
54
59
|
const kind = OBS_KIND_BY_EVENT[event.event];
|
|
@@ -2,7 +2,7 @@ import type { Decision, HarnessEvent } from "../contracts/index.ts";
|
|
|
2
2
|
import { coreFacade } from "../core/index.ts";
|
|
3
3
|
import type { Handler, HandlerContext } from "./run.ts";
|
|
4
4
|
import { main } from "./run.ts";
|
|
5
|
-
import { obsConfigFor, readModelFromToolInput, subagentSpawnInput } from "./support.ts";
|
|
5
|
+
import { currentGitSha, obsConfigFor, readModelFromToolInput, subagentSpawnInput } from "./support.ts";
|
|
6
6
|
|
|
7
7
|
const READONLY_BLOCKED_TOOLS = new Set(["Write", "Delete", "Shell"]);
|
|
8
8
|
|
|
@@ -43,6 +43,33 @@ function recordShellDecisionIfShell(event: HarnessEvent, ctx: HandlerContext, de
|
|
|
43
43
|
}
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
+
/**
|
|
47
|
+
* why the sha is read here and not in `run.ts`: `git rev-parse` is a process spawn, and this fires on every tool
|
|
48
|
+
* call. It is asked for only once a rule has actually fired, so an operator who declared nothing pays nothing
|
|
49
|
+
* ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
50
|
+
*/
|
|
51
|
+
async function rulesDecision(event: HarnessEvent, ctx: HandlerContext): Promise<Decision> {
|
|
52
|
+
const config = ctx.policy.rules;
|
|
53
|
+
const trigger = { event: event.event, toolName: event.toolName, command: event.command };
|
|
54
|
+
const dryRun = coreFacade.rules.decideAction(event.projectDir, config, trigger, {
|
|
55
|
+
sha: null,
|
|
56
|
+
sessionKey: event.sessionKey,
|
|
57
|
+
mode: ctx.policy.mode,
|
|
58
|
+
});
|
|
59
|
+
if (dryRun.outcomes.length === 0) {
|
|
60
|
+
return { kind: "abstain" };
|
|
61
|
+
}
|
|
62
|
+
// why twice: the first pass answers whether any rule fired at all, which costs no git. Only then is the sha
|
|
63
|
+
// worth a process, and the second pass is the one whose verdict counts.
|
|
64
|
+
const sha = await currentGitSha(event.projectDir);
|
|
65
|
+
const verdict = coreFacade.rules.decideAction(event.projectDir, config, trigger, {
|
|
66
|
+
sha,
|
|
67
|
+
sessionKey: event.sessionKey,
|
|
68
|
+
mode: ctx.policy.mode,
|
|
69
|
+
});
|
|
70
|
+
return verdict.decision;
|
|
71
|
+
}
|
|
72
|
+
|
|
46
73
|
function handleShellBefore(event: HarnessEvent, ctx: HandlerContext): Decision {
|
|
47
74
|
const { policy } = ctx;
|
|
48
75
|
const decision = coreFacade.shellPolicy.evaluateShellCommand({
|
|
@@ -108,10 +135,10 @@ async function handleToolBefore(event: HarnessEvent, ctx: HandlerContext): Promi
|
|
|
108
135
|
return { kind: "allow" };
|
|
109
136
|
}
|
|
110
137
|
|
|
111
|
-
export const toolBeforeHandler: Handler = (
|
|
138
|
+
export const toolBeforeHandler: Handler = async (
|
|
112
139
|
event: HarnessEvent,
|
|
113
140
|
ctx: HandlerContext,
|
|
114
|
-
):
|
|
141
|
+
): Promise<Decision> => {
|
|
115
142
|
// invariant: the floor runs first and reads no policy, so no config value and no agent edit can
|
|
116
143
|
// reach a decision before it.
|
|
117
144
|
const floor = coreFacade.floor.evaluateFloor({
|
|
@@ -159,6 +186,20 @@ export const toolBeforeHandler: Handler = (
|
|
|
159
186
|
}
|
|
160
187
|
}
|
|
161
188
|
|
|
189
|
+
/**
|
|
190
|
+
* The operator's own rules, after the floor and after the integrity check, because both are unconditional and a
|
|
191
|
+
* rail comes second ([/decisions/ad-077.md](/decisions/ad-077.md)).
|
|
192
|
+
*
|
|
193
|
+
* invariant: `deny` and `ask` answer here; `follow-up` and `warn` abstain and are the stop rail's business. With
|
|
194
|
+
* the capability off, or with no rule files, `decideAction` reads two directory entries and abstains — which is
|
|
195
|
+
* what keeps a machine that never opted in byte-identical to before ([/decisions/ad-100.md](/decisions/ad-100.md)).
|
|
196
|
+
*/
|
|
197
|
+
const rulesVerdict = await rulesDecision(event, ctx);
|
|
198
|
+
if (rulesVerdict.kind !== "abstain") {
|
|
199
|
+
recordShellDecisionIfShell(event, ctx, rulesVerdict);
|
|
200
|
+
return rulesVerdict;
|
|
201
|
+
}
|
|
202
|
+
|
|
162
203
|
switch (event.event) {
|
|
163
204
|
case "shell.before":
|
|
164
205
|
return handleShellBefore(event, ctx);
|