@deftai/directive-core 0.77.0 → 0.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/doctor/main.d.ts +1 -0
- package/dist/doctor/main.js +37 -0
- package/dist/doctor/types.d.ts +3 -0
- package/dist/eval/triggers.d.ts +76 -0
- package/dist/eval/triggers.js +259 -0
- package/dist/eval-triggers-relocation/evaluate.d.ts +36 -0
- package/dist/eval-triggers-relocation/evaluate.js +115 -0
- package/dist/eval-triggers-relocation/index.d.ts +2 -0
- package/dist/eval-triggers-relocation/index.js +2 -0
- package/dist/hooks/dispatcher.d.ts +43 -0
- package/dist/hooks/dispatcher.js +171 -0
- package/dist/hooks/index.d.ts +3 -0
- package/dist/hooks/index.js +3 -0
- package/dist/hooks/scope.d.ts +12 -0
- package/dist/hooks/scope.js +46 -0
- package/dist/hooks/tools.d.ts +4 -0
- package/dist/hooks/tools.js +23 -0
- package/dist/init-deposit/agent-hooks.d.ts +22 -0
- package/dist/init-deposit/agent-hooks.js +228 -0
- package/dist/init-deposit/hygiene.js +3 -0
- package/dist/init-deposit/index.d.ts +1 -0
- package/dist/init-deposit/index.js +1 -0
- package/dist/init-deposit/init-deposit.js +2 -0
- package/dist/init-deposit/refresh.js +2 -0
- package/dist/intake/reconcile-issues.d.ts +1 -0
- package/dist/intake/reconcile-issues.js +51 -49
- package/dist/release/build-dist.js +1 -0
- package/dist/release-e2e/greenfield-python-free-smoke-cli.js +40 -1
- package/dist/release-e2e/greenfield-python-free-smoke.d.ts +2 -0
- package/dist/release-e2e/greenfield-python-free-smoke.js +28 -8
- package/dist/session/verify-session-ritual.d.ts +16 -0
- package/dist/session/verify-session-ritual.js +63 -0
- package/dist/triage/help/registry-data.d.ts +12 -1
- package/dist/triage/help/registry-data.js +22 -1
- package/dist/verify-env/agent-hooks.d.ts +11 -0
- package/dist/verify-env/agent-hooks.js +45 -0
- package/dist/verify-env/command-spawn.d.ts +5 -0
- package/dist/verify-env/command-spawn.js +28 -9
- package/dist/verify-env/index.d.ts +1 -0
- package/dist/verify-env/index.js +1 -0
- package/package.json +15 -3
package/dist/doctor/main.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { type ResolutionFacts } from "../resolution/index.js";
|
|
|
2
2
|
import { createPlainSink } from "./output.js";
|
|
3
3
|
import type { DoctorSeams, Finding, ResolutionSummary } from "./types.js";
|
|
4
4
|
export declare function cmdDoctor(args: readonly string[], seams?: DoctorSeams): number;
|
|
5
|
+
export declare function runAgentHooksHealthCheck(projectRoot: string, consumerContext: boolean, sink: ReturnType<typeof createPlainSink>, addFinding: (finding: Finding) => void, seams: DoctorSeams): void;
|
|
5
6
|
/**
|
|
6
7
|
* Never emit a bare `task ...` remediation in a project without Taskfile wiring
|
|
7
8
|
* (#2267). The `directive` surface always works; `task deft:X` is optional and
|
package/dist/doctor/main.js
CHANGED
|
@@ -6,6 +6,7 @@ import { describeShadowedPlanExtension, detectShadowedPlanExtensions, } from "..
|
|
|
6
6
|
import { loadProjectDefinition } from "../policy/resolve.js";
|
|
7
7
|
import { checkLocalEngineIntegrity, classify, detectPackageManager, evaluateSkew, reconcileVersions, plan as resolvePlan, } from "../resolution/index.js";
|
|
8
8
|
import { resolveUserMdPath } from "../user-config/resolve-user-md.js";
|
|
9
|
+
import { evaluateAgentHooks } from "../verify-env/agent-hooks.js";
|
|
9
10
|
import { agentsRefreshPlan, hasV3ManagedMarker } from "./agents-md.js";
|
|
10
11
|
import { runChecks } from "./checks.js";
|
|
11
12
|
import { CONSUMER_FRAMEWORK_DIRS, EXPECTED_CONTENT_DIRS, EXPECTED_FRAMEWORK_DIRS, NETWORK_DISCLOSURE_LINE, PAYLOAD_STALENESS_OFFLINE_SKIP_MESSAGE, TASKFILE_INCLUDE_SNIPPET, UV_INSTALL_URL, } from "./constants.js";
|
|
@@ -162,6 +163,11 @@ export function cmdDoctor(args, seams = {}) {
|
|
|
162
163
|
if (!jsonMode) {
|
|
163
164
|
sink.blank();
|
|
164
165
|
}
|
|
166
|
+
sink.info("Checking agent-host hook registration...");
|
|
167
|
+
runAgentHooksHealthCheck(projectRoot, consumerContext, sink, addFinding, seams);
|
|
168
|
+
if (!jsonMode) {
|
|
169
|
+
sink.blank();
|
|
170
|
+
}
|
|
165
171
|
// #2182: payload-staleness is the only doctor check that can reach a
|
|
166
172
|
// registry (git verifies the pin; npm compares stable release availability).
|
|
167
173
|
// It stays in the OFFLINE
|
|
@@ -332,6 +338,37 @@ export function cmdDoctor(args, seams = {}) {
|
|
|
332
338
|
sink.finalWarn(`System check completed with ${warningCount} warning(s).`);
|
|
333
339
|
return 0;
|
|
334
340
|
}
|
|
341
|
+
export function runAgentHooksHealthCheck(projectRoot, consumerContext, sink, addFinding, seams) {
|
|
342
|
+
const checkName = "agent-hooks-registration";
|
|
343
|
+
if (!consumerContext) {
|
|
344
|
+
const reason = "maintainer source checkout; project hook deposit is consumer-only";
|
|
345
|
+
sink.info(`${checkName}: skip -- ${reason}`);
|
|
346
|
+
addFinding({ severity: "skip", message: reason, check: checkName, status: "skip" });
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
try {
|
|
350
|
+
const result = (seams.evaluateAgentHooks ?? evaluateAgentHooks)(projectRoot);
|
|
351
|
+
if (result.code === 0) {
|
|
352
|
+
sink.success(`${checkName}: healthy`);
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
const message = `${checkName}: ${result.message.replace(/\s+/g, " ").trim()}`;
|
|
356
|
+
sink.warn(message);
|
|
357
|
+
addFinding({
|
|
358
|
+
severity: "warning",
|
|
359
|
+
message,
|
|
360
|
+
check: checkName,
|
|
361
|
+
status: result.code === 2 ? "unavailable" : "non-functional",
|
|
362
|
+
registrations: result.registrations,
|
|
363
|
+
suggestion: "deft update",
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
catch (cause) {
|
|
367
|
+
const message = `${checkName}: probe failed -- ${String(cause)}`;
|
|
368
|
+
sink.warn(message);
|
|
369
|
+
addFinding({ severity: "warning", message, check: checkName, suggestion: "deft update" });
|
|
370
|
+
}
|
|
371
|
+
}
|
|
335
372
|
function runInstallIntegrityChecks(projectRoot, sink, addFinding, seams) {
|
|
336
373
|
if (runningInsideDeftRepo(projectRoot, seams)) {
|
|
337
374
|
sink.info("Skipping install-integrity checks -- running inside the deft framework repo (no install manifest in the source checkout).");
|
package/dist/doctor/types.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { ShadowedPlanExtension } from "../policy/plan-extensions.js";
|
|
|
3
3
|
import type { EngineProbeResult } from "../resolution/classify.js";
|
|
4
4
|
import type { ResolutionMode } from "../resolution/index.js";
|
|
5
5
|
import type { ResolveUserMdResult } from "../user-config/resolve-user-md.js";
|
|
6
|
+
import type { AgentHookHealthResult } from "../verify-env/agent-hooks.js";
|
|
6
7
|
export declare const EXIT_CLEAN = 0;
|
|
7
8
|
export declare const EXIT_DRIFT = 1;
|
|
8
9
|
export declare const EXIT_CONFIG_ERROR = 2;
|
|
@@ -135,5 +136,7 @@ export interface DoctorSeams {
|
|
|
135
136
|
* its `plan` object; returns [] when no project definition is present.
|
|
136
137
|
*/
|
|
137
138
|
readonly detectPlanExtensionShadows?: (projectRoot: string) => readonly ShadowedPlanExtension[];
|
|
139
|
+
/** Read-only agent-host hook registration probe (#2438). */
|
|
140
|
+
readonly evaluateAgentHooks?: (projectRoot: string) => AgentHookHealthResult;
|
|
138
141
|
}
|
|
139
142
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/** Relative path to skill-pi-trigger-eval compatible case rows (#1586). */
|
|
2
|
+
export declare const TRIGGER_CASES_REL = "evals/trigger-cases.jsonl";
|
|
3
|
+
/** Default Skills Index source for AGENTS.md skill routing (#1586). */
|
|
4
|
+
export declare const SKILLS_INDEX_REL = "REFERENCES.md";
|
|
5
|
+
/** Parsed Skills Index row. */
|
|
6
|
+
export interface SkillsIndexEntry {
|
|
7
|
+
readonly skillId: string;
|
|
8
|
+
readonly triggers: readonly string[];
|
|
9
|
+
}
|
|
10
|
+
/** One row in evals/trigger-cases.jsonl (skill-pi-trigger-eval eval-set + skill id). */
|
|
11
|
+
export interface TriggerCaseRow {
|
|
12
|
+
readonly id: string;
|
|
13
|
+
readonly query: string;
|
|
14
|
+
readonly skill: string;
|
|
15
|
+
readonly should_trigger: boolean;
|
|
16
|
+
}
|
|
17
|
+
/** Outcome for a single trigger case. */
|
|
18
|
+
export interface TriggerCaseResult {
|
|
19
|
+
readonly id: string;
|
|
20
|
+
readonly query: string;
|
|
21
|
+
readonly skill: string;
|
|
22
|
+
readonly should_trigger: boolean;
|
|
23
|
+
readonly actual: boolean;
|
|
24
|
+
readonly pass: boolean;
|
|
25
|
+
readonly matchedTrigger?: string;
|
|
26
|
+
readonly winnerSkill?: string;
|
|
27
|
+
}
|
|
28
|
+
/** Aggregate eval:triggers report. */
|
|
29
|
+
export interface TriggerEvalReport {
|
|
30
|
+
readonly total: number;
|
|
31
|
+
readonly passed: number;
|
|
32
|
+
readonly failed: number;
|
|
33
|
+
readonly pass_rate: number | null;
|
|
34
|
+
readonly results: readonly TriggerCaseResult[];
|
|
35
|
+
readonly coverageErrors: readonly string[];
|
|
36
|
+
}
|
|
37
|
+
export interface RunTriggerEvalOptions {
|
|
38
|
+
readonly projectRoot?: string;
|
|
39
|
+
readonly casesPath?: string;
|
|
40
|
+
readonly indexPath?: string;
|
|
41
|
+
readonly cases?: readonly TriggerCaseRow[];
|
|
42
|
+
readonly indexText?: string;
|
|
43
|
+
}
|
|
44
|
+
export interface RunTriggerEvalResult {
|
|
45
|
+
readonly code: 0 | 1 | 2;
|
|
46
|
+
readonly report: TriggerEvalReport | null;
|
|
47
|
+
readonly message: string;
|
|
48
|
+
}
|
|
49
|
+
/** Normalize user text for deterministic substring trigger matching. */
|
|
50
|
+
export declare function normalizeTriggerText(text: string): string;
|
|
51
|
+
/** Collapse embedded newlines before interpolating user/data into log lines. */
|
|
52
|
+
export declare function sanitizeTriggerLogLine(text: string): string;
|
|
53
|
+
/** Extract backtick trigger phrases from a Skills Index triggers cell. */
|
|
54
|
+
export declare function parseTriggerCell(cell: string): string[];
|
|
55
|
+
/** Parse the Skills Index markdown table in REFERENCES.md. */
|
|
56
|
+
export declare function parseSkillsIndex(markdown: string): SkillsIndexEntry[];
|
|
57
|
+
interface TriggerMatch {
|
|
58
|
+
readonly skillId: string;
|
|
59
|
+
readonly trigger: string;
|
|
60
|
+
}
|
|
61
|
+
/** Find the winning skill for a query using longest-trigger-wins disambiguation. */
|
|
62
|
+
export declare function resolveTriggerWinner(query: string, index: readonly SkillsIndexEntry[]): TriggerMatch | null;
|
|
63
|
+
/** True when the Skills Index routing would send *query* to *skillId*. */
|
|
64
|
+
export declare function wouldRouteToSkill(query: string, skillId: string, index: readonly SkillsIndexEntry[]): boolean;
|
|
65
|
+
/** Validate one JSONL row against the skill-pi-trigger-eval row contract. */
|
|
66
|
+
export declare function parseTriggerCaseRow(raw: unknown, lineNumber: number): TriggerCaseRow | string;
|
|
67
|
+
/** Load evals/trigger-cases.jsonl. */
|
|
68
|
+
export declare function loadTriggerCases(path: string): TriggerCaseRow[] | {
|
|
69
|
+
error: string;
|
|
70
|
+
};
|
|
71
|
+
/** Verify at least two positive and one negative case per indexed skill. */
|
|
72
|
+
export declare function validateTriggerCoverage(cases: readonly TriggerCaseRow[], index: readonly SkillsIndexEntry[]): string[];
|
|
73
|
+
/** Run deterministic trigger routing eval (offline skill-pi-trigger-eval counterpart). */
|
|
74
|
+
export declare function runTriggerEval(options?: RunTriggerEvalOptions): RunTriggerEvalResult;
|
|
75
|
+
export {};
|
|
76
|
+
//# sourceMappingURL=triggers.d.ts.map
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
/** Relative path to skill-pi-trigger-eval compatible case rows (#1586). */
|
|
4
|
+
export const TRIGGER_CASES_REL = "evals/trigger-cases.jsonl";
|
|
5
|
+
/** Default Skills Index source for AGENTS.md skill routing (#1586). */
|
|
6
|
+
export const SKILLS_INDEX_REL = "REFERENCES.md";
|
|
7
|
+
/** Normalize user text for deterministic substring trigger matching. */
|
|
8
|
+
export function normalizeTriggerText(text) {
|
|
9
|
+
return text.toLowerCase().replace(/\s+/g, " ").trim();
|
|
10
|
+
}
|
|
11
|
+
/** Collapse embedded newlines before interpolating user/data into log lines. */
|
|
12
|
+
export function sanitizeTriggerLogLine(text) {
|
|
13
|
+
return text.replace(/\r?\n/g, " ").trim();
|
|
14
|
+
}
|
|
15
|
+
/** Extract backtick trigger phrases from a Skills Index triggers cell. */
|
|
16
|
+
export function parseTriggerCell(cell) {
|
|
17
|
+
const triggers = [];
|
|
18
|
+
for (const match of cell.matchAll(/`([^`]+)`/g)) {
|
|
19
|
+
const phrase = normalizeTriggerText(match[1] ?? "");
|
|
20
|
+
if (phrase.length > 0) {
|
|
21
|
+
triggers.push(phrase);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return triggers;
|
|
25
|
+
}
|
|
26
|
+
/** Parse the Skills Index markdown table in REFERENCES.md. */
|
|
27
|
+
export function parseSkillsIndex(markdown) {
|
|
28
|
+
const entries = [];
|
|
29
|
+
for (const line of markdown.split(/\r?\n/)) {
|
|
30
|
+
if (!line.startsWith("| [deft-directive-")) {
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
const columns = line.split("|").map((col) => col.trim());
|
|
34
|
+
if (columns.length < 5) {
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
const linkCell = columns[1] ?? "";
|
|
38
|
+
const triggersCell = columns[3] ?? "";
|
|
39
|
+
// Avoid nested character-class regex (CodeQL js/polynomial-redos on \[([^\]]+)\]).
|
|
40
|
+
const open = linkCell.indexOf("[");
|
|
41
|
+
const close = open === -1 ? -1 : linkCell.indexOf("]", open + 1);
|
|
42
|
+
if (open === -1 || close === -1) {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const skillId = linkCell.slice(open + 1, close).trim();
|
|
46
|
+
if (skillId.length === 0) {
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const triggers = parseTriggerCell(triggersCell);
|
|
50
|
+
if (triggers.length === 0) {
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
entries.push({ skillId, triggers });
|
|
54
|
+
}
|
|
55
|
+
return entries;
|
|
56
|
+
}
|
|
57
|
+
/** Find the winning skill for a query using longest-trigger-wins disambiguation. */
|
|
58
|
+
export function resolveTriggerWinner(query, index) {
|
|
59
|
+
const normalized = normalizeTriggerText(query);
|
|
60
|
+
let best = null;
|
|
61
|
+
for (const entry of index) {
|
|
62
|
+
for (const trigger of entry.triggers) {
|
|
63
|
+
if (!normalized.includes(trigger)) {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (best === null ||
|
|
67
|
+
trigger.length > best.trigger.length ||
|
|
68
|
+
(trigger.length === best.trigger.length && entry.skillId.localeCompare(best.skillId) < 0)) {
|
|
69
|
+
best = { skillId: entry.skillId, trigger };
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return best;
|
|
74
|
+
}
|
|
75
|
+
/** True when the Skills Index routing would send *query* to *skillId*. */
|
|
76
|
+
export function wouldRouteToSkill(query, skillId, index) {
|
|
77
|
+
const winner = resolveTriggerWinner(query, index);
|
|
78
|
+
return winner !== null && winner.skillId === skillId;
|
|
79
|
+
}
|
|
80
|
+
/** Validate one JSONL row against the skill-pi-trigger-eval row contract. */
|
|
81
|
+
export function parseTriggerCaseRow(raw, lineNumber) {
|
|
82
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
|
|
83
|
+
return `line ${lineNumber}: expected a JSON object`;
|
|
84
|
+
}
|
|
85
|
+
const record = raw;
|
|
86
|
+
const id = record.id;
|
|
87
|
+
const query = record.query;
|
|
88
|
+
const skill = record.skill;
|
|
89
|
+
const shouldTrigger = record.should_trigger;
|
|
90
|
+
if (typeof id !== "string" || id.trim().length === 0) {
|
|
91
|
+
return `line ${lineNumber}: id must be a non-empty string`;
|
|
92
|
+
}
|
|
93
|
+
if (typeof query !== "string" || query.trim().length === 0) {
|
|
94
|
+
return `line ${lineNumber}: query must be a non-empty string`;
|
|
95
|
+
}
|
|
96
|
+
if (typeof skill !== "string" || !skill.startsWith("deft-directive-")) {
|
|
97
|
+
return `line ${lineNumber}: skill must be a deft-directive-* id`;
|
|
98
|
+
}
|
|
99
|
+
if (typeof shouldTrigger !== "boolean") {
|
|
100
|
+
return `line ${lineNumber}: should_trigger must be true or false`;
|
|
101
|
+
}
|
|
102
|
+
return {
|
|
103
|
+
id: id.trim(),
|
|
104
|
+
query: query.trim(),
|
|
105
|
+
skill: skill.trim(),
|
|
106
|
+
should_trigger: shouldTrigger,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
/** Load evals/trigger-cases.jsonl. */
|
|
110
|
+
export function loadTriggerCases(path) {
|
|
111
|
+
if (!existsSync(path)) {
|
|
112
|
+
return { error: `missing trigger cases at ${path}` };
|
|
113
|
+
}
|
|
114
|
+
const text = readFileSync(path, "utf8");
|
|
115
|
+
const rows = [];
|
|
116
|
+
const lines = text.split(/\r?\n/);
|
|
117
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
118
|
+
const line = lines[i]?.trim() ?? "";
|
|
119
|
+
if (line.length === 0 || line.startsWith("#")) {
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
let parsed;
|
|
123
|
+
try {
|
|
124
|
+
parsed = JSON.parse(line);
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
return { error: `line ${i + 1}: invalid JSON` };
|
|
128
|
+
}
|
|
129
|
+
const row = parseTriggerCaseRow(parsed, i + 1);
|
|
130
|
+
if (typeof row === "string") {
|
|
131
|
+
return { error: row };
|
|
132
|
+
}
|
|
133
|
+
rows.push(row);
|
|
134
|
+
}
|
|
135
|
+
if (rows.length === 0) {
|
|
136
|
+
return { error: "trigger-cases.jsonl has no cases" };
|
|
137
|
+
}
|
|
138
|
+
return rows;
|
|
139
|
+
}
|
|
140
|
+
/** Verify at least two positive and one negative case per indexed skill. */
|
|
141
|
+
export function validateTriggerCoverage(cases, index) {
|
|
142
|
+
const errors = [];
|
|
143
|
+
const bySkill = new Map();
|
|
144
|
+
for (const entry of index) {
|
|
145
|
+
bySkill.set(entry.skillId, { positive: 0, negative: 0 });
|
|
146
|
+
}
|
|
147
|
+
for (const row of cases) {
|
|
148
|
+
const bucket = bySkill.get(row.skill);
|
|
149
|
+
if (bucket === undefined) {
|
|
150
|
+
errors.push(`case '${row.id}' references unknown skill '${row.skill}'`);
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
if (row.should_trigger) {
|
|
154
|
+
bucket.positive += 1;
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
bucket.negative += 1;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
for (const [skillId, counts] of bySkill) {
|
|
161
|
+
if (counts.positive < 2) {
|
|
162
|
+
errors.push(`${skillId}: need at least 2 positive trigger cases (have ${counts.positive})`);
|
|
163
|
+
}
|
|
164
|
+
if (counts.negative < 1) {
|
|
165
|
+
errors.push(`${skillId}: need at least 1 negative trigger case (have ${counts.negative})`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return errors;
|
|
169
|
+
}
|
|
170
|
+
/** Run deterministic trigger routing eval (offline skill-pi-trigger-eval counterpart). */
|
|
171
|
+
export function runTriggerEval(options = {}) {
|
|
172
|
+
const projectRoot = resolve(options.projectRoot ?? process.cwd());
|
|
173
|
+
const casesPath = options.casesPath ?? resolve(projectRoot, TRIGGER_CASES_REL);
|
|
174
|
+
const indexPath = options.indexPath ?? resolve(projectRoot, SKILLS_INDEX_REL);
|
|
175
|
+
let cases;
|
|
176
|
+
if (options.cases !== undefined) {
|
|
177
|
+
cases = [...options.cases];
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
const loaded = loadTriggerCases(casesPath);
|
|
181
|
+
if ("error" in loaded) {
|
|
182
|
+
return { code: 2, report: null, message: `eval:triggers: ${loaded.error}` };
|
|
183
|
+
}
|
|
184
|
+
cases = loaded;
|
|
185
|
+
}
|
|
186
|
+
let indexText;
|
|
187
|
+
if (options.indexText !== undefined) {
|
|
188
|
+
indexText = options.indexText;
|
|
189
|
+
}
|
|
190
|
+
else if (!existsSync(indexPath)) {
|
|
191
|
+
return { code: 2, report: null, message: `eval:triggers: missing ${SKILLS_INDEX_REL}` };
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
indexText = readFileSync(indexPath, "utf8");
|
|
195
|
+
}
|
|
196
|
+
const index = parseSkillsIndex(indexText);
|
|
197
|
+
if (index.length === 0) {
|
|
198
|
+
return {
|
|
199
|
+
code: 2,
|
|
200
|
+
report: null,
|
|
201
|
+
message: `eval:triggers: no Skills Index rows parsed from ${SKILLS_INDEX_REL}`,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
const coverageErrors = validateTriggerCoverage(cases, index);
|
|
205
|
+
const results = [];
|
|
206
|
+
for (const row of cases) {
|
|
207
|
+
const winner = resolveTriggerWinner(row.query, index);
|
|
208
|
+
const actual = wouldRouteToSkill(row.query, row.skill, index);
|
|
209
|
+
results.push({
|
|
210
|
+
id: row.id,
|
|
211
|
+
query: row.query,
|
|
212
|
+
skill: row.skill,
|
|
213
|
+
should_trigger: row.should_trigger,
|
|
214
|
+
actual,
|
|
215
|
+
pass: actual === row.should_trigger,
|
|
216
|
+
matchedTrigger: winner?.trigger,
|
|
217
|
+
winnerSkill: winner?.skillId,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
const passed = results.filter((result) => result.pass).length;
|
|
221
|
+
const failed = results.length - passed;
|
|
222
|
+
const report = {
|
|
223
|
+
total: results.length,
|
|
224
|
+
passed,
|
|
225
|
+
failed,
|
|
226
|
+
pass_rate: results.length > 0 ? passed / results.length : null,
|
|
227
|
+
results,
|
|
228
|
+
coverageErrors,
|
|
229
|
+
};
|
|
230
|
+
if (coverageErrors.length > 0) {
|
|
231
|
+
const detail = coverageErrors.map((err) => ` - ${sanitizeTriggerLogLine(err)}`).join("\n");
|
|
232
|
+
return {
|
|
233
|
+
code: 1,
|
|
234
|
+
report,
|
|
235
|
+
message: `eval:triggers: trigger coverage incomplete\n${detail}`,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
if (failed > 0) {
|
|
239
|
+
const detail = results
|
|
240
|
+
.filter((result) => !result.pass)
|
|
241
|
+
.slice(0, 8)
|
|
242
|
+
.map((result) => ` - ${sanitizeTriggerLogLine(result.id)}: expected ${result.should_trigger ? "trigger" : "no-trigger"} ` +
|
|
243
|
+
`for ${sanitizeTriggerLogLine(result.skill)}; winner=${sanitizeTriggerLogLine(result.winnerSkill ?? "none")}`)
|
|
244
|
+
.join("\n");
|
|
245
|
+
return {
|
|
246
|
+
code: 1,
|
|
247
|
+
report,
|
|
248
|
+
message: `eval:triggers: ${failed}/${results.length} trigger cases failed ` +
|
|
249
|
+
`(offline skill-pi-trigger-eval routing against ${SKILLS_INDEX_REL})\n${detail}`,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
return {
|
|
253
|
+
code: 0,
|
|
254
|
+
report,
|
|
255
|
+
message: `eval:triggers: ${passed}/${results.length} trigger cases passed ` +
|
|
256
|
+
`(offline skill-pi-trigger-eval routing against ${SKILLS_INDEX_REL})`,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
//# sourceMappingURL=triggers.js.map
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export type OutputStream = "stdout" | "stderr" | "none";
|
|
2
|
+
/** Glob patterns that classify a change as skill-routing / trigger eval relevant (#1586). */
|
|
3
|
+
export declare const TRIGGER_ROUTING_PATH_PATTERNS: readonly ["AGENTS.md", "REFERENCES.md", "evals/trigger-cases.jsonl", "content/templates/agents-entry.md", "packages/core/src/eval/triggers.ts"];
|
|
4
|
+
export interface EvaluateResult {
|
|
5
|
+
readonly code: 0 | 1 | 2;
|
|
6
|
+
readonly message: string;
|
|
7
|
+
readonly stream: OutputStream;
|
|
8
|
+
readonly skipped?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export interface EvaluateOptions {
|
|
11
|
+
readonly projectRoot?: string;
|
|
12
|
+
readonly baseRef?: string;
|
|
13
|
+
readonly staged?: boolean;
|
|
14
|
+
readonly paths?: readonly string[];
|
|
15
|
+
readonly quiet?: boolean;
|
|
16
|
+
}
|
|
17
|
+
/** True when *path* matches a trigger-routing home. */
|
|
18
|
+
export declare function isTriggerRoutingPath(path: string): boolean;
|
|
19
|
+
/** Classify a path list for trigger-routing coverage. */
|
|
20
|
+
export declare function classifyTriggerRoutingPaths(paths: readonly string[]): {
|
|
21
|
+
readonly isTriggerRouting: boolean;
|
|
22
|
+
readonly matchedPaths: readonly string[];
|
|
23
|
+
};
|
|
24
|
+
/** Collect changed paths from git (base ref or staged index). */
|
|
25
|
+
export declare function collectChangedPaths(projectRoot: string, options: {
|
|
26
|
+
baseRef?: string;
|
|
27
|
+
staged?: boolean;
|
|
28
|
+
}): string[] | {
|
|
29
|
+
error: string;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Conditional gate: run eval:triggers when AGENTS.md / REFERENCES.md / trigger
|
|
33
|
+
* cases change (#1586). Skips (exit 0) when the diff does not touch routing homes.
|
|
34
|
+
*/
|
|
35
|
+
export declare function evaluate(options?: EvaluateOptions): EvaluateResult;
|
|
36
|
+
//# sourceMappingURL=evaluate.d.ts.map
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { runTriggerEval } from "../eval/triggers.js";
|
|
4
|
+
import { matchAny } from "../orchestration/pathspec.js";
|
|
5
|
+
/** Glob patterns that classify a change as skill-routing / trigger eval relevant (#1586). */
|
|
6
|
+
export const TRIGGER_ROUTING_PATH_PATTERNS = [
|
|
7
|
+
"AGENTS.md",
|
|
8
|
+
"REFERENCES.md",
|
|
9
|
+
"evals/trigger-cases.jsonl",
|
|
10
|
+
"content/templates/agents-entry.md",
|
|
11
|
+
"packages/core/src/eval/triggers.ts",
|
|
12
|
+
];
|
|
13
|
+
/** True when *path* matches a trigger-routing home. */
|
|
14
|
+
export function isTriggerRoutingPath(path) {
|
|
15
|
+
return matchAny(TRIGGER_ROUTING_PATH_PATTERNS, path);
|
|
16
|
+
}
|
|
17
|
+
/** Classify a path list for trigger-routing coverage. */
|
|
18
|
+
export function classifyTriggerRoutingPaths(paths) {
|
|
19
|
+
const matchedPaths = paths.filter(isTriggerRoutingPath);
|
|
20
|
+
return { isTriggerRouting: matchedPaths.length > 0, matchedPaths };
|
|
21
|
+
}
|
|
22
|
+
function gitNameOnlyDiff(projectRoot, args) {
|
|
23
|
+
try {
|
|
24
|
+
const stdout = execFileSync("git", ["-C", projectRoot, "diff", "--name-only", ...args], {
|
|
25
|
+
encoding: "utf8",
|
|
26
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
27
|
+
});
|
|
28
|
+
return stdout
|
|
29
|
+
.split("\n")
|
|
30
|
+
.map((line) => line.trim())
|
|
31
|
+
.filter((line) => line.length > 0);
|
|
32
|
+
}
|
|
33
|
+
catch (err) {
|
|
34
|
+
return { error: String(err) };
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/** Collect changed paths from git (base ref or staged index). */
|
|
38
|
+
export function collectChangedPaths(projectRoot, options) {
|
|
39
|
+
if (options.staged) {
|
|
40
|
+
return gitNameOnlyDiff(projectRoot, ["--cached"]);
|
|
41
|
+
}
|
|
42
|
+
if (options.baseRef !== undefined && options.baseRef.length > 0) {
|
|
43
|
+
return gitNameOnlyDiff(projectRoot, [options.baseRef, "HEAD"]);
|
|
44
|
+
}
|
|
45
|
+
return [];
|
|
46
|
+
}
|
|
47
|
+
function formatSkipMessage() {
|
|
48
|
+
return ("✓ verify:eval-triggers-relocation: no trigger-routing paths in diff " +
|
|
49
|
+
`(checked ${TRIGGER_ROUTING_PATH_PATTERNS.length} patterns).`);
|
|
50
|
+
}
|
|
51
|
+
function formatPassMessage(matchedPaths, summary) {
|
|
52
|
+
return (`✓ verify:eval-triggers-relocation: eval:triggers OK ` +
|
|
53
|
+
`(routing paths: ${matchedPaths.join(", ")}; ${summary}).`);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Conditional gate: run eval:triggers when AGENTS.md / REFERENCES.md / trigger
|
|
57
|
+
* cases change (#1586). Skips (exit 0) when the diff does not touch routing homes.
|
|
58
|
+
*/
|
|
59
|
+
export function evaluate(options = {}) {
|
|
60
|
+
const projectRoot = resolve(options.projectRoot ?? process.cwd());
|
|
61
|
+
const quiet = options.quiet ?? false;
|
|
62
|
+
let paths;
|
|
63
|
+
if (options.paths !== undefined) {
|
|
64
|
+
paths = [...options.paths];
|
|
65
|
+
}
|
|
66
|
+
else if (options.baseRef !== undefined || options.staged) {
|
|
67
|
+
const collected = collectChangedPaths(projectRoot, {
|
|
68
|
+
baseRef: options.baseRef,
|
|
69
|
+
staged: options.staged,
|
|
70
|
+
});
|
|
71
|
+
if ("error" in collected) {
|
|
72
|
+
return {
|
|
73
|
+
code: 2,
|
|
74
|
+
message: "❌ verify:eval-triggers-relocation: could not read git diff " +
|
|
75
|
+
`(project_root=${projectRoot}): ${collected.error}`,
|
|
76
|
+
stream: "stderr",
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
paths = collected;
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
return { code: 0, message: "", stream: "none", skipped: true };
|
|
83
|
+
}
|
|
84
|
+
const { isTriggerRouting, matchedPaths } = classifyTriggerRoutingPaths(paths);
|
|
85
|
+
if (!isTriggerRouting) {
|
|
86
|
+
if (quiet) {
|
|
87
|
+
return { code: 0, message: "", stream: "none", skipped: true };
|
|
88
|
+
}
|
|
89
|
+
return {
|
|
90
|
+
code: 0,
|
|
91
|
+
message: formatSkipMessage(),
|
|
92
|
+
stream: "stdout",
|
|
93
|
+
skipped: true,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
const evalResult = runTriggerEval({ projectRoot });
|
|
97
|
+
if (evalResult.code !== 0) {
|
|
98
|
+
return {
|
|
99
|
+
code: evalResult.code,
|
|
100
|
+
message: `❌ verify:eval-triggers-relocation: eval:triggers failed on trigger-routing PR (#1586).\n` +
|
|
101
|
+
` Matched routing paths: ${matchedPaths.join(", ")}\n` +
|
|
102
|
+
` ${evalResult.message.replace(/\n/g, "\n ")}`,
|
|
103
|
+
stream: "stderr",
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
if (quiet) {
|
|
107
|
+
return { code: 0, message: "", stream: "none" };
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
code: 0,
|
|
111
|
+
message: formatPassMessage(matchedPaths, evalResult.message),
|
|
112
|
+
stream: "stdout",
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=evaluate.js.map
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { type VerifyResult } from "../session/verify-session-ritual.js";
|
|
2
|
+
import { type ActiveScopeInspection } from "./scope.js";
|
|
3
|
+
export { DIRECT_WRITE_TOOL_NAMES, isDirectWriteTool } from "./tools.js";
|
|
4
|
+
export declare const HOOK_HOSTS: readonly ["claude", "grok", "cursor"];
|
|
5
|
+
export type HookHost = (typeof HOOK_HOSTS)[number];
|
|
6
|
+
export declare const HOOK_EVENTS: readonly ["session.start", "tool.before"];
|
|
7
|
+
export type HookEvent = (typeof HOOK_EVENTS)[number];
|
|
8
|
+
export type HookVerdict = "allow" | "deny";
|
|
9
|
+
export type HookDecisionCode = "session-start" | "session-start-degraded" | "not-direct-write" | "invalid-input" | "ritual-not-ready" | "scope-not-ready" | "write-ready";
|
|
10
|
+
export interface HookDecision {
|
|
11
|
+
readonly verdict: HookVerdict;
|
|
12
|
+
readonly code: HookDecisionCode;
|
|
13
|
+
readonly event: HookEvent;
|
|
14
|
+
readonly host: HookHost;
|
|
15
|
+
readonly toolName: string | null;
|
|
16
|
+
readonly projectRoot: string;
|
|
17
|
+
readonly message: string;
|
|
18
|
+
readonly scopePath: string | null;
|
|
19
|
+
}
|
|
20
|
+
export interface HookDispatchInput {
|
|
21
|
+
readonly host: HookHost;
|
|
22
|
+
readonly event: HookEvent;
|
|
23
|
+
readonly projectRoot: string;
|
|
24
|
+
readonly payload: unknown;
|
|
25
|
+
}
|
|
26
|
+
export interface HookPolicySeams {
|
|
27
|
+
readonly inspectRitual?: (projectRoot: string) => VerifyResult;
|
|
28
|
+
readonly inspectScope?: (projectRoot: string) => ActiveScopeInspection;
|
|
29
|
+
readonly sessionStart?: (projectRoot: string) => {
|
|
30
|
+
code: number;
|
|
31
|
+
stdout: string;
|
|
32
|
+
stderr: string;
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
export declare function hookToolName(payload: unknown): string | null;
|
|
36
|
+
export declare function projectRootFromHookPayload(payload: unknown, fallback: string): string;
|
|
37
|
+
export declare function isHookHost(value: string): value is HookHost;
|
|
38
|
+
export declare function isHookEvent(value: string): value is HookEvent;
|
|
39
|
+
/** Decide a normalized event using only the P0 direct-write policy. */
|
|
40
|
+
export declare function decideHook(input: HookDispatchInput, seams?: HookPolicySeams): HookDecision;
|
|
41
|
+
/** Render only authoritative denials; allow preserves the host's own permission flow. */
|
|
42
|
+
export declare function renderHostDecision(host: HookHost, decision: HookDecision): string;
|
|
43
|
+
//# sourceMappingURL=dispatcher.d.ts.map
|