@deftai/directive-core 0.76.0 → 0.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents-md-budget/evaluate.d.ts +8 -1
- package/dist/agents-md-budget/evaluate.js +54 -19
- package/dist/content-contracts/skills/skill-frontmatter.js +4 -0
- package/dist/doctor/constants.js +3 -3
- package/dist/doctor/index.d.ts +1 -0
- package/dist/doctor/index.js +1 -0
- package/dist/doctor/main.d.ts +1 -0
- package/dist/doctor/main.js +39 -1
- package/dist/doctor/payload-staleness.js +62 -63
- package/dist/doctor/release-availability.d.ts +28 -0
- package/dist/doctor/release-availability.js +35 -0
- package/dist/doctor/types.d.ts +3 -0
- package/dist/eval/crud-telemetry.d.ts +5 -4
- package/dist/eval/crud-telemetry.js +9 -5
- package/dist/eval/health.d.ts +5 -4
- package/dist/eval/health.js +24 -16
- package/dist/eval/readback.js +2 -8
- package/dist/eval/triggers.d.ts +76 -0
- package/dist/eval/triggers.js +259 -0
- package/dist/eval-triggers-relocation/evaluate.d.ts +36 -0
- package/dist/eval-triggers-relocation/evaluate.js +115 -0
- package/dist/eval-triggers-relocation/index.d.ts +2 -0
- package/dist/eval-triggers-relocation/index.js +2 -0
- package/dist/hooks/dispatcher.d.ts +43 -0
- package/dist/hooks/dispatcher.js +171 -0
- package/dist/hooks/index.d.ts +3 -0
- package/dist/hooks/index.js +3 -0
- package/dist/hooks/scope.d.ts +12 -0
- package/dist/hooks/scope.js +46 -0
- package/dist/hooks/tools.d.ts +4 -0
- package/dist/hooks/tools.js +23 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/init-deposit/agent-hooks.d.ts +22 -0
- package/dist/init-deposit/agent-hooks.js +228 -0
- package/dist/init-deposit/hygiene.js +3 -0
- package/dist/init-deposit/index.d.ts +1 -0
- package/dist/init-deposit/index.js +1 -0
- package/dist/init-deposit/init-deposit.js +2 -0
- package/dist/init-deposit/refresh.js +2 -0
- package/dist/intake/reconcile-issues.d.ts +1 -0
- package/dist/intake/reconcile-issues.js +51 -49
- package/dist/metrics/index.d.ts +2 -0
- package/dist/metrics/index.js +2 -0
- package/dist/metrics/resolve-metrics-home.d.ts +50 -0
- package/dist/metrics/resolve-metrics-home.js +125 -0
- package/dist/plan-sequence/index.d.ts +3 -0
- package/dist/plan-sequence/index.js +3 -0
- package/dist/plan-sequence/store.d.ts +6 -0
- package/dist/plan-sequence/store.js +29 -0
- package/dist/plan-sequence/types.d.ts +79 -0
- package/dist/plan-sequence/types.js +220 -0
- package/dist/release/build-dist.js +1 -0
- package/dist/release/version.d.ts +2 -0
- package/dist/release/version.js +42 -0
- package/dist/release-e2e/greenfield-python-free-smoke-cli.js +40 -1
- package/dist/release-e2e/greenfield-python-free-smoke.d.ts +2 -0
- package/dist/release-e2e/greenfield-python-free-smoke.js +28 -8
- package/dist/release-e2e/npm-ops.js +20 -6
- package/dist/session/verify-session-ritual.d.ts +16 -0
- package/dist/session/verify-session-ritual.js +63 -0
- package/dist/triage/help/registry-data.d.ts +12 -1
- package/dist/triage/help/registry-data.js +22 -1
- package/dist/verify-env/agent-hooks.d.ts +11 -0
- package/dist/verify-env/agent-hooks.js +45 -0
- package/dist/verify-env/command-spawn.d.ts +29 -0
- package/dist/verify-env/command-spawn.js +97 -0
- package/dist/verify-env/index.d.ts +2 -0
- package/dist/verify-env/index.js +2 -0
- package/package.json +19 -3
package/dist/eval/health.js
CHANGED
|
@@ -3,7 +3,8 @@ import { dirname, resolve } from "node:path";
|
|
|
3
3
|
import { agentsRefreshPlan } from "../doctor/agents-md.js";
|
|
4
4
|
import { evaluate as evaluateEncoding } from "../encoding/evaluate.js";
|
|
5
5
|
import { readCorePackageVersion } from "../engine-version.js";
|
|
6
|
-
import {
|
|
6
|
+
import { resolveProjectDefinitionPath } from "../layout/resolve.js";
|
|
7
|
+
import { healthMetricsHistoryPath } from "../metrics/resolve-metrics-home.js";
|
|
7
8
|
import { readPlanPolicy } from "../policy/plan-extensions.js";
|
|
8
9
|
import { classifyOnboarding, detectPriorState } from "../triage/welcome/prior-state.js";
|
|
9
10
|
import { validateLinks } from "../validate-content/index.js";
|
|
@@ -11,7 +12,8 @@ import { evaluateConformance } from "../vbrief-validate/conformance.js";
|
|
|
11
12
|
import { validateWipCapOnPlan } from "../vbrief-validate/plan-hooks.js";
|
|
12
13
|
import { evaluateContentManifest } from "../verify-source/content-manifest.js";
|
|
13
14
|
export const HEALTH_SCHEMA_VERSION = 1;
|
|
14
|
-
|
|
15
|
+
/** Relative path under the resolved metrics home for eval:health history (#2545). */
|
|
16
|
+
export const HEALTH_HISTORY_REL = "health/health-history.jsonl";
|
|
15
17
|
function sanitizeOneLine(value) {
|
|
16
18
|
return value.replace(/\r?\n/g, " ");
|
|
17
19
|
}
|
|
@@ -99,9 +101,9 @@ function probeContentManifest(projectRoot) {
|
|
|
99
101
|
detail: result.code === 0 ? undefined : result.message.split("\n")[0],
|
|
100
102
|
};
|
|
101
103
|
}
|
|
102
|
-
/** Absolute path to the versioned health history ledger. */
|
|
104
|
+
/** Absolute path to the versioned health history ledger (#2545). */
|
|
103
105
|
export function healthHistoryPath(projectRoot) {
|
|
104
|
-
return
|
|
106
|
+
return healthMetricsHistoryPath(projectRoot);
|
|
105
107
|
}
|
|
106
108
|
/** Detect the canonical wipCap unsatisfiable-nudge contradiction (#1694). */
|
|
107
109
|
export function detectWipCapUnsatisfiableNudge(projectRoot) {
|
|
@@ -154,9 +156,12 @@ export function computeHealthScore(gates, contradictions) {
|
|
|
154
156
|
const penalty = contradictions.length * 15;
|
|
155
157
|
return Math.max(0, base - penalty);
|
|
156
158
|
}
|
|
157
|
-
/** Append one health run to the versioned ledger (#1703 Tier 0). */
|
|
159
|
+
/** Append one health run to the versioned ledger (#1703 Tier 0 / #2545). */
|
|
158
160
|
export function persistHealthRun(projectRoot, report) {
|
|
159
161
|
const path = healthHistoryPath(projectRoot);
|
|
162
|
+
if (path === null) {
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
160
165
|
mkdirSync(dirname(path), { recursive: true });
|
|
161
166
|
appendFileSync(path, `${JSON.stringify(report)}\n`, "utf8");
|
|
162
167
|
}
|
|
@@ -223,17 +228,20 @@ export function evaluateHealth(options = {}) {
|
|
|
223
228
|
contradictions,
|
|
224
229
|
};
|
|
225
230
|
if (persist) {
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
231
|
+
const ledgerPath = healthHistoryPath(projectRoot);
|
|
232
|
+
if (ledgerPath !== null) {
|
|
233
|
+
try {
|
|
234
|
+
persistHealthRun(projectRoot, report);
|
|
235
|
+
}
|
|
236
|
+
catch (err) {
|
|
237
|
+
const persistError = `eval:health: failed to persist health history: ${String(err)}`;
|
|
238
|
+
const healthy = score === 100 && contradictions.length === 0;
|
|
239
|
+
return {
|
|
240
|
+
code: healthy ? 0 : 1,
|
|
241
|
+
report,
|
|
242
|
+
message: `${formatHumanReport(report)}\n${persistError}`,
|
|
243
|
+
};
|
|
244
|
+
}
|
|
237
245
|
}
|
|
238
246
|
}
|
|
239
247
|
const healthy = score === 100 && contradictions.length === 0;
|
package/dist/eval/readback.js
CHANGED
|
@@ -28,14 +28,8 @@ function parseJsonObjectOrNull(text) {
|
|
|
28
28
|
return null;
|
|
29
29
|
}
|
|
30
30
|
function readHealthHistoryRows(projectRoot) {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
path = healthHistoryPath(projectRoot);
|
|
34
|
-
}
|
|
35
|
-
catch {
|
|
36
|
-
return []; // No xbrief/ layout; no health history.
|
|
37
|
-
}
|
|
38
|
-
if (!existsSync(path)) {
|
|
31
|
+
const path = healthHistoryPath(projectRoot);
|
|
32
|
+
if (path === null || !existsSync(path)) {
|
|
39
33
|
return [];
|
|
40
34
|
}
|
|
41
35
|
let text;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/** Relative path to skill-pi-trigger-eval compatible case rows (#1586). */
|
|
2
|
+
export declare const TRIGGER_CASES_REL = "evals/trigger-cases.jsonl";
|
|
3
|
+
/** Default Skills Index source for AGENTS.md skill routing (#1586). */
|
|
4
|
+
export declare const SKILLS_INDEX_REL = "REFERENCES.md";
|
|
5
|
+
/** Parsed Skills Index row. */
|
|
6
|
+
export interface SkillsIndexEntry {
|
|
7
|
+
readonly skillId: string;
|
|
8
|
+
readonly triggers: readonly string[];
|
|
9
|
+
}
|
|
10
|
+
/** One row in evals/trigger-cases.jsonl (skill-pi-trigger-eval eval-set + skill id). */
|
|
11
|
+
export interface TriggerCaseRow {
|
|
12
|
+
readonly id: string;
|
|
13
|
+
readonly query: string;
|
|
14
|
+
readonly skill: string;
|
|
15
|
+
readonly should_trigger: boolean;
|
|
16
|
+
}
|
|
17
|
+
/** Outcome for a single trigger case. */
|
|
18
|
+
export interface TriggerCaseResult {
|
|
19
|
+
readonly id: string;
|
|
20
|
+
readonly query: string;
|
|
21
|
+
readonly skill: string;
|
|
22
|
+
readonly should_trigger: boolean;
|
|
23
|
+
readonly actual: boolean;
|
|
24
|
+
readonly pass: boolean;
|
|
25
|
+
readonly matchedTrigger?: string;
|
|
26
|
+
readonly winnerSkill?: string;
|
|
27
|
+
}
|
|
28
|
+
/** Aggregate eval:triggers report. */
|
|
29
|
+
export interface TriggerEvalReport {
|
|
30
|
+
readonly total: number;
|
|
31
|
+
readonly passed: number;
|
|
32
|
+
readonly failed: number;
|
|
33
|
+
readonly pass_rate: number | null;
|
|
34
|
+
readonly results: readonly TriggerCaseResult[];
|
|
35
|
+
readonly coverageErrors: readonly string[];
|
|
36
|
+
}
|
|
37
|
+
export interface RunTriggerEvalOptions {
|
|
38
|
+
readonly projectRoot?: string;
|
|
39
|
+
readonly casesPath?: string;
|
|
40
|
+
readonly indexPath?: string;
|
|
41
|
+
readonly cases?: readonly TriggerCaseRow[];
|
|
42
|
+
readonly indexText?: string;
|
|
43
|
+
}
|
|
44
|
+
export interface RunTriggerEvalResult {
|
|
45
|
+
readonly code: 0 | 1 | 2;
|
|
46
|
+
readonly report: TriggerEvalReport | null;
|
|
47
|
+
readonly message: string;
|
|
48
|
+
}
|
|
49
|
+
/** Normalize user text for deterministic substring trigger matching. */
|
|
50
|
+
export declare function normalizeTriggerText(text: string): string;
|
|
51
|
+
/** Collapse embedded newlines before interpolating user/data into log lines. */
|
|
52
|
+
export declare function sanitizeTriggerLogLine(text: string): string;
|
|
53
|
+
/** Extract backtick trigger phrases from a Skills Index triggers cell. */
|
|
54
|
+
export declare function parseTriggerCell(cell: string): string[];
|
|
55
|
+
/** Parse the Skills Index markdown table in REFERENCES.md. */
|
|
56
|
+
export declare function parseSkillsIndex(markdown: string): SkillsIndexEntry[];
|
|
57
|
+
interface TriggerMatch {
|
|
58
|
+
readonly skillId: string;
|
|
59
|
+
readonly trigger: string;
|
|
60
|
+
}
|
|
61
|
+
/** Find the winning skill for a query using longest-trigger-wins disambiguation. */
|
|
62
|
+
export declare function resolveTriggerWinner(query: string, index: readonly SkillsIndexEntry[]): TriggerMatch | null;
|
|
63
|
+
/** True when the Skills Index routing would send *query* to *skillId*. */
|
|
64
|
+
export declare function wouldRouteToSkill(query: string, skillId: string, index: readonly SkillsIndexEntry[]): boolean;
|
|
65
|
+
/** Validate one JSONL row against the skill-pi-trigger-eval row contract. */
|
|
66
|
+
export declare function parseTriggerCaseRow(raw: unknown, lineNumber: number): TriggerCaseRow | string;
|
|
67
|
+
/** Load evals/trigger-cases.jsonl. */
|
|
68
|
+
export declare function loadTriggerCases(path: string): TriggerCaseRow[] | {
|
|
69
|
+
error: string;
|
|
70
|
+
};
|
|
71
|
+
/** Verify at least two positive and one negative case per indexed skill. */
|
|
72
|
+
export declare function validateTriggerCoverage(cases: readonly TriggerCaseRow[], index: readonly SkillsIndexEntry[]): string[];
|
|
73
|
+
/** Run deterministic trigger routing eval (offline skill-pi-trigger-eval counterpart). */
|
|
74
|
+
export declare function runTriggerEval(options?: RunTriggerEvalOptions): RunTriggerEvalResult;
|
|
75
|
+
export {};
|
|
76
|
+
//# sourceMappingURL=triggers.d.ts.map
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
/** Relative path to skill-pi-trigger-eval compatible case rows (#1586). */
|
|
4
|
+
export const TRIGGER_CASES_REL = "evals/trigger-cases.jsonl";
|
|
5
|
+
/** Default Skills Index source for AGENTS.md skill routing (#1586). */
|
|
6
|
+
export const SKILLS_INDEX_REL = "REFERENCES.md";
|
|
7
|
+
/** Normalize user text for deterministic substring trigger matching. */
|
|
8
|
+
export function normalizeTriggerText(text) {
|
|
9
|
+
return text.toLowerCase().replace(/\s+/g, " ").trim();
|
|
10
|
+
}
|
|
11
|
+
/** Collapse embedded newlines before interpolating user/data into log lines. */
|
|
12
|
+
export function sanitizeTriggerLogLine(text) {
|
|
13
|
+
return text.replace(/\r?\n/g, " ").trim();
|
|
14
|
+
}
|
|
15
|
+
/** Extract backtick trigger phrases from a Skills Index triggers cell. */
|
|
16
|
+
export function parseTriggerCell(cell) {
|
|
17
|
+
const triggers = [];
|
|
18
|
+
for (const match of cell.matchAll(/`([^`]+)`/g)) {
|
|
19
|
+
const phrase = normalizeTriggerText(match[1] ?? "");
|
|
20
|
+
if (phrase.length > 0) {
|
|
21
|
+
triggers.push(phrase);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return triggers;
|
|
25
|
+
}
|
|
26
|
+
/** Parse the Skills Index markdown table in REFERENCES.md. */
|
|
27
|
+
export function parseSkillsIndex(markdown) {
|
|
28
|
+
const entries = [];
|
|
29
|
+
for (const line of markdown.split(/\r?\n/)) {
|
|
30
|
+
if (!line.startsWith("| [deft-directive-")) {
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
const columns = line.split("|").map((col) => col.trim());
|
|
34
|
+
if (columns.length < 5) {
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
const linkCell = columns[1] ?? "";
|
|
38
|
+
const triggersCell = columns[3] ?? "";
|
|
39
|
+
// Avoid nested character-class regex (CodeQL js/polynomial-redos on \[([^\]]+)\]).
|
|
40
|
+
const open = linkCell.indexOf("[");
|
|
41
|
+
const close = open === -1 ? -1 : linkCell.indexOf("]", open + 1);
|
|
42
|
+
if (open === -1 || close === -1) {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const skillId = linkCell.slice(open + 1, close).trim();
|
|
46
|
+
if (skillId.length === 0) {
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const triggers = parseTriggerCell(triggersCell);
|
|
50
|
+
if (triggers.length === 0) {
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
entries.push({ skillId, triggers });
|
|
54
|
+
}
|
|
55
|
+
return entries;
|
|
56
|
+
}
|
|
57
|
+
/** Find the winning skill for a query using longest-trigger-wins disambiguation. */
|
|
58
|
+
export function resolveTriggerWinner(query, index) {
|
|
59
|
+
const normalized = normalizeTriggerText(query);
|
|
60
|
+
let best = null;
|
|
61
|
+
for (const entry of index) {
|
|
62
|
+
for (const trigger of entry.triggers) {
|
|
63
|
+
if (!normalized.includes(trigger)) {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (best === null ||
|
|
67
|
+
trigger.length > best.trigger.length ||
|
|
68
|
+
(trigger.length === best.trigger.length && entry.skillId.localeCompare(best.skillId) < 0)) {
|
|
69
|
+
best = { skillId: entry.skillId, trigger };
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return best;
|
|
74
|
+
}
|
|
75
|
+
/** True when the Skills Index routing would send *query* to *skillId*. */
|
|
76
|
+
export function wouldRouteToSkill(query, skillId, index) {
|
|
77
|
+
const winner = resolveTriggerWinner(query, index);
|
|
78
|
+
return winner !== null && winner.skillId === skillId;
|
|
79
|
+
}
|
|
80
|
+
/** Validate one JSONL row against the skill-pi-trigger-eval row contract. */
|
|
81
|
+
export function parseTriggerCaseRow(raw, lineNumber) {
|
|
82
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
|
|
83
|
+
return `line ${lineNumber}: expected a JSON object`;
|
|
84
|
+
}
|
|
85
|
+
const record = raw;
|
|
86
|
+
const id = record.id;
|
|
87
|
+
const query = record.query;
|
|
88
|
+
const skill = record.skill;
|
|
89
|
+
const shouldTrigger = record.should_trigger;
|
|
90
|
+
if (typeof id !== "string" || id.trim().length === 0) {
|
|
91
|
+
return `line ${lineNumber}: id must be a non-empty string`;
|
|
92
|
+
}
|
|
93
|
+
if (typeof query !== "string" || query.trim().length === 0) {
|
|
94
|
+
return `line ${lineNumber}: query must be a non-empty string`;
|
|
95
|
+
}
|
|
96
|
+
if (typeof skill !== "string" || !skill.startsWith("deft-directive-")) {
|
|
97
|
+
return `line ${lineNumber}: skill must be a deft-directive-* id`;
|
|
98
|
+
}
|
|
99
|
+
if (typeof shouldTrigger !== "boolean") {
|
|
100
|
+
return `line ${lineNumber}: should_trigger must be true or false`;
|
|
101
|
+
}
|
|
102
|
+
return {
|
|
103
|
+
id: id.trim(),
|
|
104
|
+
query: query.trim(),
|
|
105
|
+
skill: skill.trim(),
|
|
106
|
+
should_trigger: shouldTrigger,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
/** Load evals/trigger-cases.jsonl. */
|
|
110
|
+
export function loadTriggerCases(path) {
|
|
111
|
+
if (!existsSync(path)) {
|
|
112
|
+
return { error: `missing trigger cases at ${path}` };
|
|
113
|
+
}
|
|
114
|
+
const text = readFileSync(path, "utf8");
|
|
115
|
+
const rows = [];
|
|
116
|
+
const lines = text.split(/\r?\n/);
|
|
117
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
118
|
+
const line = lines[i]?.trim() ?? "";
|
|
119
|
+
if (line.length === 0 || line.startsWith("#")) {
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
let parsed;
|
|
123
|
+
try {
|
|
124
|
+
parsed = JSON.parse(line);
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
return { error: `line ${i + 1}: invalid JSON` };
|
|
128
|
+
}
|
|
129
|
+
const row = parseTriggerCaseRow(parsed, i + 1);
|
|
130
|
+
if (typeof row === "string") {
|
|
131
|
+
return { error: row };
|
|
132
|
+
}
|
|
133
|
+
rows.push(row);
|
|
134
|
+
}
|
|
135
|
+
if (rows.length === 0) {
|
|
136
|
+
return { error: "trigger-cases.jsonl has no cases" };
|
|
137
|
+
}
|
|
138
|
+
return rows;
|
|
139
|
+
}
|
|
140
|
+
/** Verify at least two positive and one negative case per indexed skill. */
|
|
141
|
+
export function validateTriggerCoverage(cases, index) {
|
|
142
|
+
const errors = [];
|
|
143
|
+
const bySkill = new Map();
|
|
144
|
+
for (const entry of index) {
|
|
145
|
+
bySkill.set(entry.skillId, { positive: 0, negative: 0 });
|
|
146
|
+
}
|
|
147
|
+
for (const row of cases) {
|
|
148
|
+
const bucket = bySkill.get(row.skill);
|
|
149
|
+
if (bucket === undefined) {
|
|
150
|
+
errors.push(`case '${row.id}' references unknown skill '${row.skill}'`);
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
if (row.should_trigger) {
|
|
154
|
+
bucket.positive += 1;
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
bucket.negative += 1;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
for (const [skillId, counts] of bySkill) {
|
|
161
|
+
if (counts.positive < 2) {
|
|
162
|
+
errors.push(`${skillId}: need at least 2 positive trigger cases (have ${counts.positive})`);
|
|
163
|
+
}
|
|
164
|
+
if (counts.negative < 1) {
|
|
165
|
+
errors.push(`${skillId}: need at least 1 negative trigger case (have ${counts.negative})`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return errors;
|
|
169
|
+
}
|
|
170
|
+
/** Run deterministic trigger routing eval (offline skill-pi-trigger-eval counterpart). */
|
|
171
|
+
export function runTriggerEval(options = {}) {
|
|
172
|
+
const projectRoot = resolve(options.projectRoot ?? process.cwd());
|
|
173
|
+
const casesPath = options.casesPath ?? resolve(projectRoot, TRIGGER_CASES_REL);
|
|
174
|
+
const indexPath = options.indexPath ?? resolve(projectRoot, SKILLS_INDEX_REL);
|
|
175
|
+
let cases;
|
|
176
|
+
if (options.cases !== undefined) {
|
|
177
|
+
cases = [...options.cases];
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
const loaded = loadTriggerCases(casesPath);
|
|
181
|
+
if ("error" in loaded) {
|
|
182
|
+
return { code: 2, report: null, message: `eval:triggers: ${loaded.error}` };
|
|
183
|
+
}
|
|
184
|
+
cases = loaded;
|
|
185
|
+
}
|
|
186
|
+
let indexText;
|
|
187
|
+
if (options.indexText !== undefined) {
|
|
188
|
+
indexText = options.indexText;
|
|
189
|
+
}
|
|
190
|
+
else if (!existsSync(indexPath)) {
|
|
191
|
+
return { code: 2, report: null, message: `eval:triggers: missing ${SKILLS_INDEX_REL}` };
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
indexText = readFileSync(indexPath, "utf8");
|
|
195
|
+
}
|
|
196
|
+
const index = parseSkillsIndex(indexText);
|
|
197
|
+
if (index.length === 0) {
|
|
198
|
+
return {
|
|
199
|
+
code: 2,
|
|
200
|
+
report: null,
|
|
201
|
+
message: `eval:triggers: no Skills Index rows parsed from ${SKILLS_INDEX_REL}`,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
const coverageErrors = validateTriggerCoverage(cases, index);
|
|
205
|
+
const results = [];
|
|
206
|
+
for (const row of cases) {
|
|
207
|
+
const winner = resolveTriggerWinner(row.query, index);
|
|
208
|
+
const actual = wouldRouteToSkill(row.query, row.skill, index);
|
|
209
|
+
results.push({
|
|
210
|
+
id: row.id,
|
|
211
|
+
query: row.query,
|
|
212
|
+
skill: row.skill,
|
|
213
|
+
should_trigger: row.should_trigger,
|
|
214
|
+
actual,
|
|
215
|
+
pass: actual === row.should_trigger,
|
|
216
|
+
matchedTrigger: winner?.trigger,
|
|
217
|
+
winnerSkill: winner?.skillId,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
const passed = results.filter((result) => result.pass).length;
|
|
221
|
+
const failed = results.length - passed;
|
|
222
|
+
const report = {
|
|
223
|
+
total: results.length,
|
|
224
|
+
passed,
|
|
225
|
+
failed,
|
|
226
|
+
pass_rate: results.length > 0 ? passed / results.length : null,
|
|
227
|
+
results,
|
|
228
|
+
coverageErrors,
|
|
229
|
+
};
|
|
230
|
+
if (coverageErrors.length > 0) {
|
|
231
|
+
const detail = coverageErrors.map((err) => ` - ${sanitizeTriggerLogLine(err)}`).join("\n");
|
|
232
|
+
return {
|
|
233
|
+
code: 1,
|
|
234
|
+
report,
|
|
235
|
+
message: `eval:triggers: trigger coverage incomplete\n${detail}`,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
if (failed > 0) {
|
|
239
|
+
const detail = results
|
|
240
|
+
.filter((result) => !result.pass)
|
|
241
|
+
.slice(0, 8)
|
|
242
|
+
.map((result) => ` - ${sanitizeTriggerLogLine(result.id)}: expected ${result.should_trigger ? "trigger" : "no-trigger"} ` +
|
|
243
|
+
`for ${sanitizeTriggerLogLine(result.skill)}; winner=${sanitizeTriggerLogLine(result.winnerSkill ?? "none")}`)
|
|
244
|
+
.join("\n");
|
|
245
|
+
return {
|
|
246
|
+
code: 1,
|
|
247
|
+
report,
|
|
248
|
+
message: `eval:triggers: ${failed}/${results.length} trigger cases failed ` +
|
|
249
|
+
`(offline skill-pi-trigger-eval routing against ${SKILLS_INDEX_REL})\n${detail}`,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
return {
|
|
253
|
+
code: 0,
|
|
254
|
+
report,
|
|
255
|
+
message: `eval:triggers: ${passed}/${results.length} trigger cases passed ` +
|
|
256
|
+
`(offline skill-pi-trigger-eval routing against ${SKILLS_INDEX_REL})`,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
//# sourceMappingURL=triggers.js.map
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export type OutputStream = "stdout" | "stderr" | "none";
|
|
2
|
+
/** Glob patterns that classify a change as skill-routing / trigger eval relevant (#1586). */
|
|
3
|
+
export declare const TRIGGER_ROUTING_PATH_PATTERNS: readonly ["AGENTS.md", "REFERENCES.md", "evals/trigger-cases.jsonl", "content/templates/agents-entry.md", "packages/core/src/eval/triggers.ts"];
|
|
4
|
+
export interface EvaluateResult {
|
|
5
|
+
readonly code: 0 | 1 | 2;
|
|
6
|
+
readonly message: string;
|
|
7
|
+
readonly stream: OutputStream;
|
|
8
|
+
readonly skipped?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export interface EvaluateOptions {
|
|
11
|
+
readonly projectRoot?: string;
|
|
12
|
+
readonly baseRef?: string;
|
|
13
|
+
readonly staged?: boolean;
|
|
14
|
+
readonly paths?: readonly string[];
|
|
15
|
+
readonly quiet?: boolean;
|
|
16
|
+
}
|
|
17
|
+
/** True when *path* matches a trigger-routing home. */
|
|
18
|
+
export declare function isTriggerRoutingPath(path: string): boolean;
|
|
19
|
+
/** Classify a path list for trigger-routing coverage. */
|
|
20
|
+
export declare function classifyTriggerRoutingPaths(paths: readonly string[]): {
|
|
21
|
+
readonly isTriggerRouting: boolean;
|
|
22
|
+
readonly matchedPaths: readonly string[];
|
|
23
|
+
};
|
|
24
|
+
/** Collect changed paths from git (base ref or staged index). */
|
|
25
|
+
export declare function collectChangedPaths(projectRoot: string, options: {
|
|
26
|
+
baseRef?: string;
|
|
27
|
+
staged?: boolean;
|
|
28
|
+
}): string[] | {
|
|
29
|
+
error: string;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Conditional gate: run eval:triggers when AGENTS.md / REFERENCES.md / trigger
|
|
33
|
+
* cases change (#1586). Skips (exit 0) when the diff does not touch routing homes.
|
|
34
|
+
*/
|
|
35
|
+
export declare function evaluate(options?: EvaluateOptions): EvaluateResult;
|
|
36
|
+
//# sourceMappingURL=evaluate.d.ts.map
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { runTriggerEval } from "../eval/triggers.js";
|
|
4
|
+
import { matchAny } from "../orchestration/pathspec.js";
|
|
5
|
+
/** Glob patterns that classify a change as skill-routing / trigger eval relevant (#1586). */
|
|
6
|
+
export const TRIGGER_ROUTING_PATH_PATTERNS = [
|
|
7
|
+
"AGENTS.md",
|
|
8
|
+
"REFERENCES.md",
|
|
9
|
+
"evals/trigger-cases.jsonl",
|
|
10
|
+
"content/templates/agents-entry.md",
|
|
11
|
+
"packages/core/src/eval/triggers.ts",
|
|
12
|
+
];
|
|
13
|
+
/** True when *path* matches a trigger-routing home. */
|
|
14
|
+
export function isTriggerRoutingPath(path) {
|
|
15
|
+
return matchAny(TRIGGER_ROUTING_PATH_PATTERNS, path);
|
|
16
|
+
}
|
|
17
|
+
/** Classify a path list for trigger-routing coverage. */
|
|
18
|
+
export function classifyTriggerRoutingPaths(paths) {
|
|
19
|
+
const matchedPaths = paths.filter(isTriggerRoutingPath);
|
|
20
|
+
return { isTriggerRouting: matchedPaths.length > 0, matchedPaths };
|
|
21
|
+
}
|
|
22
|
+
function gitNameOnlyDiff(projectRoot, args) {
|
|
23
|
+
try {
|
|
24
|
+
const stdout = execFileSync("git", ["-C", projectRoot, "diff", "--name-only", ...args], {
|
|
25
|
+
encoding: "utf8",
|
|
26
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
27
|
+
});
|
|
28
|
+
return stdout
|
|
29
|
+
.split("\n")
|
|
30
|
+
.map((line) => line.trim())
|
|
31
|
+
.filter((line) => line.length > 0);
|
|
32
|
+
}
|
|
33
|
+
catch (err) {
|
|
34
|
+
return { error: String(err) };
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/** Collect changed paths from git (base ref or staged index). */
|
|
38
|
+
export function collectChangedPaths(projectRoot, options) {
|
|
39
|
+
if (options.staged) {
|
|
40
|
+
return gitNameOnlyDiff(projectRoot, ["--cached"]);
|
|
41
|
+
}
|
|
42
|
+
if (options.baseRef !== undefined && options.baseRef.length > 0) {
|
|
43
|
+
return gitNameOnlyDiff(projectRoot, [options.baseRef, "HEAD"]);
|
|
44
|
+
}
|
|
45
|
+
return [];
|
|
46
|
+
}
|
|
47
|
+
function formatSkipMessage() {
|
|
48
|
+
return ("✓ verify:eval-triggers-relocation: no trigger-routing paths in diff " +
|
|
49
|
+
`(checked ${TRIGGER_ROUTING_PATH_PATTERNS.length} patterns).`);
|
|
50
|
+
}
|
|
51
|
+
function formatPassMessage(matchedPaths, summary) {
|
|
52
|
+
return (`✓ verify:eval-triggers-relocation: eval:triggers OK ` +
|
|
53
|
+
`(routing paths: ${matchedPaths.join(", ")}; ${summary}).`);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Conditional gate: run eval:triggers when AGENTS.md / REFERENCES.md / trigger
|
|
57
|
+
* cases change (#1586). Skips (exit 0) when the diff does not touch routing homes.
|
|
58
|
+
*/
|
|
59
|
+
export function evaluate(options = {}) {
|
|
60
|
+
const projectRoot = resolve(options.projectRoot ?? process.cwd());
|
|
61
|
+
const quiet = options.quiet ?? false;
|
|
62
|
+
let paths;
|
|
63
|
+
if (options.paths !== undefined) {
|
|
64
|
+
paths = [...options.paths];
|
|
65
|
+
}
|
|
66
|
+
else if (options.baseRef !== undefined || options.staged) {
|
|
67
|
+
const collected = collectChangedPaths(projectRoot, {
|
|
68
|
+
baseRef: options.baseRef,
|
|
69
|
+
staged: options.staged,
|
|
70
|
+
});
|
|
71
|
+
if ("error" in collected) {
|
|
72
|
+
return {
|
|
73
|
+
code: 2,
|
|
74
|
+
message: "❌ verify:eval-triggers-relocation: could not read git diff " +
|
|
75
|
+
`(project_root=${projectRoot}): ${collected.error}`,
|
|
76
|
+
stream: "stderr",
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
paths = collected;
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
return { code: 0, message: "", stream: "none", skipped: true };
|
|
83
|
+
}
|
|
84
|
+
const { isTriggerRouting, matchedPaths } = classifyTriggerRoutingPaths(paths);
|
|
85
|
+
if (!isTriggerRouting) {
|
|
86
|
+
if (quiet) {
|
|
87
|
+
return { code: 0, message: "", stream: "none", skipped: true };
|
|
88
|
+
}
|
|
89
|
+
return {
|
|
90
|
+
code: 0,
|
|
91
|
+
message: formatSkipMessage(),
|
|
92
|
+
stream: "stdout",
|
|
93
|
+
skipped: true,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
const evalResult = runTriggerEval({ projectRoot });
|
|
97
|
+
if (evalResult.code !== 0) {
|
|
98
|
+
return {
|
|
99
|
+
code: evalResult.code,
|
|
100
|
+
message: `❌ verify:eval-triggers-relocation: eval:triggers failed on trigger-routing PR (#1586).\n` +
|
|
101
|
+
` Matched routing paths: ${matchedPaths.join(", ")}\n` +
|
|
102
|
+
` ${evalResult.message.replace(/\n/g, "\n ")}`,
|
|
103
|
+
stream: "stderr",
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
if (quiet) {
|
|
107
|
+
return { code: 0, message: "", stream: "none" };
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
code: 0,
|
|
111
|
+
message: formatPassMessage(matchedPaths, evalResult.message),
|
|
112
|
+
stream: "stdout",
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=evaluate.js.map
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { type VerifyResult } from "../session/verify-session-ritual.js";
|
|
2
|
+
import { type ActiveScopeInspection } from "./scope.js";
|
|
3
|
+
export { DIRECT_WRITE_TOOL_NAMES, isDirectWriteTool } from "./tools.js";
|
|
4
|
+
export declare const HOOK_HOSTS: readonly ["claude", "grok", "cursor"];
|
|
5
|
+
export type HookHost = (typeof HOOK_HOSTS)[number];
|
|
6
|
+
export declare const HOOK_EVENTS: readonly ["session.start", "tool.before"];
|
|
7
|
+
export type HookEvent = (typeof HOOK_EVENTS)[number];
|
|
8
|
+
export type HookVerdict = "allow" | "deny";
|
|
9
|
+
export type HookDecisionCode = "session-start" | "session-start-degraded" | "not-direct-write" | "invalid-input" | "ritual-not-ready" | "scope-not-ready" | "write-ready";
|
|
10
|
+
export interface HookDecision {
|
|
11
|
+
readonly verdict: HookVerdict;
|
|
12
|
+
readonly code: HookDecisionCode;
|
|
13
|
+
readonly event: HookEvent;
|
|
14
|
+
readonly host: HookHost;
|
|
15
|
+
readonly toolName: string | null;
|
|
16
|
+
readonly projectRoot: string;
|
|
17
|
+
readonly message: string;
|
|
18
|
+
readonly scopePath: string | null;
|
|
19
|
+
}
|
|
20
|
+
export interface HookDispatchInput {
|
|
21
|
+
readonly host: HookHost;
|
|
22
|
+
readonly event: HookEvent;
|
|
23
|
+
readonly projectRoot: string;
|
|
24
|
+
readonly payload: unknown;
|
|
25
|
+
}
|
|
26
|
+
export interface HookPolicySeams {
|
|
27
|
+
readonly inspectRitual?: (projectRoot: string) => VerifyResult;
|
|
28
|
+
readonly inspectScope?: (projectRoot: string) => ActiveScopeInspection;
|
|
29
|
+
readonly sessionStart?: (projectRoot: string) => {
|
|
30
|
+
code: number;
|
|
31
|
+
stdout: string;
|
|
32
|
+
stderr: string;
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
export declare function hookToolName(payload: unknown): string | null;
|
|
36
|
+
export declare function projectRootFromHookPayload(payload: unknown, fallback: string): string;
|
|
37
|
+
export declare function isHookHost(value: string): value is HookHost;
|
|
38
|
+
export declare function isHookEvent(value: string): value is HookEvent;
|
|
39
|
+
/** Decide a normalized event using only the P0 direct-write policy. */
|
|
40
|
+
export declare function decideHook(input: HookDispatchInput, seams?: HookPolicySeams): HookDecision;
|
|
41
|
+
/** Render only authoritative denials; allow preserves the host's own permission flow. */
|
|
42
|
+
export declare function renderHostDecision(host: HookHost, decision: HookDecision): string;
|
|
43
|
+
//# sourceMappingURL=dispatcher.d.ts.map
|