@jean.gnc/harness-kit 0.12.7 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -0
- package/dist/cli.js +13 -1
- package/dist/cli.js.map +1 -1
- package/dist/eval/capture.d.ts +23 -0
- package/dist/eval/capture.d.ts.map +1 -0
- package/dist/eval/capture.js +79 -0
- package/dist/eval/capture.js.map +1 -0
- package/dist/eval/cases.d.ts +10 -2
- package/dist/eval/cases.d.ts.map +1 -1
- package/dist/eval/cases.js +9 -3
- package/dist/eval/cases.js.map +1 -1
- package/dist/eval/detect.d.ts +1 -0
- package/dist/eval/detect.d.ts.map +1 -1
- package/dist/eval/detect.js +1 -1
- package/dist/eval/detect.js.map +1 -1
- package/dist/eval/grade-deterministic.d.ts +9 -0
- package/dist/eval/grade-deterministic.d.ts.map +1 -0
- package/dist/eval/grade-deterministic.js +87 -0
- package/dist/eval/grade-deterministic.js.map +1 -0
- package/dist/eval/grade-judge.d.ts +12 -0
- package/dist/eval/grade-judge.d.ts.map +1 -0
- package/dist/eval/grade-judge.js +14 -0
- package/dist/eval/grade-judge.js.map +1 -0
- package/dist/eval/grade.d.ts +5 -0
- package/dist/eval/grade.d.ts.map +1 -0
- package/dist/eval/grade.js +25 -0
- package/dist/eval/grade.js.map +1 -0
- package/dist/eval/index.d.ts +4 -0
- package/dist/eval/index.d.ts.map +1 -1
- package/dist/eval/index.js +27 -5
- package/dist/eval/index.js.map +1 -1
- package/dist/eval/judge.d.ts +26 -0
- package/dist/eval/judge.d.ts.map +1 -0
- package/dist/eval/judge.js +55 -0
- package/dist/eval/judge.js.map +1 -0
- package/dist/eval/report.d.ts +5 -1
- package/dist/eval/report.d.ts.map +1 -1
- package/dist/eval/report.js +66 -13
- package/dist/eval/report.js.map +1 -1
- package/dist/eval/runner.d.ts +13 -5
- package/dist/eval/runner.d.ts.map +1 -1
- package/dist/eval/runner.js +105 -31
- package/dist/eval/runner.js.map +1 -1
- package/dist/eval/schema.d.ts +644 -29
- package/dist/eval/schema.d.ts.map +1 -1
- package/dist/eval/schema.js +57 -6
- package/dist/eval/schema.js.map +1 -1
- package/dist/eval/score.d.ts +8 -0
- package/dist/eval/score.d.ts.map +1 -1
- package/dist/eval/score.js +17 -0
- package/dist/eval/score.js.map +1 -1
- package/dist/skill/includes.d.ts +4 -0
- package/dist/skill/includes.d.ts.map +1 -1
- package/dist/skill/includes.js +38 -32
- package/dist/skill/includes.js.map +1 -1
- package/package.json +2 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyBxB,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAKf,CAAC;AAEH,QAAA,MAAM,
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyBxB,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAKf,CAAC;AAEH,QAAA,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAmBb,CAAC;AAOH,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;IAGf,CAAC;AAEH,QAAA,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAGV,CAAC;AAWH,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAAkE,CAAC;AAEpF,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAON,CAAC;AAEZ,eAAO,MAAM,KAAK,iCAAkC,CAAC;AAcrD,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAA2D,CAAC;AAEvF,MAAM,MAAM,IAAI,GAAG,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;AAC1C,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACtD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,CAAC;AAClD,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACtD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACtD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,WAAW,GAAG,WAAW,CAAC;AACjD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD,wBAAgB,cAAc,CAAC,WAAW,EAAE,WAAW,GAAG,SAAS,MAAM,EAAE,CAM1E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,GAAG,SAAS,MAAM,EAAE,CAGxE"}
|
package/dist/eval/schema.js
CHANGED
|
@@ -22,21 +22,67 @@ const Expectation = z.union([
|
|
|
22
22
|
PathExpectation,
|
|
23
23
|
NoSkillExpectation,
|
|
24
24
|
]);
|
|
25
|
-
const
|
|
25
|
+
const Assertion = z.discriminatedUnion("kind", [
|
|
26
|
+
z.object({
|
|
27
|
+
kind: z.literal("outputMatches"),
|
|
28
|
+
pattern: z.string().min(1),
|
|
29
|
+
regex: z.boolean().default(false),
|
|
30
|
+
}),
|
|
31
|
+
z.object({
|
|
32
|
+
kind: z.literal("outputExcludes"),
|
|
33
|
+
pattern: z.string().min(1),
|
|
34
|
+
regex: z.boolean().default(false),
|
|
35
|
+
}),
|
|
36
|
+
z.object({ kind: z.literal("usedTool"), tool: z.string().min(1) }),
|
|
37
|
+
z.object({ kind: z.literal("didNotUseTool"), tool: z.string().min(1) }),
|
|
38
|
+
z.object({
|
|
39
|
+
kind: z.literal("wroteFile"),
|
|
40
|
+
path: z.string().min(1),
|
|
41
|
+
contentMatches: z.string().min(1).optional(),
|
|
42
|
+
regex: z.boolean().default(false),
|
|
43
|
+
}),
|
|
44
|
+
]);
|
|
45
|
+
const Dimension = z.object({
|
|
46
|
+
dimension: z.string().min(1),
|
|
47
|
+
criterion: z.string().min(1),
|
|
48
|
+
});
|
|
49
|
+
const CombineRule = z.discriminatedUnion("combine", [
|
|
50
|
+
z.object({ combine: z.literal("all") }),
|
|
51
|
+
z.object({ combine: z.literal("fraction"), threshold: z.number().min(0).max(1) }),
|
|
52
|
+
]);
|
|
53
|
+
const Rubric = z.object({
|
|
54
|
+
dimensions: z.array(Dimension).min(1),
|
|
55
|
+
combine: CombineRule.default({ combine: "all" }),
|
|
56
|
+
});
|
|
57
|
+
const CommonCaseFields = {
|
|
26
58
|
id: z.string().min(1),
|
|
27
59
|
prompt: z.string().min(1),
|
|
28
|
-
expect: Expectation,
|
|
29
60
|
cwd: z.string().optional(),
|
|
30
61
|
runs: z.number().int().positive().optional(),
|
|
31
62
|
threshold: z.number().min(0).max(1).optional(),
|
|
32
63
|
note: z.string().optional(),
|
|
33
|
-
}
|
|
64
|
+
};
|
|
65
|
+
const RoutingCase = z.object({ ...CommonCaseFields, expect: Expectation }).strict();
|
|
66
|
+
const SolvingCase = z
|
|
67
|
+
.object({
|
|
68
|
+
...CommonCaseFields,
|
|
69
|
+
expectSkill: FqId.optional(),
|
|
70
|
+
assert: z.array(Assertion).default([]),
|
|
71
|
+
rubric: Rubric.optional(),
|
|
72
|
+
})
|
|
73
|
+
.strict();
|
|
34
74
|
export const TIERS = ["routing", "solving"];
|
|
35
|
-
|
|
75
|
+
const RoutingFile = z.object({
|
|
76
|
+
suite: z.string().min(1),
|
|
77
|
+
tier: z.literal("routing"),
|
|
78
|
+
cases: z.array(RoutingCase).min(1),
|
|
79
|
+
});
|
|
80
|
+
const SolvingFile = z.object({
|
|
36
81
|
suite: z.string().min(1),
|
|
37
|
-
tier: z.
|
|
38
|
-
cases: z.array(
|
|
82
|
+
tier: z.literal("solving"),
|
|
83
|
+
cases: z.array(SolvingCase).min(1),
|
|
39
84
|
});
|
|
85
|
+
export const CaseFileSchema = z.discriminatedUnion("tier", [RoutingFile, SolvingFile]);
|
|
40
86
|
export function expectedSkills(expectation) {
|
|
41
87
|
if ("noSkill" in expectation)
|
|
42
88
|
return [];
|
|
@@ -47,4 +93,9 @@ export function expectedSkills(expectation) {
|
|
|
47
93
|
return [...expectation.anyOf, ...forbidden];
|
|
48
94
|
return [...expectation.path, ...forbidden];
|
|
49
95
|
}
|
|
96
|
+
export function caseExpectedSkills(evalCase) {
|
|
97
|
+
if ("expect" in evalCase)
|
|
98
|
+
return expectedSkills(evalCase.expect);
|
|
99
|
+
return evalCase.expectSkill ? [evalCase.expectSkill] : [];
|
|
100
|
+
}
|
|
50
101
|
//# sourceMappingURL=schema.js.map
|
package/dist/eval/schema.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAElC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,4BAA4B,CAAC,CAAC;AAEnE,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,KAAK,EAAE,IAAI;IACX,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;CACzB,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC;IAC1B,gBAAgB;IAChB,gBAAgB;IAChB,eAAe;IACf,kBAAkB;CACnB,CAAC,CAAC;AAEH,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAElC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,4BAA4B,CAAC,CAAC;AAEnE,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,KAAK,EAAE,IAAI;IACX,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;CACzB,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC;IAC1B,gBAAgB;IAChB,gBAAgB;IAChB,eAAe;IACf,kBAAkB;CACnB,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;IAC7C,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC;QAChC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;KAClC,CAAC;IACF,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;QACjC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;KAClC,CAAC;IACF,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;QAC5B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACvB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC5C,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;KAClC,CAAC;CACH,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;IACzB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CAC7B,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,kBAAkB,CAAC,SAAS,EAAE;IAClD,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;IACvC,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;CAClF,CAAC,CAAC;AAEH,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IACtB,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACrC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;CACjD,CAAC,CAAC;AAEH,MAAM,gBAAgB,GAAG;IACvB,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC5B,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AAEpF,MAAM,WAAW,GAAG,CAAC;KAClB,MAAM,CAAC;IACN,GAAG,gBAAgB;IACnB,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE;IAC5B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACtC,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE;CAC1B,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,SAAS,CAAU,CAAC;AAErD,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IAC1B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACnC,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IAC1B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACnC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC;AAYvF,MAAM,UAAU,cAAc,CAAC,WAAwB;IACrD,IAAI,SAAS,IAAI,WAAW;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,IAAI,EAAE,CAAC;IACxC,IAAI,OAAO,IAAI,WAAW;QAAE,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,CAAC;IACrE,IAAI,OAAO,IAAI,WAAW;QAAE,OAAO,CAAC,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,GAAG,SAAS,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,QAAkB;IACnD,IAAI,QAAQ,IAAI,QAAQ;QAAE,OAAO,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjE,OAAO,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5D,CAAC"}
|
package/dist/eval/score.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { DetectionResult } from "./detect.js";
|
|
2
|
+
import type { AssertionResult } from "./grade-deterministic.js";
|
|
3
|
+
import type { RubricResult } from "./grade-judge.js";
|
|
2
4
|
import type { Expectation } from "./schema.js";
|
|
3
5
|
export interface CaseScore {
|
|
4
6
|
readonly matched: number;
|
|
@@ -10,4 +12,10 @@ export interface CaseScore {
|
|
|
10
12
|
}
|
|
11
13
|
export declare function matchesExpectation(expectation: Expectation, run: DetectionResult): boolean;
|
|
12
14
|
export declare function scoreCase(expectation: Expectation, runs: readonly DetectionResult[], threshold?: number): CaseScore;
|
|
15
|
+
export interface SolvingRunResult {
|
|
16
|
+
readonly assertions: readonly AssertionResult[];
|
|
17
|
+
readonly rubric: RubricResult | null;
|
|
18
|
+
}
|
|
19
|
+
export declare function solvingRunPassed(run: SolvingRunResult): boolean;
|
|
20
|
+
export declare function scoreSolving(perRun: readonly SolvingRunResult[], threshold?: number): CaseScore;
|
|
13
21
|
//# sourceMappingURL=score.d.ts.map
|
package/dist/eval/score.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/eval/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI/C,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjD;AAED,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAa1F;AAED,wBAAgB,SAAS,CACvB,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,SAAS,eAAe,EAAE,EAChC,SAAS,SAAoB,GAC5B,SAAS,CAWX"}
|
|
1
|
+
{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/eval/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI/C,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjD;AAED,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAa1F;AAED,wBAAgB,SAAS,CACvB,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,SAAS,eAAe,EAAE,EAChC,SAAS,SAAoB,GAC5B,SAAS,CAWX;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IAChD,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;CACtC;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAI/D;AAED,wBAAgB,YAAY,CAC1B,MAAM,EAAE,SAAS,gBAAgB,EAAE,EACnC,SAAS,SAAoB,GAC5B,SAAS,CAWX"}
|
package/dist/eval/score.js
CHANGED
|
@@ -25,6 +25,23 @@ export function scoreCase(expectation, runs, threshold = DEFAULT_THRESHOLD) {
|
|
|
25
25
|
histogram: histogramOf(runs),
|
|
26
26
|
};
|
|
27
27
|
}
|
|
28
|
+
export function solvingRunPassed(run) {
|
|
29
|
+
const assertionsPass = run.assertions.every((a) => a.pass);
|
|
30
|
+
const rubricPass = run.rubric === null || run.rubric.pass;
|
|
31
|
+
return assertionsPass && rubricPass;
|
|
32
|
+
}
|
|
33
|
+
export function scoreSolving(perRun, threshold = DEFAULT_THRESHOLD) {
|
|
34
|
+
const matched = perRun.filter(solvingRunPassed).length;
|
|
35
|
+
const triggerRate = perRun.length === 0 ? 0 : matched / perRun.length;
|
|
36
|
+
return {
|
|
37
|
+
matched,
|
|
38
|
+
runs: perRun.length,
|
|
39
|
+
triggerRate,
|
|
40
|
+
threshold,
|
|
41
|
+
pass: triggerRate >= threshold,
|
|
42
|
+
histogram: new Map(),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
28
45
|
function violatesNot(expectation, run) {
|
|
29
46
|
const forbidden = "noSkill" in expectation ? undefined : expectation.not;
|
|
30
47
|
if (!forbidden || forbidden.length === 0)
|
package/dist/eval/score.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/eval/score.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/eval/score.ts"],"names":[],"mappings":"AAKA,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAW9B,MAAM,UAAU,kBAAkB,CAAC,WAAwB,EAAE,GAAoB;IAC/E,IAAI,WAAW,CAAC,WAAW,EAAE,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAEhD,IAAI,SAAS,IAAI,WAAW,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,IAAI,WAAW,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,UAAU,KAAK,WAAW,CAAC,KAAK,CAAC;IAC9C,CAAC;IACD,IAAI,OAAO,IAAI,WAAW,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,UAAU,KAAK,IAAI,IAAI,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,oBAAoB,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,WAAwB,EACxB,IAAgC,EAChC,SAAS,GAAG,iBAAiB;IAE7B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,kBAAkB,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAClF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAClE,OAAO;QACL,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,WAAW;QACX,SAAS;QACT,IAAI,EAAE,WAAW,IAAI,SAAS;QAC9B,SAAS,EAAE,WAAW,CAAC,IAAI,CAAC;KAC7B,CAAC;AACJ,CAAC;AAOD,MAAM,UAAU,gBAAgB,CAAC,GAAqB;IACpD,MAAM,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;IAC1D,OAAO,cAAc,IAAI,UAAU,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,MAAmC,EACnC,SAAS,GAAG,iBAAiB;IAE7B,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC;IACvD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;IACtE,OAAO;QACL,OAAO;QACP,IAAI,EAAE,MAAM,CAAC,MAAM;QACnB,WAAW;QACX,SAAS;QACT,IAAI,EAAE,WAAW,IAAI,SAAS;QAC9B,SAAS,EAAE,IAAI,GAAG,EAAE;KACrB,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,WAAwB,EAAE,GAAoB;IACjE,MAAM,SAAS,GAAG,SAAS,IAAI,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC;IACzE,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvD,OAAO,GAAG,CAAC,UAAU,KAAK,IAAI,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAyB,EAAE,QAA2B;IAClF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,CAAC,CAAC;QACvC,IAAI,MAAM,KAAK,MAAM,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;IAC5C,CAAC;IACD,OAAO,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC;AAClC,CAAC;AAED,SAAS,WAAW,CAAC,IAAgC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,GAAG,CAAC,UAAU,IAAI,YAAY,CAAC;QAC3C,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/skill/includes.d.ts
CHANGED
|
@@ -14,6 +14,10 @@ export type IncludeError = {
|
|
|
14
14
|
readonly tag: "include-unknown-root";
|
|
15
15
|
readonly raw: string;
|
|
16
16
|
readonly name: string;
|
|
17
|
+
} | {
|
|
18
|
+
readonly tag: "include-anchor-without-path";
|
|
19
|
+
readonly raw: string;
|
|
20
|
+
readonly name: string;
|
|
17
21
|
} | {
|
|
18
22
|
readonly tag: "include-not-md";
|
|
19
23
|
readonly path: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"includes.d.ts","sourceRoot":"","sources":["../../src/skill/includes.ts"],"names":[],"mappings":"AAIA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,YAAY,GACpB;IAAE,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACpE;IAAE,QAAQ,CAAC,GAAG,EAAE,iBAAiB,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACjF;IACE,QAAQ,CAAC,GAAG,EAAE,uBAAuB,CAAC;IACtC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;CACnC,GACD;IAAE,QAAQ,CAAC,GAAG,EAAE,sBAAsB,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACrF;IAAE,QAAQ,CAAC,GAAG,EAAE,gBAAgB,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACzD;IAAE,QAAQ,CAAC,GAAG,EAAE,kBAAkB,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAC1D;IAAE,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5D,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAClD;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,gBAAgB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAChD;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,YAAY,GAClB,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,SAAS,YAAY,EAAE,CAAC,CAAC,CAMxD;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"includes.d.ts","sourceRoot":"","sources":["../../src/skill/includes.ts"],"names":[],"mappings":"AAIA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,YAAY,GACpB;IAAE,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACpE;IAAE,QAAQ,CAAC,GAAG,EAAE,iBAAiB,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACjF;IACE,QAAQ,CAAC,GAAG,EAAE,uBAAuB,CAAC;IACtC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;CACnC,GACD;IAAE,QAAQ,CAAC,GAAG,EAAE,sBAAsB,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACrF;IAAE,QAAQ,CAAC,GAAG,EAAE,6BAA6B,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC5F;IAAE,QAAQ,CAAC,GAAG,EAAE,gBAAgB,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACzD;IAAE,QAAQ,CAAC,GAAG,EAAE,kBAAkB,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAC1D;IAAE,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5D,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAClD;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,gBAAgB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAChD;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,YAAY,GAClB,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,SAAS,YAAY,EAAE,CAAC,CAAC,CAMxD;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAmB9D"}
|
package/dist/skill/includes.js
CHANGED
|
@@ -20,6 +20,8 @@ export function formatIncludeError(error) {
|
|
|
20
20
|
return `include path "${error.path}" escapes all allowed roots (${error.roots.join(", ")})`;
|
|
21
21
|
case "include-unknown-root":
|
|
22
22
|
return `include references unknown root "@${error.name}" in ${error.raw}`;
|
|
23
|
+
case "include-anchor-without-path":
|
|
24
|
+
return `include "@${error.name}" is missing a path (expected @${error.name}/<file>.md) in ${error.raw}`;
|
|
23
25
|
case "include-not-md":
|
|
24
26
|
return `include only supports .md files (got ${error.path})`;
|
|
25
27
|
case "include-absolute":
|
|
@@ -30,16 +32,18 @@ export function formatIncludeError(error) {
|
|
|
30
32
|
}
|
|
31
33
|
function resolveTarget(rel, fromFile, roots) {
|
|
32
34
|
if (!rel.startsWith("@")) {
|
|
33
|
-
return { target: resolve(dirname(fromFile), rel) };
|
|
35
|
+
return { kind: "target", target: resolve(dirname(fromFile), rel) };
|
|
34
36
|
}
|
|
35
37
|
const slash = rel.indexOf("/");
|
|
36
|
-
const name = slash === -1 ? rel.slice(1) : rel.slice(1, slash);
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
return { unknownRoot: name };
|
|
38
|
+
const [name, sub] = slash === -1 ? [rel.slice(1), ""] : [rel.slice(1, slash), rel.slice(slash + 1)];
|
|
39
|
+
const root = roots.named[name];
|
|
40
|
+
if (root === undefined) {
|
|
41
|
+
return { kind: "unknown-root", name };
|
|
41
42
|
}
|
|
42
|
-
|
|
43
|
+
if (sub === "") {
|
|
44
|
+
return { kind: "anchor-without-path", name };
|
|
45
|
+
}
|
|
46
|
+
return { kind: "target", target: resolve(root, sub) };
|
|
43
47
|
}
|
|
44
48
|
function isInsideAnyRoot(target, allowed) {
|
|
45
49
|
return allowed.some((root) => {
|
|
@@ -47,6 +51,29 @@ function isInsideAnyRoot(target, allowed) {
|
|
|
47
51
|
return rel !== "" && !rel.startsWith("..") && !isAbsolute(rel);
|
|
48
52
|
});
|
|
49
53
|
}
|
|
54
|
+
function resolveInclude(token, fromFile, roots, allowed) {
|
|
55
|
+
if (token.value === null) {
|
|
56
|
+
return err({ tag: "include-empty", raw: token.raw });
|
|
57
|
+
}
|
|
58
|
+
const rel = token.value.trim();
|
|
59
|
+
if (isAbsolute(rel)) {
|
|
60
|
+
return err({ tag: "include-absolute", raw: token.raw });
|
|
61
|
+
}
|
|
62
|
+
const resolution = resolveTarget(rel, fromFile, roots);
|
|
63
|
+
if (resolution.kind === "unknown-root") {
|
|
64
|
+
return err({ tag: "include-unknown-root", raw: token.raw, name: resolution.name });
|
|
65
|
+
}
|
|
66
|
+
if (resolution.kind === "anchor-without-path") {
|
|
67
|
+
return err({ tag: "include-anchor-without-path", raw: token.raw, name: resolution.name });
|
|
68
|
+
}
|
|
69
|
+
if (!rel.endsWith(".md")) {
|
|
70
|
+
return err({ tag: "include-not-md", path: rel });
|
|
71
|
+
}
|
|
72
|
+
if (!isInsideAnyRoot(resolution.target, allowed)) {
|
|
73
|
+
return err({ tag: "include-escapes-roots", path: rel, roots: allowed });
|
|
74
|
+
}
|
|
75
|
+
return ok({ target: resolution.target, rel });
|
|
76
|
+
}
|
|
50
77
|
async function expand(body, fromFile, roots, chain, errors, resolved) {
|
|
51
78
|
const tokens = parsePlaceholders(body).filter((t) => t.prefix === "include");
|
|
52
79
|
if (tokens.length === 0)
|
|
@@ -57,34 +84,13 @@ async function expand(body, fromFile, roots, chain, errors, resolved) {
|
|
|
57
84
|
for (const token of tokens) {
|
|
58
85
|
out += body.slice(cursor, token.start);
|
|
59
86
|
cursor = token.end;
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
continue;
|
|
64
|
-
}
|
|
65
|
-
const rel = token.value.trim();
|
|
66
|
-
if (isAbsolute(rel)) {
|
|
67
|
-
errors.push({ tag: "include-absolute", raw: token.raw });
|
|
68
|
-
out += token.raw;
|
|
69
|
-
continue;
|
|
70
|
-
}
|
|
71
|
-
if (!rel.endsWith(".md")) {
|
|
72
|
-
errors.push({ tag: "include-not-md", path: rel });
|
|
73
|
-
out += token.raw;
|
|
74
|
-
continue;
|
|
75
|
-
}
|
|
76
|
-
const resolution = resolveTarget(rel, fromFile, roots);
|
|
77
|
-
if ("unknownRoot" in resolution) {
|
|
78
|
-
errors.push({ tag: "include-unknown-root", raw: token.raw, name: resolution.unknownRoot });
|
|
79
|
-
out += token.raw;
|
|
80
|
-
continue;
|
|
81
|
-
}
|
|
82
|
-
const { target } = resolution;
|
|
83
|
-
if (!isInsideAnyRoot(target, allowed)) {
|
|
84
|
-
errors.push({ tag: "include-escapes-roots", path: rel, roots: allowed });
|
|
87
|
+
const resolution = resolveInclude(token, fromFile, roots, allowed);
|
|
88
|
+
if (!resolution.ok) {
|
|
89
|
+
errors.push(resolution.error);
|
|
85
90
|
out += token.raw;
|
|
86
91
|
continue;
|
|
87
92
|
}
|
|
93
|
+
const { target, rel } = resolution.value;
|
|
88
94
|
if (chain.includes(target)) {
|
|
89
95
|
errors.push({ tag: "include-cycle", chain: [...chain, target] });
|
|
90
96
|
out += token.raw;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"includes.js","sourceRoot":"","sources":["../../src/skill/includes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"includes.js","sourceRoot":"","sources":["../../src/skill/includes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,cAAc,CAAC;AA0BpD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAY,EACZ,QAAgB,EAChB,KAAmB;IAEnB,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC5F,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1C,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAmB;IACpD,QAAQ,KAAK,CAAC,GAAG,EAAE,CAAC;QAClB,KAAK,eAAe;YAClB,OAAO,kBAAkB,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,KAAK,iBAAiB;YACpB,OAAO,6BAA6B,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC,IAAI,GAAG,CAAC;QACxE,KAAK,uBAAuB;YAC1B,OAAO,iBAAiB,KAAK,CAAC,IAAI,gCAAgC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAC9F,KAAK,sBAAsB;YACzB,OAAO,qCAAqC,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,GAAG,EAAE,CAAC;QAC5E,KAAK,6BAA6B;YAChC,OAAO,aAAa,KAAK,CAAC,IAAI,kCAAkC,KAAK,CAAC,IAAI,kBAAkB,KAAK,CAAC,GAAG,EAAE,CAAC;QAC1G,KAAK,gBAAgB;YACnB,OAAO,wCAAwC,KAAK,CAAC,IAAI,GAAG,CAAC;QAC/D,KAAK,kBAAkB;YACrB,OAAO,sCAAsC,KAAK,CAAC,GAAG,GAAG,CAAC;QAC5D,KAAK,eAAe;YAClB,OAAO,gDAAgD,KAAK,CAAC,GAAG,GAAG,CAAC;IACxE,CAAC;AACH,CAAC;AAOD,SAAS,aAAa,CAAC,GAAW,EAAE,QAAgB,EAAE,KAAmB;IACvE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IACrE,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,GACf,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IAClF,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;IACxC,CAAC;IACD,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC;IAC/C,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;AACxD,CAAC;AAED,SAAS,eAAe,CAAC,MAAc,EAAE,OAA0B;IACjE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnC,OAAO,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;AACL,CAAC;AAOD,SAAS,cAAc,CACrB,KAA8D,EAC9D,QAAgB,EAChB,KAAmB,EACnB,OAA0B;IAE1B,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QACzB,OAAO,GAAG,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC/B,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC,EAAE,GAAG,EAAE,kBAAkB,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACvD,IAAI,UAAU,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QACvC,OAAO,GAAG,CAAC,EAAE,GAAG,EAAE,sBAAsB,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,UAAU,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;QAC9C,OAAO,GAAG,CAAC,EAAE,GAAG,EAAE,6BAA6B,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5F,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,GAAG,CAAC,EAAE,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;QACjD,OAAO,GAAG,CAAC,EAAE,GAAG,EAAE,uBAAuB,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;AAChD,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,IAAY,EACZ,QAAgB,EAChB,KAAmB,EACnB,KAAwB,EACxB,MAAsB,EACtB,QAAqB;IAErB,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;IAC7E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAErC,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5D,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC;QAEnB,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACnE,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC9B,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC;YACjB,SAAS;QACX,CAAC;QACD,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC;QACzC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;YACjE,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC;YACjB,SAAS;QACX,CAAC;QACD,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnE,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC;YACjB,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACxF,GAAG,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACpE,CAAC;IACD,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1B,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jean.gnc/harness-kit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Build your own multi-agent harness: typed toolkit for authoring plugins (skills, agents, commands, hooks) and shipping them to Claude Code and Codex from a single source tree.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -68,6 +68,7 @@
|
|
|
68
68
|
]
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
+
"@anthropic-ai/sdk": "^0.100.1",
|
|
71
72
|
"citty": "^0.2.2",
|
|
72
73
|
"js-yaml": "^4.1.1",
|
|
73
74
|
"markdownlint-cli2": "^0.18.1",
|