dsh-continual-evolve 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +149 -19
- package/README.zh.md +64 -21
- package/lib/apply.js +2 -0
- package/lib/approval.d.ts +19 -0
- package/lib/auto.d.ts +61 -1
- package/lib/auto.js +108 -2
- package/lib/benchmark.d.ts +14 -0
- package/lib/command.js +234 -5
- package/lib/evaluate.d.ts +36 -7
- package/lib/evaluate.js +157 -43
- package/lib/fate.d.ts +126 -0
- package/lib/fate.js +338 -0
- package/lib/index.d.ts +26 -0
- package/lib/index.js +18 -2
- package/lib/mount.js +5 -0
- package/lib/planner.d.ts +8 -1
- package/lib/planner.js +27 -0
- package/lib/render.js +2 -1
- package/lib/review.d.ts +1 -1
- package/lib/review.js +17 -0
- package/lib/score.d.ts +22 -4
- package/lib/score.js +48 -7
- package/lib/skill.d.ts +10 -2
- package/lib/skill.js +34 -2
- package/lib/skillquality.d.ts +81 -0
- package/lib/skillquality.js +311 -0
- package/lib/tool.js +6 -3
- package/lib/types.d.ts +31 -0
- package/lib/types.js +19 -0
- package/lib/validate.js +25 -1
- package/lib/wrapup.d.ts +210 -0
- package/lib/wrapup.js +439 -0
- package/package.json +24 -14
package/lib/score.js
CHANGED
|
@@ -1,8 +1,22 @@
|
|
|
1
|
-
export const DEFAULT_AGGREGATE = {
|
|
2
|
-
|
|
1
|
+
export const DEFAULT_AGGREGATE = {
|
|
2
|
+
passThreshold: 60,
|
|
3
|
+
regressionTolerance: 0,
|
|
4
|
+
maxFailedCells: 0,
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Aggregate raw cells into code-owned per-case means + overall mean.
|
|
8
|
+
* Failure-cell protocol (gap A2): failed cells are EXCLUDED from means and
|
|
9
|
+
* counted separately — a crashed unit can never silently drag the mean down
|
|
10
|
+
* like a zero. A case whose cells all failed reports null (no mean).
|
|
11
|
+
*/
|
|
3
12
|
export function aggregate(cells) {
|
|
4
13
|
const byCase = new Map();
|
|
14
|
+
let failed = 0;
|
|
5
15
|
for (const cell of cells) {
|
|
16
|
+
if (cell.status === "failed") {
|
|
17
|
+
failed += 1;
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
6
20
|
const list = byCase.get(cell.caseId) ?? [];
|
|
7
21
|
list.push(clampScore(cell.score));
|
|
8
22
|
byCase.set(cell.caseId, list);
|
|
@@ -12,7 +26,12 @@ export function aggregate(cells) {
|
|
|
12
26
|
perCase[caseId] = mean(scores);
|
|
13
27
|
}
|
|
14
28
|
const all = [...byCase.values()].flat();
|
|
15
|
-
return {
|
|
29
|
+
return {
|
|
30
|
+
...perCase,
|
|
31
|
+
overall: all.length > 0 ? mean(all) : null,
|
|
32
|
+
failed,
|
|
33
|
+
total: cells.length,
|
|
34
|
+
};
|
|
16
35
|
}
|
|
17
36
|
export function entryFromCells(label, cells, refinementId) {
|
|
18
37
|
const aggr = aggregate(cells);
|
|
@@ -29,31 +48,53 @@ export function entryFromCells(label, cells, refinementId) {
|
|
|
29
48
|
export function decisionReport(reference, candidate, decision) {
|
|
30
49
|
const lines = [`overall: ${reference.overall ?? "?"} → ${candidate.overall ?? "?"}`];
|
|
31
50
|
for (const [caseId, refScore] of Object.entries(reference.aggregate)) {
|
|
32
|
-
if (caseId === "overall" || refScore === null)
|
|
51
|
+
if (caseId === "overall" || caseId === "failed" || caseId === "total" || refScore === null)
|
|
33
52
|
continue;
|
|
34
53
|
const candScore = candidate.aggregate[caseId];
|
|
35
|
-
|
|
54
|
+
const failedMark = isCaseFailed(reference, caseId) || isCaseFailed(candidate, caseId) ? " (failed)" : "";
|
|
55
|
+
lines.push(` ${caseId}: ${refScore} → ${candScore ?? "?"}${failedMark}`);
|
|
56
|
+
}
|
|
57
|
+
const refFailed = reference.aggregate.failed ?? 0;
|
|
58
|
+
const candFailed = candidate.aggregate.failed ?? 0;
|
|
59
|
+
if (refFailed > 0 || candFailed > 0) {
|
|
60
|
+
lines.push(`failed cells: reference ${refFailed}/${reference.aggregate.total ?? 0} · candidate ${candFailed}/${candidate.aggregate.total ?? 0}`);
|
|
36
61
|
}
|
|
37
62
|
lines.push(decision.accepted
|
|
38
63
|
? "DECISION: ACCEPTED — overall improved, no regression"
|
|
39
64
|
: `DECISION: REJECTED — ${decision.reasons.join("; ")}`);
|
|
40
65
|
return lines;
|
|
41
66
|
}
|
|
67
|
+
function isCaseFailed(entry, caseId) {
|
|
68
|
+
return entry.cells.some((cell) => cell.caseId === caseId && cell.status === "failed");
|
|
69
|
+
}
|
|
42
70
|
/**
|
|
43
71
|
* Non-regressive acceptance rule (Self-Harness style):
|
|
44
72
|
* the candidate is accepted iff its overall mean is STRICTLY higher than the
|
|
45
|
-
* reference
|
|
73
|
+
* reference, no case regresses by more than `regressionTolerance` points,
|
|
74
|
+
* and neither side has more failed cells than `maxFailedCells` (failure-cell
|
|
75
|
+
* protocol, gap A2 — a partial/invalid round is never accepted).
|
|
46
76
|
*/
|
|
47
77
|
export function decide(reference, candidate, opts) {
|
|
48
78
|
const reasons = [];
|
|
49
79
|
if (reference.overall === null || candidate.overall === null) {
|
|
50
80
|
return { accepted: false, reasons: ["reference or candidate evaluation is incomplete"] };
|
|
51
81
|
}
|
|
82
|
+
const refFailed = reference.aggregate.failed ?? 0;
|
|
83
|
+
const candFailed = candidate.aggregate.failed ?? 0;
|
|
84
|
+
if (refFailed > opts.maxFailedCells) {
|
|
85
|
+
reasons.push(`reference has ${refFailed} failed cells (max ${opts.maxFailedCells})`);
|
|
86
|
+
}
|
|
87
|
+
if (candFailed > opts.maxFailedCells) {
|
|
88
|
+
reasons.push(`candidate has ${candFailed} failed cells (max ${opts.maxFailedCells})`);
|
|
89
|
+
}
|
|
90
|
+
if (reasons.length > 0) {
|
|
91
|
+
return { accepted: false, reasons };
|
|
92
|
+
}
|
|
52
93
|
if (candidate.overall <= reference.overall) {
|
|
53
94
|
reasons.push(`overall not improved: ${candidate.overall} <= ${reference.overall}`);
|
|
54
95
|
}
|
|
55
96
|
for (const [caseId, refScore] of Object.entries(reference.aggregate)) {
|
|
56
|
-
if (caseId === "overall" || refScore === null)
|
|
97
|
+
if (caseId === "overall" || caseId === "failed" || caseId === "total" || refScore === null)
|
|
57
98
|
continue;
|
|
58
99
|
const candScore = candidate.aggregate[caseId];
|
|
59
100
|
if (candScore === null || candScore === undefined) {
|
package/lib/skill.d.ts
CHANGED
|
@@ -5,6 +5,14 @@ export declare function skillNameOf(id: string): string;
|
|
|
5
5
|
export declare function skillDir(skillsRoot: string, id: string): string;
|
|
6
6
|
/** Render a harness skill entry as a discoverable SKILL.md document. */
|
|
7
7
|
export declare function renderSkillMarkdown(entry: HarnessEntry): string;
|
|
8
|
-
/**
|
|
9
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Apply the skill-kind edits of an applied refinement to the skills root.
|
|
10
|
+
* Returns materialization warnings (rendered-SKILL.md mechanical problems
|
|
11
|
+
* and dangling resource references) — the file is still written, but the
|
|
12
|
+
* caller should surface them: a rendered file that fails the platform's
|
|
13
|
+
* frontmatter rules would be IGNORED by the skill loader, and a body
|
|
14
|
+
* referencing resources the entry does not ship would load with broken
|
|
15
|
+
* links.
|
|
16
|
+
*/
|
|
17
|
+
export declare function syncSkillsFromResult(skillsRoot: string, result: RefinementResult): string[];
|
|
10
18
|
//# sourceMappingURL=skill.d.ts.map
|
package/lib/skill.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
11
11
|
import { join, resolve, sep } from "node:path";
|
|
12
|
+
import { skillResourceRefs, validateRenderedSkill } from "./skillquality.js";
|
|
12
13
|
/** Convert a harness entry id (underscore slug) to a kebab-case skill name. */
|
|
13
14
|
export function skillNameOf(id) {
|
|
14
15
|
return id.toLowerCase().replace(/_/g, "-");
|
|
@@ -44,8 +45,17 @@ export function renderSkillMarkdown(entry) {
|
|
|
44
45
|
}
|
|
45
46
|
return `${lines.join("\n").trimEnd()}\n`;
|
|
46
47
|
}
|
|
47
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* Apply the skill-kind edits of an applied refinement to the skills root.
|
|
50
|
+
* Returns materialization warnings (rendered-SKILL.md mechanical problems
|
|
51
|
+
* and dangling resource references) — the file is still written, but the
|
|
52
|
+
* caller should surface them: a rendered file that fails the platform's
|
|
53
|
+
* frontmatter rules would be IGNORED by the skill loader, and a body
|
|
54
|
+
* referencing resources the entry does not ship would load with broken
|
|
55
|
+
* links.
|
|
56
|
+
*/
|
|
48
57
|
export function syncSkillsFromResult(skillsRoot, result) {
|
|
58
|
+
const warnings = [];
|
|
49
59
|
for (const edit of result.appliedEdits) {
|
|
50
60
|
if (edit.kind !== "skill" || !edit.applied)
|
|
51
61
|
continue;
|
|
@@ -53,15 +63,37 @@ export function syncSkillsFromResult(skillsRoot, result) {
|
|
|
53
63
|
removeSkill(skillsRoot, edit.id);
|
|
54
64
|
continue;
|
|
55
65
|
}
|
|
56
|
-
writeSkill(skillsRoot, edit.after);
|
|
66
|
+
warnings.push(...writeSkill(skillsRoot, edit.after));
|
|
57
67
|
}
|
|
68
|
+
return warnings;
|
|
58
69
|
}
|
|
70
|
+
/** Write one skill entry as a SKILL.md; returns materialization warnings. */
|
|
59
71
|
function writeSkill(skillsRoot, entry) {
|
|
60
72
|
const dir = skillDir(skillsRoot, entry.id);
|
|
61
73
|
mkdirSync(dir, { recursive: true });
|
|
62
74
|
const temp = join(dir, `SKILL.md.${process.pid}.tmp`);
|
|
63
75
|
writeFileSync(temp, renderSkillMarkdown(entry), "utf8");
|
|
64
76
|
renameSync(temp, join(dir, "SKILL.md"));
|
|
77
|
+
return materializationWarnings(dir, entry);
|
|
78
|
+
}
|
|
79
|
+
/** Post-write checks on the exact file that landed on disk. */
|
|
80
|
+
function materializationWarnings(dir, entry) {
|
|
81
|
+
const warnings = [];
|
|
82
|
+
for (const problem of validateRenderedSkill(entry)) {
|
|
83
|
+
warnings.push(`skill ${entry.id}: rendered SKILL.md would be ignored by the platform: ${problem}`);
|
|
84
|
+
}
|
|
85
|
+
const root = resolve(dir);
|
|
86
|
+
for (const ref of skillResourceRefs(entry.content)) {
|
|
87
|
+
const target = resolve(root, ref);
|
|
88
|
+
if (target !== root && !target.startsWith(`${root}${sep}`)) {
|
|
89
|
+
warnings.push(`skill ${entry.id}: body resource reference escapes the skill directory: ${ref}`);
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
if (!existsSync(target)) {
|
|
93
|
+
warnings.push(`skill ${entry.id}: body references missing resource ${ref} (expected at ${target})`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return warnings;
|
|
65
97
|
}
|
|
66
98
|
function removeSkill(skillsRoot, id) {
|
|
67
99
|
const dir = skillDir(skillsRoot, id);
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import type { HarnessEntry } from "./types.js";
|
|
2
|
+
import { skillNameOf } from "./skill.js";
|
|
3
|
+
/** Relative location of the skill-creator template facts. */
|
|
4
|
+
export declare const SKILL_CREATOR_TEMPLATE_REL: string;
|
|
5
|
+
/**
|
|
6
|
+
* Read the skill-creator template facts
|
|
7
|
+
* (`<skillsRoot>/skill-creator/references/template.md`; facts distilled
|
|
8
|
+
* from the official deepseek-harness skills). Returns null when the skills
|
|
9
|
+
* are not installed — callers fall back to the builtin distilled guide.
|
|
10
|
+
* Reading is a runtime reference, never a copy: template updates in the
|
|
11
|
+
* skill are picked up automatically.
|
|
12
|
+
*/
|
|
13
|
+
export declare function readSkillCreatorTemplate(skillsRoot: string): string | null;
|
|
14
|
+
/**
|
|
15
|
+
* Builtin distilled skill-quality guide (fallback when the skill-creator
|
|
16
|
+
* template is not installed). Condenses the template facts — frontmatter
|
|
17
|
+
* schema, the 7 structural features, paragraph skeleton, and the
|
|
18
|
+
* no-duplication / real-trigger rules — so a planner still authors skills
|
|
19
|
+
* to the standard on installs without the skill-creator / skill-audit
|
|
20
|
+
* skills.
|
|
21
|
+
*/
|
|
22
|
+
export declare const BUILTIN_SKILL_QUALITY_GUIDE = "DSH skill quality standard (distilled by the author from the official deepseek-harness 11 skills; the full facts live in <skillsRoot>/skill-creator/references/template.md when installed):\n\nFrontmatter schema (platform-enforced; violations make the platform IGNORE the whole file):\n- name: required, kebab-case only (^[a-z0-9]+(?:-[a-z0-9]+)*$)\n- description: required, non-empty; write \"use when / do not use when\" routing so the model can select it correctly\n- invocation booleans accept true/false/yes/no/on/off/1/0; legacy camelCase keys (disableModelInvocation / modelInvocable / userInvocable) are rejected\n- whenToUse (optional): non-empty string; metadata (optional): object\n\nThe 7 structural features of the official deepseek-harness skills:\n1. Frontmatter is routing metadata, not a summary (description = when to use / when not to use)\n2. Opens with a boundary declaration (guidance, not a script; mechanical flow skills may omit the disclaimer)\n3. Prerequisites + exclusions: explicit required input, stop when missing (report the required input and stop), excluded scenarios\n4. Layered information: Sources of truth (link only, do not re-summarize) -> numbered blocking requirements -> manual checks -> verification commands -> report format; all executable, no slogans\n5. Skill interlinks: reference a single source of truth instead of duplicating it\n6. Verifiable completion criteria: explicit verification commands and report format\n7. Real use + iteration: a real trigger scenario must exist; calibration conclusions distill into references/\n\nParagraph skeleton (writing order): frontmatter -> H1 + boundary declaration -> Sources of truth -> numbered requirements / workflow (full commands) -> exclusions / stop conditions -> verification and report.\n\nCreation rules: only create a skill for a REAL trigger scenario (who, in what real task, what signal) grounded in the trajectory \u2014 never invent one to pad the store; do not duplicate the official 11 skills or existing entries; skill bodies should be a SKILL.md document (this is what materializes under <skillsRoot>/<kebab-name>/SKILL.md).";
|
|
23
|
+
export interface SkillQualityGuide {
|
|
24
|
+
/** Where the guide text came from: the on-disk template or the builtin guide. */
|
|
25
|
+
source: "template" | "builtin";
|
|
26
|
+
text: string;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* The quality guide handed to the planner: the skill-creator template facts
|
|
30
|
+
* when the skills are installed, otherwise the builtin distilled guide.
|
|
31
|
+
* Never throws — a missing/unreadable template degrades to the builtin.
|
|
32
|
+
*/
|
|
33
|
+
export declare function skillQualityGuide(skillsRoot: string | undefined): SkillQualityGuide;
|
|
34
|
+
/** Split frontmatter out of a raw SKILL.md. Returns { yaml, body } or null when delimiters are missing. */
|
|
35
|
+
export declare function splitFrontmatter(raw: string): {
|
|
36
|
+
yaml: string;
|
|
37
|
+
body: string;
|
|
38
|
+
} | null;
|
|
39
|
+
/**
|
|
40
|
+
* Mechanical frontmatter validation of a rendered SKILL.md, mirroring
|
|
41
|
+
* `skill-creator/scripts/validate-frontmatter.mjs` (and the platform's
|
|
42
|
+
* skill-filesystem rules): delimiter structure, name kebab-case, non-empty
|
|
43
|
+
* description, invocation-boolean spellings, legacy camelCase key rejection,
|
|
44
|
+
* whenToUse/metadata types. Returns human-readable problems; an empty array
|
|
45
|
+
* means the file would load.
|
|
46
|
+
*/
|
|
47
|
+
export declare function validateRenderedSkillMarkdown(markdown: string): string[];
|
|
48
|
+
/**
|
|
49
|
+
* Mechanical validation of a skill entry's raw `content` (the SKILL.md body
|
|
50
|
+
* that materializes under the generated frontmatter). Code-enforced at
|
|
51
|
+
* apply time so a bad entry never reaches the store:
|
|
52
|
+
* - empty content is rejected;
|
|
53
|
+
* - content must not open with a `---` block: the materializer generates
|
|
54
|
+
* its own frontmatter, and a second frontmatter in the body would be
|
|
55
|
+
* parsed instead of the generated one (the platform reads the FIRST
|
|
56
|
+
* closing `---`), so the file could be ignored or routed wrongly;
|
|
57
|
+
* - resource references (`references/…`, `scripts/…`) must be skill-local
|
|
58
|
+
* relative paths — parent-relative (`../`) or absolute targets escape the
|
|
59
|
+
* skill directory and are rejected.
|
|
60
|
+
* Returns human-readable problems; an empty array means the content is
|
|
61
|
+
* mechanically acceptable.
|
|
62
|
+
*/
|
|
63
|
+
export declare function validateSkillEntryContent(content: string): string[];
|
|
64
|
+
/**
|
|
65
|
+
* Validate the FULL rendered SKILL.md of an entry (generated frontmatter +
|
|
66
|
+
* body) — the exact bytes that materialize on disk. Used as the final
|
|
67
|
+
* code-enforced line after materialization; problems here mean the platform
|
|
68
|
+
* would refuse to load the file.
|
|
69
|
+
*/
|
|
70
|
+
export declare function validateRenderedSkill(entry: HarnessEntry): string[];
|
|
71
|
+
/**
|
|
72
|
+
* Resource references (`references/…`, `scripts/…`) found in a skill body —
|
|
73
|
+
* the same scanning policy as validate-frontmatter.mjs: markdown link
|
|
74
|
+
* targets starting with the category, plus backticked/prose paths carrying
|
|
75
|
+
* a filename extension. Used after materialization to warn about dangling
|
|
76
|
+
* references (a body referencing a resource the entry never ships).
|
|
77
|
+
*/
|
|
78
|
+
export declare function skillResourceRefs(content: string): string[];
|
|
79
|
+
/** Kebab-case name under which the entry materializes (exported for diagnostics). */
|
|
80
|
+
export { skillNameOf };
|
|
81
|
+
//# sourceMappingURL=skillquality.d.ts.map
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Skill-quality integration: makes the DSH skill quality standard — carried
|
|
3
|
+
* by the author-distilled skills skill-creator / skill-audit (distilled
|
|
4
|
+
* from the official deepseek-harness 11 skills, facts verified against
|
|
5
|
+
* deepseek-harness 47f9438) — usable INSIDE the self-evolution loop.
|
|
6
|
+
*
|
|
7
|
+
* The planner and review gate are raw `ctx.llm` calls — they do not live in
|
|
8
|
+
* an agent session, so they cannot load skills through the `skill` tool.
|
|
9
|
+
* The skill-creator / skill-audit skills stay the single source of truth on
|
|
10
|
+
* disk; this module only:
|
|
11
|
+
*
|
|
12
|
+
* 1. reads the template facts at runtime
|
|
13
|
+
* (`<skillsRoot>/skill-creator/references/template.md`, 85 lines) and
|
|
14
|
+
* hands them to the planner as a `<skill_quality_standard>` block —
|
|
15
|
+
* the on-disk template wins, a built-in distilled guide is the fallback
|
|
16
|
+
* for installs without these skills;
|
|
17
|
+
* 2. code-enforces the mechanical frontmatter rules of
|
|
18
|
+
* `skill-creator/scripts/validate-frontmatter.mjs` (the platform would
|
|
19
|
+
* IGNORE a file that fails them), so a skill entry can never materialize
|
|
20
|
+
* a SKILL.md the platform refuses to load.
|
|
21
|
+
*/
|
|
22
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
23
|
+
import { join } from "node:path";
|
|
24
|
+
import { renderSkillMarkdown, skillNameOf } from "./skill.js";
|
|
25
|
+
/** Skill-name regex the platform enforces (skill-filesystem). */
|
|
26
|
+
const NAME_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
27
|
+
const TRUE_WORDS = new Set(["true", "yes", "on", "1"]);
|
|
28
|
+
const FALSE_WORDS = new Set(["false", "no", "off", "0"]);
|
|
29
|
+
const LEGACY_KEYS = ["disableModelInvocation", "modelInvocable", "userInvocable"];
|
|
30
|
+
const CANONICAL_KEYS = {
|
|
31
|
+
disableModelInvocation: "disable-model-invocation",
|
|
32
|
+
modelInvocable: "disable-model-invocation",
|
|
33
|
+
userInvocable: "user-invocable",
|
|
34
|
+
};
|
|
35
|
+
/** Relative location of the skill-creator template facts. */
|
|
36
|
+
export const SKILL_CREATOR_TEMPLATE_REL = join("skill-creator", "references", "template.md");
|
|
37
|
+
/**
|
|
38
|
+
* Read the skill-creator template facts
|
|
39
|
+
* (`<skillsRoot>/skill-creator/references/template.md`; facts distilled
|
|
40
|
+
* from the official deepseek-harness skills). Returns null when the skills
|
|
41
|
+
* are not installed — callers fall back to the builtin distilled guide.
|
|
42
|
+
* Reading is a runtime reference, never a copy: template updates in the
|
|
43
|
+
* skill are picked up automatically.
|
|
44
|
+
*/
|
|
45
|
+
export function readSkillCreatorTemplate(skillsRoot) {
|
|
46
|
+
const path = join(skillsRoot, SKILL_CREATOR_TEMPLATE_REL);
|
|
47
|
+
try {
|
|
48
|
+
if (!existsSync(path))
|
|
49
|
+
return null;
|
|
50
|
+
const text = readFileSync(path, "utf8");
|
|
51
|
+
return text.trim().length > 0 ? text : null;
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Builtin distilled skill-quality guide (fallback when the skill-creator
|
|
59
|
+
* template is not installed). Condenses the template facts — frontmatter
|
|
60
|
+
* schema, the 7 structural features, paragraph skeleton, and the
|
|
61
|
+
* no-duplication / real-trigger rules — so a planner still authors skills
|
|
62
|
+
* to the standard on installs without the skill-creator / skill-audit
|
|
63
|
+
* skills.
|
|
64
|
+
*/
|
|
65
|
+
export const BUILTIN_SKILL_QUALITY_GUIDE = `DSH skill quality standard (distilled by the author from the official deepseek-harness 11 skills; the full facts live in <skillsRoot>/skill-creator/references/template.md when installed):
|
|
66
|
+
|
|
67
|
+
Frontmatter schema (platform-enforced; violations make the platform IGNORE the whole file):
|
|
68
|
+
- name: required, kebab-case only (^[a-z0-9]+(?:-[a-z0-9]+)*$)
|
|
69
|
+
- description: required, non-empty; write "use when / do not use when" routing so the model can select it correctly
|
|
70
|
+
- invocation booleans accept true/false/yes/no/on/off/1/0; legacy camelCase keys (disableModelInvocation / modelInvocable / userInvocable) are rejected
|
|
71
|
+
- whenToUse (optional): non-empty string; metadata (optional): object
|
|
72
|
+
|
|
73
|
+
The 7 structural features of the official deepseek-harness skills:
|
|
74
|
+
1. Frontmatter is routing metadata, not a summary (description = when to use / when not to use)
|
|
75
|
+
2. Opens with a boundary declaration (guidance, not a script; mechanical flow skills may omit the disclaimer)
|
|
76
|
+
3. Prerequisites + exclusions: explicit required input, stop when missing (report the required input and stop), excluded scenarios
|
|
77
|
+
4. Layered information: Sources of truth (link only, do not re-summarize) -> numbered blocking requirements -> manual checks -> verification commands -> report format; all executable, no slogans
|
|
78
|
+
5. Skill interlinks: reference a single source of truth instead of duplicating it
|
|
79
|
+
6. Verifiable completion criteria: explicit verification commands and report format
|
|
80
|
+
7. Real use + iteration: a real trigger scenario must exist; calibration conclusions distill into references/
|
|
81
|
+
|
|
82
|
+
Paragraph skeleton (writing order): frontmatter -> H1 + boundary declaration -> Sources of truth -> numbered requirements / workflow (full commands) -> exclusions / stop conditions -> verification and report.
|
|
83
|
+
|
|
84
|
+
Creation rules: only create a skill for a REAL trigger scenario (who, in what real task, what signal) grounded in the trajectory — never invent one to pad the store; do not duplicate the official 11 skills or existing entries; skill bodies should be a SKILL.md document (this is what materializes under <skillsRoot>/<kebab-name>/SKILL.md).`;
|
|
85
|
+
/**
|
|
86
|
+
* The quality guide handed to the planner: the skill-creator template facts
|
|
87
|
+
* when the skills are installed, otherwise the builtin distilled guide.
|
|
88
|
+
* Never throws — a missing/unreadable template degrades to the builtin.
|
|
89
|
+
*/
|
|
90
|
+
export function skillQualityGuide(skillsRoot) {
|
|
91
|
+
if (skillsRoot) {
|
|
92
|
+
const template = readSkillCreatorTemplate(skillsRoot);
|
|
93
|
+
if (template !== null) {
|
|
94
|
+
return {
|
|
95
|
+
source: "template",
|
|
96
|
+
text: `The skill-creator template facts (distilled from the official deepseek-harness 11 skills, verified against deepseek-harness 47f9438; single source of truth, read from <skillsRoot>/skill-creator/references/template.md):\n\n${template}`,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return { source: "builtin", text: BUILTIN_SKILL_QUALITY_GUIDE };
|
|
101
|
+
}
|
|
102
|
+
/** Split frontmatter out of a raw SKILL.md. Returns { yaml, body } or null when delimiters are missing. */
|
|
103
|
+
export function splitFrontmatter(raw) {
|
|
104
|
+
const lines = raw.split(/\r?\n/);
|
|
105
|
+
if (lines[0] !== "---")
|
|
106
|
+
return null;
|
|
107
|
+
const close = lines.indexOf("---", 1);
|
|
108
|
+
if (close < 0)
|
|
109
|
+
return null;
|
|
110
|
+
return { yaml: lines.slice(1, close).join("\n"), body: lines.slice(close + 1).join("\n") };
|
|
111
|
+
}
|
|
112
|
+
/** Parse one scalar in the YAML subset the platform's schema keys use. */
|
|
113
|
+
function parseScalar(raw, lineNo) {
|
|
114
|
+
const value = raw.trim();
|
|
115
|
+
if (value === "")
|
|
116
|
+
return "";
|
|
117
|
+
if (value.startsWith("'")) {
|
|
118
|
+
if (!value.endsWith("'"))
|
|
119
|
+
throw new Error(`unterminated single-quoted scalar at line ${lineNo}`);
|
|
120
|
+
return value.slice(1, -1).replace(/''/g, "'");
|
|
121
|
+
}
|
|
122
|
+
if (value.startsWith('"')) {
|
|
123
|
+
if (!value.endsWith('"'))
|
|
124
|
+
throw new Error(`unterminated double-quoted scalar at line ${lineNo}`);
|
|
125
|
+
return value
|
|
126
|
+
.slice(1, -1)
|
|
127
|
+
.replace(/\\n/g, "\n")
|
|
128
|
+
.replace(/\\t/g, "\t")
|
|
129
|
+
.replace(/\\"/g, '"')
|
|
130
|
+
.replace(/\\\\/g, "\\");
|
|
131
|
+
}
|
|
132
|
+
return value;
|
|
133
|
+
}
|
|
134
|
+
/** Minimal YAML-subset parser for the flat schema the platform reads (mirrors validate-frontmatter.mjs). */
|
|
135
|
+
function parseMiniYaml(text) {
|
|
136
|
+
const data = {};
|
|
137
|
+
const lines = text.split("\n");
|
|
138
|
+
let i = 0;
|
|
139
|
+
while (i < lines.length) {
|
|
140
|
+
const line = lines[i] ?? "";
|
|
141
|
+
const lineNo = i + 1;
|
|
142
|
+
i += 1;
|
|
143
|
+
const trimmed = line.trim();
|
|
144
|
+
if (trimmed === "" || trimmed.startsWith("#"))
|
|
145
|
+
continue;
|
|
146
|
+
if (line.length - line.trimStart().length > 0) {
|
|
147
|
+
throw new Error(`unsupported indented construct at line ${lineNo}: ${trimmed}`);
|
|
148
|
+
}
|
|
149
|
+
const match = /^([A-Za-z0-9_-]+):(?:\s+(.*))?$/.exec(trimmed);
|
|
150
|
+
if (!match)
|
|
151
|
+
throw new Error(`unparseable line ${lineNo}: ${trimmed}`);
|
|
152
|
+
const key = match[1] ?? "";
|
|
153
|
+
let value = match[2] ?? "";
|
|
154
|
+
if (value === "|" || value === ">") {
|
|
155
|
+
const block = [];
|
|
156
|
+
while (i < lines.length && (lines[i] ?? "").trim() !== "" && (lines[i] ?? "").startsWith(" ")) {
|
|
157
|
+
block.push(lines[i] ?? "");
|
|
158
|
+
i += 1;
|
|
159
|
+
}
|
|
160
|
+
data[key] = block.join("\n");
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
if (value === "" && i < lines.length && (lines[i] ?? "").startsWith(" ") && (lines[i] ?? "").trim() !== "") {
|
|
164
|
+
const nested = {};
|
|
165
|
+
while (i < lines.length && (lines[i] ?? "").trim() !== "" && (lines[i] ?? "").startsWith(" ")) {
|
|
166
|
+
const nm = /^([A-Za-z0-9_-]+):(?:\s+(.*))?$/.exec((lines[i] ?? "").trim());
|
|
167
|
+
if (!nm)
|
|
168
|
+
throw new Error(`unparseable nested line ${i + 1}: ${(lines[i] ?? "").trim()}`);
|
|
169
|
+
nested[nm[1] ?? ""] = parseScalar(nm[2] ?? "", i + 1);
|
|
170
|
+
i += 1;
|
|
171
|
+
}
|
|
172
|
+
data[key] = nested;
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
data[key] = parseScalar(value, lineNo);
|
|
176
|
+
}
|
|
177
|
+
return data;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Mechanical frontmatter validation of a rendered SKILL.md, mirroring
|
|
181
|
+
* `skill-creator/scripts/validate-frontmatter.mjs` (and the platform's
|
|
182
|
+
* skill-filesystem rules): delimiter structure, name kebab-case, non-empty
|
|
183
|
+
* description, invocation-boolean spellings, legacy camelCase key rejection,
|
|
184
|
+
* whenToUse/metadata types. Returns human-readable problems; an empty array
|
|
185
|
+
* means the file would load.
|
|
186
|
+
*/
|
|
187
|
+
export function validateRenderedSkillMarkdown(markdown) {
|
|
188
|
+
const problems = [];
|
|
189
|
+
const split = splitFrontmatter(markdown);
|
|
190
|
+
if (!split) {
|
|
191
|
+
return [
|
|
192
|
+
"missing YAML frontmatter (first line `---` with a closing `---`) — platform would IGNORE this file",
|
|
193
|
+
];
|
|
194
|
+
}
|
|
195
|
+
let data;
|
|
196
|
+
try {
|
|
197
|
+
data = parseMiniYaml(split.yaml);
|
|
198
|
+
}
|
|
199
|
+
catch (cause) {
|
|
200
|
+
return [`invalid YAML frontmatter: ${cause instanceof Error ? cause.message : String(cause)} — platform would IGNORE this file`];
|
|
201
|
+
}
|
|
202
|
+
const name = typeof data["name"] === "string" && data["name"].length > 0 ? data["name"] : undefined;
|
|
203
|
+
if (name === undefined) {
|
|
204
|
+
problems.push("frontmatter requires non-empty `name` — platform would IGNORE this file");
|
|
205
|
+
}
|
|
206
|
+
else if (!NAME_RE.test(name)) {
|
|
207
|
+
problems.push(`invalid skill name "${name}" (must match ${NAME_RE}) — platform would IGNORE this file`);
|
|
208
|
+
}
|
|
209
|
+
const description = typeof data["description"] === "string" && data["description"].length > 0 ? data["description"] : undefined;
|
|
210
|
+
if (description === undefined) {
|
|
211
|
+
problems.push("frontmatter requires non-empty `description` — platform would IGNORE this file");
|
|
212
|
+
}
|
|
213
|
+
for (const legacy of LEGACY_KEYS) {
|
|
214
|
+
if (Object.hasOwn(data, legacy)) {
|
|
215
|
+
problems.push(`legacy key "${legacy}" is unsupported; use "${CANONICAL_KEYS[legacy]}" — platform would IGNORE this file`);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
for (const key of ["disable-model-invocation", "user-invocable"]) {
|
|
219
|
+
if (!Object.hasOwn(data, key))
|
|
220
|
+
continue;
|
|
221
|
+
const value = data[key];
|
|
222
|
+
if (typeof value === "boolean")
|
|
223
|
+
continue;
|
|
224
|
+
const word = String(value).toLowerCase();
|
|
225
|
+
if (TRUE_WORDS.has(word) || FALSE_WORDS.has(word))
|
|
226
|
+
continue;
|
|
227
|
+
problems.push(`frontmatter field "${key}" must be a boolean (accepted: true/false/yes/no/on/off/1/0), got ${JSON.stringify(value)} — platform would IGNORE this file`);
|
|
228
|
+
}
|
|
229
|
+
if (Object.hasOwn(data, "whenToUse") && !(typeof data["whenToUse"] === "string" && data["whenToUse"].length > 0)) {
|
|
230
|
+
problems.push("`whenToUse` must be a non-empty string when present");
|
|
231
|
+
}
|
|
232
|
+
if (Object.hasOwn(data, "metadata") && (typeof data["metadata"] !== "object" || data["metadata"] === null || Array.isArray(data["metadata"]))) {
|
|
233
|
+
problems.push("`metadata` must be an object when present");
|
|
234
|
+
}
|
|
235
|
+
return problems;
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Mechanical validation of a skill entry's raw `content` (the SKILL.md body
|
|
239
|
+
* that materializes under the generated frontmatter). Code-enforced at
|
|
240
|
+
* apply time so a bad entry never reaches the store:
|
|
241
|
+
* - empty content is rejected;
|
|
242
|
+
* - content must not open with a `---` block: the materializer generates
|
|
243
|
+
* its own frontmatter, and a second frontmatter in the body would be
|
|
244
|
+
* parsed instead of the generated one (the platform reads the FIRST
|
|
245
|
+
* closing `---`), so the file could be ignored or routed wrongly;
|
|
246
|
+
* - resource references (`references/…`, `scripts/…`) must be skill-local
|
|
247
|
+
* relative paths — parent-relative (`../`) or absolute targets escape the
|
|
248
|
+
* skill directory and are rejected.
|
|
249
|
+
* Returns human-readable problems; an empty array means the content is
|
|
250
|
+
* mechanically acceptable.
|
|
251
|
+
*/
|
|
252
|
+
export function validateSkillEntryContent(content) {
|
|
253
|
+
const problems = [];
|
|
254
|
+
const trimmed = content.trim();
|
|
255
|
+
if (trimmed.length === 0) {
|
|
256
|
+
problems.push("skill content is empty");
|
|
257
|
+
return problems;
|
|
258
|
+
}
|
|
259
|
+
if (trimmed.startsWith("---")) {
|
|
260
|
+
problems.push("skill content must not start with a `---` frontmatter block (the materializer generates frontmatter from id/title; a body-level `---` would shadow it and the platform could IGNORE the file)");
|
|
261
|
+
}
|
|
262
|
+
for (const match of trimmed.matchAll(/(?<![\w])(references|scripts)\/[^\s)]+/g)) {
|
|
263
|
+
const ref = match[0] ?? "";
|
|
264
|
+
if (ref.startsWith("../") || ref.includes("/../") || ref.startsWith("/") || /^[a-zA-Z]:[\\/]/.test(ref)) {
|
|
265
|
+
problems.push(`skill content resource reference escapes the skill directory: ${ref}`);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return problems;
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Validate the FULL rendered SKILL.md of an entry (generated frontmatter +
|
|
272
|
+
* body) — the exact bytes that materialize on disk. Used as the final
|
|
273
|
+
* code-enforced line after materialization; problems here mean the platform
|
|
274
|
+
* would refuse to load the file.
|
|
275
|
+
*/
|
|
276
|
+
export function validateRenderedSkill(entry) {
|
|
277
|
+
return validateRenderedSkillMarkdown(renderSkillMarkdown(entry));
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Resource references (`references/…`, `scripts/…`) found in a skill body —
|
|
281
|
+
* the same scanning policy as validate-frontmatter.mjs: markdown link
|
|
282
|
+
* targets starting with the category, plus backticked/prose paths carrying
|
|
283
|
+
* a filename extension. Used after materialization to warn about dangling
|
|
284
|
+
* references (a body referencing a resource the entry never ships).
|
|
285
|
+
*/
|
|
286
|
+
export function skillResourceRefs(content) {
|
|
287
|
+
const refs = new Set();
|
|
288
|
+
for (const match of content.matchAll(/\[[^\]]*\]\(([^)]+)\)/g)) {
|
|
289
|
+
const target = ((match[1] ?? "").trim().split(/\s+/)[0] ?? "").trim();
|
|
290
|
+
if (/^(references|scripts)\/[\w./-]+$/.test(target) && !target.startsWith("../")) {
|
|
291
|
+
refs.add(target);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
const stripped = content.replace(/\[[^\]]*\]\([^)]*\)/g, "");
|
|
295
|
+
for (const match of stripped.matchAll(/(?<![\w])(references|scripts)\/[\w./-]+\.\w+/g)) {
|
|
296
|
+
const path = match[0] ?? "";
|
|
297
|
+
// Cross-skill interlinks (`../skill-creator/...`) resolve against the
|
|
298
|
+
// sibling skill's directory, not this one — skip references whose
|
|
299
|
+
// prose prefix walks up a directory (mirrors validate-frontmatter.mjs).
|
|
300
|
+
let cursor = (match.index ?? 0) - 1;
|
|
301
|
+
while (cursor >= 0 && /[\w./-]/.test(stripped[cursor] ?? ""))
|
|
302
|
+
cursor -= 1;
|
|
303
|
+
if (!stripped.slice(cursor + 1, match.index ?? 0).includes("..")) {
|
|
304
|
+
refs.add(path);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
return [...refs];
|
|
308
|
+
}
|
|
309
|
+
/** Kebab-case name under which the entry materializes (exported for diagnostics). */
|
|
310
|
+
export { skillNameOf };
|
|
311
|
+
//# sourceMappingURL=skillquality.js.map
|
package/lib/tool.js
CHANGED
|
@@ -37,14 +37,15 @@ export function registerEvolveTools(ctx, engine, opts) {
|
|
|
37
37
|
}));
|
|
38
38
|
ctx.tools.register(defineTool({
|
|
39
39
|
name: "evolve_add",
|
|
40
|
-
description: "Create one harness entry (prompt/memory/skill/subagent).
|
|
40
|
+
description: "Create one harness entry (prompt/memory/skill/subagent). Executable skills require reference {type:python, import, callable} and an arguments contract; guidance skills (skill_kind=guidance) are SKILL.md documents — recurring multi-step workflows — and must NOT carry a reference. Snapshot, version, and history are handled automatically.",
|
|
41
41
|
parameters: {
|
|
42
42
|
kind: { type: "string", enum: ["prompt", "memory", "skill", "subagent"], required: true, description: "Entry kind." },
|
|
43
43
|
title: { type: "string", required: true, description: "Stable title." },
|
|
44
44
|
content: { type: "string", required: true, description: "Entry body." },
|
|
45
45
|
path: { type: "string", description: "Optional grouping path." },
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
skill_kind: { type: "string", enum: ["executable", "guidance"], description: "For skills: executable (python reference, default) or guidance (SKILL.md document, no reference)." },
|
|
47
|
+
reference: { type: "object", additionalProperties: true, description: "For executable skills: {type:'python', import, callable}." },
|
|
48
|
+
arguments: { type: "object", additionalProperties: true, description: "For executable skills: accepted input contract." },
|
|
48
49
|
global: { type: "boolean", description: "Set true to write the cross-session store (requires human approval; only for durable, reusable lessons)." },
|
|
49
50
|
},
|
|
50
51
|
output: {
|
|
@@ -64,6 +65,8 @@ export function registerEvolveTools(ctx, engine, opts) {
|
|
|
64
65
|
};
|
|
65
66
|
if (args.path !== undefined)
|
|
66
67
|
edit.path = args.path;
|
|
68
|
+
if (args.skill_kind !== undefined)
|
|
69
|
+
edit.skill_kind = args.skill_kind;
|
|
67
70
|
if (args.reference !== undefined)
|
|
68
71
|
edit.reference = args.reference;
|
|
69
72
|
if (args.arguments !== undefined)
|
package/lib/types.d.ts
CHANGED
|
@@ -10,6 +10,14 @@
|
|
|
10
10
|
*/
|
|
11
11
|
/** What a harness entry can be. */
|
|
12
12
|
export type RefinementKind = "prompt" | "memory" | "skill" | "subagent";
|
|
13
|
+
/**
|
|
14
|
+
* Skill-entry form: `executable` skills carry a python reference contract
|
|
15
|
+
* and can be hot-mounted as tools; `guidance` skills are SKILL.md documents
|
|
16
|
+
* (no python reference) that materialize as discoverable skills for the
|
|
17
|
+
* `skill` tool — the form for recurring multi-step workflows. Absent means
|
|
18
|
+
* `executable` (backwards compatible with pre-guidance stores).
|
|
19
|
+
*/
|
|
20
|
+
export type SkillKind = "executable" | "guidance";
|
|
13
21
|
/** How an entry changes. */
|
|
14
22
|
export type RefinementAction = "create" | "update" | "delete" | "archive";
|
|
15
23
|
/** Where an entry lives: session-scoped or cross-session. */
|
|
@@ -30,6 +38,25 @@ export declare const SOURCE_SEQS_KEY = "sourceSeqs";
|
|
|
30
38
|
* the entry can be restored (unarchive) or rolled back like any other edit.
|
|
31
39
|
*/
|
|
32
40
|
export declare const ARCHIVED_AT_KEY = "archivedAt";
|
|
41
|
+
/**
|
|
42
|
+
* Metadata key stamped on a LOCAL entry that was promoted to the global
|
|
43
|
+
* store by a session wrap-up: the id of the global entry it became. Present
|
|
44
|
+
* means the entry's lifecycle is finished — it must not be offered for
|
|
45
|
+
* promotion again (the global copy is the live one, the local copy is a
|
|
46
|
+
* restorable trace).
|
|
47
|
+
*/
|
|
48
|
+
export declare const PROMOTED_TO_KEY = "promotedTo";
|
|
49
|
+
/**
|
|
50
|
+
* Metadata key recording when a local entry was promoted to the global
|
|
51
|
+
* store (companion of {@link PROMOTED_TO_KEY}).
|
|
52
|
+
*/
|
|
53
|
+
export declare const PROMOTED_AT_KEY = "promotedAt";
|
|
54
|
+
/**
|
|
55
|
+
* Metadata key stamped on a GLOBAL entry created by a session wrap-up
|
|
56
|
+
* promotion: `<sessionId>:<localEntryId>` — the反向 provenance link from the
|
|
57
|
+
* cross-session copy back to the session it was distilled from.
|
|
58
|
+
*/
|
|
59
|
+
export declare const SOURCED_FROM_KEY = "sourcedFromLocal";
|
|
33
60
|
/**
|
|
34
61
|
* True when the entry is archived (hidden from injection, restorable).
|
|
35
62
|
* Absent or empty archivedAt means the entry is active.
|
|
@@ -58,6 +85,8 @@ export interface HarnessEntry {
|
|
|
58
85
|
reference: Record<string, unknown>;
|
|
59
86
|
/** Skill entries declare their accepted inputs here. */
|
|
60
87
|
arguments: Record<string, unknown>;
|
|
88
|
+
/** Skill form: "executable" (default) or "guidance" (SKILL.md document). */
|
|
89
|
+
skill_kind?: SkillKind;
|
|
61
90
|
metadata: Record<string, unknown>;
|
|
62
91
|
source: "evolve";
|
|
63
92
|
created_at: string;
|
|
@@ -91,6 +120,8 @@ export interface RefinementEdit {
|
|
|
91
120
|
path?: string;
|
|
92
121
|
reference?: Record<string, unknown>;
|
|
93
122
|
arguments?: Record<string, unknown>;
|
|
123
|
+
/** Skill form: "guidance" for SKILL.md document skills; absent = executable. */
|
|
124
|
+
skill_kind?: SkillKind;
|
|
94
125
|
metadata?: Record<string, unknown>;
|
|
95
126
|
reason?: string;
|
|
96
127
|
}
|