kc-beta 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +81 -0
- package/LICENSE-COMMERCIAL.md +125 -0
- package/README.md +21 -3
- package/package.json +14 -5
- package/src/agent/context-window.js +9 -12
- package/src/agent/context.js +14 -1
- package/src/agent/document-parser.js +169 -0
- package/src/agent/engine.js +499 -20
- package/src/agent/history/event-history.js +222 -0
- package/src/agent/llm-client.js +55 -0
- package/src/agent/message-utils.js +63 -0
- package/src/agent/pipelines/_milestone-derive.js +511 -0
- package/src/agent/pipelines/base.js +21 -0
- package/src/agent/pipelines/distillation.js +28 -15
- package/src/agent/pipelines/extraction.js +103 -36
- package/src/agent/pipelines/finalization.js +178 -11
- package/src/agent/pipelines/index.js +6 -1
- package/src/agent/pipelines/initializer.js +74 -8
- package/src/agent/pipelines/production-qc.js +31 -44
- package/src/agent/pipelines/skill-authoring.js +152 -80
- package/src/agent/pipelines/skill-testing.js +67 -23
- package/src/agent/retry.js +10 -2
- package/src/agent/scheduler.js +14 -2
- package/src/agent/session-state.js +35 -2
- package/src/agent/skill-loader.js +13 -7
- package/src/agent/skill-validator.js +163 -0
- package/src/agent/task-manager.js +61 -5
- package/src/agent/tools/_workflow-result-schema.js +249 -0
- package/src/agent/tools/document-chunk.js +21 -9
- package/src/agent/tools/phase-advance.js +52 -6
- package/src/agent/tools/release.js +51 -9
- package/src/agent/tools/rule-catalog.js +11 -1
- package/src/agent/tools/workflow-run.js +9 -4
- package/src/agent/tools/workspace-file.js +32 -0
- package/src/agent/workspace.js +61 -0
- package/src/cli/components.js +64 -14
- package/src/cli/index.js +62 -3
- package/src/cli/meme.js +26 -25
- package/src/config.js +65 -22
- package/src/model-tiers.json +48 -0
- package/src/providers.js +87 -0
- package/template/release/v1/README.md.tmpl +108 -0
- package/template/release/v1/catalog.json.tmpl +4 -0
- package/template/release/v1/kc_runtime/__init__.py +11 -0
- package/template/release/v1/kc_runtime/confidence.py +63 -0
- package/template/release/v1/kc_runtime/doc_parser.py +127 -0
- package/template/release/v1/manifest.json.tmpl +11 -0
- package/template/release/v1/render_dashboard.py +117 -0
- package/template/release/v1/run.py +212 -0
- package/template/release/v1/serve.sh +17 -0
- package/template/skills/en/meta-meta/skill-authoring/SKILL.md +19 -0
- package/template/skills/en/meta-meta/work-decomposition/SKILL.md +266 -0
- package/template/skills/en/skill-creator/SKILL.md +1 -1
- package/template/skills/zh/meta-meta/skill-authoring/SKILL.md +19 -0
- package/template/skills/zh/meta-meta/work-decomposition/SKILL.md +264 -0
- package/template/skills/zh/skill-creator/SKILL.md +1 -1
|
@@ -2,6 +2,8 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { Phase, PipelineEvent } from "./index.js";
|
|
4
4
|
import { Pipeline } from "./base.js";
|
|
5
|
+
import { SkillValidator } from "../skill-validator.js";
|
|
6
|
+
import { deriveSkillAuthoringMilestones } from "./_milestone-derive.js";
|
|
5
7
|
|
|
6
8
|
export class SkillAuthoringPipeline extends Pipeline {
|
|
7
9
|
/**
|
|
@@ -16,6 +18,13 @@ export class SkillAuthoringPipeline extends Pipeline {
|
|
|
16
18
|
super();
|
|
17
19
|
this._workspace = workspace;
|
|
18
20
|
this._taskManager = taskManager;
|
|
21
|
+
// v0.6.2 I2: skill validator catches malformed check_r###.py at the
|
|
22
|
+
// skill_authoring exit boundary instead of silently passing the
|
|
23
|
+
// phase and breaking in production_qc (E2E #4 unified_qc.py
|
|
24
|
+
// SyntaxError went undiagnosed for hours).
|
|
25
|
+
this._validator = new SkillValidator();
|
|
26
|
+
this._validationFailures = [];
|
|
27
|
+
this._validationSkipped = false;
|
|
19
28
|
this.totalRules = [];
|
|
20
29
|
this.skillsAuthored = [];
|
|
21
30
|
this.skillsWithScripts = [];
|
|
@@ -41,83 +50,22 @@ export class SkillAuthoringPipeline extends Pipeline {
|
|
|
41
50
|
}
|
|
42
51
|
|
|
43
52
|
_scanSkills() {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
//
|
|
47
|
-
//
|
|
48
|
-
// (
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
this.ruleIdsCovered = new Set();
|
|
54
|
-
const dir = path.join(this._workspace.cwd, "rule_skills");
|
|
55
|
-
if (!fs.existsSync(dir)) return;
|
|
56
|
-
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
57
|
-
if (!e.isDirectory() || e.name.startsWith("__")) continue;
|
|
58
|
-
const skillPath = path.join(dir, e.name);
|
|
59
|
-
if (fs.existsSync(path.join(skillPath, "SKILL.md")) || fs.readdirSync(skillPath).some((f) => f.endsWith(".py"))) {
|
|
60
|
-
this.skillsAuthored.push(e.name);
|
|
61
|
-
}
|
|
62
|
-
const scriptsDir = path.join(skillPath, "scripts");
|
|
63
|
-
if (fs.existsSync(scriptsDir) && fs.readdirSync(scriptsDir).length > 0) {
|
|
64
|
-
this.skillsWithScripts.push(e.name);
|
|
65
|
-
}
|
|
66
|
-
this._walkForRuleIds(skillPath);
|
|
67
|
-
}
|
|
53
|
+
// v0.7.0 A1: route through filesystem-derived milestone helper. The
|
|
54
|
+
// helper centralizes the ruleId extraction patterns (R### dirs,
|
|
55
|
+
// check_r###.py, range dirs R078_R128, grouped check_r###_r###.py)
|
|
56
|
+
// and recognizes both root-level check_*.py AND scripts/check*.py
|
|
57
|
+
// (per A6 — XM E2E #5 used scripts/ subdir).
|
|
58
|
+
const m = deriveSkillAuthoringMilestones(this._workspace);
|
|
59
|
+
this.skillsAuthored = [...m.skillsAuthored];
|
|
60
|
+
this.skillsWithScripts = [...m.skillsWithScripts];
|
|
61
|
+
this.ruleIdsCovered = new Set(m.ruleIdsCovered);
|
|
68
62
|
}
|
|
69
63
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
* - Grouped script: check_r002_r007.py → covers R002 through R007
|
|
76
|
-
*/
|
|
77
|
-
_walkForRuleIds(skillDir) {
|
|
78
|
-
const dirName = path.basename(skillDir);
|
|
79
|
-
const dirMatch = dirName.match(/^R0*(\d+)$/i);
|
|
80
|
-
if (dirMatch) this.ruleIdsCovered.add(`R${String(parseInt(dirMatch[1], 10)).padStart(3, "0")}`);
|
|
81
|
-
|
|
82
|
-
const walk = (d) => {
|
|
83
|
-
let entries;
|
|
84
|
-
try { entries = fs.readdirSync(d, { withFileTypes: true }); }
|
|
85
|
-
catch { return; }
|
|
86
|
-
for (const e of entries) {
|
|
87
|
-
if (e.name.startsWith(".")) continue;
|
|
88
|
-
const p = path.join(d, e.name);
|
|
89
|
-
if (e.isDirectory()) { walk(p); continue; }
|
|
90
|
-
// Per-rule: check_r014.py
|
|
91
|
-
const single = e.name.match(/check_r0*(\d+)\.py$/i);
|
|
92
|
-
if (single) {
|
|
93
|
-
this.ruleIdsCovered.add(`R${String(parseInt(single[1], 10)).padStart(3, "0")}`);
|
|
94
|
-
continue;
|
|
95
|
-
}
|
|
96
|
-
// Grouped: check_r002_r007.py, check_r002-r007.py, check_r59_r77.py
|
|
97
|
-
const grouped = e.name.match(/check_r0*(\d+)[_-]+r0*(\d+)\.py$/i);
|
|
98
|
-
if (grouped) {
|
|
99
|
-
const lo = parseInt(grouped[1], 10);
|
|
100
|
-
const hi = parseInt(grouped[2], 10);
|
|
101
|
-
for (let n = lo; n <= hi; n++) {
|
|
102
|
-
this.ruleIdsCovered.add(`R${String(n).padStart(3, "0")}`);
|
|
103
|
-
}
|
|
104
|
-
continue;
|
|
105
|
-
}
|
|
106
|
-
// Directory names that encode ranges: R078_R128/
|
|
107
|
-
// handled by caller passing skillDir
|
|
108
|
-
}
|
|
109
|
-
};
|
|
110
|
-
// Also handle dirs named like R078_R128/
|
|
111
|
-
const rangeDir = dirName.match(/^R0*(\d+)[_-]R0*(\d+)$/i);
|
|
112
|
-
if (rangeDir) {
|
|
113
|
-
const lo = parseInt(rangeDir[1], 10);
|
|
114
|
-
const hi = parseInt(rangeDir[2], 10);
|
|
115
|
-
for (let n = lo; n <= hi; n++) {
|
|
116
|
-
this.ruleIdsCovered.add(`R${String(n).padStart(3, "0")}`);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
walk(skillDir);
|
|
120
|
-
}
|
|
64
|
+
// v0.7.0 A1: ruleId extraction moved to _milestone-derive.js
|
|
65
|
+
// (deriveSkillAuthoringMilestones). Pattern recognition is identical
|
|
66
|
+
// — single rule (R014, check_r014.py), grouped scripts
|
|
67
|
+
// (check_r002_r007.py), range dirs (R078_R128). Kept as a single
|
|
68
|
+
// canonical implementation rather than duplicating across pipelines.
|
|
121
69
|
|
|
122
70
|
describeState() {
|
|
123
71
|
this._scanWorkspace();
|
|
@@ -128,15 +76,37 @@ export class SkillAuthoringPipeline extends Pipeline {
|
|
|
128
76
|
"## Phase: SKILL_AUTHORING\n" +
|
|
129
77
|
"Write verification skills for each extracted rule. Skills are first-class " +
|
|
130
78
|
"deliverables — they may serve as the production solution when worker LLM " +
|
|
131
|
-
"workflows are insufficient. Follow
|
|
132
|
-
"BUILD mode.\n\n" +
|
|
79
|
+
"workflows are insufficient. Follow the canonical skill-folder layout " +
|
|
80
|
+
"(below). This is BUILD mode.\n\n" +
|
|
81
|
+
// v0.7.0 D1: inline the canonical folder structure spec so the
|
|
82
|
+
// agent sees it in every system prompt of this phase. E2E #5
|
|
83
|
+
// showed three of four contestants ignored the meta-meta spec
|
|
84
|
+
// because it required navigating to read the SKILL.md file
|
|
85
|
+
// separately. Inlining costs ~250 tokens and dramatically improves
|
|
86
|
+
// first-attempt structural compliance.
|
|
87
|
+
"### Canonical skill folder layout\n" +
|
|
88
|
+
"```\n" +
|
|
89
|
+
"rule_skills/\n" +
|
|
90
|
+
" R014/ # one dir per rule (or grouped range)\n" +
|
|
91
|
+
" SKILL.md # YAML frontmatter (name+description) + methodology\n" +
|
|
92
|
+
" check_r014.py # entry point: def check_rule|verify|check|evaluate(...)\n" +
|
|
93
|
+
" references/regulation.md # verbatim regulation text (optional)\n" +
|
|
94
|
+
" references/interpretation.md # edge-case notes (optional)\n" +
|
|
95
|
+
" assets/test_cases.json # annotated samples + expected verdicts (optional)\n" +
|
|
96
|
+
"```\n" +
|
|
97
|
+
"Validator-accepted alternatives: `scripts/check_r###.py` (under scripts/) " +
|
|
98
|
+
"instead of root-level. SKILL.md filename is case-insensitive (skill.md " +
|
|
99
|
+
"is also accepted). The check.py just needs a top-level `def` at module " +
|
|
100
|
+
"level — entry-point name does not have to match a strict pattern.\n\n" +
|
|
133
101
|
// D2: soft granularity nudge
|
|
134
102
|
"**Granularity preference:** 1 rule = 1 skill directory. Group rules into " +
|
|
135
103
|
"the same file ONLY when they share evidence and fail together (e.g. " +
|
|
136
104
|
"siblings from the same required-fields table). When grouping, name the " +
|
|
137
105
|
"file with the range: `check_r002_r007.py`. Downstream consumers " +
|
|
138
|
-
"(workflow-run, dashboards) count rule coverage by parsing
|
|
139
|
-
"so the file-naming matters
|
|
106
|
+
"(workflow-run, dashboards, release tool) count rule coverage by parsing " +
|
|
107
|
+
"these names, so the file-naming matters. (Read `meta-meta/work-decomposition` " +
|
|
108
|
+
"for the full grouping/ordering decision framework + PATTERNS.md memory " +
|
|
109
|
+
"discipline.)\n\n" +
|
|
140
110
|
"**Do not write to rules/catalog.json via sandbox_exec.** Use the " +
|
|
141
111
|
"`rule_catalog` tool for any catalog edits — sandbox_exec bypasses the " +
|
|
142
112
|
"workspace file lock and races with parallel workers."
|
|
@@ -152,6 +122,18 @@ export class SkillAuthoringPipeline extends Pipeline {
|
|
|
152
122
|
(failedT > 0 ? ` (+${failedT} failed)` : "");
|
|
153
123
|
}
|
|
154
124
|
}
|
|
125
|
+
// v0.6.2 I2: validation status (only meaningful after first
|
|
126
|
+
// exitCriteriaMet call populates _validationFailures)
|
|
127
|
+
let validationLine = "";
|
|
128
|
+
if (this._validationSkipped) {
|
|
129
|
+
validationLine = `\n- Skill validation: SKIPPED (python3 not on PATH — install to enable)`;
|
|
130
|
+
} else if (this._validationFailures.length > 0) {
|
|
131
|
+
const f = this._validationFailures.slice(0, 5).map(({ filePath, error }) =>
|
|
132
|
+
`\n - ${path.relative(this._workspace.cwd, filePath)}: ${error.split("\n")[0]}`,
|
|
133
|
+
).join("");
|
|
134
|
+
validationLine = `\n- Skills failing validation (${this._validationFailures.length}):${f}` +
|
|
135
|
+
(this._validationFailures.length > 5 ? `\n - … and ${this._validationFailures.length - 5} more` : "");
|
|
136
|
+
}
|
|
155
137
|
parts.push(
|
|
156
138
|
`### Progress (rule-id coverage, D2)\n` +
|
|
157
139
|
`- Total rules in catalog: ${total}\n` +
|
|
@@ -159,6 +141,7 @@ export class SkillAuthoringPipeline extends Pipeline {
|
|
|
159
141
|
`- Skill directories authored: ${this.skillsAuthored.length}\n` +
|
|
160
142
|
`- Skills with scripts/: ${this.skillsWithScripts.length}` +
|
|
161
143
|
taskLine +
|
|
144
|
+
validationLine +
|
|
162
145
|
(uncovered.length > 0
|
|
163
146
|
? `\n- Missing coverage (${uncovered.length}): ${uncovered.slice(0, 15).join(", ")}${uncovered.length > 15 ? "…" : ""}`
|
|
164
147
|
: ""),
|
|
@@ -173,7 +156,38 @@ export class SkillAuthoringPipeline extends Pipeline {
|
|
|
173
156
|
onToolResult(toolName, toolInput, result) {
|
|
174
157
|
if (result.isError) return null;
|
|
175
158
|
const wasReady = this.exitCriteriaMet();
|
|
176
|
-
|
|
159
|
+
const writeToSkill = toolName === "workspace_file" &&
|
|
160
|
+
toolInput?.operation === "write" &&
|
|
161
|
+
(toolInput.path || "").includes("rule_skills/");
|
|
162
|
+
if (writeToSkill) {
|
|
163
|
+
this._scanSkills();
|
|
164
|
+
// v0.7.0 A4: validate this specific file immediately if it looks
|
|
165
|
+
// like a check.py. Surfaces syntax/entry-point issues in the next
|
|
166
|
+
// describeState rather than waiting for the phase boundary —
|
|
167
|
+
// E2E #5 had skill_authoring force-bypassed before exitCriteriaMet
|
|
168
|
+
// ever fired, so the v0.6.2 boundary-only validator never ran in
|
|
169
|
+
// practice.
|
|
170
|
+
const p = toolInput.path || "";
|
|
171
|
+
if (/\/check[_a-zA-Z0-9-]*\.py$/i.test(p) && /^rule_skills\//.test(p)) {
|
|
172
|
+
const abs = path.join(this._workspace.cwd, p);
|
|
173
|
+
// Invalidate any stale mtime cache entry for this path then
|
|
174
|
+
// re-validate. Folds the result into _validationFailures so
|
|
175
|
+
// describeState picks it up.
|
|
176
|
+
this._validator.invalidate(abs);
|
|
177
|
+
const r = this._validator.validateFile(abs);
|
|
178
|
+
if (!r.ok) {
|
|
179
|
+
// Replace any prior failure record for this path
|
|
180
|
+
this._validationFailures = this._validationFailures.filter(
|
|
181
|
+
(f) => f.filePath !== abs,
|
|
182
|
+
);
|
|
183
|
+
this._validationFailures.push({ filePath: abs, error: r.error || "unknown" });
|
|
184
|
+
} else {
|
|
185
|
+
this._validationFailures = this._validationFailures.filter(
|
|
186
|
+
(f) => f.filePath !== abs,
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
177
191
|
if (!wasReady && this.exitCriteriaMet()) {
|
|
178
192
|
return new PipelineEvent({ type: "phase_ready", message: "Skill authoring complete. Ready for SKILL_TESTING.", nextPhase: Phase.SKILL_TESTING });
|
|
179
193
|
}
|
|
@@ -204,9 +218,67 @@ export class SkillAuthoringPipeline extends Pipeline {
|
|
|
204
218
|
if (completed + failed < total) return false;
|
|
205
219
|
}
|
|
206
220
|
}
|
|
221
|
+
// v0.6.2 I2: skill validator — every check_r###.py must parse and
|
|
222
|
+
// expose an entry point. Catches the unified_qc.py-style monolith
|
|
223
|
+
// and other malformed scripts before they break in production_qc.
|
|
224
|
+
// mtime cache keeps this O(1) in steady state. Failures preserved
|
|
225
|
+
// in this._validationFailures for describeState rendering.
|
|
226
|
+
const checkFiles = this._collectCheckScripts();
|
|
227
|
+
const v = this._validator.validateAll(checkFiles);
|
|
228
|
+
this._validationFailures = v.failures;
|
|
229
|
+
this._validationSkipped = v.skipped;
|
|
230
|
+
if (!v.ok) return false;
|
|
207
231
|
return this.skillsWithScripts.length >= Math.max(1, this.skillsAuthored.length * 0.5);
|
|
208
232
|
}
|
|
209
233
|
|
|
234
|
+
/**
|
|
235
|
+
* v0.6.2 I2: gather every check_r###.py path under rule_skills/. Used by
|
|
236
|
+
* the skill validator. Walks one level into each skill directory.
|
|
237
|
+
*/
|
|
238
|
+
/**
|
|
239
|
+
* v0.6.3 (#74): SKILL_AUTHORING writes per-rule check scripts under
|
|
240
|
+
* rule_skills/. Workflow runs against production samples or distillation
|
|
241
|
+
* outputs are later-phase work.
|
|
242
|
+
*/
|
|
243
|
+
phaseMisfitHint(toolName, toolInput, result) {
|
|
244
|
+
if (result?.isError) return null;
|
|
245
|
+
const exitText = this.exitCriteriaMet()
|
|
246
|
+
? "Skill-authoring exit criteria are MET — call phase_advance(to=\"skill_testing\") to proceed."
|
|
247
|
+
: "Skill-authoring not yet complete (see describeState).";
|
|
248
|
+
|
|
249
|
+
if (toolName === "workspace_file" && toolInput?.operation === "write") {
|
|
250
|
+
const p = toolInput.path || "";
|
|
251
|
+
if (p.startsWith("workflows/")) {
|
|
252
|
+
return `Writing under workflows/ is DISTILLATION-phase work, but engine is in SKILL_AUTHORING. ${exitText}`;
|
|
253
|
+
}
|
|
254
|
+
if (p.startsWith("output/results/")) {
|
|
255
|
+
return `Writing under output/results/ is PRODUCTION_QC-phase work, but engine is in SKILL_AUTHORING. ${exitText}`;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
return null;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
_collectCheckScripts() {
|
|
263
|
+
const out = [];
|
|
264
|
+
const dir = path.join(this._workspace.cwd, "rule_skills");
|
|
265
|
+
if (!fs.existsSync(dir)) return out;
|
|
266
|
+
const walk = (d) => {
|
|
267
|
+
let entries;
|
|
268
|
+
try { entries = fs.readdirSync(d, { withFileTypes: true }); } catch { return; }
|
|
269
|
+
for (const e of entries) {
|
|
270
|
+
if (e.name.startsWith(".") || e.name.startsWith("__")) continue;
|
|
271
|
+
const p = path.join(d, e.name);
|
|
272
|
+
if (e.isDirectory()) { walk(p); continue; }
|
|
273
|
+
if (e.isFile() && /^check_r[\d_-]+\.py$/i.test(e.name)) {
|
|
274
|
+
out.push(p);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
};
|
|
278
|
+
walk(dir);
|
|
279
|
+
return out;
|
|
280
|
+
}
|
|
281
|
+
|
|
210
282
|
exportState() {
|
|
211
283
|
return {
|
|
212
284
|
totalRules: this.totalRules,
|
|
@@ -2,6 +2,7 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { Phase, PipelineEvent } from "./index.js";
|
|
4
4
|
import { Pipeline } from "./base.js";
|
|
5
|
+
import { deriveSkillAuthoringMilestones, deriveSkillTestingMilestones } from "./_milestone-derive.js";
|
|
5
6
|
|
|
6
7
|
export class SkillTestingPipeline extends Pipeline {
|
|
7
8
|
constructor(workspace) {
|
|
@@ -33,35 +34,48 @@ export class SkillTestingPipeline extends Pipeline {
|
|
|
33
34
|
}
|
|
34
35
|
|
|
35
36
|
_loadSkills() {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
if (e.isDirectory() && !e.name.startsWith("__")) {
|
|
41
|
-
const p = path.join(dir, e.name);
|
|
42
|
-
if (fs.existsSync(path.join(p, "SKILL.md")) || fs.readdirSync(p).some((f) => f.endsWith(".py"))) {
|
|
43
|
-
this.skillsToTest.push(e.name);
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
}
|
|
37
|
+
// v0.7.0 A1: route through filesystem-derived helper (skill_authoring's
|
|
38
|
+
// skillsAuthored is the canonical "what skills exist" view).
|
|
39
|
+
const m = deriveSkillAuthoringMilestones(this._workspace);
|
|
40
|
+
this.skillsToTest = [...m.skillsAuthored];
|
|
47
41
|
}
|
|
48
42
|
|
|
49
43
|
_loadTestResults() {
|
|
50
44
|
this.skillsTested = {};
|
|
51
45
|
this.skillsPassing = [];
|
|
46
|
+
|
|
47
|
+
// Layer 1 (canonical schema): output/<rule_id>.json with `accuracy` field.
|
|
48
|
+
// Carries the actual numeric threshold check.
|
|
52
49
|
const outDir = path.join(this._workspace.cwd, "output");
|
|
53
|
-
if (
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
50
|
+
if (fs.existsSync(outDir)) {
|
|
51
|
+
for (const f of fs.readdirSync(outDir).filter((f) => f.endsWith(".json"))) {
|
|
52
|
+
try {
|
|
53
|
+
const data = JSON.parse(fs.readFileSync(path.join(outDir, f), "utf-8"));
|
|
54
|
+
if (data.accuracy != null) {
|
|
55
|
+
const ruleId = data.rule_id || path.parse(f).name;
|
|
56
|
+
const acc = parseFloat(data.accuracy);
|
|
57
|
+
this.skillsTested[ruleId] = Math.max(this.skillsTested[ruleId] || 0, acc);
|
|
58
|
+
}
|
|
59
|
+
} catch { /* skip */ }
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Layer 2 (helper-derived floor): per-skill test_results/, tests/, or
|
|
64
|
+
// assets/test_cases.json count as "tested" even without an accuracy
|
|
65
|
+
// reading. Without this floor, agents who tested via sandbox_exec
|
|
66
|
+
// (no accuracy JSON written) showed skillsTested={} despite real
|
|
67
|
+
// testing — exactly the E2E #5 GLM case.
|
|
68
|
+
const m = deriveSkillTestingMilestones(this._workspace);
|
|
69
|
+
for (const id of m.skillsTested) {
|
|
70
|
+
// Test artifact present but no numeric accuracy → record as tested
|
|
71
|
+
// at threshold value (just-passing). The agent can revise via
|
|
72
|
+
// canonical-schema JSON if needed.
|
|
73
|
+
if (!(id in this.skillsTested)) this.skillsTested[id] = this._accuracyThreshold;
|
|
63
74
|
}
|
|
64
|
-
|
|
75
|
+
|
|
76
|
+
this.skillsPassing = Object.entries(this.skillsTested)
|
|
77
|
+
.filter(([, acc]) => acc >= this._accuracyThreshold)
|
|
78
|
+
.map(([id]) => id);
|
|
65
79
|
}
|
|
66
80
|
|
|
67
81
|
_loadEvolutionLog() {
|
|
@@ -104,7 +118,37 @@ export class SkillTestingPipeline extends Pipeline {
|
|
|
104
118
|
exitCriteriaMet() {
|
|
105
119
|
const total = this.skillsToTest.length;
|
|
106
120
|
if (!total) return false;
|
|
107
|
-
|
|
121
|
+
// v0.7.0 H/C2 fix: previous gate `skillsPassing.length >= total * threshold`
|
|
122
|
+
// was multiplying *count* by accuracy threshold (default 0.9), so 9/10
|
|
123
|
+
// failing skills could still pass the gate. The intent is "every
|
|
124
|
+
// skill passes its per-skill threshold" — count parity, not weighted.
|
|
125
|
+
// (Fraction-of-skills fallbacks belong in optional config, not the
|
|
126
|
+
// default exit criterion.)
|
|
127
|
+
return Object.keys(this.skillsTested).length >= total &&
|
|
128
|
+
this.skillsPassing.length >= total;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* v0.6.3 (#74): SKILL_TESTING runs check scripts against test samples and
|
|
133
|
+
* measures accuracy. Writing distillation outputs or production results
|
|
134
|
+
* here means phase boundaries got skipped.
|
|
135
|
+
*/
|
|
136
|
+
phaseMisfitHint(toolName, toolInput, result) {
|
|
137
|
+
if (result?.isError) return null;
|
|
138
|
+
const exitText = this.exitCriteriaMet()
|
|
139
|
+
? "Skill-testing exit criteria are MET — call phase_advance(to=\"distillation\")."
|
|
140
|
+
: "Skill-testing not yet complete.";
|
|
141
|
+
|
|
142
|
+
if (toolName === "workspace_file" && toolInput?.operation === "write") {
|
|
143
|
+
const p = toolInput.path || "";
|
|
144
|
+
if (p.startsWith("workflows/")) {
|
|
145
|
+
return `Writing under workflows/ is DISTILLATION-phase work, but engine is in SKILL_TESTING. ${exitText}`;
|
|
146
|
+
}
|
|
147
|
+
if (p.startsWith("output/results/")) {
|
|
148
|
+
return `Writing under output/results/ is PRODUCTION_QC-phase work, but engine is in SKILL_TESTING. ${exitText}`;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return null;
|
|
108
152
|
}
|
|
109
153
|
|
|
110
154
|
exportState() {
|
package/src/agent/retry.js
CHANGED
|
@@ -1,9 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Retry wrapper with exponential backoff and jitter.
|
|
3
3
|
* Designed for LLM API calls — retries transient errors, fails fast on auth/validation errors.
|
|
4
|
+
*
|
|
5
|
+
* v0.6.3.1: KC_MAX_RETRIES env override. Default 10 attempts ≈ 5 min of
|
|
6
|
+
* exponential backoff (1+2+4+8+16+32+60+60+60+60s). E2E #5 surfaced a
|
|
7
|
+
* Tencent outage that lasted longer than the default; setting
|
|
8
|
+
* KC_MAX_RETRIES=20 buys ~15 more min before the engine gives up.
|
|
4
9
|
*/
|
|
5
|
-
|
|
6
|
-
const
|
|
10
|
+
const MAX_RETRIES = (() => {
|
|
11
|
+
const raw = parseInt(process.env.KC_MAX_RETRIES || "", 10);
|
|
12
|
+
if (Number.isFinite(raw) && raw >= 0 && raw <= 50) return raw;
|
|
13
|
+
return 10;
|
|
14
|
+
})();
|
|
7
15
|
const INITIAL_DELAY_MS = 1000;
|
|
8
16
|
const MAX_DELAY_MS = 60000;
|
|
9
17
|
const BACKOFF_MULTIPLIER = 2;
|
package/src/agent/scheduler.js
CHANGED
|
@@ -222,14 +222,26 @@ export class Scheduler {
|
|
|
222
222
|
}
|
|
223
223
|
|
|
224
224
|
/**
|
|
225
|
-
* Count of files directly under input/ (excluding subdirs like archived/
|
|
225
|
+
* Count of files directly under input/ (excluding subdirs like archived/
|
|
226
|
+
* and v0.7.0 F3 agent-scratch marker .kc-scratch/).
|
|
227
|
+
*
|
|
228
|
+
* Background: E2E #5 DS surfaced "📥 4 new file(s) pending in input/"
|
|
229
|
+
* when the agent's sandbox_exec had dropped 4 test fixtures into
|
|
230
|
+
* input/ during smoke-testing. The user assumed external arrivals.
|
|
231
|
+
* The scheduler never had a way to disambiguate.
|
|
232
|
+
*
|
|
233
|
+
* v0.7.0 F3: agent-side scratch writes go under input/.kc-scratch/
|
|
234
|
+
* (a sidecar dir, hidden by the standard "starts with ." filter).
|
|
235
|
+
* The banner counts only top-level non-hidden files, which is what
|
|
236
|
+
* external arrivals actually look like (schedule_fetch drops files
|
|
237
|
+
* directly into input/ root).
|
|
226
238
|
*/
|
|
227
239
|
pendingInputCount() {
|
|
228
240
|
const dir = path.join(this._workspace.cwd, "input");
|
|
229
241
|
if (!fs.existsSync(dir)) return 0;
|
|
230
242
|
try {
|
|
231
243
|
return fs.readdirSync(dir, { withFileTypes: true })
|
|
232
|
-
.filter((e) => e.isFile())
|
|
244
|
+
.filter((e) => e.isFile() && !e.name.startsWith("."))
|
|
233
245
|
.length;
|
|
234
246
|
} catch {
|
|
235
247
|
return 0;
|
|
@@ -12,9 +12,14 @@ export class SessionState {
|
|
|
12
12
|
* @param {string} workspacePath - Session workspace directory
|
|
13
13
|
* @param {object} [opts]
|
|
14
14
|
* @param {string} [opts.statePath] - Override absolute path (used for sub-agent isolation, Bug 2)
|
|
15
|
+
* @param {Workspace} [opts.workspace] - v0.6.2 J3: optional workspace ref so
|
|
16
|
+
* save() can acquire a sync file lock on session-state.json. Without it
|
|
17
|
+
* (subagents, tests), save() falls back to lock-free writes — same
|
|
18
|
+
* behavior as pre-v0.6.2.
|
|
15
19
|
*/
|
|
16
20
|
constructor(workspacePath, opts = {}) {
|
|
17
21
|
this._path = opts.statePath || path.join(workspacePath, "session-state.json");
|
|
22
|
+
this._workspace = opts.workspace || null;
|
|
18
23
|
}
|
|
19
24
|
|
|
20
25
|
/**
|
|
@@ -46,7 +51,18 @@ export class SessionState {
|
|
|
46
51
|
pipelineMilestones: this._extractMilestones(engine.pipelines),
|
|
47
52
|
};
|
|
48
53
|
|
|
49
|
-
|
|
54
|
+
// v0.6.2 J3: acquire sync file lock if workspace ref available.
|
|
55
|
+
// session-state.json is in SHARED_COORDINATION_PATHS — concurrent
|
|
56
|
+
// writers (parallel ralph-loop workers + main saveState ticks)
|
|
57
|
+
// could otherwise interleave and corrupt the JSON.
|
|
58
|
+
const write = () => {
|
|
59
|
+
fs.writeFileSync(this._path, JSON.stringify(state, null, 2), "utf-8");
|
|
60
|
+
};
|
|
61
|
+
if (this._workspace?.withSyncFileLock) {
|
|
62
|
+
this._workspace.withSyncFileLock("session-state.json", write);
|
|
63
|
+
} else {
|
|
64
|
+
write();
|
|
65
|
+
}
|
|
50
66
|
}
|
|
51
67
|
|
|
52
68
|
/**
|
|
@@ -54,7 +70,24 @@ export class SessionState {
|
|
|
54
70
|
* @returns {object} The persisted state
|
|
55
71
|
*/
|
|
56
72
|
load() {
|
|
57
|
-
|
|
73
|
+
const raw = this._loadRaw() || {};
|
|
74
|
+
// v0.6.3: phase value renamed "extraction" → "rule_extraction" to
|
|
75
|
+
// disambiguate from data/entity extraction inside skills. Migrate old
|
|
76
|
+
// session-state on read so resumed workspaces don't end up in a phase
|
|
77
|
+
// the engine doesn't recognize. Idempotent — already-renamed values
|
|
78
|
+
// pass through unchanged.
|
|
79
|
+
if (raw.currentPhase === "extraction") raw.currentPhase = "rule_extraction";
|
|
80
|
+
if (raw.pipelineMilestones?.extraction && !raw.pipelineMilestones.rule_extraction) {
|
|
81
|
+
raw.pipelineMilestones.rule_extraction = raw.pipelineMilestones.extraction;
|
|
82
|
+
delete raw.pipelineMilestones.extraction;
|
|
83
|
+
}
|
|
84
|
+
if (Array.isArray(raw.phaseSummaries)) {
|
|
85
|
+
for (const s of raw.phaseSummaries) {
|
|
86
|
+
if (s?.fromPhase === "extraction") s.fromPhase = "rule_extraction";
|
|
87
|
+
if (s?.toPhase === "extraction") s.toPhase = "rule_extraction";
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return raw;
|
|
58
91
|
}
|
|
59
92
|
|
|
60
93
|
/**
|
|
@@ -17,22 +17,28 @@ const BUNDLED_SKILLS_DIR = path.resolve(__dirname, "../../template/skills");
|
|
|
17
17
|
// to default to always-visible.
|
|
18
18
|
const PHASE_RELEVANT_SKILLS = {
|
|
19
19
|
"bootstrap-workspace": ["bootstrap"],
|
|
20
|
-
"rule-extraction": ["bootstrap", "
|
|
21
|
-
"rule-graph": ["
|
|
22
|
-
"task-decomposition": ["
|
|
20
|
+
"rule-extraction": ["bootstrap", "rule_extraction"],
|
|
21
|
+
"rule-graph": ["rule_extraction", "skill_authoring"],
|
|
22
|
+
"task-decomposition": ["rule_extraction", "skill_authoring", "distillation"],
|
|
23
|
+
// v0.7.0 B1: work-decomposition teaches the system-level decomposition
|
|
24
|
+
// discipline (ordering, grouping, difficulty triage, PATTERNS.md memory).
|
|
25
|
+
// Distinct from task-decomposition (per-rule sub-tasks). Loaded on
|
|
26
|
+
// rule_extraction → skill_authoring transition where the agent owns
|
|
27
|
+
// the TaskBoard.
|
|
28
|
+
"work-decomposition": ["rule_extraction", "skill_authoring"],
|
|
23
29
|
"skill-authoring": ["skill_authoring", "skill_testing"],
|
|
24
30
|
"skill-to-workflow": ["distillation"],
|
|
25
31
|
"evolution-loop": ["skill_testing", "distillation", "production_qc"],
|
|
26
|
-
"version-control": ["bootstrap", "
|
|
32
|
+
"version-control": ["bootstrap", "rule_extraction", "skill_authoring", "skill_testing", "distillation", "production_qc", "finalization"],
|
|
27
33
|
"quality-control": ["production_qc", "finalization"],
|
|
28
34
|
"confidence-system": ["distillation", "production_qc"],
|
|
29
35
|
"dashboard-reporting": ["production_qc", "finalization"],
|
|
30
36
|
"cross-document-verification": ["production_qc"],
|
|
31
37
|
"corner-case-management": ["skill_testing", "distillation", "production_qc"],
|
|
32
|
-
"data-sensibility": ["
|
|
38
|
+
"data-sensibility": ["rule_extraction", "skill_authoring"],
|
|
33
39
|
"entity-extraction": ["skill_authoring", "distillation"],
|
|
34
|
-
"document-parsing": ["bootstrap", "
|
|
35
|
-
"document-chunking": ["bootstrap", "
|
|
40
|
+
"document-parsing": ["bootstrap", "rule_extraction", "skill_authoring"],
|
|
41
|
+
"document-chunking": ["bootstrap", "rule_extraction"],
|
|
36
42
|
"tree-processing": ["skill_authoring", "skill_testing"],
|
|
37
43
|
"compliance-judgment": ["skill_authoring", "skill_testing", "production_qc"],
|
|
38
44
|
"skill-creator": ["skill_authoring"],
|