kc-beta 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/kc-beta.js +14 -2
- package/package.json +1 -1
- package/src/agent/context-window.js +151 -0
- package/src/agent/context.js +8 -4
- package/src/agent/engine.js +261 -8
- package/src/agent/event-log.js +111 -0
- package/src/agent/llm-client.js +352 -59
- package/src/agent/pipelines/base.js +6 -0
- package/src/agent/pipelines/distillation.js +18 -0
- package/src/agent/pipelines/extraction.js +21 -0
- package/src/agent/pipelines/initializer.js +75 -14
- package/src/agent/pipelines/production-qc.js +19 -0
- package/src/agent/pipelines/skill-authoring.js +14 -0
- package/src/agent/pipelines/skill-testing.js +20 -0
- package/src/agent/retry.js +83 -0
- package/src/agent/session-state.js +79 -0
- package/src/agent/skill-loader.js +13 -1
- package/src/agent/token-counter.js +62 -0
- package/src/agent/tools/document-parse.js +104 -21
- package/src/agent/tools/document-search.js +24 -8
- package/src/agent/tools/sandbox-exec.js +16 -5
- package/src/agent/tools/web-search.js +107 -0
- package/src/agent/tools/worker-llm-call.js +14 -5
- package/src/agent/tools/workspace-file.js +47 -20
- package/src/agent/workspace.js +24 -1
- package/src/cli/components.js +24 -5
- package/src/cli/config.js +340 -0
- package/src/cli/index.js +113 -11
- package/src/cli/onboard.js +216 -53
- package/src/config.js +63 -10
- package/src/model-tiers.json +153 -0
- package/src/providers.js +367 -0
- package/template/AGENT.md +20 -0
- package/template/skills/en/meta/compliance-judgment/SKILL.md +10 -42
- package/template/skills/en/meta/document-chunking/SKILL.md +32 -0
- package/template/skills/en/meta/document-parsing/SKILL.md +11 -18
- package/template/skills/en/meta/entity-extraction/SKILL.md +13 -28
- package/template/skills/en/meta/tree-processing/SKILL.md +19 -1
- package/template/skills/en/meta-meta/auto-model-selection/SKILL.md +53 -0
- package/template/skills/en/meta-meta/pdf-review-dashboard/SKILL.md +57 -0
- package/template/skills/en/meta-meta/pdf-review-dashboard/scripts/generate_review.js +262 -0
- package/template/skills/en/meta-meta/rule-extraction/SKILL.md +24 -1
- package/template/skills/en/meta-meta/skill-authoring/SKILL.md +6 -0
- package/template/skills/en/meta-meta/skill-to-workflow/SKILL.md +4 -0
- package/template/skills/zh/meta/compliance-judgment/SKILL.md +41 -262
- package/template/skills/zh/meta/document-chunking/SKILL.md +32 -0
- package/template/skills/zh/meta/document-parsing/SKILL.md +65 -132
- package/template/skills/zh/meta/entity-extraction/SKILL.md +68 -230
- package/template/skills/zh/meta/tree-processing/SKILL.md +82 -194
- package/template/skills/zh/meta-meta/auto-model-selection/SKILL.md +51 -0
- package/template/skills/zh/meta-meta/pdf-review-dashboard/SKILL.md +55 -0
- package/template/skills/zh/meta-meta/pdf-review-dashboard/scripts/generate_review.js +262 -0
- package/template/skills/zh/meta-meta/rule-extraction/SKILL.md +79 -164
- package/template/skills/zh/meta-meta/skill-authoring/SKILL.md +64 -185
- package/template/skills/zh/meta-meta/skill-to-workflow/SKILL.md +95 -216
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import os from "node:os";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
4
5
|
import { Phase, PipelineEvent } from "./index.js";
|
|
5
6
|
import { Pipeline } from "./base.js";
|
|
6
7
|
|
|
8
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
9
|
+
const AGENT_MD_TEMPLATE = path.resolve(__dirname, "../../../template/AGENT.md");
|
|
10
|
+
|
|
7
11
|
const REQUIRED_DIRS = ["rules", "samples", "input", "output", "logs", "workflows", "rule_skills"];
|
|
8
12
|
|
|
9
13
|
const DEFAULT_ENV = `# === KC Agent Project Configuration ===
|
|
@@ -11,9 +15,9 @@ const DEFAULT_ENV = `# === KC Agent Project Configuration ===
|
|
|
11
15
|
# Language: en | zh
|
|
12
16
|
LANGUAGE=en
|
|
13
17
|
|
|
14
|
-
# ===
|
|
15
|
-
|
|
16
|
-
|
|
18
|
+
# === LLM API ===
|
|
19
|
+
LLM_API_KEY=
|
|
20
|
+
LLM_BASE_URL=https://api.siliconflow.cn/v1
|
|
17
21
|
|
|
18
22
|
# === Worker LLM Tiers (highest capability to lowest) ===
|
|
19
23
|
TIER1=Pro/zai-org/GLM-5, Pro/moonshotai/Kimi-K2.5
|
|
@@ -53,8 +57,8 @@ export class ProjectInitializer extends Pipeline {
|
|
|
53
57
|
if (!fs.existsSync(envPath)) {
|
|
54
58
|
let envContent = DEFAULT_ENV;
|
|
55
59
|
const gc = this._loadGlobalConfig();
|
|
56
|
-
if (gc.api_key) envContent = envContent.replace("
|
|
57
|
-
if (gc.base_url) envContent = envContent.replace("
|
|
60
|
+
if (gc.api_key) envContent = envContent.replace("LLM_API_KEY=", `LLM_API_KEY=${gc.api_key}`);
|
|
61
|
+
if (gc.base_url) envContent = envContent.replace("LLM_BASE_URL=https://api.siliconflow.cn/v1", `LLM_BASE_URL=${gc.base_url}`);
|
|
58
62
|
if (gc.accuracy_threshold) {
|
|
59
63
|
envContent = envContent.replace("SKILL_ACCURACY=0.9", `SKILL_ACCURACY=${gc.accuracy_threshold}`);
|
|
60
64
|
envContent = envContent.replace("WORKFLOW_ACCURACY=0.9", `WORKFLOW_ACCURACY=${gc.accuracy_threshold}`);
|
|
@@ -74,6 +78,12 @@ export class ProjectInitializer extends Pipeline {
|
|
|
74
78
|
fs.writeFileSync(manifestPath, JSON.stringify({ version: "0.1.0", entries: [] }, null, 2), "utf-8");
|
|
75
79
|
}
|
|
76
80
|
|
|
81
|
+
// AGENT.md — per-project context (agent can modify)
|
|
82
|
+
const agentMdPath = path.join(this._workspace.cwd, "AGENT.md");
|
|
83
|
+
if (!fs.existsSync(agentMdPath) && fs.existsSync(AGENT_MD_TEMPLATE)) {
|
|
84
|
+
fs.copyFileSync(AGENT_MD_TEMPLATE, agentMdPath);
|
|
85
|
+
}
|
|
86
|
+
|
|
77
87
|
this.workspaceCreated = true;
|
|
78
88
|
this._checkRegulations();
|
|
79
89
|
this._checkSamples();
|
|
@@ -81,22 +91,48 @@ export class ProjectInitializer extends Pipeline {
|
|
|
81
91
|
}
|
|
82
92
|
|
|
83
93
|
_checkRegulations() {
|
|
94
|
+
// Check workspace rules/
|
|
84
95
|
const dir = path.join(this._workspace.cwd, "rules");
|
|
85
|
-
if (
|
|
86
|
-
|
|
96
|
+
if (fs.existsSync(dir) && fs.readdirSync(dir, { withFileTypes: true }).some((e) => e.isFile())) {
|
|
97
|
+
this.hasRegulations = true; return;
|
|
98
|
+
}
|
|
99
|
+
// Check project dir rules/ (case-insensitive)
|
|
100
|
+
if (this._workspace.projectDir) {
|
|
101
|
+
for (const name of ["rules", "Rules", "RULES", "regulations", "Regulations"]) {
|
|
102
|
+
const pdir = path.join(this._workspace.projectDir, name);
|
|
103
|
+
if (fs.existsSync(pdir) && fs.statSync(pdir).isDirectory() &&
|
|
104
|
+
fs.readdirSync(pdir, { withFileTypes: true }).some((e) => e.isFile())) {
|
|
105
|
+
this.hasRegulations = true; return;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
this.hasRegulations = false;
|
|
87
110
|
}
|
|
88
111
|
|
|
89
112
|
_checkSamples() {
|
|
113
|
+
// Check workspace samples/
|
|
90
114
|
const dir = path.join(this._workspace.cwd, "samples");
|
|
91
|
-
if (
|
|
92
|
-
|
|
115
|
+
if (fs.existsSync(dir) && fs.readdirSync(dir, { withFileTypes: true }).some((e) => e.isFile())) {
|
|
116
|
+
this.hasSamples = true; return;
|
|
117
|
+
}
|
|
118
|
+
// Check project dir samples/ (case-insensitive)
|
|
119
|
+
if (this._workspace.projectDir) {
|
|
120
|
+
for (const name of ["samples", "Samples", "SAMPLES", "sample", "Sample"]) {
|
|
121
|
+
const pdir = path.join(this._workspace.projectDir, name);
|
|
122
|
+
if (fs.existsSync(pdir) && fs.statSync(pdir).isDirectory() &&
|
|
123
|
+
fs.readdirSync(pdir, { withFileTypes: true }).some((e) => e.isFile())) {
|
|
124
|
+
this.hasSamples = true; return;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
this.hasSamples = false;
|
|
93
129
|
}
|
|
94
130
|
|
|
95
131
|
_checkConfig() {
|
|
96
132
|
const envPath = path.join(this._workspace.cwd, ".env");
|
|
97
133
|
if (fs.existsSync(envPath)) {
|
|
98
134
|
for (const line of fs.readFileSync(envPath, "utf-8").split("\n")) {
|
|
99
|
-
if (line.startsWith("SILICONFLOW_API_KEY=") && line.split("=")[1].trim()) {
|
|
135
|
+
if ((line.startsWith("LLM_API_KEY=") || line.startsWith("SILICONFLOW_API_KEY=")) && line.split("=")[1].trim()) {
|
|
100
136
|
this.configReady = true; return;
|
|
101
137
|
}
|
|
102
138
|
}
|
|
@@ -115,10 +151,13 @@ export class ProjectInitializer extends Pipeline {
|
|
|
115
151
|
const completed = [], pending = [];
|
|
116
152
|
if (this.workspaceCreated) completed.push("Workspace structure created"); else pending.push("Workspace structure");
|
|
117
153
|
if (this.configReady) completed.push("API keys configured"); else pending.push("API keys (check .env)");
|
|
118
|
-
if (this.hasRegulations) completed.push("Regulation documents
|
|
119
|
-
if (this.hasSamples) completed.push("Sample documents
|
|
154
|
+
if (this.hasRegulations) completed.push("Regulation documents found"); else pending.push("Regulation documents (add to rules/ in workspace or project dir)");
|
|
155
|
+
if (this.hasSamples) completed.push("Sample documents found"); else pending.push("Sample documents (add to samples/ in workspace or project dir)");
|
|
120
156
|
|
|
121
157
|
const parts = ["## Phase: BOOTSTRAP\nSet up the workspace and understand the developer user's verification scenario. Bundled methodology skills are available in the workspace skills/ directory."];
|
|
158
|
+
if (this._workspace.projectDir) {
|
|
159
|
+
parts.push(`**Project directory:** ${this._workspace.projectDir}\nUse scope="project" to read files from the user's project folder.`);
|
|
160
|
+
}
|
|
122
161
|
if (completed.length) parts.push("### Done\n" + completed.map((c) => `- [x] ${c}`).join("\n"));
|
|
123
162
|
if (pending.length) parts.push("### Needed\n" + pending.map((p) => `- [ ] ${p}`).join("\n"));
|
|
124
163
|
|
|
@@ -135,14 +174,20 @@ export class ProjectInitializer extends Pipeline {
|
|
|
135
174
|
if (toolName === "workspace_file") {
|
|
136
175
|
const op = toolInput.operation || "";
|
|
137
176
|
const p = toolInput.path || "";
|
|
138
|
-
|
|
177
|
+
const scope = toolInput.scope || "workspace";
|
|
178
|
+
if (op === "write" && scope === "workspace") {
|
|
139
179
|
if (p.startsWith("rules/")) this.hasRegulations = true;
|
|
140
180
|
else if (p.startsWith("samples/")) this.hasSamples = true;
|
|
141
181
|
else if (p === ".env") this._checkConfig();
|
|
142
|
-
} else if (op === "list") {
|
|
182
|
+
} else if (op === "list" || op === "read") {
|
|
183
|
+
// Re-check after any list/read — project dir files may satisfy criteria
|
|
143
184
|
this._checkRegulations();
|
|
144
185
|
this._checkSamples();
|
|
145
186
|
}
|
|
187
|
+
} else if (toolName === "document_parse") {
|
|
188
|
+
// Parsing a document from project dir counts as having files
|
|
189
|
+
this._checkRegulations();
|
|
190
|
+
this._checkSamples();
|
|
146
191
|
}
|
|
147
192
|
|
|
148
193
|
if (!wasReady && this.exitCriteriaMet()) {
|
|
@@ -154,4 +199,20 @@ export class ProjectInitializer extends Pipeline {
|
|
|
154
199
|
exitCriteriaMet() {
|
|
155
200
|
return this.workspaceCreated && this.configReady && this.hasRegulations && this.hasSamples;
|
|
156
201
|
}
|
|
202
|
+
|
|
203
|
+
exportState() {
|
|
204
|
+
return {
|
|
205
|
+
workspaceCreated: this.workspaceCreated,
|
|
206
|
+
configReady: this.configReady,
|
|
207
|
+
hasRegulations: this.hasRegulations,
|
|
208
|
+
hasSamples: this.hasSamples,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
importState(data) {
|
|
213
|
+
if (data.workspaceCreated) this.workspaceCreated = true;
|
|
214
|
+
if (data.configReady) this.configReady = true;
|
|
215
|
+
if (data.hasRegulations) this.hasRegulations = true;
|
|
216
|
+
if (data.hasSamples) this.hasSamples = true;
|
|
217
|
+
}
|
|
157
218
|
}
|
|
@@ -94,4 +94,23 @@ export class ProductionQCPipeline extends Pipeline {
|
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
exitCriteriaMet() { return this.monitoringPhase === "stable"; }
|
|
97
|
+
|
|
98
|
+
exportState() {
|
|
99
|
+
return {
|
|
100
|
+
batchesProcessed: this.batchesProcessed,
|
|
101
|
+
totalDocuments: this.totalDocuments,
|
|
102
|
+
documentsReviewed: this.documentsReviewed,
|
|
103
|
+
monitoringPhase: this.monitoringPhase,
|
|
104
|
+
accuracyByRule: this.accuracyByRule,
|
|
105
|
+
issuesCount: this.issuesFound.length,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
importState(data) {
|
|
110
|
+
if (typeof data.batchesProcessed === "number" && data.batchesProcessed > this.batchesProcessed) this.batchesProcessed = data.batchesProcessed;
|
|
111
|
+
if (typeof data.totalDocuments === "number" && data.totalDocuments > this.totalDocuments) this.totalDocuments = data.totalDocuments;
|
|
112
|
+
if (typeof data.documentsReviewed === "number" && data.documentsReviewed > this.documentsReviewed) this.documentsReviewed = data.documentsReviewed;
|
|
113
|
+
if (data.monitoringPhase) this.monitoringPhase = data.monitoringPhase;
|
|
114
|
+
if (data.accuracyByRule && typeof data.accuracyByRule === "object") Object.assign(this.accuracyByRule, data.accuracyByRule);
|
|
115
|
+
}
|
|
97
116
|
}
|
|
@@ -77,4 +77,18 @@ export class SkillAuthoringPipeline extends Pipeline {
|
|
|
77
77
|
if (!this.totalRules.length) return false;
|
|
78
78
|
return this.skillsAuthored.length >= this.totalRules.length && this.skillsWithScripts.length >= this.skillsAuthored.length * 0.5;
|
|
79
79
|
}
|
|
80
|
+
|
|
81
|
+
exportState() {
|
|
82
|
+
return {
|
|
83
|
+
totalRules: this.totalRules,
|
|
84
|
+
skillsAuthored: this.skillsAuthored,
|
|
85
|
+
skillsWithScripts: this.skillsWithScripts,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
importState(data) {
|
|
90
|
+
if (Array.isArray(data.totalRules) && data.totalRules.length > this.totalRules.length) this.totalRules = data.totalRules;
|
|
91
|
+
if (Array.isArray(data.skillsAuthored) && data.skillsAuthored.length > this.skillsAuthored.length) this.skillsAuthored = data.skillsAuthored;
|
|
92
|
+
if (Array.isArray(data.skillsWithScripts) && data.skillsWithScripts.length > this.skillsWithScripts.length) this.skillsWithScripts = data.skillsWithScripts;
|
|
93
|
+
}
|
|
80
94
|
}
|
|
@@ -106,4 +106,24 @@ export class SkillTestingPipeline extends Pipeline {
|
|
|
106
106
|
if (!total) return false;
|
|
107
107
|
return Object.keys(this.skillsTested).length >= total && this.skillsPassing.length >= total * this._accuracyThreshold;
|
|
108
108
|
}
|
|
109
|
+
|
|
110
|
+
exportState() {
|
|
111
|
+
return {
|
|
112
|
+
skillsToTest: this.skillsToTest,
|
|
113
|
+
skillsTested: this.skillsTested,
|
|
114
|
+
skillsPassing: this.skillsPassing,
|
|
115
|
+
iterationCount: this.iterationCount,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
importState(data) {
|
|
120
|
+
if (typeof data.iterationCount === "number" && data.iterationCount > this.iterationCount) this.iterationCount = data.iterationCount;
|
|
121
|
+
if (Array.isArray(data.skillsToTest) && data.skillsToTest.length > this.skillsToTest.length) this.skillsToTest = data.skillsToTest;
|
|
122
|
+
if (Array.isArray(data.skillsPassing) && data.skillsPassing.length > this.skillsPassing.length) this.skillsPassing = data.skillsPassing;
|
|
123
|
+
if (data.skillsTested && typeof data.skillsTested === "object") {
|
|
124
|
+
for (const [k, v] of Object.entries(data.skillsTested)) {
|
|
125
|
+
if (!this.skillsTested[k] || v > this.skillsTested[k]) this.skillsTested[k] = v;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
109
129
|
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retry wrapper with exponential backoff and jitter.
|
|
3
|
+
* Designed for LLM API calls — retries transient errors, fails fast on auth/validation errors.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
const MAX_RETRIES = 10;
|
|
7
|
+
const INITIAL_DELAY_MS = 1000;
|
|
8
|
+
const MAX_DELAY_MS = 60000;
|
|
9
|
+
const BACKOFF_MULTIPLIER = 2;
|
|
10
|
+
const JITTER_FRACTION = 0.2;
|
|
11
|
+
|
|
12
|
+
const RETRYABLE_STATUS = new Set([408, 429, 500, 502, 503, 504, 520, 522, 524]);
|
|
13
|
+
const NON_RETRYABLE_STATUS = new Set([400, 401, 403, 404, 422]);
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Determine if an error is retryable.
|
|
17
|
+
* @param {Error} err
|
|
18
|
+
* @returns {boolean}
|
|
19
|
+
*/
|
|
20
|
+
function isRetryable(err) {
|
|
21
|
+
if (err.status) {
|
|
22
|
+
if (NON_RETRYABLE_STATUS.has(err.status)) return false;
|
|
23
|
+
if (RETRYABLE_STATUS.has(err.status)) return true;
|
|
24
|
+
}
|
|
25
|
+
// Network errors (ECONNRESET, ETIMEDOUT, fetch TypeError, AbortError)
|
|
26
|
+
const msg = err.message || "";
|
|
27
|
+
if (/ECONNRESET|ETIMEDOUT|ENOTFOUND|ECONNREFUSED|UND_ERR|fetch failed|network|socket hang up/i.test(msg)) {
|
|
28
|
+
return true;
|
|
29
|
+
}
|
|
30
|
+
if (err.name === "AbortError" || err.name === "TimeoutError") return true;
|
|
31
|
+
// If we have a status code and it's not in our known sets, retry server errors (5xx)
|
|
32
|
+
if (err.status && err.status >= 500) return true;
|
|
33
|
+
// Unknown errors without status — retry conservatively
|
|
34
|
+
return !err.status;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Calculate delay for a given attempt using exponential backoff with jitter.
|
|
39
|
+
* @param {number} attempt - 0-indexed attempt number
|
|
40
|
+
* @param {number|null} retryAfterSec - Retry-After header value in seconds
|
|
41
|
+
* @returns {number} Delay in milliseconds
|
|
42
|
+
*/
|
|
43
|
+
function calculateDelay(attempt, retryAfterSec) {
|
|
44
|
+
if (retryAfterSec && retryAfterSec > 0) {
|
|
45
|
+
return Math.min(retryAfterSec * 1000, MAX_DELAY_MS);
|
|
46
|
+
}
|
|
47
|
+
const base = Math.min(INITIAL_DELAY_MS * Math.pow(BACKOFF_MULTIPLIER, attempt), MAX_DELAY_MS);
|
|
48
|
+
const jitter = base * JITTER_FRACTION * Math.random();
|
|
49
|
+
return base + jitter;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Execute an async function with retry logic.
|
|
54
|
+
*
|
|
55
|
+
* @param {() => Promise<any>} fn - The async function to execute. Should throw with
|
|
56
|
+
* an error that has `.status` and optionally `.retryAfter` properties on failure.
|
|
57
|
+
* @returns {Promise<any>} The successful result
|
|
58
|
+
* @throws {Error} The last error after all retries exhausted, or a non-retryable error immediately
|
|
59
|
+
*/
|
|
60
|
+
export async function withRetry(fn) {
|
|
61
|
+
let lastError;
|
|
62
|
+
|
|
63
|
+
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
64
|
+
try {
|
|
65
|
+
return await fn();
|
|
66
|
+
} catch (err) {
|
|
67
|
+
lastError = err;
|
|
68
|
+
|
|
69
|
+
if (!isRetryable(err)) throw err;
|
|
70
|
+
if (attempt === MAX_RETRIES) break;
|
|
71
|
+
|
|
72
|
+
const retryAfterSec = err.retryAfter ? parseFloat(err.retryAfter) : null;
|
|
73
|
+
const delay = calculateDelay(attempt, retryAfterSec);
|
|
74
|
+
|
|
75
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const wrapper = new Error(`LLM API call failed after ${MAX_RETRIES + 1} attempts: ${lastError.message}`);
|
|
80
|
+
wrapper.cause = lastError;
|
|
81
|
+
wrapper.status = lastError.status;
|
|
82
|
+
throw wrapper;
|
|
83
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Persists session state (phase, pipeline milestones, phase summaries)
|
|
6
|
+
* to enable cross-session resume.
|
|
7
|
+
*
|
|
8
|
+
* Stored as: workspace/{sessionId}/session-state.json
|
|
9
|
+
*/
|
|
10
|
+
export class SessionState {
|
|
11
|
+
/**
|
|
12
|
+
* @param {string} workspacePath - Session workspace directory
|
|
13
|
+
*/
|
|
14
|
+
constructor(workspacePath) {
|
|
15
|
+
this._path = path.join(workspacePath, "session-state.json");
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Whether a session state file exists */
|
|
19
|
+
get exists() {
|
|
20
|
+
return fs.existsSync(this._path);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Save engine state to disk.
|
|
25
|
+
* @param {import('./engine.js').AgentEngine} engine
|
|
26
|
+
*/
|
|
27
|
+
save(engine) {
|
|
28
|
+
const state = {
|
|
29
|
+
version: 1,
|
|
30
|
+
sessionId: engine.workspace.sessionId,
|
|
31
|
+
currentPhase: engine.currentPhase,
|
|
32
|
+
projectDir: engine.workspace.projectDir || null,
|
|
33
|
+
phaseSummaries: engine._phaseSummaries || [],
|
|
34
|
+
lastEventSeq: engine.eventLog?.currentSeq || 0,
|
|
35
|
+
createdAt: this._loadRaw()?.createdAt || new Date().toISOString(),
|
|
36
|
+
updatedAt: new Date().toISOString(),
|
|
37
|
+
pipelineMilestones: this._extractMilestones(engine.pipelines),
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
fs.writeFileSync(this._path, JSON.stringify(state, null, 2), "utf-8");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Load session state from disk.
|
|
45
|
+
* @returns {object} The persisted state
|
|
46
|
+
*/
|
|
47
|
+
load() {
|
|
48
|
+
return this._loadRaw() || {};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Read raw file contents.
|
|
53
|
+
*/
|
|
54
|
+
_loadRaw() {
|
|
55
|
+
if (!this.exists) return null;
|
|
56
|
+
try {
|
|
57
|
+
return JSON.parse(fs.readFileSync(this._path, "utf-8"));
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Serialize pipeline milestones for persistence.
|
|
65
|
+
* @param {object} pipelines - Map of phase -> pipeline instance
|
|
66
|
+
* @returns {object}
|
|
67
|
+
*/
|
|
68
|
+
_extractMilestones(pipelines) {
|
|
69
|
+
const milestones = {};
|
|
70
|
+
for (const [phase, pipeline] of Object.entries(pipelines)) {
|
|
71
|
+
if (pipeline?.exportState) {
|
|
72
|
+
try {
|
|
73
|
+
milestones[phase] = pipeline.exportState();
|
|
74
|
+
} catch { /* skip if not implemented */ }
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return milestones;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -130,7 +130,19 @@ export class SkillLoader {
|
|
|
130
130
|
|
|
131
131
|
const frontmatter = match[1];
|
|
132
132
|
const name = frontmatter.match(/^name:\s*(.+)$/m)?.[1]?.trim() || "";
|
|
133
|
-
|
|
133
|
+
|
|
134
|
+
// Handle both single-line and multi-line (YAML >) descriptions
|
|
135
|
+
let description = "";
|
|
136
|
+
const descMatch = frontmatter.match(/^description:\s*(.+)$/m);
|
|
137
|
+
if (descMatch && descMatch[1].trim() === ">") {
|
|
138
|
+
// Multi-line: capture indented lines after "description: >"
|
|
139
|
+
const multiMatch = frontmatter.match(/^description:\s*>\s*\n((?:[ \t]+.+\n?)*)/m);
|
|
140
|
+
if (multiMatch) {
|
|
141
|
+
description = multiMatch[1].replace(/^[ \t]+/gm, "").replace(/\n/g, " ").trim();
|
|
142
|
+
}
|
|
143
|
+
} else if (descMatch) {
|
|
144
|
+
description = descMatch[1].trim();
|
|
145
|
+
}
|
|
134
146
|
return { name, description };
|
|
135
147
|
} catch {
|
|
136
148
|
return {};
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight token estimation without external dependencies.
|
|
3
|
+
* Uses character-based heuristics: ~4 chars per token for Latin text,
|
|
4
|
+
* ~1.5 tokens per CJK character.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
// CJK Unified Ideographs and extensions
|
|
8
|
+
const CJK_REGEX = /[\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]/g;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Estimate the number of tokens in a string.
|
|
12
|
+
* @param {string} text
|
|
13
|
+
* @returns {number}
|
|
14
|
+
*/
|
|
15
|
+
export function estimateTokens(text) {
|
|
16
|
+
if (!text) return 0;
|
|
17
|
+
const cjkMatches = text.match(CJK_REGEX);
|
|
18
|
+
const cjkCount = cjkMatches ? cjkMatches.length : 0;
|
|
19
|
+
const nonCjkLength = text.length - cjkCount;
|
|
20
|
+
return Math.ceil(nonCjkLength / 4) + Math.ceil(cjkCount * 1.5);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Estimate total tokens for an array of OpenAI-format messages.
|
|
25
|
+
* Accounts for per-message overhead (~4 tokens for role/formatting).
|
|
26
|
+
* @param {Array<object>} messages
|
|
27
|
+
* @returns {number}
|
|
28
|
+
*/
|
|
29
|
+
export function estimateMessagesTokens(messages) {
|
|
30
|
+
let total = 0;
|
|
31
|
+
for (const msg of messages) {
|
|
32
|
+
total += 4; // role + formatting overhead
|
|
33
|
+
if (typeof msg.content === "string") {
|
|
34
|
+
total += estimateTokens(msg.content);
|
|
35
|
+
} else if (Array.isArray(msg.content)) {
|
|
36
|
+
// Anthropic-style content blocks
|
|
37
|
+
for (const block of msg.content) {
|
|
38
|
+
if (block.text) total += estimateTokens(block.text);
|
|
39
|
+
if (block.content) total += estimateTokens(block.content);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if (msg.tool_calls) {
|
|
43
|
+
for (const tc of msg.tool_calls) {
|
|
44
|
+
total += estimateTokens(tc.function?.name || "");
|
|
45
|
+
total += estimateTokens(tc.function?.arguments || "");
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return total;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Format a token count for display (e.g., "45.2k").
|
|
54
|
+
* @param {number} tokens
|
|
55
|
+
* @returns {string}
|
|
56
|
+
*/
|
|
57
|
+
export function formatTokenCount(tokens) {
|
|
58
|
+
if (tokens >= 1000) {
|
|
59
|
+
return (tokens / 1000).toFixed(1) + "k";
|
|
60
|
+
}
|
|
61
|
+
return tokens.toString();
|
|
62
|
+
}
|
|
@@ -12,13 +12,13 @@ const MIN_CHARS_PER_PAGE = 50;
|
|
|
12
12
|
* Level 3: OCR models via SiliconFlow — fallback via vision models
|
|
13
13
|
*/
|
|
14
14
|
export class DocumentParseTool extends BaseTool {
|
|
15
|
-
constructor(workspace, { mineruApiUrl, mineruApiKey,
|
|
15
|
+
constructor(workspace, { mineruApiUrl, mineruApiKey, llmApiKey, llmBaseUrl, ocrModel } = {}) {
|
|
16
16
|
super();
|
|
17
17
|
this._workspace = workspace;
|
|
18
18
|
this._mineruApiUrl = mineruApiUrl || "";
|
|
19
19
|
this._mineruApiKey = mineruApiKey || "";
|
|
20
|
-
this.
|
|
21
|
-
this.
|
|
20
|
+
this._vlmApiKey = llmApiKey || "";
|
|
21
|
+
this._vlmBaseUrl = (llmBaseUrl || "").replace(/\/+$/, "");
|
|
22
22
|
this._ocrModel = ocrModel || "";
|
|
23
23
|
}
|
|
24
24
|
|
|
@@ -36,13 +36,18 @@ export class DocumentParseTool extends BaseTool {
|
|
|
36
36
|
return {
|
|
37
37
|
type: "object",
|
|
38
38
|
properties: {
|
|
39
|
-
path: { type: "string", description: "Relative path to the document
|
|
39
|
+
path: { type: "string", description: "Relative path to the document" },
|
|
40
40
|
pages: { type: "string", description: "Page range to extract, e.g. '1-5', '3', '10-20'. Omit for all pages." },
|
|
41
41
|
force_method: {
|
|
42
42
|
type: "string",
|
|
43
|
-
enum: ["pdfjs", "mineru", "ocr"],
|
|
43
|
+
enum: ["pdfjs", "vlm", "mineru", "ocr"],
|
|
44
44
|
description: "Force a specific parsing method, skipping the escalation chain.",
|
|
45
45
|
},
|
|
46
|
+
scope: {
|
|
47
|
+
type: "string",
|
|
48
|
+
enum: ["workspace", "project"],
|
|
49
|
+
description: "Which directory to find the file in. 'workspace' (default) or 'project' (user's project folder).",
|
|
50
|
+
},
|
|
46
51
|
},
|
|
47
52
|
required: ["path"],
|
|
48
53
|
};
|
|
@@ -52,11 +57,19 @@ export class DocumentParseTool extends BaseTool {
|
|
|
52
57
|
const pathStr = input.path || "";
|
|
53
58
|
const pages = input.pages;
|
|
54
59
|
const force = input.force_method;
|
|
60
|
+
const scope = input.scope || "workspace";
|
|
55
61
|
|
|
56
62
|
if (!pathStr) return new ToolResult("No path provided", true);
|
|
63
|
+
if (scope === "project" && !this._workspace.projectDir) {
|
|
64
|
+
return new ToolResult("No project directory available", true);
|
|
65
|
+
}
|
|
57
66
|
|
|
58
67
|
let resolved;
|
|
59
|
-
try {
|
|
68
|
+
try {
|
|
69
|
+
resolved = scope === "project"
|
|
70
|
+
? this._workspace.resolveProjectPath(pathStr)
|
|
71
|
+
: this._workspace.resolvePath(pathStr);
|
|
72
|
+
}
|
|
60
73
|
catch (e) { return new ToolResult(e.message, true); }
|
|
61
74
|
|
|
62
75
|
if (!fs.existsSync(resolved) || !fs.statSync(resolved).isFile()) {
|
|
@@ -76,13 +89,21 @@ export class DocumentParseTool extends BaseTool {
|
|
|
76
89
|
if (force) return this._runMethod(force, resolved, pageRange);
|
|
77
90
|
|
|
78
91
|
// Escalation chain
|
|
79
|
-
// Level 1: pdfjs-dist
|
|
92
|
+
// Level 1: pdfjs-dist (free, local text extraction)
|
|
80
93
|
let result = await this._tryPdfjs(resolved, pageRange);
|
|
81
94
|
if (result && this._qualityOk(result)) {
|
|
82
95
|
return new ToolResult(this._formatOutput(result, "pdfjs", resolved));
|
|
83
96
|
}
|
|
84
97
|
|
|
85
|
-
// Level 2:
|
|
98
|
+
// Level 2: Provider VLM (vision model via API — more convenient than local OCR)
|
|
99
|
+
if (this._vlmApiKey && this._ocrModel) {
|
|
100
|
+
result = await this._tryVlm(resolved, pageRange);
|
|
101
|
+
if (result && this._qualityOk(result)) {
|
|
102
|
+
return new ToolResult(this._formatOutput(result, "vlm", resolved));
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Level 3: MineRU API (optional fallback)
|
|
86
107
|
if (this._mineruApiUrl) {
|
|
87
108
|
result = await this._tryMineru(resolved, pageRange);
|
|
88
109
|
if (result && this._qualityOk(result)) {
|
|
@@ -90,12 +111,6 @@ export class DocumentParseTool extends BaseTool {
|
|
|
90
111
|
}
|
|
91
112
|
}
|
|
92
113
|
|
|
93
|
-
// Level 3: OCR via SiliconFlow
|
|
94
|
-
if (this._sfApiKey && this._ocrModel) {
|
|
95
|
-
result = await this._tryOcr(resolved, pageRange);
|
|
96
|
-
if (result) return new ToolResult(this._formatOutput(result, "ocr", resolved));
|
|
97
|
-
}
|
|
98
|
-
|
|
99
114
|
if (result) return new ToolResult(this._formatOutput(result, "pdfjs (low quality)", resolved));
|
|
100
115
|
|
|
101
116
|
return new ToolResult(
|
|
@@ -108,7 +123,7 @@ export class DocumentParseTool extends BaseTool {
|
|
|
108
123
|
let result;
|
|
109
124
|
if (method === "pdfjs") result = await this._tryPdfjs(filePath, pageRange);
|
|
110
125
|
else if (method === "mineru") result = await this._tryMineru(filePath, pageRange);
|
|
111
|
-
else if (method === "ocr") result = await this.
|
|
126
|
+
else if (method === "ocr" || method === "vlm") result = await this._tryVlm(filePath, pageRange);
|
|
112
127
|
else return new ToolResult(`Unknown method: ${method}`, true);
|
|
113
128
|
|
|
114
129
|
if (result) return new ToolResult(this._formatOutput(result, method, filePath));
|
|
@@ -145,12 +160,80 @@ export class DocumentParseTool extends BaseTool {
|
|
|
145
160
|
return null;
|
|
146
161
|
}
|
|
147
162
|
|
|
148
|
-
async
|
|
149
|
-
//
|
|
150
|
-
//
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
163
|
+
async _tryVlm(filePath, pageRange) {
|
|
164
|
+
// Send page images to a VLM provider for OCR/interpretation.
|
|
165
|
+
// Renders PDF pages to PNG via pdfjs canvas, then sends base64 to VLM API.
|
|
166
|
+
if (!this._vlmApiKey || !this._ocrModel || !this._vlmBaseUrl) return null;
|
|
167
|
+
|
|
168
|
+
try {
|
|
169
|
+
const pdfjsLib = await import("pdfjs-dist/legacy/build/pdf.mjs");
|
|
170
|
+
const data = new Uint8Array(fs.readFileSync(filePath));
|
|
171
|
+
const doc = await pdfjsLib.getDocument({ data, useSystemFonts: true }).promise;
|
|
172
|
+
|
|
173
|
+
const start = pageRange ? pageRange[0] : 0;
|
|
174
|
+
const end = pageRange ? pageRange[1] : doc.numPages - 1;
|
|
175
|
+
const pages = [];
|
|
176
|
+
|
|
177
|
+
for (let i = Math.max(0, start); i <= Math.min(end, doc.numPages - 1); i++) {
|
|
178
|
+
const page = await doc.getPage(i + 1);
|
|
179
|
+
const viewport = page.getViewport({ scale: 2.0 }); // Higher res for OCR
|
|
180
|
+
|
|
181
|
+
// Use OffscreenCanvas or node-canvas if available, otherwise skip
|
|
182
|
+
let imageBase64;
|
|
183
|
+
try {
|
|
184
|
+
// In Node.js, pdfjs can render to a canvas-like object
|
|
185
|
+
// We'll use the simpler approach: convert page to image via the API
|
|
186
|
+
const { createCanvas } = await import("canvas").catch(() => ({ createCanvas: null }));
|
|
187
|
+
if (!createCanvas) {
|
|
188
|
+
// No canvas available — fall back to sending raw text content hint + page number
|
|
189
|
+
pages.push(`--- Page ${i + 1} (VLM) ---`);
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
const canvas = createCanvas(viewport.width, viewport.height);
|
|
193
|
+
const ctx = canvas.getContext("2d");
|
|
194
|
+
await page.render({ canvasContext: ctx, viewport }).promise;
|
|
195
|
+
imageBase64 = canvas.toBuffer("image/png").toString("base64");
|
|
196
|
+
} catch {
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (!imageBase64) continue;
|
|
201
|
+
|
|
202
|
+
// Call VLM API with the page image
|
|
203
|
+
const baseUrl = this._vlmBaseUrl.replace(/\/+$/, "");
|
|
204
|
+
const resp = await fetch(`${baseUrl}/chat/completions`, {
|
|
205
|
+
method: "POST",
|
|
206
|
+
headers: {
|
|
207
|
+
"Content-Type": "application/json",
|
|
208
|
+
"Authorization": `Bearer ${this._vlmApiKey}`,
|
|
209
|
+
},
|
|
210
|
+
body: JSON.stringify({
|
|
211
|
+
model: this._ocrModel,
|
|
212
|
+
messages: [
|
|
213
|
+
{ role: "system", content: "Extract all text from this document page. Preserve structure: headings, paragraphs, tables (as markdown), lists. Output clean text only." },
|
|
214
|
+
{ role: "user", content: [
|
|
215
|
+
{ type: "image_url", image_url: { url: `data:image/png;base64,${imageBase64}` } },
|
|
216
|
+
{ type: "text", text: "Extract all text from this page." },
|
|
217
|
+
]},
|
|
218
|
+
],
|
|
219
|
+
max_tokens: 4096,
|
|
220
|
+
}),
|
|
221
|
+
signal: AbortSignal.timeout(60000),
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
if (resp.ok) {
|
|
225
|
+
const result = await resp.json();
|
|
226
|
+
const text = result.choices?.[0]?.message?.content || "";
|
|
227
|
+
if (text.trim()) {
|
|
228
|
+
pages.push(`--- Page ${i + 1} ---\n${text.trim()}`);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
return pages.length > 0 ? pages.join("\n\n") : null;
|
|
234
|
+
} catch {
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
154
237
|
}
|
|
155
238
|
|
|
156
239
|
_qualityOk(text) {
|