kc-beta 0.1.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/bin/kc-beta.js +14 -2
  2. package/package.json +1 -1
  3. package/src/agent/context-window.js +151 -0
  4. package/src/agent/context.js +8 -4
  5. package/src/agent/engine.js +261 -8
  6. package/src/agent/event-log.js +111 -0
  7. package/src/agent/llm-client.js +352 -59
  8. package/src/agent/pipelines/base.js +6 -0
  9. package/src/agent/pipelines/distillation.js +18 -0
  10. package/src/agent/pipelines/extraction.js +21 -0
  11. package/src/agent/pipelines/initializer.js +75 -14
  12. package/src/agent/pipelines/production-qc.js +19 -0
  13. package/src/agent/pipelines/skill-authoring.js +14 -0
  14. package/src/agent/pipelines/skill-testing.js +20 -0
  15. package/src/agent/retry.js +83 -0
  16. package/src/agent/session-state.js +79 -0
  17. package/src/agent/skill-loader.js +13 -1
  18. package/src/agent/token-counter.js +62 -0
  19. package/src/agent/tools/document-parse.js +104 -21
  20. package/src/agent/tools/document-search.js +24 -8
  21. package/src/agent/tools/sandbox-exec.js +16 -5
  22. package/src/agent/tools/web-search.js +107 -0
  23. package/src/agent/tools/worker-llm-call.js +14 -5
  24. package/src/agent/tools/workspace-file.js +47 -20
  25. package/src/agent/workspace.js +24 -1
  26. package/src/cli/components.js +24 -5
  27. package/src/cli/config.js +340 -0
  28. package/src/cli/index.js +113 -11
  29. package/src/cli/onboard.js +216 -53
  30. package/src/config.js +63 -10
  31. package/src/model-tiers.json +153 -0
  32. package/src/providers.js +367 -0
  33. package/template/AGENT.md +20 -0
  34. package/template/skills/en/meta/compliance-judgment/SKILL.md +10 -42
  35. package/template/skills/en/meta/document-chunking/SKILL.md +32 -0
  36. package/template/skills/en/meta/document-parsing/SKILL.md +11 -18
  37. package/template/skills/en/meta/entity-extraction/SKILL.md +13 -28
  38. package/template/skills/en/meta/tree-processing/SKILL.md +19 -1
  39. package/template/skills/en/meta-meta/auto-model-selection/SKILL.md +53 -0
  40. package/template/skills/en/meta-meta/pdf-review-dashboard/SKILL.md +57 -0
  41. package/template/skills/en/meta-meta/pdf-review-dashboard/scripts/generate_review.js +262 -0
  42. package/template/skills/en/meta-meta/rule-extraction/SKILL.md +24 -1
  43. package/template/skills/en/meta-meta/skill-authoring/SKILL.md +6 -0
  44. package/template/skills/en/meta-meta/skill-to-workflow/SKILL.md +4 -0
  45. package/template/skills/zh/meta/compliance-judgment/SKILL.md +41 -262
  46. package/template/skills/zh/meta/document-chunking/SKILL.md +32 -0
  47. package/template/skills/zh/meta/document-parsing/SKILL.md +65 -132
  48. package/template/skills/zh/meta/entity-extraction/SKILL.md +68 -230
  49. package/template/skills/zh/meta/tree-processing/SKILL.md +82 -194
  50. package/template/skills/zh/meta-meta/auto-model-selection/SKILL.md +51 -0
  51. package/template/skills/zh/meta-meta/pdf-review-dashboard/SKILL.md +55 -0
  52. package/template/skills/zh/meta-meta/pdf-review-dashboard/scripts/generate_review.js +262 -0
  53. package/template/skills/zh/meta-meta/rule-extraction/SKILL.md +79 -164
  54. package/template/skills/zh/meta-meta/skill-authoring/SKILL.md +64 -185
  55. package/template/skills/zh/meta-meta/skill-to-workflow/SKILL.md +95 -216
@@ -1,9 +1,13 @@
1
1
  import fs from "node:fs";
2
2
  import path from "node:path";
3
3
  import os from "node:os";
4
+ import { fileURLToPath } from "node:url";
4
5
  import { Phase, PipelineEvent } from "./index.js";
5
6
  import { Pipeline } from "./base.js";
6
7
 
8
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
9
+ const AGENT_MD_TEMPLATE = path.resolve(__dirname, "../../../template/AGENT.md");
10
+
7
11
  const REQUIRED_DIRS = ["rules", "samples", "input", "output", "logs", "workflows", "rule_skills"];
8
12
 
9
13
  const DEFAULT_ENV = `# === KC Agent Project Configuration ===
@@ -11,9 +15,9 @@ const DEFAULT_ENV = `# === KC Agent Project Configuration ===
11
15
  # Language: en | zh
12
16
  LANGUAGE=en
13
17
 
14
- # === Worker LLM API (SiliconFlow) ===
15
- SILICONFLOW_API_KEY=
16
- SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1
18
+ # === LLM API ===
19
+ LLM_API_KEY=
20
+ LLM_BASE_URL=https://api.siliconflow.cn/v1
17
21
 
18
22
  # === Worker LLM Tiers (highest capability to lowest) ===
19
23
  TIER1=Pro/zai-org/GLM-5, Pro/moonshotai/Kimi-K2.5
@@ -53,8 +57,8 @@ export class ProjectInitializer extends Pipeline {
53
57
  if (!fs.existsSync(envPath)) {
54
58
  let envContent = DEFAULT_ENV;
55
59
  const gc = this._loadGlobalConfig();
56
- if (gc.api_key) envContent = envContent.replace("SILICONFLOW_API_KEY=", `SILICONFLOW_API_KEY=${gc.api_key}`);
57
- if (gc.base_url) envContent = envContent.replace("SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1", `SILICONFLOW_BASE_URL=${gc.base_url}`);
60
+ if (gc.api_key) envContent = envContent.replace("LLM_API_KEY=", `LLM_API_KEY=${gc.api_key}`);
61
+ if (gc.base_url) envContent = envContent.replace("LLM_BASE_URL=https://api.siliconflow.cn/v1", `LLM_BASE_URL=${gc.base_url}`);
58
62
  if (gc.accuracy_threshold) {
59
63
  envContent = envContent.replace("SKILL_ACCURACY=0.9", `SKILL_ACCURACY=${gc.accuracy_threshold}`);
60
64
  envContent = envContent.replace("WORKFLOW_ACCURACY=0.9", `WORKFLOW_ACCURACY=${gc.accuracy_threshold}`);
@@ -74,6 +78,12 @@ export class ProjectInitializer extends Pipeline {
74
78
  fs.writeFileSync(manifestPath, JSON.stringify({ version: "0.1.0", entries: [] }, null, 2), "utf-8");
75
79
  }
76
80
 
81
+ // AGENT.md — per-project context (agent can modify)
82
+ const agentMdPath = path.join(this._workspace.cwd, "AGENT.md");
83
+ if (!fs.existsSync(agentMdPath) && fs.existsSync(AGENT_MD_TEMPLATE)) {
84
+ fs.copyFileSync(AGENT_MD_TEMPLATE, agentMdPath);
85
+ }
86
+
77
87
  this.workspaceCreated = true;
78
88
  this._checkRegulations();
79
89
  this._checkSamples();
@@ -81,22 +91,48 @@ export class ProjectInitializer extends Pipeline {
81
91
  }
82
92
 
83
93
  _checkRegulations() {
94
+ // Check workspace rules/
84
95
  const dir = path.join(this._workspace.cwd, "rules");
85
- if (!fs.existsSync(dir)) { this.hasRegulations = false; return; }
86
- this.hasRegulations = fs.readdirSync(dir, { withFileTypes: true }).some((e) => e.isFile());
96
+ if (fs.existsSync(dir) && fs.readdirSync(dir, { withFileTypes: true }).some((e) => e.isFile())) {
97
+ this.hasRegulations = true; return;
98
+ }
99
+ // Check project dir rules/ (case-insensitive)
100
+ if (this._workspace.projectDir) {
101
+ for (const name of ["rules", "Rules", "RULES", "regulations", "Regulations"]) {
102
+ const pdir = path.join(this._workspace.projectDir, name);
103
+ if (fs.existsSync(pdir) && fs.statSync(pdir).isDirectory() &&
104
+ fs.readdirSync(pdir, { withFileTypes: true }).some((e) => e.isFile())) {
105
+ this.hasRegulations = true; return;
106
+ }
107
+ }
108
+ }
109
+ this.hasRegulations = false;
87
110
  }
88
111
 
89
112
  _checkSamples() {
113
+ // Check workspace samples/
90
114
  const dir = path.join(this._workspace.cwd, "samples");
91
- if (!fs.existsSync(dir)) { this.hasSamples = false; return; }
92
- this.hasSamples = fs.readdirSync(dir, { withFileTypes: true }).some((e) => e.isFile());
115
+ if (fs.existsSync(dir) && fs.readdirSync(dir, { withFileTypes: true }).some((e) => e.isFile())) {
116
+ this.hasSamples = true; return;
117
+ }
118
+ // Check project dir samples/ (case-insensitive)
119
+ if (this._workspace.projectDir) {
120
+ for (const name of ["samples", "Samples", "SAMPLES", "sample", "Sample"]) {
121
+ const pdir = path.join(this._workspace.projectDir, name);
122
+ if (fs.existsSync(pdir) && fs.statSync(pdir).isDirectory() &&
123
+ fs.readdirSync(pdir, { withFileTypes: true }).some((e) => e.isFile())) {
124
+ this.hasSamples = true; return;
125
+ }
126
+ }
127
+ }
128
+ this.hasSamples = false;
93
129
  }
94
130
 
95
131
  _checkConfig() {
96
132
  const envPath = path.join(this._workspace.cwd, ".env");
97
133
  if (fs.existsSync(envPath)) {
98
134
  for (const line of fs.readFileSync(envPath, "utf-8").split("\n")) {
99
- if (line.startsWith("SILICONFLOW_API_KEY=") && line.split("=")[1].trim()) {
135
+ if ((line.startsWith("LLM_API_KEY=") || line.startsWith("SILICONFLOW_API_KEY=")) && line.split("=")[1].trim()) {
100
136
  this.configReady = true; return;
101
137
  }
102
138
  }
@@ -115,10 +151,13 @@ export class ProjectInitializer extends Pipeline {
115
151
  const completed = [], pending = [];
116
152
  if (this.workspaceCreated) completed.push("Workspace structure created"); else pending.push("Workspace structure");
117
153
  if (this.configReady) completed.push("API keys configured"); else pending.push("API keys (check .env)");
118
- if (this.hasRegulations) completed.push("Regulation documents in rules/"); else pending.push("Regulation documents in rules/");
119
- if (this.hasSamples) completed.push("Sample documents in samples/"); else pending.push("Sample documents in samples/");
154
+ if (this.hasRegulations) completed.push("Regulation documents found"); else pending.push("Regulation documents (add to rules/ in workspace or project dir)");
155
+ if (this.hasSamples) completed.push("Sample documents found"); else pending.push("Sample documents (add to samples/ in workspace or project dir)");
120
156
 
121
157
  const parts = ["## Phase: BOOTSTRAP\nSet up the workspace and understand the developer user's verification scenario. Bundled methodology skills are available in the workspace skills/ directory."];
158
+ if (this._workspace.projectDir) {
159
+ parts.push(`**Project directory:** ${this._workspace.projectDir}\nUse scope="project" to read files from the user's project folder.`);
160
+ }
122
161
  if (completed.length) parts.push("### Done\n" + completed.map((c) => `- [x] ${c}`).join("\n"));
123
162
  if (pending.length) parts.push("### Needed\n" + pending.map((p) => `- [ ] ${p}`).join("\n"));
124
163
 
@@ -135,14 +174,20 @@ export class ProjectInitializer extends Pipeline {
135
174
  if (toolName === "workspace_file") {
136
175
  const op = toolInput.operation || "";
137
176
  const p = toolInput.path || "";
138
- if (op === "write") {
177
+ const scope = toolInput.scope || "workspace";
178
+ if (op === "write" && scope === "workspace") {
139
179
  if (p.startsWith("rules/")) this.hasRegulations = true;
140
180
  else if (p.startsWith("samples/")) this.hasSamples = true;
141
181
  else if (p === ".env") this._checkConfig();
142
- } else if (op === "list") {
182
+ } else if (op === "list" || op === "read") {
183
+ // Re-check after any list/read — project dir files may satisfy criteria
143
184
  this._checkRegulations();
144
185
  this._checkSamples();
145
186
  }
187
+ } else if (toolName === "document_parse") {
188
+ // Parsing a document from project dir counts as having files
189
+ this._checkRegulations();
190
+ this._checkSamples();
146
191
  }
147
192
 
148
193
  if (!wasReady && this.exitCriteriaMet()) {
@@ -154,4 +199,20 @@ export class ProjectInitializer extends Pipeline {
154
199
  exitCriteriaMet() {
155
200
  return this.workspaceCreated && this.configReady && this.hasRegulations && this.hasSamples;
156
201
  }
202
+
203
+ exportState() {
204
+ return {
205
+ workspaceCreated: this.workspaceCreated,
206
+ configReady: this.configReady,
207
+ hasRegulations: this.hasRegulations,
208
+ hasSamples: this.hasSamples,
209
+ };
210
+ }
211
+
212
+ importState(data) {
213
+ if (data.workspaceCreated) this.workspaceCreated = true;
214
+ if (data.configReady) this.configReady = true;
215
+ if (data.hasRegulations) this.hasRegulations = true;
216
+ if (data.hasSamples) this.hasSamples = true;
217
+ }
157
218
  }
@@ -94,4 +94,23 @@ export class ProductionQCPipeline extends Pipeline {
94
94
  }
95
95
 
96
96
  exitCriteriaMet() { return this.monitoringPhase === "stable"; }
97
+
98
+ exportState() {
99
+ return {
100
+ batchesProcessed: this.batchesProcessed,
101
+ totalDocuments: this.totalDocuments,
102
+ documentsReviewed: this.documentsReviewed,
103
+ monitoringPhase: this.monitoringPhase,
104
+ accuracyByRule: this.accuracyByRule,
105
+ issuesCount: this.issuesFound.length,
106
+ };
107
+ }
108
+
109
+ importState(data) {
110
+ if (typeof data.batchesProcessed === "number" && data.batchesProcessed > this.batchesProcessed) this.batchesProcessed = data.batchesProcessed;
111
+ if (typeof data.totalDocuments === "number" && data.totalDocuments > this.totalDocuments) this.totalDocuments = data.totalDocuments;
112
+ if (typeof data.documentsReviewed === "number" && data.documentsReviewed > this.documentsReviewed) this.documentsReviewed = data.documentsReviewed;
113
+ if (data.monitoringPhase) this.monitoringPhase = data.monitoringPhase;
114
+ if (data.accuracyByRule && typeof data.accuracyByRule === "object") Object.assign(this.accuracyByRule, data.accuracyByRule);
115
+ }
97
116
  }
@@ -77,4 +77,18 @@ export class SkillAuthoringPipeline extends Pipeline {
77
77
  if (!this.totalRules.length) return false;
78
78
  return this.skillsAuthored.length >= this.totalRules.length && this.skillsWithScripts.length >= this.skillsAuthored.length * 0.5;
79
79
  }
80
+
81
+ exportState() {
82
+ return {
83
+ totalRules: this.totalRules,
84
+ skillsAuthored: this.skillsAuthored,
85
+ skillsWithScripts: this.skillsWithScripts,
86
+ };
87
+ }
88
+
89
+ importState(data) {
90
+ if (Array.isArray(data.totalRules) && data.totalRules.length > this.totalRules.length) this.totalRules = data.totalRules;
91
+ if (Array.isArray(data.skillsAuthored) && data.skillsAuthored.length > this.skillsAuthored.length) this.skillsAuthored = data.skillsAuthored;
92
+ if (Array.isArray(data.skillsWithScripts) && data.skillsWithScripts.length > this.skillsWithScripts.length) this.skillsWithScripts = data.skillsWithScripts;
93
+ }
80
94
  }
@@ -106,4 +106,24 @@ export class SkillTestingPipeline extends Pipeline {
106
106
  if (!total) return false;
107
107
  return Object.keys(this.skillsTested).length >= total && this.skillsPassing.length >= total * this._accuracyThreshold;
108
108
  }
109
+
110
+ exportState() {
111
+ return {
112
+ skillsToTest: this.skillsToTest,
113
+ skillsTested: this.skillsTested,
114
+ skillsPassing: this.skillsPassing,
115
+ iterationCount: this.iterationCount,
116
+ };
117
+ }
118
+
119
+ importState(data) {
120
+ if (typeof data.iterationCount === "number" && data.iterationCount > this.iterationCount) this.iterationCount = data.iterationCount;
121
+ if (Array.isArray(data.skillsToTest) && data.skillsToTest.length > this.skillsToTest.length) this.skillsToTest = data.skillsToTest;
122
+ if (Array.isArray(data.skillsPassing) && data.skillsPassing.length > this.skillsPassing.length) this.skillsPassing = data.skillsPassing;
123
+ if (data.skillsTested && typeof data.skillsTested === "object") {
124
+ for (const [k, v] of Object.entries(data.skillsTested)) {
125
+ if (!this.skillsTested[k] || v > this.skillsTested[k]) this.skillsTested[k] = v;
126
+ }
127
+ }
128
+ }
109
129
  }
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Retry wrapper with exponential backoff and jitter.
3
+ * Designed for LLM API calls — retries transient errors, fails fast on auth/validation errors.
4
+ */
5
+
6
+ const MAX_RETRIES = 10;
7
+ const INITIAL_DELAY_MS = 1000;
8
+ const MAX_DELAY_MS = 60000;
9
+ const BACKOFF_MULTIPLIER = 2;
10
+ const JITTER_FRACTION = 0.2;
11
+
12
+ const RETRYABLE_STATUS = new Set([408, 429, 500, 502, 503, 504, 520, 522, 524]);
13
+ const NON_RETRYABLE_STATUS = new Set([400, 401, 403, 404, 422]);
14
+
15
+ /**
16
+ * Determine if an error is retryable.
17
+ * @param {Error} err
18
+ * @returns {boolean}
19
+ */
20
+ function isRetryable(err) {
21
+ if (err.status) {
22
+ if (NON_RETRYABLE_STATUS.has(err.status)) return false;
23
+ if (RETRYABLE_STATUS.has(err.status)) return true;
24
+ }
25
+ // Network errors (ECONNRESET, ETIMEDOUT, fetch TypeError, AbortError)
26
+ const msg = err.message || "";
27
+ if (/ECONNRESET|ETIMEDOUT|ENOTFOUND|ECONNREFUSED|UND_ERR|fetch failed|network|socket hang up/i.test(msg)) {
28
+ return true;
29
+ }
30
+ if (err.name === "AbortError" || err.name === "TimeoutError") return true;
31
+ // If we have a status code and it's not in our known sets, retry server errors (5xx)
32
+ if (err.status && err.status >= 500) return true;
33
+ // Unknown errors without status — retry conservatively
34
+ return !err.status;
35
+ }
36
+
37
+ /**
38
+ * Calculate delay for a given attempt using exponential backoff with jitter.
39
+ * @param {number} attempt - 0-indexed attempt number
40
+ * @param {number|null} retryAfterSec - Retry-After header value in seconds
41
+ * @returns {number} Delay in milliseconds
42
+ */
43
+ function calculateDelay(attempt, retryAfterSec) {
44
+ if (retryAfterSec && retryAfterSec > 0) {
45
+ return Math.min(retryAfterSec * 1000, MAX_DELAY_MS);
46
+ }
47
+ const base = Math.min(INITIAL_DELAY_MS * Math.pow(BACKOFF_MULTIPLIER, attempt), MAX_DELAY_MS);
48
+ const jitter = base * JITTER_FRACTION * Math.random();
49
+ return base + jitter;
50
+ }
51
+
52
+ /**
53
+ * Execute an async function with retry logic.
54
+ *
55
+ * @param {() => Promise<any>} fn - The async function to execute. Should throw with
56
+ * an error that has `.status` and optionally `.retryAfter` properties on failure.
57
+ * @returns {Promise<any>} The successful result
58
+ * @throws {Error} The last error after all retries exhausted, or a non-retryable error immediately
59
+ */
60
+ export async function withRetry(fn) {
61
+ let lastError;
62
+
63
+ for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
64
+ try {
65
+ return await fn();
66
+ } catch (err) {
67
+ lastError = err;
68
+
69
+ if (!isRetryable(err)) throw err;
70
+ if (attempt === MAX_RETRIES) break;
71
+
72
+ const retryAfterSec = err.retryAfter ? parseFloat(err.retryAfter) : null;
73
+ const delay = calculateDelay(attempt, retryAfterSec);
74
+
75
+ await new Promise((resolve) => setTimeout(resolve, delay));
76
+ }
77
+ }
78
+
79
+ const wrapper = new Error(`LLM API call failed after ${MAX_RETRIES + 1} attempts: ${lastError.message}`);
80
+ wrapper.cause = lastError;
81
+ wrapper.status = lastError.status;
82
+ throw wrapper;
83
+ }
@@ -0,0 +1,79 @@
1
+ import fs from "node:fs";
2
+ import path from "node:path";
3
+
4
+ /**
5
+ * Persists session state (phase, pipeline milestones, phase summaries)
6
+ * to enable cross-session resume.
7
+ *
8
+ * Stored as: workspace/{sessionId}/session-state.json
9
+ */
10
+ export class SessionState {
11
+ /**
12
+ * @param {string} workspacePath - Session workspace directory
13
+ */
14
+ constructor(workspacePath) {
15
+ this._path = path.join(workspacePath, "session-state.json");
16
+ }
17
+
18
+ /** Whether a session state file exists */
19
+ get exists() {
20
+ return fs.existsSync(this._path);
21
+ }
22
+
23
+ /**
24
+ * Save engine state to disk.
25
+ * @param {import('./engine.js').AgentEngine} engine
26
+ */
27
+ save(engine) {
28
+ const state = {
29
+ version: 1,
30
+ sessionId: engine.workspace.sessionId,
31
+ currentPhase: engine.currentPhase,
32
+ projectDir: engine.workspace.projectDir || null,
33
+ phaseSummaries: engine._phaseSummaries || [],
34
+ lastEventSeq: engine.eventLog?.currentSeq || 0,
35
+ createdAt: this._loadRaw()?.createdAt || new Date().toISOString(),
36
+ updatedAt: new Date().toISOString(),
37
+ pipelineMilestones: this._extractMilestones(engine.pipelines),
38
+ };
39
+
40
+ fs.writeFileSync(this._path, JSON.stringify(state, null, 2), "utf-8");
41
+ }
42
+
43
+ /**
44
+ * Load session state from disk.
45
+ * @returns {object} The persisted state
46
+ */
47
+ load() {
48
+ return this._loadRaw() || {};
49
+ }
50
+
51
+ /**
52
+ * Read raw file contents.
53
+ */
54
+ _loadRaw() {
55
+ if (!this.exists) return null;
56
+ try {
57
+ return JSON.parse(fs.readFileSync(this._path, "utf-8"));
58
+ } catch {
59
+ return null;
60
+ }
61
+ }
62
+
63
+ /**
64
+ * Serialize pipeline milestones for persistence.
65
+ * @param {object} pipelines - Map of phase -> pipeline instance
66
+ * @returns {object}
67
+ */
68
+ _extractMilestones(pipelines) {
69
+ const milestones = {};
70
+ for (const [phase, pipeline] of Object.entries(pipelines)) {
71
+ if (pipeline?.exportState) {
72
+ try {
73
+ milestones[phase] = pipeline.exportState();
74
+ } catch { /* skip if not implemented */ }
75
+ }
76
+ }
77
+ return milestones;
78
+ }
79
+ }
@@ -130,7 +130,19 @@ export class SkillLoader {
130
130
 
131
131
  const frontmatter = match[1];
132
132
  const name = frontmatter.match(/^name:\s*(.+)$/m)?.[1]?.trim() || "";
133
- const description = frontmatter.match(/^description:\s*(.+)$/m)?.[1]?.trim() || "";
133
+
134
+ // Handle both single-line and multi-line (YAML >) descriptions
135
+ let description = "";
136
+ const descMatch = frontmatter.match(/^description:\s*(.+)$/m);
137
+ if (descMatch && descMatch[1].trim() === ">") {
138
+ // Multi-line: capture indented lines after "description: >"
139
+ const multiMatch = frontmatter.match(/^description:\s*>\s*\n((?:[ \t]+.+\n?)*)/m);
140
+ if (multiMatch) {
141
+ description = multiMatch[1].replace(/^[ \t]+/gm, "").replace(/\n/g, " ").trim();
142
+ }
143
+ } else if (descMatch) {
144
+ description = descMatch[1].trim();
145
+ }
134
146
  return { name, description };
135
147
  } catch {
136
148
  return {};
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Lightweight token estimation without external dependencies.
3
+ * Uses character-based heuristics: ~4 chars per token for Latin text,
4
+ * ~1.5 tokens per CJK character.
5
+ */
6
+
7
+ // CJK Unified Ideographs and extensions
8
+ const CJK_REGEX = /[\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]/g;
9
+
10
+ /**
11
+ * Estimate the number of tokens in a string.
12
+ * @param {string} text
13
+ * @returns {number}
14
+ */
15
+ export function estimateTokens(text) {
16
+ if (!text) return 0;
17
+ const cjkMatches = text.match(CJK_REGEX);
18
+ const cjkCount = cjkMatches ? cjkMatches.length : 0;
19
+ const nonCjkLength = text.length - cjkCount;
20
+ return Math.ceil(nonCjkLength / 4) + Math.ceil(cjkCount * 1.5);
21
+ }
22
+
23
+ /**
24
+ * Estimate total tokens for an array of OpenAI-format messages.
25
+ * Accounts for per-message overhead (~4 tokens for role/formatting).
26
+ * @param {Array<object>} messages
27
+ * @returns {number}
28
+ */
29
+ export function estimateMessagesTokens(messages) {
30
+ let total = 0;
31
+ for (const msg of messages) {
32
+ total += 4; // role + formatting overhead
33
+ if (typeof msg.content === "string") {
34
+ total += estimateTokens(msg.content);
35
+ } else if (Array.isArray(msg.content)) {
36
+ // Anthropic-style content blocks
37
+ for (const block of msg.content) {
38
+ if (block.text) total += estimateTokens(block.text);
39
+ if (block.content) total += estimateTokens(block.content);
40
+ }
41
+ }
42
+ if (msg.tool_calls) {
43
+ for (const tc of msg.tool_calls) {
44
+ total += estimateTokens(tc.function?.name || "");
45
+ total += estimateTokens(tc.function?.arguments || "");
46
+ }
47
+ }
48
+ }
49
+ return total;
50
+ }
51
+
52
+ /**
53
+ * Format a token count for display (e.g., "45.2k").
54
+ * @param {number} tokens
55
+ * @returns {string}
56
+ */
57
+ export function formatTokenCount(tokens) {
58
+ if (tokens >= 1000) {
59
+ return (tokens / 1000).toFixed(1) + "k";
60
+ }
61
+ return tokens.toString();
62
+ }
@@ -12,13 +12,13 @@ const MIN_CHARS_PER_PAGE = 50;
12
12
  * Level 3: OCR models via SiliconFlow — fallback via vision models
13
13
  */
14
14
  export class DocumentParseTool extends BaseTool {
15
- constructor(workspace, { mineruApiUrl, mineruApiKey, siliconflowApiKey, siliconflowBaseUrl, ocrModel } = {}) {
15
+ constructor(workspace, { mineruApiUrl, mineruApiKey, llmApiKey, llmBaseUrl, ocrModel } = {}) {
16
16
  super();
17
17
  this._workspace = workspace;
18
18
  this._mineruApiUrl = mineruApiUrl || "";
19
19
  this._mineruApiKey = mineruApiKey || "";
20
- this._sfApiKey = siliconflowApiKey || "";
21
- this._sfBaseUrl = siliconflowBaseUrl || "https://api.siliconflow.cn/v1";
20
+ this._vlmApiKey = llmApiKey || "";
21
+ this._vlmBaseUrl = (llmBaseUrl || "").replace(/\/+$/, "");
22
22
  this._ocrModel = ocrModel || "";
23
23
  }
24
24
 
@@ -36,13 +36,18 @@ export class DocumentParseTool extends BaseTool {
36
36
  return {
37
37
  type: "object",
38
38
  properties: {
39
- path: { type: "string", description: "Relative path to the document in the workspace" },
39
+ path: { type: "string", description: "Relative path to the document" },
40
40
  pages: { type: "string", description: "Page range to extract, e.g. '1-5', '3', '10-20'. Omit for all pages." },
41
41
  force_method: {
42
42
  type: "string",
43
- enum: ["pdfjs", "mineru", "ocr"],
43
+ enum: ["pdfjs", "vlm", "mineru", "ocr"],
44
44
  description: "Force a specific parsing method, skipping the escalation chain.",
45
45
  },
46
+ scope: {
47
+ type: "string",
48
+ enum: ["workspace", "project"],
49
+ description: "Which directory to find the file in. 'workspace' (default) or 'project' (user's project folder).",
50
+ },
46
51
  },
47
52
  required: ["path"],
48
53
  };
@@ -52,11 +57,19 @@ export class DocumentParseTool extends BaseTool {
52
57
  const pathStr = input.path || "";
53
58
  const pages = input.pages;
54
59
  const force = input.force_method;
60
+ const scope = input.scope || "workspace";
55
61
 
56
62
  if (!pathStr) return new ToolResult("No path provided", true);
63
+ if (scope === "project" && !this._workspace.projectDir) {
64
+ return new ToolResult("No project directory available", true);
65
+ }
57
66
 
58
67
  let resolved;
59
- try { resolved = this._workspace.resolvePath(pathStr); }
68
+ try {
69
+ resolved = scope === "project"
70
+ ? this._workspace.resolveProjectPath(pathStr)
71
+ : this._workspace.resolvePath(pathStr);
72
+ }
60
73
  catch (e) { return new ToolResult(e.message, true); }
61
74
 
62
75
  if (!fs.existsSync(resolved) || !fs.statSync(resolved).isFile()) {
@@ -76,13 +89,21 @@ export class DocumentParseTool extends BaseTool {
76
89
  if (force) return this._runMethod(force, resolved, pageRange);
77
90
 
78
91
  // Escalation chain
79
- // Level 1: pdfjs-dist
92
+ // Level 1: pdfjs-dist (free, local text extraction)
80
93
  let result = await this._tryPdfjs(resolved, pageRange);
81
94
  if (result && this._qualityOk(result)) {
82
95
  return new ToolResult(this._formatOutput(result, "pdfjs", resolved));
83
96
  }
84
97
 
85
- // Level 2: MineRU API
98
+ // Level 2: Provider VLM (vision model via API — more convenient than local OCR)
99
+ if (this._vlmApiKey && this._ocrModel) {
100
+ result = await this._tryVlm(resolved, pageRange);
101
+ if (result && this._qualityOk(result)) {
102
+ return new ToolResult(this._formatOutput(result, "vlm", resolved));
103
+ }
104
+ }
105
+
106
+ // Level 3: MineRU API (optional fallback)
86
107
  if (this._mineruApiUrl) {
87
108
  result = await this._tryMineru(resolved, pageRange);
88
109
  if (result && this._qualityOk(result)) {
@@ -90,12 +111,6 @@ export class DocumentParseTool extends BaseTool {
90
111
  }
91
112
  }
92
113
 
93
- // Level 3: OCR via SiliconFlow
94
- if (this._sfApiKey && this._ocrModel) {
95
- result = await this._tryOcr(resolved, pageRange);
96
- if (result) return new ToolResult(this._formatOutput(result, "ocr", resolved));
97
- }
98
-
99
114
  if (result) return new ToolResult(this._formatOutput(result, "pdfjs (low quality)", resolved));
100
115
 
101
116
  return new ToolResult(
@@ -108,7 +123,7 @@ export class DocumentParseTool extends BaseTool {
108
123
  let result;
109
124
  if (method === "pdfjs") result = await this._tryPdfjs(filePath, pageRange);
110
125
  else if (method === "mineru") result = await this._tryMineru(filePath, pageRange);
111
- else if (method === "ocr") result = await this._tryOcr(filePath, pageRange);
126
+ else if (method === "ocr" || method === "vlm") result = await this._tryVlm(filePath, pageRange);
112
127
  else return new ToolResult(`Unknown method: ${method}`, true);
113
128
 
114
129
  if (result) return new ToolResult(this._formatOutput(result, method, filePath));
@@ -145,12 +160,80 @@ export class DocumentParseTool extends BaseTool {
145
160
  return null;
146
161
  }
147
162
 
148
- async _tryOcr(filePath, pageRange) {
149
- // OCR requires sending page images to a vision model API.
150
- // Without a native image renderer, we delegate to the agent
151
- // to use sandbox_exec for custom OCR pipelines.
152
- // For now, return null to signal OCR is not available natively.
153
- return null;
163
+ async _tryVlm(filePath, pageRange) {
164
+ // Send page images to a VLM provider for OCR/interpretation.
165
+ // Renders PDF pages to PNG via pdfjs canvas, then sends base64 to VLM API.
166
+ if (!this._vlmApiKey || !this._ocrModel || !this._vlmBaseUrl) return null;
167
+
168
+ try {
169
+ const pdfjsLib = await import("pdfjs-dist/legacy/build/pdf.mjs");
170
+ const data = new Uint8Array(fs.readFileSync(filePath));
171
+ const doc = await pdfjsLib.getDocument({ data, useSystemFonts: true }).promise;
172
+
173
+ const start = pageRange ? pageRange[0] : 0;
174
+ const end = pageRange ? pageRange[1] : doc.numPages - 1;
175
+ const pages = [];
176
+
177
+ for (let i = Math.max(0, start); i <= Math.min(end, doc.numPages - 1); i++) {
178
+ const page = await doc.getPage(i + 1);
179
+ const viewport = page.getViewport({ scale: 2.0 }); // Higher res for OCR
180
+
181
+ // Use OffscreenCanvas or node-canvas if available, otherwise skip
182
+ let imageBase64;
183
+ try {
184
+ // In Node.js, pdfjs can render to a canvas-like object
185
+ // We'll use the simpler approach: convert page to image via the API
186
+ const { createCanvas } = await import("canvas").catch(() => ({ createCanvas: null }));
187
+ if (!createCanvas) {
188
+ // No canvas available — fall back to sending raw text content hint + page number
189
+ pages.push(`--- Page ${i + 1} (VLM) ---`);
190
+ continue;
191
+ }
192
+ const canvas = createCanvas(viewport.width, viewport.height);
193
+ const ctx = canvas.getContext("2d");
194
+ await page.render({ canvasContext: ctx, viewport }).promise;
195
+ imageBase64 = canvas.toBuffer("image/png").toString("base64");
196
+ } catch {
197
+ continue;
198
+ }
199
+
200
+ if (!imageBase64) continue;
201
+
202
+ // Call VLM API with the page image
203
+ const baseUrl = this._vlmBaseUrl.replace(/\/+$/, "");
204
+ const resp = await fetch(`${baseUrl}/chat/completions`, {
205
+ method: "POST",
206
+ headers: {
207
+ "Content-Type": "application/json",
208
+ "Authorization": `Bearer ${this._vlmApiKey}`,
209
+ },
210
+ body: JSON.stringify({
211
+ model: this._ocrModel,
212
+ messages: [
213
+ { role: "system", content: "Extract all text from this document page. Preserve structure: headings, paragraphs, tables (as markdown), lists. Output clean text only." },
214
+ { role: "user", content: [
215
+ { type: "image_url", image_url: { url: `data:image/png;base64,${imageBase64}` } },
216
+ { type: "text", text: "Extract all text from this page." },
217
+ ]},
218
+ ],
219
+ max_tokens: 4096,
220
+ }),
221
+ signal: AbortSignal.timeout(60000),
222
+ });
223
+
224
+ if (resp.ok) {
225
+ const result = await resp.json();
226
+ const text = result.choices?.[0]?.message?.content || "";
227
+ if (text.trim()) {
228
+ pages.push(`--- Page ${i + 1} ---\n${text.trim()}`);
229
+ }
230
+ }
231
+ }
232
+
233
+ return pages.length > 0 ? pages.join("\n\n") : null;
234
+ } catch {
235
+ return null;
236
+ }
154
237
  }
155
238
 
156
239
  _qualityOk(text) {