godot-daedalus_backend 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -0
- package/bin/godot-daedalus-backend.js +4 -0
- package/bin/godot-daedalus-mcp.js +4 -0
- package/bin/godot-daedalus-terminal-mcp.js +4 -0
- package/bin/run-tsx-entry.js +26 -0
- package/package.json +54 -0
- package/scripts/deepseek-tokenizer-server.py +54 -0
- package/src/app-paths.ts +36 -0
- package/src/main.ts +21 -0
- package/src/mcp/content-length-protocol.ts +68 -0
- package/src/mcp/custom-mcp-config-store.ts +397 -0
- package/src/mcp/godot-diagnostics-bridge.ts +1298 -0
- package/src/mcp/godot-editor-bridge.ts +307 -0
- package/src/mcp/godot-mcp-server.ts +3484 -0
- package/src/mcp/godot-paths.ts +151 -0
- package/src/mcp/godot-project-settings.ts +233 -0
- package/src/mcp/godot-tool-registration.ts +46 -0
- package/src/mcp/mcp-config.ts +48 -0
- package/src/mcp/mcp-host.ts +393 -0
- package/src/mcp/mcp-session.ts +81 -0
- package/src/mcp/terminal-mcp-server.ts +576 -0
- package/src/mcp/tscn-tools.ts +302 -0
- package/src/mcp/types.ts +12 -0
- package/src/ping-client.ts +24 -0
- package/src/prompts/registry.ts +97 -0
- package/src/prompts/templates/backend-helper.md +25 -0
- package/src/prompts/templates/gdscript-reviewer.md +19 -0
- package/src/prompts/templates/godot-assistant.md +225 -0
- package/src/prompts/templates/scene-architect.md +15 -0
- package/src/prompts/templates/session-compressor.md +33 -0
- package/src/protocol/schema.ts +486 -0
- package/src/protocol/types.ts +77 -0
- package/src/providers/deepseek-agent.ts +1014 -0
- package/src/providers/deepseek-client.ts +114 -0
- package/src/providers/deepseek-dsml-tools.ts +90 -0
- package/src/providers/deepseek-loose-tools.ts +450 -0
- package/src/providers/provider-config-store.ts +164 -0
- package/src/server/client-session.ts +93 -0
- package/src/server/request-dispatcher.ts +74 -0
- package/src/server/response-helpers.ts +33 -0
- package/src/server/send-json.ts +8 -0
- package/src/server/websocket-server.ts +3997 -0
- package/src/session/session-compressor.ts +68 -0
- package/src/session/session-store.ts +669 -0
- package/src/skills/registry.ts +180 -0
- package/src/skills/templates/backend-helper.md +12 -0
- package/src/skills/templates/file-creator.md +14 -0
- package/src/skills/templates/gdscript-review.md +12 -0
- package/src/skills/templates/godot-project-init.md +29 -0
- package/src/skills/templates/scene-builder.md +12 -0
- package/src/tokens/deepseek-tokenizer-counter.ts +233 -0
- package/src/tokens/model-profiles.ts +38 -0
- package/src/tokens/token-counter-factory.ts +52 -0
- package/src/tokens/token-counter.ts +22 -0
- package/src/tools/approval-gateway.ts +111 -0
- package/src/tools/llm-tools.ts +1415 -0
- package/src/tools/tool-dispatcher.ts +147 -0
- package/src/tools/tool-event-describer.ts +387 -0
- package/src/tools/tool-idempotency.ts +373 -0
- package/src/tools/tool-policy-table.ts +61 -0
- package/src/tools/tool-policy.ts +73 -0
- package/src/workflow/llm-planner.ts +407 -0
- package/src/workflow/planner.ts +201 -0
- package/src/workflow/runner.ts +141 -0
- package/src/workflow/types.ts +69 -0
- package/src/workspace/registry.ts +104 -0
- package/src/workspace/types.ts +7 -0
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import type { PromptId } from "../protocol/types.js";
|
|
4
|
+
import { CUSTOM_MCP_TOOLS_SENTINEL } from "../tools/llm-tools.js";
|
|
5
|
+
|
|
6
|
+
export const skillIds = [
|
|
7
|
+
"godot.project_init",
|
|
8
|
+
"gdscript.review",
|
|
9
|
+
"scene.builder",
|
|
10
|
+
"file.creator",
|
|
11
|
+
"backend.helper"
|
|
12
|
+
] as const;
|
|
13
|
+
|
|
14
|
+
export type SkillId = typeof skillIds[number];
|
|
15
|
+
|
|
16
|
+
export type Skill = {
|
|
17
|
+
id: SkillId;
|
|
18
|
+
name: string;
|
|
19
|
+
description: string;
|
|
20
|
+
promptPath: string;
|
|
21
|
+
defaultPromptId?: PromptId;
|
|
22
|
+
allowedTools: string[];
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
const READ_TOOLS: string[] = [
|
|
26
|
+
"mcp_godot_get_project_summary",
|
|
27
|
+
"mcp_godot_list_project_files",
|
|
28
|
+
"mcp_godot_list_scenes",
|
|
29
|
+
"mcp_godot_list_scripts",
|
|
30
|
+
"mcp_godot_read_text_file",
|
|
31
|
+
"mcp_godot_search_text",
|
|
32
|
+
"mcp_godot_get_project_log_config",
|
|
33
|
+
"mcp_godot_list_project_logs",
|
|
34
|
+
"mcp_godot_read_project_log",
|
|
35
|
+
"mcp_godot_get_project_settings",
|
|
36
|
+
"mcp_godot_get_editor_config_summary",
|
|
37
|
+
"mcp_godot_get_editor_settings",
|
|
38
|
+
"mcp_godot_list_editor_config_files",
|
|
39
|
+
"mcp_godot_read_editor_config_file",
|
|
40
|
+
"mcp_godot_get_editor_project_state",
|
|
41
|
+
"mcp_godot_get_recent_projects",
|
|
42
|
+
"mcp_godot_inspect_scene_tree",
|
|
43
|
+
"mcp_godot_editor_get_context",
|
|
44
|
+
"mcp_godot_editor_get_selected_nodes",
|
|
45
|
+
"mcp_godot_editor_inspect_node",
|
|
46
|
+
"mcp_godot_lsp_get_status",
|
|
47
|
+
"mcp_godot_lsp_get_file_diagnostics",
|
|
48
|
+
"mcp_godot_lsp_get_document_symbols",
|
|
49
|
+
"mcp_godot_lsp_hover",
|
|
50
|
+
"mcp_godot_lsp_goto_definition",
|
|
51
|
+
"mcp_godot_dap_get_status",
|
|
52
|
+
"mcp_godot_dap_get_last_error",
|
|
53
|
+
"mcp_godot_dap_get_stack_trace",
|
|
54
|
+
"mcp_godot_dap_get_variables",
|
|
55
|
+
CUSTOM_MCP_TOOLS_SENTINEL
|
|
56
|
+
];
|
|
57
|
+
|
|
58
|
+
const FILE_CREATE_TOOLS: string[] = [
|
|
59
|
+
"mcp_godot_create_text_file"
|
|
60
|
+
];
|
|
61
|
+
|
|
62
|
+
const VERIFY_TOOLS: string[] = [
|
|
63
|
+
"mcp_godot_lsp_get_file_diagnostics",
|
|
64
|
+
"mcp_terminal_get_capabilities",
|
|
65
|
+
"mcp_terminal_run_safe_preset"
|
|
66
|
+
];
|
|
67
|
+
|
|
68
|
+
const TERMINAL_WRITE_TOOLS: string[] = [
|
|
69
|
+
"mcp_terminal_run_write_preset",
|
|
70
|
+
"mcp_terminal_run_godot_scene_script"
|
|
71
|
+
];
|
|
72
|
+
|
|
73
|
+
const FILE_EDIT_TOOLS: string[] = [
|
|
74
|
+
"mcp_godot_overwrite_text_file",
|
|
75
|
+
"mcp_godot_replace_text_in_file",
|
|
76
|
+
"mcp_godot_propose_set_project_setting",
|
|
77
|
+
"mcp_godot_set_project_setting",
|
|
78
|
+
"mcp_godot_propose_unset_project_setting",
|
|
79
|
+
"mcp_godot_unset_project_setting"
|
|
80
|
+
];
|
|
81
|
+
|
|
82
|
+
const SCENE_WRITE_TOOLS: string[] = [
|
|
83
|
+
"mcp_godot_create_scene",
|
|
84
|
+
"mcp_godot_add_node_to_scene",
|
|
85
|
+
"mcp_godot_attach_script_to_node",
|
|
86
|
+
"mcp_godot_connect_signal_in_scene",
|
|
87
|
+
"mcp_godot_apply_scene_patch",
|
|
88
|
+
"mcp_godot_editor_apply_scene_patch"
|
|
89
|
+
];
|
|
90
|
+
|
|
91
|
+
const skills: Record<SkillId, Skill> = {
|
|
92
|
+
"godot.project_init": {
|
|
93
|
+
id: "godot.project_init",
|
|
94
|
+
name: "Godot Project Init",
|
|
95
|
+
description: "Inspect the Godot project and create an AGENTS.md project guide.",
|
|
96
|
+
promptPath: "src/skills/templates/godot-project-init.md",
|
|
97
|
+
defaultPromptId: "godot.assistant",
|
|
98
|
+
allowedTools: [...READ_TOOLS, ...FILE_CREATE_TOOLS, ...FILE_EDIT_TOOLS, ...VERIFY_TOOLS, ...TERMINAL_WRITE_TOOLS]
|
|
99
|
+
},
|
|
100
|
+
"gdscript.review": {
|
|
101
|
+
id: "gdscript.review",
|
|
102
|
+
name: "GDScript Review",
|
|
103
|
+
description: "Review GDScript for type safety, Godot lifecycle issues, signals, and style.",
|
|
104
|
+
promptPath: "src/skills/templates/gdscript-review.md",
|
|
105
|
+
defaultPromptId: "gdscript.reviewer",
|
|
106
|
+
allowedTools: [...READ_TOOLS, ...VERIFY_TOOLS]
|
|
107
|
+
},
|
|
108
|
+
"scene.builder": {
|
|
109
|
+
id: "scene.builder",
|
|
110
|
+
name: "Scene Builder",
|
|
111
|
+
description: "Plan Godot scene structures and node responsibilities.",
|
|
112
|
+
promptPath: "src/skills/templates/scene-builder.md",
|
|
113
|
+
defaultPromptId: "scene.architect",
|
|
114
|
+
allowedTools: [...READ_TOOLS, ...SCENE_WRITE_TOOLS, ...FILE_CREATE_TOOLS, ...VERIFY_TOOLS]
|
|
115
|
+
},
|
|
116
|
+
"file.creator": {
|
|
117
|
+
id: "file.creator",
|
|
118
|
+
name: "File Creator",
|
|
119
|
+
description: "Create new project files through approval-gated tools.",
|
|
120
|
+
promptPath: "src/skills/templates/file-creator.md",
|
|
121
|
+
defaultPromptId: "godot.assistant",
|
|
122
|
+
allowedTools: [...READ_TOOLS, ...FILE_CREATE_TOOLS, ...FILE_EDIT_TOOLS, ...VERIFY_TOOLS, ...TERMINAL_WRITE_TOOLS]
|
|
123
|
+
},
|
|
124
|
+
"backend.helper": {
|
|
125
|
+
id: "backend.helper",
|
|
126
|
+
name: "Backend Helper",
|
|
127
|
+
description: "Work on the TypeScript WebSocket/MCP backend.",
|
|
128
|
+
promptPath: "src/skills/templates/backend-helper.md",
|
|
129
|
+
defaultPromptId: "backend.helper",
|
|
130
|
+
allowedTools: [...READ_TOOLS, ...VERIFY_TOOLS]
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
const skillPromptCache: Map<SkillId, string> = new Map();
|
|
135
|
+
|
|
136
|
+
export function listSkills(): Skill[] {
|
|
137
|
+
return Object.values(skills);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export function isSkillId(value: string): value is SkillId {
|
|
141
|
+
return (skillIds as readonly string[]).includes(value);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export function getSkill(skillId: SkillId): Skill {
|
|
145
|
+
return skills[skillId];
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export async function loadSkillPrompt(skillId: SkillId): Promise<string> {
|
|
149
|
+
const cachedPrompt: string | undefined = skillPromptCache.get(skillId);
|
|
150
|
+
if (cachedPrompt !== undefined) {
|
|
151
|
+
return cachedPrompt;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const skill: Skill = getSkill(skillId);
|
|
155
|
+
const content: string = await readFile(resolve(process.cwd(), skill.promptPath), "utf8");
|
|
156
|
+
const trimmedContent: string = content.trim();
|
|
157
|
+
skillPromptCache.set(skillId, trimmedContent);
|
|
158
|
+
return trimmedContent;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export async function composeSkillPrompt(skillId: SkillId | undefined): Promise<string> {
|
|
162
|
+
if (skillId === undefined) {
|
|
163
|
+
return "";
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const skill: Skill = getSkill(skillId);
|
|
167
|
+
const prompt: string = await loadSkillPrompt(skillId);
|
|
168
|
+
return [
|
|
169
|
+
"## 当前激活 Skill",
|
|
170
|
+
`- ID: ${skill.id}`,
|
|
171
|
+
`- 名称: ${skill.name}`,
|
|
172
|
+
`- 描述: ${skill.description}`,
|
|
173
|
+
"- 允许工具:",
|
|
174
|
+
...skill.allowedTools
|
|
175
|
+
.filter((toolName: string): boolean => toolName !== CUSTOM_MCP_TOOLS_SENTINEL)
|
|
176
|
+
.map((toolName: string): string => ` - ${toolName}`),
|
|
177
|
+
"",
|
|
178
|
+
prompt
|
|
179
|
+
].join("\n");
|
|
180
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# File Creator Skill
|
|
2
|
+
|
|
3
|
+
你负责创建新文件。必须遵守 Runtime 文件能力和审批规则。
|
|
4
|
+
|
|
5
|
+
流程:
|
|
6
|
+
|
|
7
|
+
1. 明确文件路径和用途。
|
|
8
|
+
2. 确保路径在项目内,且不是 `.godot/`、`addons/` 或隐藏目录。
|
|
9
|
+
3. 创建或修改文件时调用实际写入工具,不要把 `propose_*` 当作实现结果。
|
|
10
|
+
4. 修改 `project.godot` 项目设置时,先读取当前设置并调用 `propose_*` 预览,再调用实际 set/unset 工具。
|
|
11
|
+
5. 实际写入工具需要审批时,后端会暂停本轮并向客户端发送审批事件;不要继续总结成“已完成”。
|
|
12
|
+
6. 文件创建或修改成功后,读取文件确认内容。
|
|
13
|
+
|
|
14
|
+
不能覆盖、删除或伪造文件修改结果。
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Godot Project Init Skill
|
|
2
|
+
|
|
3
|
+
你的任务是初始化当前 Godot 项目的 AI 协作上下文,并生成项目根目录的 `AGENTS.md`。
|
|
4
|
+
|
|
5
|
+
执行流程:
|
|
6
|
+
|
|
7
|
+
1. 先读取项目摘要、场景列表、脚本列表和关键配置。
|
|
8
|
+
2. 如有必要,读取少量代表性 `.gd`、`.tscn`、`.tres`、`project.godot` 文件来判断项目结构。
|
|
9
|
+
3. 生成一份简洁、可执行的 `AGENTS.md`,面向后续 AI/开发者协作。
|
|
10
|
+
4. 如果项目根目录不存在 `AGENTS.md`,调用文件创建工具创建它。
|
|
11
|
+
5. 如果 `AGENTS.md` 已存在,不要覆盖;先读取它,说明已有内容,并提出是否需要用户确认后再更新。
|
|
12
|
+
|
|
13
|
+
`AGENTS.md` 应包含:
|
|
14
|
+
|
|
15
|
+
- 项目概览
|
|
16
|
+
- 目录结构与主要文件
|
|
17
|
+
- Godot/GDScript 编码规范
|
|
18
|
+
- 场景与资源修改规则
|
|
19
|
+
- 测试与验证命令
|
|
20
|
+
- AI 工具使用与审批规则
|
|
21
|
+
- 不允许修改的目录或文件
|
|
22
|
+
|
|
23
|
+
写作要求:
|
|
24
|
+
|
|
25
|
+
- 使用 Markdown。
|
|
26
|
+
- 内容应基于实际项目,不要泛泛而谈。
|
|
27
|
+
- 保持 200-600 词左右,除非项目复杂度要求更长。
|
|
28
|
+
- 不能声称文件已创建,除非工具明确返回创建成功。
|
|
29
|
+
- 如果创建工具返回需要审批,提醒用户在 Godot 客户端 Approvals 区域批准。
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { type ChildProcess, spawn } from "node:child_process";
|
|
2
|
+
import { createInterface } from "node:readline";
|
|
3
|
+
import { dirname, resolve } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import type { TokenCounter } from "./token-counter.js";
|
|
6
|
+
import type { ChatMessage } from "../protocol/types.js";
|
|
7
|
+
|
|
8
|
+
const PACKAGE_ROOT: string = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
|
9
|
+
const TOKENIZER_SCRIPT: string = resolve(PACKAGE_ROOT, "scripts/deepseek-tokenizer-server.py");
|
|
10
|
+
const DEFAULT_TOKENIZER_DIR: string = resolve(PACKAGE_ROOT, "scripts/tokenizer");
|
|
11
|
+
const START_TIMEOUT_MS: number = 30_000;
|
|
12
|
+
const REQUEST_TIMEOUT_MS: number = 10_000;
|
|
13
|
+
|
|
14
|
+
type PendingRequest = {
|
|
15
|
+
resolve: (tokens: number) => void;
|
|
16
|
+
reject: (error: Error) => void;
|
|
17
|
+
timeout: ReturnType<typeof setTimeout>;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
export class DeepSeekTokenizerCounter implements TokenCounter {
|
|
21
|
+
private process: ChildProcess | null = null;
|
|
22
|
+
private pending: Map<number, PendingRequest> = new Map();
|
|
23
|
+
private requestId: number = 0;
|
|
24
|
+
private ready: boolean = false;
|
|
25
|
+
private initPromise: Promise<void> | null = null;
|
|
26
|
+
|
|
27
|
+
async initialize(): Promise<void> {
|
|
28
|
+
if (this.initPromise) {
|
|
29
|
+
return this.initPromise;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
this.initPromise = this.startProcess();
|
|
33
|
+
return this.initPromise;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
private startProcess(): Promise<void> {
|
|
37
|
+
return new Promise<void>((resolvePromise, rejectPromise) => {
|
|
38
|
+
const tokenizerDir: string = process.env.DEEPSEEK_TOKENIZER_DIR ?? DEFAULT_TOKENIZER_DIR;
|
|
39
|
+
const pythonCmd: string = process.env.PYTHON_CMD ?? "python";
|
|
40
|
+
let startupSettled: boolean = false;
|
|
41
|
+
|
|
42
|
+
const child: ChildProcess = spawn(pythonCmd, [TOKENIZER_SCRIPT, tokenizerDir], {
|
|
43
|
+
env: {
|
|
44
|
+
...process.env,
|
|
45
|
+
PYTHONIOENCODING: "utf-8",
|
|
46
|
+
PYTHONUTF8: "1"
|
|
47
|
+
},
|
|
48
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
const startTimeout: ReturnType<typeof setTimeout> = setTimeout((): void => {
|
|
52
|
+
rejectStartup(new Error("Tokenizer startup timed out. Install tokenizers: pip install tokenizers"));
|
|
53
|
+
}, START_TIMEOUT_MS);
|
|
54
|
+
|
|
55
|
+
const resolveStartup = (): void => {
|
|
56
|
+
if (startupSettled) {
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
startupSettled = true;
|
|
61
|
+
clearTimeout(startTimeout);
|
|
62
|
+
resolvePromise();
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const rejectStartup = (error: Error): void => {
|
|
66
|
+
if (startupSettled) {
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
startupSettled = true;
|
|
71
|
+
clearTimeout(startTimeout);
|
|
72
|
+
rejectPromise(error);
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
const rl = createInterface({ input: child.stdout! });
|
|
76
|
+
|
|
77
|
+
rl.on("line", (line: string): void => {
|
|
78
|
+
const trimmed: string = line.trim();
|
|
79
|
+
if (trimmed.length === 0) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
try {
|
|
84
|
+
const response: { id?: number; ready?: boolean; tokens?: number; error?: string } = JSON.parse(trimmed) as {
|
|
85
|
+
id?: number; ready?: boolean; tokens?: number; error?: string;
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
if (response.ready === true) {
|
|
89
|
+
this.ready = true;
|
|
90
|
+
resolveStartup();
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (response.error) {
|
|
95
|
+
const error: Error = new Error(response.error);
|
|
96
|
+
if (!this.ready) {
|
|
97
|
+
rejectStartup(error);
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
if (response.id !== undefined) {
|
|
102
|
+
this.rejectPending(response.id, error);
|
|
103
|
+
} else {
|
|
104
|
+
this.rejectAll(error);
|
|
105
|
+
}
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (response.tokens !== undefined) {
|
|
110
|
+
if (response.id !== undefined) {
|
|
111
|
+
this.resolvePending(response.id, response.tokens);
|
|
112
|
+
} else {
|
|
113
|
+
this.resolveFirstPending(response.tokens);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
} catch {
|
|
117
|
+
// Non-JSON output from Python — ignore (e.g., warnings)
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
child.stderr?.on("data", (data: Buffer): void => {
|
|
122
|
+
console.error("[tokenizer]", data.toString("utf8").trimEnd());
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
child.on("error", (error: Error): void => {
|
|
126
|
+
this.ready = false;
|
|
127
|
+
this.process = null;
|
|
128
|
+
this.initPromise = null;
|
|
129
|
+
rejectStartup(error);
|
|
130
|
+
this.rejectAll(error);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
child.on("close", (): void => {
|
|
134
|
+
this.ready = false;
|
|
135
|
+
this.process = null;
|
|
136
|
+
this.initPromise = null;
|
|
137
|
+
const error: Error = new Error("Tokenizer process exited");
|
|
138
|
+
rejectStartup(error);
|
|
139
|
+
this.rejectAll(error);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
this.process = child;
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
private sendRequest(text: string): Promise<number> {
|
|
147
|
+
return new Promise<number>((resolve, reject) => {
|
|
148
|
+
if (!this.process?.stdin?.writable) {
|
|
149
|
+
reject(new Error("Tokenizer process is not writable"));
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const id: number = this.requestId;
|
|
154
|
+
this.requestId += 1;
|
|
155
|
+
|
|
156
|
+
const timeout: ReturnType<typeof setTimeout> = setTimeout((): void => {
|
|
157
|
+
this.pending.delete(id);
|
|
158
|
+
reject(new Error("Tokenizer request timed out"));
|
|
159
|
+
}, REQUEST_TIMEOUT_MS);
|
|
160
|
+
|
|
161
|
+
this.pending.set(id, { resolve, reject, timeout });
|
|
162
|
+
|
|
163
|
+
this.process.stdin.write(JSON.stringify({ id, text }) + "\n", (error: Error | null | undefined): void => {
|
|
164
|
+
if (error !== null && error !== undefined) {
|
|
165
|
+
this.rejectPending(id, error);
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
private resolvePending(id: number, tokens: number): void {
|
|
172
|
+
const pending: PendingRequest | undefined = this.pending.get(id);
|
|
173
|
+
if (pending === undefined) {
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
clearTimeout(pending.timeout);
|
|
178
|
+
pending.resolve(tokens);
|
|
179
|
+
this.pending.delete(id);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
private resolveFirstPending(tokens: number): void {
|
|
183
|
+
for (const [id, pending] of this.pending) {
|
|
184
|
+
clearTimeout(pending.timeout);
|
|
185
|
+
pending.resolve(tokens);
|
|
186
|
+
this.pending.delete(id);
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private rejectPending(id: number, error: Error): void {
|
|
192
|
+
const pending: PendingRequest | undefined = this.pending.get(id);
|
|
193
|
+
if (pending === undefined) {
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
clearTimeout(pending.timeout);
|
|
198
|
+
pending.reject(error);
|
|
199
|
+
this.pending.delete(id);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
private rejectAll(error: Error): void {
|
|
203
|
+
for (const [id, pending] of this.pending) {
|
|
204
|
+
clearTimeout(pending.timeout);
|
|
205
|
+
pending.reject(error);
|
|
206
|
+
this.pending.delete(id);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async countText(text: string): Promise<number> {
|
|
211
|
+
if (!this.ready) {
|
|
212
|
+
await this.initialize();
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
return this.sendRequest(text);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async countMessages(messages: ChatMessage[]): Promise<number> {
|
|
219
|
+
const combined: string = messages.map((m: ChatMessage): string => m.content).join("\n");
|
|
220
|
+
const baseTokens: number = await this.countText(combined);
|
|
221
|
+
return baseTokens + messages.length * 4;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async dispose(): Promise<void> {
|
|
225
|
+
if (this.process) {
|
|
226
|
+
this.process.stdin?.end();
|
|
227
|
+
this.process.kill();
|
|
228
|
+
this.process = null;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
this.ready = false;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { ModelProfile } from "../protocol/types.js";
|
|
2
|
+
|
|
3
|
+
export const DEEPSEEK_V4_FLASH: ModelProfile = {
|
|
4
|
+
provider: "deepseek",
|
|
5
|
+
model: "deepseek-v4-flash",
|
|
6
|
+
contextWindowTokens: 1_000_000,
|
|
7
|
+
maxOutputTokens: 384_000,
|
|
8
|
+
defaultOutputReserveTokens: 16_000,
|
|
9
|
+
safetyMarginTokens: 8_000,
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export const DEEPSEEK_V4_PRO: ModelProfile = {
|
|
13
|
+
provider: "deepseek",
|
|
14
|
+
model: "deepseek-v4-pro",
|
|
15
|
+
contextWindowTokens: 1_000_000,
|
|
16
|
+
maxOutputTokens: 384_000,
|
|
17
|
+
defaultOutputReserveTokens: 32_000,
|
|
18
|
+
safetyMarginTokens: 12_000,
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const MODEL_REGISTRY: Record<string, ModelProfile> = {
|
|
22
|
+
"deepseek-v4-flash": DEEPSEEK_V4_FLASH,
|
|
23
|
+
"deepseek-v4-pro": DEEPSEEK_V4_PRO,
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export function resolveModelProfile(modelName: string): ModelProfile {
|
|
27
|
+
const profile: ModelProfile | undefined = MODEL_REGISTRY[modelName];
|
|
28
|
+
|
|
29
|
+
if (!profile) {
|
|
30
|
+
throw new Error(`Unknown model: ${modelName}`);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
return profile;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function getDefaultModelProfile(): ModelProfile {
|
|
37
|
+
return DEEPSEEK_V4_FLASH;
|
|
38
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { TokenCounter } from "./token-counter.js";
|
|
2
|
+
import { ApproxTokenCounter } from "./token-counter.js";
|
|
3
|
+
import type { ChatMessage } from "../protocol/types.js";
|
|
4
|
+
|
|
5
|
+
class ResilientTokenCounter implements TokenCounter {
|
|
6
|
+
constructor(
|
|
7
|
+
private readonly primary: TokenCounter,
|
|
8
|
+
private readonly fallback: TokenCounter
|
|
9
|
+
) {}
|
|
10
|
+
|
|
11
|
+
async countText(text: string): Promise<number> {
|
|
12
|
+
try {
|
|
13
|
+
return await this.primary.countText(text);
|
|
14
|
+
} catch (error: unknown) {
|
|
15
|
+
const message: string = error instanceof Error ? error.message : String(error);
|
|
16
|
+
console.warn(`[token-counter] Precise count failed, using approximate count: ${message}`);
|
|
17
|
+
return this.fallback.countText(text);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async countMessages(messages: ChatMessage[]): Promise<number> {
|
|
22
|
+
try {
|
|
23
|
+
return await this.primary.countMessages(messages);
|
|
24
|
+
} catch (error: unknown) {
|
|
25
|
+
const message: string = error instanceof Error ? error.message : String(error);
|
|
26
|
+
console.warn(`[token-counter] Precise message count failed, using approximate count: ${message}`);
|
|
27
|
+
return this.fallback.countMessages(messages);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export async function createTokenCounter(): Promise<TokenCounter> {
|
|
33
|
+
const disableTokenizer: string | undefined = process.env.DISABLE_DEEPSEEK_TOKENIZER;
|
|
34
|
+
const fallback: ApproxTokenCounter = new ApproxTokenCounter();
|
|
35
|
+
|
|
36
|
+
if (disableTokenizer === "1" || disableTokenizer === "true") {
|
|
37
|
+
return fallback;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
try {
|
|
41
|
+
const { DeepSeekTokenizerCounter } = await import("./deepseek-tokenizer-counter.js");
|
|
42
|
+
const counter = new DeepSeekTokenizerCounter();
|
|
43
|
+
await counter.initialize();
|
|
44
|
+
console.log("[token-counter] Using DeepSeekTokenizerCounter (Python)");
|
|
45
|
+
return new ResilientTokenCounter(counter, fallback);
|
|
46
|
+
} catch (error: unknown) {
|
|
47
|
+
const message: string = error instanceof Error ? error.message : String(error);
|
|
48
|
+
console.warn(`[token-counter] DeepSeek tokenizer unavailable: ${message}`);
|
|
49
|
+
console.warn("[token-counter] Falling back to ApproxTokenCounter (char/3 estimate)");
|
|
50
|
+
return fallback;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { ChatMessage } from "../protocol/types.js";
|
|
2
|
+
|
|
3
|
+
export type TokenCounter = {
|
|
4
|
+
countText(text: string): Promise<number>;
|
|
5
|
+
countMessages(messages: ChatMessage[]): Promise<number>;
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export class ApproxTokenCounter implements TokenCounter {
|
|
9
|
+
async countText(text: string): Promise<number> {
|
|
10
|
+
return Math.max(1, Math.ceil(text.length / 3));
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
async countMessages(messages: ChatMessage[]): Promise<number> {
|
|
14
|
+
let total: number = 0;
|
|
15
|
+
|
|
16
|
+
for (const message of messages) {
|
|
17
|
+
total += await this.countText(message.content) + 4;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
return total;
|
|
21
|
+
}
|
|
22
|
+
}
|