@jwcode/cli 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +348 -0
- package/backend/.gitkeep +0 -0
- package/dist/__tests__/store.test.d.ts +1 -0
- package/dist/__tests__/store.test.js +43 -0
- package/dist/__tests__/theme.test.d.ts +1 -0
- package/dist/__tests__/theme.test.js +40 -0
- package/dist/__tests__/tokenEstimate.test.d.ts +1 -0
- package/dist/__tests__/tokenEstimate.test.js +48 -0
- package/dist/cli.js +83 -0
- package/dist/commands/index.d.ts +18 -0
- package/dist/commands/index.js +99 -0
- package/dist/components/ApprovalModal.d.ts +8 -0
- package/dist/components/ApprovalModal.js +47 -0
- package/dist/components/ChatArea.d.ts +10 -0
- package/dist/components/ChatArea.js +49 -0
- package/dist/components/CommandPalette.d.ts +6 -0
- package/dist/components/CommandPalette.js +72 -0
- package/dist/components/StatusLine.d.ts +1 -0
- package/dist/components/StatusLine.js +25 -0
- package/dist/components/TextInput.d.ts +10 -0
- package/dist/components/TextInput.js +118 -0
- package/dist/hooks/useAppState.d.ts +18 -0
- package/dist/hooks/useAppState.js +42 -0
- package/dist/hooks/useWebSocket.d.ts +3 -0
- package/dist/hooks/useWebSocket.js +8 -0
- package/package.json +43 -0
package/README.md
ADDED
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
# @jwcode/cli — JWCode TypeScript CLI
|
|
2
|
+
|
|
3
|
+
> **JWCode** 是一款 AI 编程助手 CLI 工具,基于 TypeScript + React/Ink 构建,提供交互式终端界面(TUI),与 Java 后端通过 WebSocket 通信,实现 AI 驱动的代码辅助能力。
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 📦 安装
|
|
8
|
+
|
|
9
|
+
### 前置要求
|
|
10
|
+
|
|
11
|
+
| 依赖 | 最低版本 | 说明 |
|
|
12
|
+
|------|---------|------|
|
|
13
|
+
| Node.js | 18+ | 运行时环境 (推荐 20 LTS) |
|
|
14
|
+
| npm | 9+ | 包管理器 |
|
|
15
|
+
| Java | 17+ | 后端服务 (仅 `start` 命令需要) |
|
|
16
|
+
|
|
17
|
+
### 全局安装
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
npm install -g @jwcode/cli
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
### 本地开发安装
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
# 克隆项目
|
|
27
|
+
git clone <repo-url>
|
|
28
|
+
cd ts-cli
|
|
29
|
+
|
|
30
|
+
# 安装依赖
|
|
31
|
+
npm install
|
|
32
|
+
|
|
33
|
+
# 构建
|
|
34
|
+
npm run build
|
|
35
|
+
|
|
36
|
+
# 链接到全局 (可选)
|
|
37
|
+
npm link
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## 🚀 快速开始
|
|
43
|
+
|
|
44
|
+
### 启动完整服务(后端 + TUI)
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
jwcode start
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
启动后端 Java 服务并打开交互式终端界面。
|
|
51
|
+
|
|
52
|
+
### 指定端口和工作目录
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
jwcode start -p 8080 -w /path/to/workspace
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### 仅启动 TUI 客户端(连接到已有后端)
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
jwcode run -b http://localhost:8080
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### 查看版本
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
jwcode version
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## 📖 命令参考
|
|
73
|
+
|
|
74
|
+
### `jwcode start`
|
|
75
|
+
|
|
76
|
+
启动 Java 后端服务并打开交互式 TUI 界面。
|
|
77
|
+
|
|
78
|
+
| 参数 | 别名 | 类型 | 默认值 | 说明 |
|
|
79
|
+
|------|------|------|--------|------|
|
|
80
|
+
| `--port` | `-p` | number | `17340` | 后端服务端口 |
|
|
81
|
+
| `--backend` | `-B` | boolean | `false` | 仅启动后端,不启动 TUI |
|
|
82
|
+
| `--workspace` | `-w` | string | `cwd` | 工作目录路径 |
|
|
83
|
+
|
|
84
|
+
### `jwcode run`
|
|
85
|
+
|
|
86
|
+
仅启动 TUI 客户端,连接到已有的后端服务。
|
|
87
|
+
|
|
88
|
+
| 参数 | 别名 | 类型 | 默认值 | 说明 |
|
|
89
|
+
|------|------|------|--------|------|
|
|
90
|
+
| `--backend-url` | `-b` | string | `http://localhost:17340` | 后端 WebSocket URL |
|
|
91
|
+
| `--ws-url` | `--ws` | string | 派生自 `-b` | 直接指定 WebSocket URL |
|
|
92
|
+
|
|
93
|
+
### `jwcode version`
|
|
94
|
+
|
|
95
|
+
显示 CLI 版本号和构建信息。
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
## ⚙️ 配置
|
|
100
|
+
|
|
101
|
+
### 环境变量
|
|
102
|
+
|
|
103
|
+
| 变量名 | 类型 | 默认值 | 说明 |
|
|
104
|
+
|--------|------|--------|------|
|
|
105
|
+
| `JWCODE_THEME` | `dark` / `light` | `dark` | 界面主题 |
|
|
106
|
+
| `JWCODE_THEME_COLORS` | JSON string | — | 自定义主题颜色(覆盖默认色) |
|
|
107
|
+
| `JWCODE_PORT` | number | `17340` | 默认后端端口 |
|
|
108
|
+
| `DEBUG` | `jwcode:*` | — | 启用调试日志 |
|
|
109
|
+
|
|
110
|
+
### 配置文件
|
|
111
|
+
|
|
112
|
+
配置文件位于 `~/.jwcode/config.json`,支持以下配置项:
|
|
113
|
+
|
|
114
|
+
```json
|
|
115
|
+
{
|
|
116
|
+
"theme": "dark",
|
|
117
|
+
"port": 17340,
|
|
118
|
+
"workspace": "/path/to/default/workspace"
|
|
119
|
+
}
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### 主题自定义
|
|
123
|
+
|
|
124
|
+
通过 `JWCODE_THEME_COLORS` 环境变量自定义颜色:
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
# 覆盖主题色
|
|
128
|
+
export JWCODE_THEME_COLORS='{"primary":"#00ff00","bg":"#1a1a2e"}'
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
## 🏗️ 项目架构
|
|
134
|
+
|
|
135
|
+
```
|
|
136
|
+
ts-cli/
|
|
137
|
+
├── src/
|
|
138
|
+
│ ├── main.ts # 入口文件
|
|
139
|
+
│ ├── App.tsx # React 根组件 (Ink)
|
|
140
|
+
│ ├── client.ts # WebSocket 客户端
|
|
141
|
+
│ ├── config.ts # 配置管理
|
|
142
|
+
│ ├── launcher.ts # 后端进程启动器
|
|
143
|
+
│ ├── pasteBuffer.ts # 粘贴缓冲区
|
|
144
|
+
│ ├── protocol.ts # WebSocket 消息协议
|
|
145
|
+
│ ├── store.ts # 全局状态管理
|
|
146
|
+
│ ├── theme.ts # 主题系统
|
|
147
|
+
│ ├── commands/ # CLI 命令实现
|
|
148
|
+
│ ├── components/ # React/Ink UI 组件
|
|
149
|
+
│ │ ├── SetupWizard.tsx # 安装向导
|
|
150
|
+
│ │ ├── CommandPalette.tsx # 命令面板
|
|
151
|
+
│ │ ├── StatusLine.tsx # 状态栏
|
|
152
|
+
│ │ ├── TextInput.tsx # 文本输入框
|
|
153
|
+
│ │ ├── ApprovalModal.tsx # 审批弹窗
|
|
154
|
+
│ │ ├── FilePalette.tsx # 文件选择面板
|
|
155
|
+
│ │ └── PlanTaskBoard.tsx # 计划任务面板
|
|
156
|
+
│ ├── hooks/ # React Hooks
|
|
157
|
+
│ │ ├── useMouseWheel.ts
|
|
158
|
+
│ │ ├── useStreamHandlers.ts
|
|
159
|
+
│ │ └── useWebSocket.ts
|
|
160
|
+
│ └── __tests__/ # 单元测试
|
|
161
|
+
├── backend/ # Java 后端(独立项目)
|
|
162
|
+
├── .github/workflows/ci.yml # CI 工作流配置
|
|
163
|
+
├── build.mjs # esbuild 构建脚本
|
|
164
|
+
├── package.json
|
|
165
|
+
├── tsconfig.json
|
|
166
|
+
└── proguard.conf # Java 后端混淆配置
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
### 架构流程图
|
|
170
|
+
|
|
171
|
+
```
|
|
172
|
+
┌─────────────────────────────────────────────┐
|
|
173
|
+
│ 终端用户 (Terminal) │
|
|
174
|
+
└──────────────────┬──────────────────────────┘
|
|
175
|
+
│ stdin/stdout
|
|
176
|
+
┌──────────────────▼──────────────────────────┐
|
|
177
|
+
│ @jwcode/cli (TypeScript TUI) │
|
|
178
|
+
│ ┌──────────┐ ┌──────────────────────┐ │
|
|
179
|
+
│ │ Commands │ │ React/Ink UI │ │
|
|
180
|
+
│ │ (CLI) │ │ (组件树) │ │
|
|
181
|
+
│ └─────┬────┘ └──────────┬───────────┘ │
|
|
182
|
+
│ │ │ │
|
|
183
|
+
│ ┌─────▼──────────────────▼───────────┐ │
|
|
184
|
+
│ │ JwCodeClient │ │
|
|
185
|
+
│ │ (WebSocket 客户端) │ │
|
|
186
|
+
│ └─────────────────┬──────────────────┘ │
|
|
187
|
+
└─────────────────────┼───────────────────────┘
|
|
188
|
+
│ WebSocket (ws://)
|
|
189
|
+
┌─────────────────────▼───────────────────────┐
|
|
190
|
+
│ Java 后端服务 │
|
|
191
|
+
│ (WebSocket Server + AI 引擎) │
|
|
192
|
+
└─────────────────────────────────────────────┘
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
## 🔧 开发指南
|
|
198
|
+
|
|
199
|
+
### 开发环境搭建
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
# 安装依赖
|
|
203
|
+
npm install
|
|
204
|
+
|
|
205
|
+
# 启动开发模式(自动构建 + 运行)
|
|
206
|
+
npm run go
|
|
207
|
+
|
|
208
|
+
# 或者分步执行
|
|
209
|
+
npm run build # 构建 CLI
|
|
210
|
+
node dist/cli.js run # 运行 CLI
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### 测试
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
# 运行所有测试
|
|
217
|
+
npm test
|
|
218
|
+
|
|
219
|
+
# 监听模式(开发时使用)
|
|
220
|
+
npm run test:watch
|
|
221
|
+
|
|
222
|
+
# 运行指定测试文件
|
|
223
|
+
npx vitest run src/__tests__/store.test.ts
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
### 构建
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
# 生产构建
|
|
230
|
+
npm run build
|
|
231
|
+
|
|
232
|
+
# 验证构建产物
|
|
233
|
+
node dist/cli.js version
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
构建产物输出到 `dist/cli.js`,使用 **esbuild** 打包为单个 ESM 文件,外部依赖不打包。
|
|
237
|
+
|
|
238
|
+
### 代码风格
|
|
239
|
+
|
|
240
|
+
- TypeScript: 严格模式 (`strict: true`)
|
|
241
|
+
- JSX: `react-jsx` 运行时
|
|
242
|
+
- 模块: ESM (`"type": "module"`)
|
|
243
|
+
- 缩进: 2 空格
|
|
244
|
+
- 命名规范:
|
|
245
|
+
- 变量/函数: `camelCase`
|
|
246
|
+
- 类/组件: `PascalCase`
|
|
247
|
+
- 文件: `camelCase.ts` / `PascalCase.tsx`
|
|
248
|
+
- 常量: `UPPER_SNAKE_CASE`
|
|
249
|
+
|
|
250
|
+
---
|
|
251
|
+
|
|
252
|
+
## 🧪 测试
|
|
253
|
+
|
|
254
|
+
项目使用 [Vitest](https://vitest.dev/) 作为测试框架。
|
|
255
|
+
|
|
256
|
+
### 现有测试
|
|
257
|
+
|
|
258
|
+
| 测试文件 | 覆盖模块 | 说明 |
|
|
259
|
+
|---------|---------|------|
|
|
260
|
+
| `store.test.ts` | `store.ts` | 全局状态管理单元测试 |
|
|
261
|
+
| `theme.test.ts` | `theme.ts` | 主题系统单元测试 |
|
|
262
|
+
| `pasteBuffer.test.ts` | `pasteBuffer.ts` | 粘贴缓冲区测试 |
|
|
263
|
+
| `tokenEstimate.test.ts` | 工具函数 | Token 估算测试 |
|
|
264
|
+
|
|
265
|
+
### 编写新测试
|
|
266
|
+
|
|
267
|
+
```typescript
|
|
268
|
+
// src/__tests__/example.test.ts
|
|
269
|
+
import { describe, it, expect } from 'vitest';
|
|
270
|
+
import { yourFunction } from '../yourModule.js';
|
|
271
|
+
|
|
272
|
+
describe('yourModule', () => {
|
|
273
|
+
it('should do something', () => {
|
|
274
|
+
expect(yourFunction()).toBe(expected);
|
|
275
|
+
});
|
|
276
|
+
});
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
---
|
|
280
|
+
|
|
281
|
+
## 🚢 CI/CD
|
|
282
|
+
|
|
283
|
+
项目使用 **GitHub Actions** 进行持续集成。
|
|
284
|
+
|
|
285
|
+
### CI 工作流 (`.github/workflows/ci.yml`)
|
|
286
|
+
|
|
287
|
+
| 阶段 | 操作 | 说明 |
|
|
288
|
+
|------|------|------|
|
|
289
|
+
| Checkout | `actions/checkout@v4` | 检出代码 |
|
|
290
|
+
| Setup Node | `actions/setup-node@v4` | 配置 Node.js (18/20/22) |
|
|
291
|
+
| Install | `npm ci` | 安装依赖(锁定版本) |
|
|
292
|
+
| Type Check | `npx tsc --noEmit` | TypeScript 类型检查 |
|
|
293
|
+
| Test | `npm test` | 运行单元测试 |
|
|
294
|
+
| Build | `npm run build` | 构建打包 |
|
|
295
|
+
| Verify | 检查 dist/cli.js | 验证构建产物 |
|
|
296
|
+
|
|
297
|
+
### 触发条件
|
|
298
|
+
|
|
299
|
+
- `push` 到 `main` / `master` 分支
|
|
300
|
+
- `pull_request` 到 `main` / `master` 分支
|
|
301
|
+
|
|
302
|
+
---
|
|
303
|
+
|
|
304
|
+
## ❓ 常见问题
|
|
305
|
+
|
|
306
|
+
### 1. 启动时端口被占用
|
|
307
|
+
|
|
308
|
+
```bash
|
|
309
|
+
# 指定其他端口
|
|
310
|
+
jwcode start -p 8080
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
### 2. WebSocket 连接失败
|
|
314
|
+
|
|
315
|
+
确保后端服务已启动,检查 URL 是否正确:
|
|
316
|
+
|
|
317
|
+
```bash
|
|
318
|
+
# 指定后端地址
|
|
319
|
+
jwcode run -b http://localhost:17340
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
### 3. 构建后运行报错
|
|
323
|
+
|
|
324
|
+
```bash
|
|
325
|
+
# 清理后重新构建
|
|
326
|
+
rm -rf dist
|
|
327
|
+
npm run build
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
### 4. `EPIPE` / `ECONNRESET` 错误
|
|
331
|
+
|
|
332
|
+
这是终端断开时的正常行为,不影响程序运行。项目已自动处理这些信号。
|
|
333
|
+
|
|
334
|
+
---
|
|
335
|
+
|
|
336
|
+
## 📄 许可
|
|
337
|
+
|
|
338
|
+
本项目为 **内部项目**,未经授权不得分发或修改。
|
|
339
|
+
|
|
340
|
+
---
|
|
341
|
+
|
|
342
|
+
## 🏷️ 版本历史
|
|
343
|
+
|
|
344
|
+
| 版本 | 日期 | 说明 |
|
|
345
|
+
|------|------|------|
|
|
346
|
+
| 3.0.0 | — | 当前版本,TypeScript 重写,React/Ink TUI |
|
|
347
|
+
| 2.x | — | Python 版本 CLI |
|
|
348
|
+
| 1.x | — | 初版 CLI |
|
package/backend/.gitkeep
ADDED
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { createStore } from '../store.js';
|
|
3
|
+
describe('store', () => {
|
|
4
|
+
it('creates store with initial state', () => {
|
|
5
|
+
const store = createStore({ count: 0, name: 'test' });
|
|
6
|
+
expect(store.getState().count).toBe(0);
|
|
7
|
+
expect(store.getState().name).toBe('test');
|
|
8
|
+
});
|
|
9
|
+
it('setState updates state immutably', () => {
|
|
10
|
+
const store = createStore({ count: 0, name: 'test' });
|
|
11
|
+
const prev = store.getState();
|
|
12
|
+
store.setState(s => ({ ...s, count: 5 }));
|
|
13
|
+
expect(store.getState().count).toBe(5);
|
|
14
|
+
expect(prev.count).toBe(0); // original unchanged
|
|
15
|
+
});
|
|
16
|
+
it('subscribe receives updates', () => {
|
|
17
|
+
const store = createStore({ count: 0, name: 'test' });
|
|
18
|
+
let called = false;
|
|
19
|
+
store.subscribe(() => { called = true; });
|
|
20
|
+
store.setState(s => ({ ...s, count: 10 }));
|
|
21
|
+
expect(called).toBe(true);
|
|
22
|
+
expect(store.getState().count).toBe(10);
|
|
23
|
+
});
|
|
24
|
+
it('subscribe returns unsubscribe function', () => {
|
|
25
|
+
const store = createStore({ count: 0, name: 'test' });
|
|
26
|
+
let callCount = 0;
|
|
27
|
+
const unsub = store.subscribe(() => { callCount++; });
|
|
28
|
+
store.setState(s => ({ ...s, count: 1 }));
|
|
29
|
+
expect(callCount).toBe(1);
|
|
30
|
+
unsub();
|
|
31
|
+
store.setState(s => ({ ...s, count: 2 }));
|
|
32
|
+
expect(callCount).toBe(1); // not called again
|
|
33
|
+
});
|
|
34
|
+
it('multiple subscribers all notified', () => {
|
|
35
|
+
const store = createStore({ count: 0, name: 'test' });
|
|
36
|
+
let calls = 0;
|
|
37
|
+
store.subscribe(() => { calls++; });
|
|
38
|
+
store.subscribe(() => { calls++; });
|
|
39
|
+
store.setState(s => ({ ...s, count: 42 }));
|
|
40
|
+
expect(calls).toBe(2);
|
|
41
|
+
expect(store.getState().count).toBe(42);
|
|
42
|
+
});
|
|
43
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { describe, it, expect, afterEach } from 'vitest';
|
|
2
|
+
import { getTheme, setTheme, t } from '../theme.js';
|
|
3
|
+
describe('theme', () => {
|
|
4
|
+
afterEach(() => {
|
|
5
|
+
setTheme('dark');
|
|
6
|
+
});
|
|
7
|
+
it('default theme is dark', () => {
|
|
8
|
+
const theme = getTheme();
|
|
9
|
+
expect(theme.primary).toBe('cyan');
|
|
10
|
+
expect(theme.success).toBe('green');
|
|
11
|
+
expect(theme.error).toBe('red');
|
|
12
|
+
});
|
|
13
|
+
it('setTheme changes current theme', () => {
|
|
14
|
+
setTheme('light');
|
|
15
|
+
const theme = getTheme();
|
|
16
|
+
expect(theme.muted).toBe('blackBright');
|
|
17
|
+
});
|
|
18
|
+
it('all required color keys exist', () => {
|
|
19
|
+
const required = [
|
|
20
|
+
'bg', 'text', 'muted', 'border',
|
|
21
|
+
'primary', 'success', 'warning', 'error', 'info',
|
|
22
|
+
'user', 'assistant', 'system', 'tool', 'thinking',
|
|
23
|
+
'plan', 'auto', 'connected', 'disconnected',
|
|
24
|
+
];
|
|
25
|
+
const theme = getTheme();
|
|
26
|
+
for (const key of required) {
|
|
27
|
+
expect(theme).toHaveProperty(key);
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
it('module-level t exports dark theme initially', () => {
|
|
31
|
+
expect(t.primary).toBe('cyan');
|
|
32
|
+
});
|
|
33
|
+
it('theme switches preserve all keys', () => {
|
|
34
|
+
setTheme('light');
|
|
35
|
+
const light = getTheme();
|
|
36
|
+
setTheme('dark');
|
|
37
|
+
const dark = getTheme();
|
|
38
|
+
expect(Object.keys(light)).toEqual(Object.keys(dark));
|
|
39
|
+
});
|
|
40
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
// Replicate the token estimation logic from TextInput.tsx
|
|
3
|
+
function estimateTokens(text) {
|
|
4
|
+
let cjk = 0;
|
|
5
|
+
let other = 0;
|
|
6
|
+
for (const ch of text) {
|
|
7
|
+
if (/[一-鿿㐀-䶿豈- -〿-]/.test(ch)) {
|
|
8
|
+
cjk++;
|
|
9
|
+
}
|
|
10
|
+
else {
|
|
11
|
+
other++;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
return Math.ceil(cjk / 1.5 + other / 4);
|
|
15
|
+
}
|
|
16
|
+
describe('tokenEstimate', () => {
|
|
17
|
+
it('empty string is 0', () => {
|
|
18
|
+
expect(estimateTokens('')).toBe(0);
|
|
19
|
+
});
|
|
20
|
+
it('English text: ~4 chars per token', () => {
|
|
21
|
+
// 40 English chars → ~10 tokens
|
|
22
|
+
const tokens = estimateTokens('Hello world this is a test message here');
|
|
23
|
+
expect(tokens).toBeGreaterThan(6);
|
|
24
|
+
expect(tokens).toBeLessThan(20);
|
|
25
|
+
});
|
|
26
|
+
it('Chinese text: ~1.5 chars per token', () => {
|
|
27
|
+
// 15 Chinese chars → ~10 tokens
|
|
28
|
+
const tokens = estimateTokens('这是一段中文测试文字用于验证分词估算');
|
|
29
|
+
expect(tokens).toBeGreaterThan(8);
|
|
30
|
+
expect(tokens).toBeLessThan(15);
|
|
31
|
+
});
|
|
32
|
+
it('mixed text combines both ratios', () => {
|
|
33
|
+
const tokens = estimateTokens('Hello 世界 this 测试 works');
|
|
34
|
+
expect(tokens).toBeGreaterThan(0);
|
|
35
|
+
});
|
|
36
|
+
it('code block is mostly other chars', () => {
|
|
37
|
+
const code = 'function hello() { return 42; }';
|
|
38
|
+
const tokens = estimateTokens(code);
|
|
39
|
+
expect(tokens).toBeGreaterThan(2);
|
|
40
|
+
expect(tokens).toBeLessThan(15);
|
|
41
|
+
});
|
|
42
|
+
it('100K token threshold is detectable', () => {
|
|
43
|
+
// Very rough: 400K chars ≈ 100K tokens
|
|
44
|
+
const long = 'x'.repeat(400_000);
|
|
45
|
+
const tokens = estimateTokens(long);
|
|
46
|
+
expect(tokens).toBeGreaterThanOrEqual(100_000);
|
|
47
|
+
});
|
|
48
|
+
});
|