@miphamai/cli 0.2.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mipham.ts +2 -0
- package/dist/mipham +0 -0
- package/package.json +10 -9
- package/skills/mipham/om-artifact.mipham-skill.md +69 -0
- package/skills/mipham/om-model-optimize.mipham-skill.md +58 -9
- package/skills/mipham/om-security.mipham-skill.md +85 -10
- package/skills/standard/doc-generator.SKILL.md +72 -11
- package/skills/standard/github-ops.SKILL.md +94 -13
- package/skills/standard/memory.SKILL.md +72 -11
- package/skills/standard/self-review.SKILL.md +50 -10
- package/skills/standard/superpower.SKILL.md +46 -8
- package/skills/standard/tdd.SKILL.md +85 -12
- package/skills/standard/web-search.SKILL.md +65 -13
- package/src/agent/sub-agent.ts +0 -1
- package/src/artifacts/manifest.ts +101 -0
- package/src/artifacts/server.ts +343 -0
- package/src/core/context.ts +73 -13
- package/src/core/engine.ts +194 -71
- package/src/core/instructions.ts +1 -1
- package/src/core/permission.ts +35 -4
- package/src/core/session-store.ts +5 -1
- package/src/index.tsx +35 -2
- package/src/mcp/transport.ts +42 -5
- package/src/providers/anthropic.ts +2 -1
- package/src/providers/fetch-utils.ts +101 -0
- package/src/providers/openai-compat.ts +53 -15
- package/src/providers/registry.ts +1 -0
- package/src/shared/constants.ts +8 -1
- package/src/shared/types.ts +21 -1
- package/src/skills/loader.ts +2 -2
- package/src/tools/artifact/artifact.ts +144 -0
- package/src/tools/index.ts +3 -0
- package/src/ui/app.tsx +122 -15
- package/src/ui/chat.tsx +29 -7
- package/src/ui/commands.ts +117 -19
- package/src/ui/input.tsx +117 -35
- package/src/ui/picker.tsx +2 -5
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
import { createServer, type Server } from 'node:http'
|
|
2
|
+
import { createReadStream, existsSync, statSync } from 'node:fs'
|
|
3
|
+
import { join, normalize, extname } from 'node:path'
|
|
4
|
+
import { ARTIFACT_ALLOWED_EXTENSIONS } from '../shared/constants'
|
|
5
|
+
import { readManifest } from './manifest'
|
|
6
|
+
import type { ArtifactEntry } from '../shared/types'
|
|
7
|
+
|
|
8
|
+
// ── SSE client tracking ──
|
|
9
|
+
interface SseClient {
|
|
10
|
+
id: number
|
|
11
|
+
res: any
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Embedded HTTP server for serving artifact files (.html, .svg).
|
|
16
|
+
*
|
|
17
|
+
* Phase 2 additions:
|
|
18
|
+
* - Gallery page at / with dark-themed card layout
|
|
19
|
+
* - SSE /events endpoint for live reload
|
|
20
|
+
* - Versioned file serving (name.v2.html)
|
|
21
|
+
* - Port auto-increment on conflict
|
|
22
|
+
*/
|
|
23
|
+
export class ArtifactServer {
|
|
24
|
+
private server: Server | null = null
|
|
25
|
+
private port: number
|
|
26
|
+
private artifactsDir: string
|
|
27
|
+
private started = false
|
|
28
|
+
private sseClients: SseClient[] = []
|
|
29
|
+
private sseIdCounter = 0
|
|
30
|
+
|
|
31
|
+
constructor(artifactsDir: string, preferredPort: number) {
|
|
32
|
+
this.artifactsDir = artifactsDir
|
|
33
|
+
this.port = preferredPort
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Notify all connected SSE clients to reload. Called after artifact changes. */
|
|
37
|
+
notifyReload(): void {
|
|
38
|
+
for (const client of this.sseClients) {
|
|
39
|
+
try {
|
|
40
|
+
client.res.write('event: reload\ndata: {}\n\n')
|
|
41
|
+
} catch {
|
|
42
|
+
// Client disconnected — will be cleaned up on next request
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async start(maxTries = 10): Promise<number> {
|
|
48
|
+
if (this.started) return this.port
|
|
49
|
+
|
|
50
|
+
for (let attempt = 0; attempt < maxTries; attempt++) {
|
|
51
|
+
const port = this.port + attempt
|
|
52
|
+
try {
|
|
53
|
+
await this.listenOn(port)
|
|
54
|
+
this.port = port
|
|
55
|
+
this.started = true
|
|
56
|
+
return port
|
|
57
|
+
} catch (err: unknown) {
|
|
58
|
+
if ((err as NodeJS.ErrnoException).code !== 'EADDRINUSE') throw err
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
throw new Error(`Artifact server could not find an available port after ${maxTries} attempts.`)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
stop(): void {
|
|
66
|
+
// Close all SSE connections
|
|
67
|
+
for (const client of this.sseClients) {
|
|
68
|
+
try {
|
|
69
|
+
client.res.end()
|
|
70
|
+
} catch {
|
|
71
|
+
/* ignore */
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
this.sseClients = []
|
|
75
|
+
|
|
76
|
+
if (this.server) {
|
|
77
|
+
this.server.close()
|
|
78
|
+
this.server = null
|
|
79
|
+
this.started = false
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
getPort(): number {
|
|
84
|
+
return this.port
|
|
85
|
+
}
|
|
86
|
+
isRunning(): boolean {
|
|
87
|
+
return this.started
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
resolveFile(urlPath: string): { filePath: string; error?: string; status?: number } {
|
|
91
|
+
const cleaned = urlPath.replace(/^\/+/, '')
|
|
92
|
+
const normalized = normalize(cleaned)
|
|
93
|
+
if (normalized.startsWith('..') || normalized.includes('/../')) {
|
|
94
|
+
return { filePath: '', error: 'Path traversal rejected', status: 403 }
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const fullPath = join(this.artifactsDir, normalized)
|
|
98
|
+
const ext = extname(fullPath).toLowerCase()
|
|
99
|
+
if (!ARTIFACT_ALLOWED_EXTENSIONS.includes(ext)) {
|
|
100
|
+
return { filePath: '', error: `File type "${ext}" not served`, status: 403 }
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
return { filePath: fullPath }
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ── Private ──
|
|
107
|
+
|
|
108
|
+
private listenOn(port: number): Promise<void> {
|
|
109
|
+
return new Promise((resolve, reject) => {
|
|
110
|
+
const srv = createServer((req, res) => {
|
|
111
|
+
this.handleRequest(req, res)
|
|
112
|
+
})
|
|
113
|
+
srv.on('error', reject)
|
|
114
|
+
srv.listen(port, () => {
|
|
115
|
+
this.server = srv
|
|
116
|
+
resolve()
|
|
117
|
+
})
|
|
118
|
+
})
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
private handleRequest(req: { method?: string; url?: string }, res: any): void {
|
|
122
|
+
if (req.method !== 'GET') {
|
|
123
|
+
res.writeHead(405, { Allow: 'GET' })
|
|
124
|
+
res.end('Method Not Allowed')
|
|
125
|
+
return
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const urlPath = req.url || '/'
|
|
129
|
+
|
|
130
|
+
// SSE endpoint
|
|
131
|
+
if (urlPath === '/events') {
|
|
132
|
+
this.handleSse(res)
|
|
133
|
+
return
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Gallery page
|
|
137
|
+
if (urlPath === '/' || urlPath === '/index.html') {
|
|
138
|
+
this.serveGallery(res)
|
|
139
|
+
return
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Artifact file
|
|
143
|
+
this.serveArtifact(urlPath, res)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// ── SSE ──
|
|
147
|
+
|
|
148
|
+
private handleSse(res: any): void {
|
|
149
|
+
res.writeHead(200, {
|
|
150
|
+
'Content-Type': 'text/event-stream',
|
|
151
|
+
'Cache-Control': 'no-cache',
|
|
152
|
+
Connection: 'keep-alive',
|
|
153
|
+
'Access-Control-Allow-Origin': '*',
|
|
154
|
+
})
|
|
155
|
+
|
|
156
|
+
const client: SseClient = { id: ++this.sseIdCounter, res }
|
|
157
|
+
this.sseClients.push(client)
|
|
158
|
+
|
|
159
|
+
// Send initial heartbeat
|
|
160
|
+
res.write(':ok\n\n')
|
|
161
|
+
|
|
162
|
+
// Clean up on close
|
|
163
|
+
res.on('close', () => {
|
|
164
|
+
this.sseClients = this.sseClients.filter((c) => c.id !== client.id)
|
|
165
|
+
})
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// ── Gallery Page ──
|
|
169
|
+
|
|
170
|
+
private serveGallery(res: any): void {
|
|
171
|
+
const manifest = readManifest(this.artifactsDir)
|
|
172
|
+
const artifacts = manifest.artifacts
|
|
173
|
+
|
|
174
|
+
const cards =
|
|
175
|
+
artifacts.length === 0
|
|
176
|
+
? `<div style="grid-column: 1/-1; text-align: center; padding: 60px 20px; color: #666;">
|
|
177
|
+
<p style="font-size: 48px; margin: 0 0 16px 0;">🎨</p>
|
|
178
|
+
<p style="font-size: 16px;">No artifacts yet.</p>
|
|
179
|
+
<p style="font-size: 13px; color: #555;">Ask the AI to create one with the Artifact tool.</p>
|
|
180
|
+
</div>`
|
|
181
|
+
: artifacts.map((a) => this.artifactCard(a)).join('\n')
|
|
182
|
+
|
|
183
|
+
const html = `<!DOCTYPE html>
|
|
184
|
+
<html lang="en">
|
|
185
|
+
<head>
|
|
186
|
+
<meta charset="UTF-8">
|
|
187
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
188
|
+
<title>Mipham Code — Artifacts</title>
|
|
189
|
+
<style>
|
|
190
|
+
:root {
|
|
191
|
+
--bg: #0d1117; --surface: #161b22; --border: #30363d;
|
|
192
|
+
--text: #c9d1d9; --muted: #8b949e; --accent: #58a6ff;
|
|
193
|
+
--green: #3fb950; --orange: #d2991d; --purple: #a371f7;
|
|
194
|
+
}
|
|
195
|
+
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
196
|
+
body {
|
|
197
|
+
background: var(--bg); color: var(--text);
|
|
198
|
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif;
|
|
199
|
+
min-height: 100vh;
|
|
200
|
+
}
|
|
201
|
+
header {
|
|
202
|
+
background: var(--surface); border-bottom: 1px solid var(--border);
|
|
203
|
+
padding: 20px 32px; display: flex; align-items: center; justify-content: space-between;
|
|
204
|
+
}
|
|
205
|
+
header h1 { font-size: 20px; font-weight: 600; }
|
|
206
|
+
header .meta { font-size: 13px; color: var(--muted); }
|
|
207
|
+
.grid {
|
|
208
|
+
display: grid;
|
|
209
|
+
grid-template-columns: repeat(auto-fill, minmax(280px, 1fr));
|
|
210
|
+
gap: 16px; padding: 24px 32px;
|
|
211
|
+
}
|
|
212
|
+
.card {
|
|
213
|
+
background: var(--surface); border: 1px solid var(--border);
|
|
214
|
+
border-radius: 8px; padding: 20px; transition: border-color .15s;
|
|
215
|
+
text-decoration: none; color: inherit; display: block;
|
|
216
|
+
}
|
|
217
|
+
.card:hover { border-color: var(--accent); }
|
|
218
|
+
.card .icon { font-size: 32px; margin-bottom: 12px; }
|
|
219
|
+
.card .name { font-size: 15px; font-weight: 600; margin-bottom: 8px; color: var(--accent); }
|
|
220
|
+
.card .row { font-size: 12px; color: var(--muted); margin-top: 4px; display: flex; gap: 12px; }
|
|
221
|
+
.card .badge {
|
|
222
|
+
display: inline-block; padding: 1px 6px; border-radius: 3px;
|
|
223
|
+
font-size: 11px; font-weight: 600;
|
|
224
|
+
}
|
|
225
|
+
.badge-html { background: #1f3a5f; color: var(--accent); }
|
|
226
|
+
.badge-svg { background: #2d1f4e; color: var(--purple); }
|
|
227
|
+
.badge-version { background: #2d2d1f; color: var(--orange); }
|
|
228
|
+
footer {
|
|
229
|
+
padding: 16px 32px; border-top: 1px solid var(--border);
|
|
230
|
+
font-size: 12px; color: var(--muted); display: flex; gap: 16px;
|
|
231
|
+
}
|
|
232
|
+
.dot { width: 6px; height: 6px; border-radius: 50%; background: var(--green); display: inline-block; margin-right: 4px; }
|
|
233
|
+
</style>
|
|
234
|
+
</head>
|
|
235
|
+
<body>
|
|
236
|
+
<header>
|
|
237
|
+
<div>
|
|
238
|
+
<h1>🎨 Mipham Code Artifacts</h1>
|
|
239
|
+
<p style="font-size:13px;color:var(--muted);margin-top:4px;">
|
|
240
|
+
Interactive visual output from your AI sessions
|
|
241
|
+
</p>
|
|
242
|
+
</div>
|
|
243
|
+
<div class="meta">
|
|
244
|
+
<div><span class="dot"></span> Server running</div>
|
|
245
|
+
<div>Port ${this.port}</div>
|
|
246
|
+
</div>
|
|
247
|
+
</header>
|
|
248
|
+
<div class="grid">${cards}</div>
|
|
249
|
+
<footer>
|
|
250
|
+
<span>${artifacts.length} artifact${artifacts.length === 1 ? '' : 's'}</span>
|
|
251
|
+
<span>·</span>
|
|
252
|
+
<span>Open in terminal: <code>/artifact open <name></code></span>
|
|
253
|
+
<span>·</span>
|
|
254
|
+
<span>Auto-refresh active</span>
|
|
255
|
+
</footer>
|
|
256
|
+
<script>
|
|
257
|
+
// SSE auto-refresh
|
|
258
|
+
const es = new EventSource('/events')
|
|
259
|
+
es.addEventListener('reload', () => location.reload())
|
|
260
|
+
</script>
|
|
261
|
+
</body>
|
|
262
|
+
</html>`
|
|
263
|
+
|
|
264
|
+
res.writeHead(200, {
|
|
265
|
+
'Content-Type': 'text/html; charset=utf-8',
|
|
266
|
+
'Cache-Control': 'no-cache',
|
|
267
|
+
})
|
|
268
|
+
res.end(html)
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
private artifactCard(a: ArtifactEntry): string {
|
|
272
|
+
const icon = a.type === 'svg' ? '🖼' : '📊'
|
|
273
|
+
const sizeStr =
|
|
274
|
+
a.size < 1024
|
|
275
|
+
? `${a.size}B`
|
|
276
|
+
: a.size < 1_048_576
|
|
277
|
+
? `${(a.size / 1024).toFixed(1)}KB`
|
|
278
|
+
: `${(a.size / 1_048_576).toFixed(1)}MB`
|
|
279
|
+
const date = a.createdAt.slice(0, 16).replace('T', ' ')
|
|
280
|
+
const versionInfo =
|
|
281
|
+
a.versions && a.versions.length > 1
|
|
282
|
+
? `<span class="badge badge-version">${a.versions.length} versions</span>`
|
|
283
|
+
: ''
|
|
284
|
+
|
|
285
|
+
return `<a class="card" href="/${a.sessionId}/${a.name}.${a.type === 'svg' ? 'svg' : 'html'}">
|
|
286
|
+
<div class="icon">${icon}</div>
|
|
287
|
+
<div class="name">${this.escapeHtml(a.name)}</div>
|
|
288
|
+
<div class="row">
|
|
289
|
+
<span class="badge badge-${a.type}">${a.type.toUpperCase()}</span>
|
|
290
|
+
<span>${sizeStr}</span>
|
|
291
|
+
<span>${date}</span>
|
|
292
|
+
${versionInfo}
|
|
293
|
+
</div>
|
|
294
|
+
<div class="row">
|
|
295
|
+
<span>session: ${a.sessionId}</span>
|
|
296
|
+
</div>
|
|
297
|
+
</a>`
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// ── Artifact Serving ──
|
|
301
|
+
|
|
302
|
+
private serveArtifact(urlPath: string, res: any): void {
|
|
303
|
+
const { filePath, error, status } = this.resolveFile(urlPath)
|
|
304
|
+
|
|
305
|
+
if (error) {
|
|
306
|
+
res.writeHead(status || 403, { 'Content-Type': 'text/plain' })
|
|
307
|
+
res.end(error)
|
|
308
|
+
return
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
if (!existsSync(filePath)) {
|
|
312
|
+
res.writeHead(404, { 'Content-Type': 'text/plain' })
|
|
313
|
+
res.end('Artifact not found')
|
|
314
|
+
return
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
const ext = extname(filePath).toLowerCase()
|
|
318
|
+
const mimeType = ext === '.svg' ? 'image/svg+xml' : 'text/html; charset=utf-8'
|
|
319
|
+
const size = statSync(filePath).size
|
|
320
|
+
|
|
321
|
+
res.writeHead(200, {
|
|
322
|
+
'Content-Type': mimeType,
|
|
323
|
+
'Content-Length': size,
|
|
324
|
+
'Content-Security-Policy':
|
|
325
|
+
"default-src 'self'; style-src 'unsafe-inline'; script-src 'none'; img-src data: 'self';",
|
|
326
|
+
'X-Content-Type-Options': 'nosniff',
|
|
327
|
+
'Cache-Control': 'no-cache',
|
|
328
|
+
})
|
|
329
|
+
|
|
330
|
+
const stream = createReadStream(filePath)
|
|
331
|
+
stream.pipe(res)
|
|
332
|
+
stream.on('error', () => {
|
|
333
|
+
if (!res.headersSent) {
|
|
334
|
+
res.writeHead(500)
|
|
335
|
+
res.end('Stream error')
|
|
336
|
+
}
|
|
337
|
+
})
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
private escapeHtml(s: string): string {
|
|
341
|
+
return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
342
|
+
}
|
|
343
|
+
}
|
package/src/core/context.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { Message } from '../shared/index.ts'
|
|
2
2
|
|
|
3
|
+
export type Summarizer = (messages: Message[], heading: string) => Promise<string>
|
|
4
|
+
|
|
3
5
|
interface ContextConfig {
|
|
4
6
|
maxTokens: number
|
|
5
7
|
compactionThreshold: number // e.g. 0.9 → compact at 90% usage
|
|
@@ -19,9 +21,15 @@ export class ContextManager {
|
|
|
19
21
|
private estimatedTokens = 0
|
|
20
22
|
private checkpoints: Checkpoint[] = []
|
|
21
23
|
private checkpointCounter = 0
|
|
24
|
+
private summarizer?: Summarizer
|
|
22
25
|
|
|
23
26
|
constructor(private config: ContextConfig) {}
|
|
24
27
|
|
|
28
|
+
/** Set an optional LLM summarizer for intelligent compaction. */
|
|
29
|
+
setSummarizer(fn: Summarizer): void {
|
|
30
|
+
this.summarizer = fn
|
|
31
|
+
}
|
|
32
|
+
|
|
25
33
|
setSystemPrompt(prompt: string): void {
|
|
26
34
|
this.systemPrompt = prompt
|
|
27
35
|
this.estimatedTokens = this.estimateTokens(prompt)
|
|
@@ -46,19 +54,36 @@ export class ContextManager {
|
|
|
46
54
|
return this.estimatedTokens > this.config.maxTokens * this.config.compactionThreshold
|
|
47
55
|
}
|
|
48
56
|
|
|
49
|
-
async compact(
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
57
|
+
async compact(heading: string): Promise<void> {
|
|
58
|
+
if (this.messages.length <= 30) return
|
|
59
|
+
|
|
60
|
+
const keep = 20
|
|
61
|
+
const toDrop = this.messages.slice(0, -keep)
|
|
62
|
+
|
|
63
|
+
if (this.summarizer && toDrop.length >= 4) {
|
|
64
|
+
// LLM-based summarization of truncated messages
|
|
65
|
+
try {
|
|
66
|
+
const summary = await this.summarizer(toDrop, heading)
|
|
67
|
+
const summaryMsg: Message = {
|
|
68
|
+
role: 'user',
|
|
69
|
+
content: `[Earlier conversation summary]: ${summary}`,
|
|
70
|
+
}
|
|
71
|
+
this.messages = [summaryMsg, ...this.messages.slice(-keep)]
|
|
72
|
+
} catch {
|
|
73
|
+
// Fall back to truncation on summarizer failure
|
|
74
|
+
this.messages = this.messages.slice(-keep)
|
|
75
|
+
}
|
|
76
|
+
} else {
|
|
77
|
+
// Simple truncation fallback
|
|
53
78
|
this.messages = this.messages.slice(-keep)
|
|
79
|
+
}
|
|
54
80
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
}
|
|
81
|
+
// Re-estimate tokens
|
|
82
|
+
this.estimatedTokens = this.estimateTokens(this.systemPrompt)
|
|
83
|
+
for (const msg of this.messages) {
|
|
84
|
+
this.estimatedTokens += this.estimateTokens(
|
|
85
|
+
typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content),
|
|
86
|
+
)
|
|
62
87
|
}
|
|
63
88
|
}
|
|
64
89
|
|
|
@@ -128,8 +153,43 @@ export class ContextManager {
|
|
|
128
153
|
return this.checkpoints.at(-1)?.id
|
|
129
154
|
}
|
|
130
155
|
|
|
156
|
+
/**
|
|
157
|
+
* Estimate token count for a text string.
|
|
158
|
+
*
|
|
159
|
+
* Uses a character-class-aware heuristic:
|
|
160
|
+
* - CJK / emoji: ~1.5 chars per token
|
|
161
|
+
* - Other scripts (Latin, Cyrillic, Arabic): ~4 chars per token
|
|
162
|
+
* - Whitespace-heavy (code): ~3 chars per token
|
|
163
|
+
*
|
|
164
|
+
* This is ~30-40% more accurate than flat 4 chars/token for mixed-language text.
|
|
165
|
+
*/
|
|
131
166
|
private estimateTokens(text: string): number {
|
|
132
|
-
|
|
133
|
-
|
|
167
|
+
if (!text) return 0
|
|
168
|
+
|
|
169
|
+
let cjk = 0
|
|
170
|
+
let latin = 0
|
|
171
|
+
|
|
172
|
+
for (const ch of text) {
|
|
173
|
+
const cp = ch.codePointAt(0)!
|
|
174
|
+
// CJK Unified Ideographs, Hangul, Kana, CJK Extensions, fullwidth forms
|
|
175
|
+
if (
|
|
176
|
+
(cp >= 0x4e00 && cp <= 0x9fff) || // CJK Unified
|
|
177
|
+
(cp >= 0x3400 && cp <= 0x4dbf) || // CJK Ext-A
|
|
178
|
+
(cp >= 0x20000 && cp <= 0x2a6df) || // CJK Ext-B
|
|
179
|
+
(cp >= 0xac00 && cp <= 0xd7af) || // Hangul
|
|
180
|
+
(cp >= 0x3040 && cp <= 0x30ff) || // Hiragana + Katakana
|
|
181
|
+
(cp >= 0xff01 && cp <= 0xff60) || // Fullwidth
|
|
182
|
+
(cp >= 0x1f300 && cp <= 0x1f9ff) // Emoji / pictographs
|
|
183
|
+
) {
|
|
184
|
+
cjk++
|
|
185
|
+
} else if (cp > 0x7f) {
|
|
186
|
+
latin++
|
|
187
|
+
} else {
|
|
188
|
+
latin++
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// CJK: ~1.5 chars/token, Latin/other: ~4 chars/token
|
|
193
|
+
return Math.max(1, Math.ceil(cjk / 1.5 + latin / 4))
|
|
134
194
|
}
|
|
135
195
|
}
|