dsh-context-toolkit 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +123 -2
- package/README.zh.md +124 -0
- package/client.js +334 -0
- package/cordis.patch.yml +14 -0
- package/icon.svg +5 -0
- package/index.js +1237 -0
- package/lib/runner.js +265 -0
- package/lib/store.js +243 -0
- package/locale/en.json +83 -0
- package/locale/zh.json +83 -0
- package/package.json +59 -4
package/lib/runner.js
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sandboxed code/command execution.
|
|
3
|
+
*
|
|
4
|
+
* Everything here exists for one reason: a command or script can produce
|
|
5
|
+
* megabytes of output, while the answer a conversation needs is usually a few
|
|
6
|
+
* hundred bytes. The raw bytes are captured in this process and never returned;
|
|
7
|
+
* only the measured head/tail plus the byte count crosses back.
|
|
8
|
+
*/
|
|
9
|
+
import { spawn } from 'node:child_process'
|
|
10
|
+
import { existsSync, readFileSync, statSync, writeFileSync, mkdtempSync, rmSync, mkdirSync } from 'node:fs'
|
|
11
|
+
import { tmpdir, homedir } from 'node:os'
|
|
12
|
+
import { join, extname, isAbsolute, resolve, dirname } from 'node:path'
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Locate the interpreters we can actually launch.
|
|
16
|
+
*
|
|
17
|
+
* process.execPath is NOT usable here: inside the harness it points at the host
|
|
18
|
+
* application binary, not at a scripting runtime. Each candidate is therefore
|
|
19
|
+
* probed on disk, and PATH names are kept as-is so the OS resolves them.
|
|
20
|
+
*/
|
|
21
|
+
let RUNTIME_CACHE = null
|
|
22
|
+
|
|
23
|
+
function firstExisting(paths) {
|
|
24
|
+
for (const candidate of paths) {
|
|
25
|
+
if (typeof candidate !== 'string' || !candidate) continue
|
|
26
|
+
if (!/[\\/]/.test(candidate)) return candidate // a bare name: let PATH resolve it
|
|
27
|
+
if (existsSync(candidate)) return candidate
|
|
28
|
+
}
|
|
29
|
+
return null
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function probeRuntimes() {
|
|
33
|
+
if (RUNTIME_CACHE !== null) return RUNTIME_CACHE
|
|
34
|
+
const steps = (process.env.PATH || '').split(process.platform === 'win32' ? ';' : ':').filter(Boolean)
|
|
35
|
+
const nodeNames = process.platform === 'win32' ? ['node.exe'] : ['node']
|
|
36
|
+
const fromPath = steps.flatMap((dir) => nodeNames.map((n) => join(dir, n)))
|
|
37
|
+
const node = firstExisting([
|
|
38
|
+
process.env.CTK_NODE,
|
|
39
|
+
process.env.NODE_BINARY,
|
|
40
|
+
'C:\\Program Files\\nodejs\\node.exe',
|
|
41
|
+
join(homedir(), 'AppData', 'Local', 'Programs', 'nodejs', 'node.exe'),
|
|
42
|
+
...fromPath,
|
|
43
|
+
'node',
|
|
44
|
+
])
|
|
45
|
+
const bun = firstExisting([
|
|
46
|
+
process.env.CTK_BUN,
|
|
47
|
+
join(homedir(), '.bun', 'bin', process.platform === 'win32' ? 'bun.exe' : 'bun'),
|
|
48
|
+
'bun',
|
|
49
|
+
])
|
|
50
|
+
const shell = process.platform === 'win32'
|
|
51
|
+
? firstExisting([
|
|
52
|
+
join(process.env.SystemRoot || 'C:\\Windows', 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'),
|
|
53
|
+
'powershell',
|
|
54
|
+
])
|
|
55
|
+
: firstExisting(['/bin/bash', 'bash', '/bin/sh', 'sh'])
|
|
56
|
+
const found = {}
|
|
57
|
+
if (node) { found.javascript = node; found.typescript = node }
|
|
58
|
+
if (bun) found.bun = bun
|
|
59
|
+
if (shell) found.shell = shell
|
|
60
|
+
RUNTIME_CACHE = found
|
|
61
|
+
return found
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function availableRuntimes() {
|
|
65
|
+
return Object.keys(probeRuntimes())
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The shell plan a command runner should use on this platform. */
|
|
69
|
+
export function shellPlan() {
|
|
70
|
+
if (process.platform === 'win32') {
|
|
71
|
+
const ps = probeRuntimes().shell || 'powershell'
|
|
72
|
+
return { command: ps, args: (code) => ['-NoProfile', '-NonInteractive', '-Command', code] }
|
|
73
|
+
}
|
|
74
|
+
return { command: probeRuntimes().shell || 'bash', args: (code) => ['-c', code] }
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Run one child process, capturing stdout/stderr with a hard byte cap.
|
|
79
|
+
*
|
|
80
|
+
* The cap is applied on the fly: once a stream exceeds it, further chunks are
|
|
81
|
+
* counted but not retained. That keeps memory bounded even for a runaway
|
|
82
|
+
* command, while still reporting the true total size.
|
|
83
|
+
*/
|
|
84
|
+
export function runProcess(command, args, options = {}) {
|
|
85
|
+
const {
|
|
86
|
+
cwd = process.cwd(),
|
|
87
|
+
env = {},
|
|
88
|
+
timeoutMs = 120000,
|
|
89
|
+
maxCaptureBytes = 262144,
|
|
90
|
+
shell = false,
|
|
91
|
+
input = null,
|
|
92
|
+
} = options
|
|
93
|
+
|
|
94
|
+
return new Promise((settle) => {
|
|
95
|
+
let child
|
|
96
|
+
try {
|
|
97
|
+
child = spawn(command, args, {
|
|
98
|
+
cwd,
|
|
99
|
+
shell,
|
|
100
|
+
env: childEnv(env),
|
|
101
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
102
|
+
windowsHide: true,
|
|
103
|
+
})
|
|
104
|
+
} catch (error) {
|
|
105
|
+
settle({ ok: false, error: String((error && error.message) || error), stdout: '', stderr: '', stdoutBytes: 0, stderrBytes: 0, exitCode: null, timedOut: false })
|
|
106
|
+
return
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const state = { outBytes: 0, errBytes: 0, outKept: 0, errKept: 0, outCap: false, errCap: false }
|
|
110
|
+
const outChunks = []
|
|
111
|
+
const errChunks = []
|
|
112
|
+
let timedOut = false
|
|
113
|
+
let settled = false
|
|
114
|
+
|
|
115
|
+
const timer = timeoutMs > 0
|
|
116
|
+
? setTimeout(() => {
|
|
117
|
+
timedOut = true
|
|
118
|
+
try { child.kill('SIGKILL') } catch { /* already gone */ }
|
|
119
|
+
}, timeoutMs)
|
|
120
|
+
: null
|
|
121
|
+
|
|
122
|
+
const collect = (chunk, isOut) => {
|
|
123
|
+
const size = chunk.length
|
|
124
|
+
if (isOut) {
|
|
125
|
+
state.outBytes += size
|
|
126
|
+
if (state.outKept < maxCaptureBytes) {
|
|
127
|
+
const room = maxCaptureBytes - state.outKept
|
|
128
|
+
outChunks.push(room >= size ? chunk : chunk.subarray(0, room))
|
|
129
|
+
state.outKept += Math.min(size, room)
|
|
130
|
+
} else state.outCap = true
|
|
131
|
+
} else {
|
|
132
|
+
state.errBytes += size
|
|
133
|
+
if (state.errKept < maxCaptureBytes) {
|
|
134
|
+
const room = maxCaptureBytes - state.errKept
|
|
135
|
+
errChunks.push(room >= size ? chunk : chunk.subarray(0, room))
|
|
136
|
+
state.errKept += Math.min(size, room)
|
|
137
|
+
} else state.errCap = true
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
child.stdout.on('data', (c) => collect(c, true))
|
|
142
|
+
child.stderr.on('data', (c) => collect(c, false))
|
|
143
|
+
|
|
144
|
+
const finish = (exitCode, signal, error) => {
|
|
145
|
+
if (settled) return
|
|
146
|
+
settled = true
|
|
147
|
+
if (timer) clearTimeout(timer)
|
|
148
|
+
settle({
|
|
149
|
+
ok: !timedOut && error === undefined && exitCode === 0,
|
|
150
|
+
error,
|
|
151
|
+
exitCode,
|
|
152
|
+
signal: signal || null,
|
|
153
|
+
timedOut,
|
|
154
|
+
stdout: Buffer.concat(outChunks).toString('utf8'),
|
|
155
|
+
stderr: Buffer.concat(errChunks).toString('utf8'),
|
|
156
|
+
stdoutBytes: state.outBytes,
|
|
157
|
+
stderrBytes: state.errBytes,
|
|
158
|
+
stdoutTruncated: state.outCap || state.outKept < state.outBytes,
|
|
159
|
+
stderrTruncated: state.errCap || state.errKept < state.errBytes,
|
|
160
|
+
stdoutKeptBytes: state.outKept,
|
|
161
|
+
})
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
child.on('error', (error) => finish(null, null, String((error && error.message) || error)))
|
|
165
|
+
child.on('close', (code, signal) => finish(code, signal, undefined))
|
|
166
|
+
|
|
167
|
+
if (input !== null) {
|
|
168
|
+
try { child.stdin.end(String(input)) } catch { /* close races are fine */ }
|
|
169
|
+
} else {
|
|
170
|
+
try { child.stdin.end() } catch { /* ignore */ }
|
|
171
|
+
}
|
|
172
|
+
})
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Build the environment a child process sees.
|
|
177
|
+
*
|
|
178
|
+
* The harness sandbox exports neither PATH entries for the interpreters we found
|
|
179
|
+
* nor TEMP/TMPDIR, so a plain {...process.env} spawn cannot run "node" by name
|
|
180
|
+
* even when we located the binary. This adds the directories we resolved, plus a
|
|
181
|
+
* working scratch dir, without discarding whatever the parent did export.
|
|
182
|
+
*/
|
|
183
|
+
function childEnv(extra) {
|
|
184
|
+
const env = { ...process.env, ...extra }
|
|
185
|
+
const dirs = []
|
|
186
|
+
const runtimes = probeRuntimes()
|
|
187
|
+
for (const key of ['javascript', 'bun', 'shell']) {
|
|
188
|
+
const bin = runtimes[key]
|
|
189
|
+
if (typeof bin === 'string' && /[\\/]/.test(bin)) dirs.push(dirname(bin))
|
|
190
|
+
}
|
|
191
|
+
const scratch = scratchDir()
|
|
192
|
+
dirs.push(scratch)
|
|
193
|
+
const sep = process.platform === 'win32' ? ';' : ':'
|
|
194
|
+
const existing = String(env.PATH || env.Path || '').split(sep).filter(Boolean)
|
|
195
|
+
env.PATH = [...new Set([...dirs, ...existing])].join(sep)
|
|
196
|
+
if (!env.TEMP) env.TEMP = scratch
|
|
197
|
+
if (!env.TMP) env.TMP = scratch
|
|
198
|
+
if (!env.TMPDIR) env.TMPDIR = scratch
|
|
199
|
+
|
|
200
|
+
if (process.platform === 'win32') {
|
|
201
|
+
// Measured: this sandbox exports PATHEXT as ".CPL" only and leaves COMSPEC
|
|
202
|
+
// empty. PowerShell then resolves no .exe at all — not even "cmd" — even
|
|
203
|
+
// though the file is on disk and PATH. Restoring both turns command lookup
|
|
204
|
+
// back on. Done only when the inherited value is unusable, so a normal
|
|
205
|
+
// environment is never overridden.
|
|
206
|
+
const systemRoot = env.SystemRoot || env.windir || 'C:\\Windows'
|
|
207
|
+
const wantedExt = '.COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL'
|
|
208
|
+
const currentExt = String(env.PATHEXT || '')
|
|
209
|
+
const hasExe = /\\.EXE/i.test(currentExt)
|
|
210
|
+
if (!hasExe) env.PATHEXT = wantedExt
|
|
211
|
+
if (!env.COMSPEC) env.COMSPEC = join(systemRoot, 'System32', 'cmd.exe')
|
|
212
|
+
if (!env.SystemRoot) env.SystemRoot = systemRoot
|
|
213
|
+
if (!env.windir) env.windir = systemRoot
|
|
214
|
+
}
|
|
215
|
+
return env
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Resolve a writable scratch directory.
|
|
220
|
+
*
|
|
221
|
+
* os.tmpdir() returns a bogus value ("undefined\temp") when TEMP/TMPDIR are not
|
|
222
|
+
* exported, which is exactly what happens inside the harness sandbox. So the
|
|
223
|
+
* candidates are probed in order and the first writable one wins.
|
|
224
|
+
*/
|
|
225
|
+
let SCRATCH_DIR = null
|
|
226
|
+
export function scratchDir() {
|
|
227
|
+
if (SCRATCH_DIR !== null) return SCRATCH_DIR
|
|
228
|
+
const candidates = [
|
|
229
|
+
process.env.CTK_TMP,
|
|
230
|
+
process.env.TEMP,
|
|
231
|
+
process.env.TMP,
|
|
232
|
+
process.env.TMPDIR,
|
|
233
|
+
join(homedir(), 'AppData', 'Local', 'Temp'),
|
|
234
|
+
join(homedir(), '.cache', 'dsh-context-toolkit'),
|
|
235
|
+
'/tmp',
|
|
236
|
+
dirname(tmpdir()),
|
|
237
|
+
]
|
|
238
|
+
for (const candidate of candidates) {
|
|
239
|
+
if (typeof candidate !== 'string' || !candidate.trim()) continue
|
|
240
|
+
if (/undefined/i.test(candidate)) continue
|
|
241
|
+
try {
|
|
242
|
+
mkdirSync(candidate, { recursive: true })
|
|
243
|
+
const probe = mkdtempSync(join(candidate, 'ctk-check-'))
|
|
244
|
+
rmSync(probe, { recursive: true, force: true })
|
|
245
|
+
SCRATCH_DIR = candidate
|
|
246
|
+
return SCRATCH_DIR
|
|
247
|
+
} catch { /* try the next candidate */ }
|
|
248
|
+
}
|
|
249
|
+
const fallback = process.cwd()
|
|
250
|
+
SCRATCH_DIR = fallback
|
|
251
|
+
return fallback
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** Write a script to a temp file so its real name shows up in stack traces. */
|
|
255
|
+
export function withTempScript(extension, body, fn) {
|
|
256
|
+
const dir = mkdtempSync(join(scratchDir(), 'ctk-run-'))
|
|
257
|
+
const file = join(dir, 'script' + extension)
|
|
258
|
+
writeFileSync(file, body, 'utf8')
|
|
259
|
+
return Promise.resolve()
|
|
260
|
+
.then(() => fn(file))
|
|
261
|
+
.finally(() => { try { rmSync(dir, { recursive: true, force: true }) } catch { /* best effort */ } })
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
export const SANDBOX_TMP = () => mkdtempSync(join(scratchDir(), 'ctk-'))
|
|
265
|
+
export { extname, isAbsolute, resolve, readFileSync, statSync, existsSync }
|
package/lib/store.js
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The local knowledge base.
|
|
3
|
+
*
|
|
4
|
+
* Large material is filed here and retrieved by section later, so the bytes
|
|
5
|
+
* never occupy the conversation in the meantime. The whole store rides on
|
|
6
|
+
* node:sqlite, whose FTS5 module ships with Node itself — no native build, no
|
|
7
|
+
* third-party package, which is what keeps this plugin installable anywhere.
|
|
8
|
+
*
|
|
9
|
+
* Layout: one database per workspace. Documents are split into chunks at
|
|
10
|
+
* heading boundaries so a search hit returns the relevant section rather than a
|
|
11
|
+
* whole file, and chunk text is indexed with BM25 ranking.
|
|
12
|
+
*/
|
|
13
|
+
import { DatabaseSync } from 'node:sqlite'
|
|
14
|
+
import { createHash } from 'node:crypto'
|
|
15
|
+
import { existsSync, mkdirSync, readFileSync, statSync } from 'node:fs'
|
|
16
|
+
import { join, basename, resolve } from 'node:path'
|
|
17
|
+
|
|
18
|
+
const SCHEMA_VERSION = 1
|
|
19
|
+
|
|
20
|
+
/** Split text into indexable chunks at markdown headings, keeping code fences intact. */
|
|
21
|
+
export function chunkText(text, options = {}) {
|
|
22
|
+
const { maxChars = 1800, minChars = 1 } = options
|
|
23
|
+
const source = String(text || '')
|
|
24
|
+
if (!source.trim()) return []
|
|
25
|
+
|
|
26
|
+
const lines = source.split('\n')
|
|
27
|
+
const chunks = []
|
|
28
|
+
let heading = ''
|
|
29
|
+
let buffer = []
|
|
30
|
+
let size = 0
|
|
31
|
+
let inFence = false
|
|
32
|
+
|
|
33
|
+
const flush = () => {
|
|
34
|
+
const body = buffer.join('\n').trim()
|
|
35
|
+
if (body.length >= minChars) chunks.push({ heading, text: body })
|
|
36
|
+
buffer = []
|
|
37
|
+
size = 0
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
for (const line of lines) {
|
|
41
|
+
// A fence toggles raw mode: a "## " inside a code block is not a heading.
|
|
42
|
+
if (/^```/.test(line)) inFence = !inFence
|
|
43
|
+
const isHeading = !inFence && /^#{1,6}\s+/.test(line)
|
|
44
|
+
if (isHeading && size > 0) flush()
|
|
45
|
+
if (isHeading) heading = line.replace(/^#{1,6}\s+/, '').trim()
|
|
46
|
+
buffer.push(line)
|
|
47
|
+
size += line.length + 1
|
|
48
|
+
if (size >= maxChars && !inFence) flush()
|
|
49
|
+
}
|
|
50
|
+
flush()
|
|
51
|
+
return chunks
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** A stable id for a document path, so re-indexing replaces rather than duplicates. */
|
|
55
|
+
export function documentKey(absPath) {
|
|
56
|
+
const normalized = resolve(String(absPath)).replace(/\\/g, '/').toLowerCase()
|
|
57
|
+
return createHash('sha256').update(normalized).digest('hex').slice(0, 24)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Build a readable excerpt centred on the first query term found.
|
|
62
|
+
*
|
|
63
|
+
* Slicing from the start of a chunk hides the very words that matched whenever
|
|
64
|
+
* the hit sits mid-chunk, which makes a search result useless for judging
|
|
65
|
+
* relevance. The window opens just before the match and is marked when it is
|
|
66
|
+
* not the beginning of the chunk.
|
|
67
|
+
*/
|
|
68
|
+
export function excerptAround(text, query, maxChars = 900) {
|
|
69
|
+
const body = String(text || '')
|
|
70
|
+
if (body.length <= maxChars) return body
|
|
71
|
+
const terms = String(query || '').split(/\s+/).filter((t) => t.length > 1)
|
|
72
|
+
let at = -1
|
|
73
|
+
for (const term of terms) {
|
|
74
|
+
const found = body.toLowerCase().indexOf(term.toLowerCase())
|
|
75
|
+
if (found >= 0 && (at < 0 || found < at)) at = found
|
|
76
|
+
}
|
|
77
|
+
if (at < 0) return body.slice(0, maxChars)
|
|
78
|
+
const back = Math.floor(maxChars * 0.3)
|
|
79
|
+
const start = Math.max(0, at - back)
|
|
80
|
+
const end = Math.min(body.length, start + maxChars)
|
|
81
|
+
return (start > 0 ? '…' : '') + body.slice(start, end) + (end < body.length ? '…' : '')
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function openStore(storageDir) {
|
|
85
|
+
mkdirSync(storageDir, { recursive: true })
|
|
86
|
+
const dbPath = join(storageDir, 'knowledge.db')
|
|
87
|
+
const db = new DatabaseSync(dbPath)
|
|
88
|
+
|
|
89
|
+
// The schema is created idempotently; an FTS5 virtual table backs retrieval.
|
|
90
|
+
db.exec(`
|
|
91
|
+
PRAGMA journal_mode = WAL;
|
|
92
|
+
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
|
|
93
|
+
CREATE TABLE IF NOT EXISTS documents (
|
|
94
|
+
doc_key TEXT PRIMARY KEY,
|
|
95
|
+
path TEXT NOT NULL,
|
|
96
|
+
title TEXT,
|
|
97
|
+
bytes INTEGER,
|
|
98
|
+
mtime_ms INTEGER,
|
|
99
|
+
chunks INTEGER,
|
|
100
|
+
indexed_at_ms INTEGER
|
|
101
|
+
);
|
|
102
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks USING fts5(
|
|
103
|
+
body,
|
|
104
|
+
doc_key UNINDEXED,
|
|
105
|
+
path UNINDEXED,
|
|
106
|
+
title UNINDEXED,
|
|
107
|
+
heading UNINDEXED,
|
|
108
|
+
ord UNINDEXED,
|
|
109
|
+
tokenize = 'unicode61'
|
|
110
|
+
);
|
|
111
|
+
`)
|
|
112
|
+
db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run('schema_version', String(SCHEMA_VERSION))
|
|
113
|
+
return makeApi(db, dbPath, storageDir)
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Paths that are labels rather than disk locations must survive resolve(). */
|
|
117
|
+
export function normalizeTarget(path) {
|
|
118
|
+
const value = String(path || '')
|
|
119
|
+
if (/^(text|memory|https?):/i.test(value)) return value
|
|
120
|
+
return resolve(value)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function makeApi(db, dbPath, storageDir) {
|
|
124
|
+
const insertChunk = db.prepare('INSERT INTO chunks(body, doc_key, path, title, heading, ord) VALUES (?, ?, ?, ?, ?, ?)')
|
|
125
|
+
const deleteChunks = db.prepare('DELETE FROM chunks WHERE doc_key = ?')
|
|
126
|
+
const upsertDoc = db.prepare(`INSERT OR REPLACE INTO documents
|
|
127
|
+
(doc_key, path, title, bytes, mtime_ms, chunks, indexed_at_ms) VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
|
128
|
+
const findDoc = db.prepare('SELECT * FROM documents WHERE doc_key = ?')
|
|
129
|
+
|
|
130
|
+
return {
|
|
131
|
+
/** File one document, replacing any earlier version of the same path. */
|
|
132
|
+
indexDocument({ path, title, text, mtimeMs }) {
|
|
133
|
+
const abs = normalizeTarget(path)
|
|
134
|
+
const key = documentKey(abs)
|
|
135
|
+
const body = String(text == null ? '' : text)
|
|
136
|
+
const chunks = chunkText(body)
|
|
137
|
+
const meta = findDoc.get(key)
|
|
138
|
+
const now = Date.now()
|
|
139
|
+
|
|
140
|
+
const write = db.prepare('BEGIN')
|
|
141
|
+
write.run()
|
|
142
|
+
try {
|
|
143
|
+
deleteChunks.run(key)
|
|
144
|
+
upsertDoc.run(key, abs, String(title || basename(abs)), Buffer.byteLength(body, 'utf8'),
|
|
145
|
+
Number(mtimeMs) || 0, chunks.length, now)
|
|
146
|
+
chunks.forEach((c, i) => insertChunk.run(c.text, key, abs, String(title || basename(abs)), c.heading || '', i))
|
|
147
|
+
db.prepare('COMMIT').run()
|
|
148
|
+
} catch (error) {
|
|
149
|
+
try { db.prepare('ROLLBACK').run() } catch { /* already rolled back */ }
|
|
150
|
+
throw error
|
|
151
|
+
}
|
|
152
|
+
return { key, path: abs, chunks: chunks.length, bytes: Buffer.byteLength(body, 'utf8'), replaced: Boolean(meta) }
|
|
153
|
+
},
|
|
154
|
+
|
|
155
|
+
/** Index a file straight from disk. */
|
|
156
|
+
indexFile(path, options = {}) {
|
|
157
|
+
const abs = normalizeTarget(path)
|
|
158
|
+
if (!existsSync(abs)) throw new Error('no such file: ' + abs)
|
|
159
|
+
const st = statSync(abs)
|
|
160
|
+
if (!st.isFile()) throw new Error('not a regular file: ' + abs)
|
|
161
|
+
const text = readFileSync(abs, 'utf8')
|
|
162
|
+
return this.indexDocument({ path: abs, title: options.title, text, mtimeMs: st.mtimeMs })
|
|
163
|
+
},
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Search the knowledge base.
|
|
167
|
+
*
|
|
168
|
+
* BM25 ranks the chunks; results carry the heading so a caller can decide
|
|
169
|
+
* which section is worth opening in full.
|
|
170
|
+
*/
|
|
171
|
+
search(queries, options = {}) {
|
|
172
|
+
const list = Array.isArray(queries) ? queries : [queries]
|
|
173
|
+
const { limit = 5, maxChars = 900 } = options
|
|
174
|
+
const results = []
|
|
175
|
+
for (const raw of list) {
|
|
176
|
+
const query = String(raw || '').trim()
|
|
177
|
+
if (!query) continue
|
|
178
|
+
// Quote each term so FTS5 treats user text as terms, not as operators.
|
|
179
|
+
const terms = query.split(/\s+/).filter(Boolean).map((t) => '"' + t.replace(/"/g, '""') + '"')
|
|
180
|
+
if (!terms.length) continue
|
|
181
|
+
const match = terms.join(' OR ')
|
|
182
|
+
let rows = []
|
|
183
|
+
try {
|
|
184
|
+
rows = db.prepare(`
|
|
185
|
+
SELECT body, path, title, heading, bm25(chunks) AS score
|
|
186
|
+
FROM chunks WHERE chunks MATCH ? ORDER BY score LIMIT ?
|
|
187
|
+
`).all(match, limit)
|
|
188
|
+
} catch (error) {
|
|
189
|
+
results.push({ query, error: String((error && error.message) || error), hits: [] })
|
|
190
|
+
continue
|
|
191
|
+
}
|
|
192
|
+
results.push({
|
|
193
|
+
query,
|
|
194
|
+
hits: rows.map((row) => {
|
|
195
|
+
const body = String(row.body || '')
|
|
196
|
+
return {
|
|
197
|
+
path: row.path,
|
|
198
|
+
title: row.title,
|
|
199
|
+
heading: row.heading,
|
|
200
|
+
snippet: excerptAround(body, query, maxChars),
|
|
201
|
+
score: Number(row.score),
|
|
202
|
+
}
|
|
203
|
+
}),
|
|
204
|
+
})
|
|
205
|
+
}
|
|
206
|
+
return results
|
|
207
|
+
},
|
|
208
|
+
|
|
209
|
+
/** Every document currently filed. */
|
|
210
|
+
list() {
|
|
211
|
+
return db.prepare('SELECT path, title, bytes, chunks, indexed_at_ms FROM documents ORDER BY indexed_at_ms DESC').all()
|
|
212
|
+
.map((r) => ({ path: r.path, title: r.title, bytes: r.bytes, chunks: r.chunks, indexedAt: r.indexed_at_ms }))
|
|
213
|
+
},
|
|
214
|
+
|
|
215
|
+
/** Remove documents by path, by key, or all of them. */
|
|
216
|
+
purge({ path, all = false } = {}) {
|
|
217
|
+
if (all) {
|
|
218
|
+
const count = db.prepare('SELECT COUNT(*) AS n FROM documents').get().n
|
|
219
|
+
db.exec('DELETE FROM chunks; DELETE FROM documents;')
|
|
220
|
+
return { removed: Number(count) || 0, scope: 'all' }
|
|
221
|
+
}
|
|
222
|
+
if (!path) return { removed: 0, scope: 'none' }
|
|
223
|
+
const key = documentKey(path)
|
|
224
|
+
const before = findDoc.get(key)
|
|
225
|
+
deleteChunks.run(key)
|
|
226
|
+
db.prepare('DELETE FROM documents WHERE doc_key = ?').run(key)
|
|
227
|
+
return { removed: before ? 1 : 0, scope: 'path', path: normalizeTarget(path) }
|
|
228
|
+
},
|
|
229
|
+
|
|
230
|
+
stats() {
|
|
231
|
+
const docs = db.prepare('SELECT COUNT(*) AS n FROM documents').get().n
|
|
232
|
+
const chunks = db.prepare('SELECT COUNT(*) AS n FROM chunks').get().n
|
|
233
|
+
const bytes = db.prepare('SELECT COALESCE(SUM(bytes),0) AS n FROM documents').get().n
|
|
234
|
+
return { documents: Number(docs) || 0, chunks: Number(chunks) || 0, bytes: Number(bytes) || 0, dbPath }
|
|
235
|
+
},
|
|
236
|
+
|
|
237
|
+
close() { try { db.close() } catch { /* already closed */ } },
|
|
238
|
+
_db: db,
|
|
239
|
+
_storageDir: storageDir,
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
export { SCHEMA_VERSION }
|
package/locale/en.json
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
{
|
|
2
|
+
"meta": {
|
|
3
|
+
"title": "Context Toolkit",
|
|
4
|
+
"description": "Run code in a sandbox, file large material into a local searchable knowledge base, and block the fetches and whole-file reads that flood a conversation. Ships a live context-occupancy panel."
|
|
5
|
+
},
|
|
6
|
+
"strings": {
|
|
7
|
+
"title": "Context Toolkit",
|
|
8
|
+
"description": "Run code in a sandbox, file large material into a local searchable knowledge base, and block the fetches and whole-file reads that flood a conversation. Ships a live context-occupancy panel.",
|
|
9
|
+
"hint": "These switches apply immediately, no restart: the gate is read before every tool call.",
|
|
10
|
+
"routing": "Routing gate",
|
|
11
|
+
"routingHint": "Master switch. When on, every tool call passes through the gate before running.",
|
|
12
|
+
"denyCurlWget": "Block curl / wget",
|
|
13
|
+
"denyCurlWgetHint": "Both exist to paste whole responses into the conversation; blocked here, use ctk_fetch_and_index instead.",
|
|
14
|
+
"bashNudgeMinCommandBytes": "Long-command nudge (bytes)",
|
|
15
|
+
"bashNudgeMinCommandBytesHint": "Commands longer than this get a nudge toward ctk_batch_execute. 0 disables it.",
|
|
16
|
+
"maxReadBytesBeforeAsk": "Whole-file read nudge (bytes)",
|
|
17
|
+
"maxReadBytesBeforeAskHint": "Reading a file larger than this gets a nudge toward ctk_execute_file. 0 disables it.",
|
|
18
|
+
"trustedReadBasenames": "Filenames exempt from the read nudge",
|
|
19
|
+
"trustedReadBasenamesHint": "Comma separated; these may be read whole however large they are.",
|
|
20
|
+
"compressNudgeTokens": "Occupancy warning (tokens)",
|
|
21
|
+
"compressNudgeTokensHint": "At this surface occupancy the report suggests compaction. 0 disables it.",
|
|
22
|
+
"searchLimit": "Search result limit",
|
|
23
|
+
"searchLimitHint": "How many excerpts ctk_search returns per query.",
|
|
24
|
+
"snippetChars": "Excerpt size (chars)",
|
|
25
|
+
"snippetCharsHint": "Character ceiling on each excerpt from ctk_search.",
|
|
26
|
+
"fetchMaxBytes": "Per-page fetch ceiling (bytes)",
|
|
27
|
+
"fetchMaxBytesHint": "ctk_fetch_and_index keeps at most this many bytes of each page.",
|
|
28
|
+
"storageDir": "Knowledge-base root",
|
|
29
|
+
"storageDirHint": "Empty = <DSH_HOME>/context-toolkit.",
|
|
30
|
+
"toolsTitle": "Tool switches",
|
|
31
|
+
"toolsHint": "Enable or disable individual ctk_* tools.",
|
|
32
|
+
"overridden": "overridden",
|
|
33
|
+
"reset": "reset",
|
|
34
|
+
"readOnly": "This deployment is read-only.",
|
|
35
|
+
"unavailable": "The plugin is not loaded right now; it cannot be configured.",
|
|
36
|
+
"save": "Save",
|
|
37
|
+
"saving": "Saving…",
|
|
38
|
+
"saveFailed": "This deployment did not accept those values; they are kept for you to edit.",
|
|
39
|
+
"invalidNumber": "Numbers only; blank means the default.",
|
|
40
|
+
"note": "Gate switches apply immediately; reopen the page to confirm tool-switch changes.",
|
|
41
|
+
"ctxTitle": "Context occupancy",
|
|
42
|
+
"ctxLoading": "Reading…",
|
|
43
|
+
"ctxReload": "Refresh",
|
|
44
|
+
"ctxFailed": "Read failed: ",
|
|
45
|
+
"ctxNoSession": "No measurable projection in this session yet.",
|
|
46
|
+
"ctxStale": "Snapshot from the last real measurement ({time}): a page request has no session boundary; ask ctk_context_report in the conversation for a live reading.",
|
|
47
|
+
"ctxTotal": "{n} tokens total",
|
|
48
|
+
"ctxSurface": "{n} tokens on the surface",
|
|
49
|
+
"ctxBudget": "warning at {n} tokens",
|
|
50
|
+
"ctxOver": "Above the warning threshold — route bulk output through ctk_execute / ctk_index.",
|
|
51
|
+
"ctxUnder": "Below the warning threshold; nothing to do.",
|
|
52
|
+
"ctxSystem": "System prompt",
|
|
53
|
+
"ctxTools": "Tool schemas",
|
|
54
|
+
"ctxMessages": "Conversation",
|
|
55
|
+
"ctxEstimated": " (estimated)",
|
|
56
|
+
"ctxNodes": "Biggest contributors (top {n})",
|
|
57
|
+
"ctxLedger": "This plugin's ledger",
|
|
58
|
+
"ctxLedgerLine": "The gate blocked {n} calls, about {bytes} bytes kept out of context; the sandbox kept out {sandbox} bytes; {tools} ctk_* tools registered.",
|
|
59
|
+
"ctxKnowledge": "Knowledge base: {docs} documents / {chunks} chunks ({bytes} bytes)",
|
|
60
|
+
"ctxStorage": "Knowledge base at {path}",
|
|
61
|
+
"tools.execute": "ctk_execute (run code in the sandbox, return only the answer)",
|
|
62
|
+
"tools.executeHint": "For analysis where you want the conclusion, not the raw output.",
|
|
63
|
+
"tools.executeFile": "ctk_execute_file (run code over a file, return only the answer)",
|
|
64
|
+
"tools.executeFileHint": "Read a big file, compute over it, keep its bytes in the sandbox.",
|
|
65
|
+
"tools.batchExecute": "ctk_batch_execute (many commands in one call, per-command status)",
|
|
66
|
+
"tools.batchExecuteHint": "One call for many commands; only conclusions come back.",
|
|
67
|
+
"tools.index": "ctk_index (file large material into the local knowledge base)",
|
|
68
|
+
"tools.indexHint": "Archive long docs, logs or specs, then pull sections with ctk_search.",
|
|
69
|
+
"tools.search": "ctk_search (retrieve excerpts on demand, not the whole thing)",
|
|
70
|
+
"tools.searchHint": "Prefer this for material already in the knowledge base.",
|
|
71
|
+
"tools.fetchAndIndex": "ctk_fetch_and_index (fetch a page straight into the index)",
|
|
72
|
+
"tools.fetchAndIndexHint": "When a whole page would flood the conversation; curl/wget are gated.",
|
|
73
|
+
"tools.purge": "ctk_purge (delete indexed content — permanent)",
|
|
74
|
+
"tools.purgeHint": "Clears the knowledge base; permanent, so the call must confirm:true.",
|
|
75
|
+
"tools.stats": "ctk_stats (this plugin's savings ledger and knowledge usage)",
|
|
76
|
+
"tools.statsHint": "How much this plugin saved, and how much it filed.",
|
|
77
|
+
"tools.doctor": "ctk_doctor (self-check: runtimes, scratch dir, knowledge round-trip)",
|
|
78
|
+
"tools.doctorHint": "Run it first when a child process or indexing misbehaves.",
|
|
79
|
+
"tools.contextReport": "ctk_context_report (session context occupancy report)",
|
|
80
|
+
"tools.contextReportHint": "Lets the model ask for this session context occupancy and composition.",
|
|
81
|
+
"restart": "needs a restart"
|
|
82
|
+
}
|
|
83
|
+
}
|
package/locale/zh.json
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
{
|
|
2
|
+
"meta": {
|
|
3
|
+
"title": "上下文工具箱",
|
|
4
|
+
"description": "在沙箱里跑代码、把大块材料归档成可检索的本地知识库、并挡住会灌满对话的抓取与整读。附一张实时上下文占用面板。"
|
|
5
|
+
},
|
|
6
|
+
"strings": {
|
|
7
|
+
"title": "上下文工具箱",
|
|
8
|
+
"description": "在沙箱里跑代码、把大块材料归档成可检索的本地知识库、并挡住会灌满对话的抓取与整读。附一张实时上下文占用面板。",
|
|
9
|
+
"hint": "下列开关即时生效,无需重启:门禁在每次工具调用前读取。",
|
|
10
|
+
"routing": "路由门禁",
|
|
11
|
+
"routingHint": "总开关。开启后,每次工具调用执行前都会经过门禁检查。",
|
|
12
|
+
"denyCurlWget": "硬拦 curl / wget",
|
|
13
|
+
"denyCurlWgetHint": "这两个工具的设计就是把整页内容塞进对话;拦掉,改用 ctk_fetch_and_index。",
|
|
14
|
+
"bashNudgeMinCommandBytes": "长命令引导阈值(字节)",
|
|
15
|
+
"bashNudgeMinCommandBytesHint": "命令超过这个长度时提示改用 ctk_batch_execute。填 0 表示不提示。",
|
|
16
|
+
"maxReadBytesBeforeAsk": "大文件整读引导阈值(字节)",
|
|
17
|
+
"maxReadBytesBeforeAskHint": "整读超过这个大小的文件时提示改用 ctk_execute_file。填 0 表示不提示。",
|
|
18
|
+
"trustedReadBasenames": "不受整读门禁限制的文件名",
|
|
19
|
+
"trustedReadBasenamesHint": "逗号分隔;这些文件即使很大也允许整读。",
|
|
20
|
+
"compressNudgeTokens": "占用提示阈值(tokens)",
|
|
21
|
+
"compressNudgeTokensHint": "表面占用达到此值时,占用报告给出压缩建议。填 0 表示不提示。",
|
|
22
|
+
"searchLimit": "检索返回条数上限",
|
|
23
|
+
"searchLimitHint": "ctk_search 每次查询返回的片段数上限。",
|
|
24
|
+
"snippetChars": "片段最大字符数",
|
|
25
|
+
"snippetCharsHint": "ctk_search 每个片段返回的字符上限,超出部分省略。",
|
|
26
|
+
"fetchMaxBytes": "单页抓取上限(字节)",
|
|
27
|
+
"fetchMaxBytesHint": "ctk_fetch_and_index 抓取单页的最大字节数,超出部分丢弃。",
|
|
28
|
+
"storageDir": "知识库根目录",
|
|
29
|
+
"storageDirHint": "留空 = <DSH_HOME>/context-toolkit。",
|
|
30
|
+
"toolsTitle": "工具开关",
|
|
31
|
+
"toolsHint": "逐个启用或停用 ctk_* 工具。",
|
|
32
|
+
"overridden": "已覆盖",
|
|
33
|
+
"reset": "恢复默认",
|
|
34
|
+
"readOnly": "本部署的设置为只读。",
|
|
35
|
+
"unavailable": "该插件当前未加载,暂时无法配置。",
|
|
36
|
+
"save": "保存",
|
|
37
|
+
"saving": "保存中…",
|
|
38
|
+
"saveFailed": "本部署没有接受这些值,已保留供你修改。",
|
|
39
|
+
"invalidNumber": "请填数字;留空表示使用默认值。",
|
|
40
|
+
"note": "门禁开关即时生效;工具开关改动后建议重开页面确认。",
|
|
41
|
+
"ctxTitle": "上下文情况",
|
|
42
|
+
"ctxLoading": "正在读取…",
|
|
43
|
+
"ctxReload": "刷新",
|
|
44
|
+
"ctxFailed": "读取失败:",
|
|
45
|
+
"ctxNoSession": "本会话暂无可测量的投影。",
|
|
46
|
+
"ctxStale": "以下为最近一次实测快照({time}):网页请求没有会话边界,实时值请在对话中调用 ctk_context_report。",
|
|
47
|
+
"ctxTotal": "总压力 {n} tokens",
|
|
48
|
+
"ctxSurface": "表面占用 {n} tokens",
|
|
49
|
+
"ctxBudget": "提示阈值 {n} tokens",
|
|
50
|
+
"ctxOver": "已超过提示阈值,建议把大块输出改走 ctk_execute / ctk_index。",
|
|
51
|
+
"ctxUnder": "低于提示阈值,暂不需要干预。",
|
|
52
|
+
"ctxSystem": "系统提示",
|
|
53
|
+
"ctxTools": "工具 schema",
|
|
54
|
+
"ctxMessages": "会话消息",
|
|
55
|
+
"ctxEstimated": "(估算)",
|
|
56
|
+
"ctxNodes": "占用大头(前 {n} 项)",
|
|
57
|
+
"ctxLedger": "本插件节约台账",
|
|
58
|
+
"ctxLedgerLine": "门禁已拦截 {n} 次,约 {bytes} 字节未进入上下文;沙箱保留 {sandbox} 字节;已注册 {tools} 个 ctk_* 工具。",
|
|
59
|
+
"ctxKnowledge": "知识库:{docs} 文档 / {chunks} 片段({bytes} 字节)",
|
|
60
|
+
"ctxStorage": "知识库位置:{path}",
|
|
61
|
+
"tools.execute": "ctk_execute(在沙箱里跑代码,只把结论带回来)",
|
|
62
|
+
"tools.executeHint": "用于「只想要结论、不想让原始输出占满上下文」的分析型任务。",
|
|
63
|
+
"tools.executeFile": "ctk_execute_file(对某个文件跑代码,只回结论)",
|
|
64
|
+
"tools.executeFileHint": "适合读大文件后做统计或抽取,文件原文留在沙箱里。",
|
|
65
|
+
"tools.batchExecute": "ctk_batch_execute(一次跑多条命令,逐条状态)",
|
|
66
|
+
"tools.batchExecuteHint": "多条命令一次调用,只有结论回来。",
|
|
67
|
+
"tools.index": "ctk_index(把大块材料存进本地知识库)",
|
|
68
|
+
"tools.indexHint": "把长文档、日志、规范归档,之后用 ctk_search 按需取用。",
|
|
69
|
+
"tools.search": "ctk_search(按需检索知识库,取片段而非整份)",
|
|
70
|
+
"tools.searchHint": "知识库里已有的内容优先用它检索,不要整份读进来。",
|
|
71
|
+
"tools.fetchAndIndex": "ctk_fetch_and_index(抓网页并直接并入索引)",
|
|
72
|
+
"tools.fetchAndIndexHint": "抓整页网页又不让它淹没对话时用;curl/wget 会被门禁拦掉。",
|
|
73
|
+
"tools.purge": "ctk_purge(删除已索引内容,不可恢复)",
|
|
74
|
+
"tools.purgeHint": "清理知识库用;会永久删除,调用必须带 confirm:true。",
|
|
75
|
+
"tools.stats": "ctk_stats(节约台账与知识库用量)",
|
|
76
|
+
"tools.statsHint": "想知道这个插件省了多少、归档了多少时看它。",
|
|
77
|
+
"tools.doctor": "ctk_doctor(自检:运行时、临时目录、知识库往返)",
|
|
78
|
+
"tools.doctorHint": "出现异常(子进程跑不起来、索引失败)时先跑它定位。",
|
|
79
|
+
"tools.contextReport": "ctk_context_report(会话上下文占用报告)",
|
|
80
|
+
"tools.contextReportHint": "让模型可以主动查询本会话的上下文占用与构成。",
|
|
81
|
+
"restart": "需重启"
|
|
82
|
+
}
|
|
83
|
+
}
|