dsh-reference-anything 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -22
- package/NOTICE.md +52 -25
- package/README.md +334 -285
- package/cordis.patch.yml +89 -18
- package/lib/client.js +13530 -517
- package/lib/contract.d.ts +155 -3
- package/lib/contract.js +68 -4
- package/lib/download-directory.d.ts +18 -0
- package/lib/download-directory.js +55 -0
- package/lib/errors.d.ts +4 -0
- package/lib/host.d.ts +96 -3
- package/lib/host.js +88 -5
- package/lib/index.d.ts +22 -2
- package/lib/index.js +57 -7
- package/lib/opencli.d.ts +5 -3
- package/lib/opencli.js +13 -6
- package/lib/openlist/host.d.ts +157 -0
- package/lib/openlist/host.js +873 -0
- package/lib/openlist/index.d.ts +84 -0
- package/lib/openlist/index.js +507 -0
- package/lib/render.d.ts +12 -7
- package/lib/render.js +27 -21
- package/lib/sources/cloud-drive/cache.d.ts +60 -0
- package/lib/sources/cloud-drive/cache.js +107 -0
- package/lib/sources/cloud-drive/index.d.ts +190 -0
- package/lib/sources/cloud-drive/index.js +526 -0
- package/lib/sources/cloud-drive/providers/http.d.ts +27 -0
- package/lib/sources/cloud-drive/providers/http.js +73 -0
- package/lib/sources/cloud-drive/providers/openlist.d.ts +59 -0
- package/lib/sources/cloud-drive/providers/openlist.js +535 -0
- package/lib/sources/cloud-drive/registry.d.ts +53 -0
- package/lib/sources/cloud-drive/registry.js +101 -0
- package/lib/sources/cloud-drive/types.d.ts +169 -0
- package/lib/sources/cloud-drive/types.js +17 -0
- package/lib/sources/file.d.ts +2 -0
- package/lib/sources/file.js +2 -0
- package/lib/sources/local-agent/adapters/claude-code.d.ts +16 -0
- package/lib/sources/local-agent/adapters/claude-code.js +171 -0
- package/lib/sources/local-agent/adapters/codex.d.ts +20 -0
- package/lib/sources/local-agent/adapters/codex.js +204 -0
- package/lib/sources/local-agent/adapters/cursor.d.ts +23 -0
- package/lib/sources/local-agent/adapters/cursor.js +121 -0
- package/lib/sources/local-agent/adapters/gemini-cli.d.ts +20 -0
- package/lib/sources/local-agent/adapters/gemini-cli.js +141 -0
- package/lib/sources/local-agent/adapters/grokbuild.d.ts +22 -0
- package/lib/sources/local-agent/adapters/grokbuild.js +76 -0
- package/lib/sources/local-agent/adapters/hermes.d.ts +19 -0
- package/lib/sources/local-agent/adapters/hermes.js +85 -0
- package/lib/sources/local-agent/adapters/kimi.d.ts +28 -0
- package/lib/sources/local-agent/adapters/kimi.js +261 -0
- package/lib/sources/local-agent/adapters/openclaw.d.ts +21 -0
- package/lib/sources/local-agent/adapters/openclaw.js +120 -0
- package/lib/sources/local-agent/adapters/opencode.d.ts +44 -0
- package/lib/sources/local-agent/adapters/opencode.js +236 -0
- package/lib/sources/local-agent/adapters/pi.d.ts +26 -0
- package/lib/sources/local-agent/adapters/pi.js +242 -0
- package/lib/sources/local-agent/adapters/qoder.d.ts +19 -0
- package/lib/sources/local-agent/adapters/qoder.js +93 -0
- package/lib/sources/local-agent/adapters/reasonix.d.ts +23 -0
- package/lib/sources/local-agent/adapters/reasonix.js +90 -0
- package/lib/sources/local-agent/adapters/shared.d.ts +163 -0
- package/lib/sources/local-agent/adapters/shared.js +415 -0
- package/lib/sources/local-agent/adapters/sqlite-shared.d.ts +92 -0
- package/lib/sources/local-agent/adapters/sqlite-shared.js +215 -0
- package/lib/sources/local-agent/adapters/zcode.d.ts +41 -0
- package/lib/sources/local-agent/adapters/zcode.js +159 -0
- package/lib/sources/local-agent/index.d.ts +222 -0
- package/lib/sources/local-agent/index.js +656 -0
- package/lib/sources/local-agent/page.d.ts +55 -0
- package/lib/sources/local-agent/page.js +83 -0
- package/lib/sources/local-agent/path.d.ts +8 -0
- package/lib/sources/local-agent/path.js +43 -0
- package/lib/sources/local-agent/registry.d.ts +68 -0
- package/lib/sources/local-agent/registry.js +116 -0
- package/lib/sources/local-agent/scan.d.ts +175 -0
- package/lib/sources/local-agent/scan.js +472 -0
- package/lib/sources/local-agent/sqlite.d.ts +45 -0
- package/lib/sources/local-agent/sqlite.js +113 -0
- package/lib/sources/local-agent/store.d.ts +93 -0
- package/lib/sources/local-agent/store.js +113 -0
- package/lib/sources/local-agent/types.d.ts +338 -0
- package/lib/sources/local-agent/types.js +20 -0
- package/lib/sources/web-chat/index.d.ts +5 -1
- package/lib/sources/web-chat/index.js +47 -8
- package/lib/store/spec.d.ts +50 -0
- package/lib/store/spec.js +2 -2
- package/lib/sync/index.d.ts +1 -0
- package/lib/sync/index.js +28 -8
- package/lib/tool.d.ts +5 -0
- package/lib/tool.js +418 -45
- package/lib/typert.js +13 -1
- package/lib/types.d.ts +28 -2
- package/lib/update.d.ts +6 -0
- package/lib/update.js +29 -1
- package/lib/web.js +5 -1
- package/lib/wire.d.ts +80 -0
- package/lib/wire.js +30 -1
- package/opencli-plugin/chatgpt.js +30 -27
- package/opencli-plugin/claude.js +12 -8
- package/opencli-plugin/common.js +266 -187
- package/opencli-plugin/deepseek.js +8 -5
- package/opencli-plugin/gemini.js +42 -38
- package/opencli-plugin/grok.js +22 -21
- package/opencli-plugin/kimi.js +34 -31
- package/opencli-plugin/opencli-plugin.json +6 -6
- package/opencli-plugin/package.json +8 -8
- package/package.json +200 -185
|
@@ -0,0 +1,472 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Filesystem access for local agent transcripts.
|
|
3
|
+
*
|
|
4
|
+
* Everything that touches the disk lives here, so the adapters stay pure and
|
|
5
|
+
* testable against inline fixtures. Two facts about the corpus shape every
|
|
6
|
+
* function below, both measured rather than assumed:
|
|
7
|
+
*
|
|
8
|
+
* - **Transcripts are large.** 541 Claude files totalling 734 MB and 212 Codex
|
|
9
|
+
* rollouts totalling 2.24 GB sit on the machine this was written against, the
|
|
10
|
+
* largest single rollout being 192 MB. Nothing here may read a whole file.
|
|
11
|
+
* - **Lines are large and wildly uneven.** In one 21 MB rollout: p50 = 812 B,
|
|
12
|
+
* p90 = 14 KB, p99 = 1.4 MB, max = 4.0 MB. A fixed 32 KiB probe window
|
|
13
|
+
* recovers exactly one complete line there — which is why probes escalate by
|
|
14
|
+
* *lines wanted* under a byte ceiling, and why the streaming pass budgets in
|
|
15
|
+
* bytes rather than lines.
|
|
16
|
+
*
|
|
17
|
+
* @module dsh-reference-anything/local-agent/scan
|
|
18
|
+
*/
|
|
19
|
+
import { createReadStream } from 'node:fs';
|
|
20
|
+
import { open, readdir, stat } from 'node:fs/promises';
|
|
21
|
+
import { join, relative } from 'node:path';
|
|
22
|
+
import { ReferenceAnythingError } from "../../errors.js";
|
|
23
|
+
import { withDatabase } from "./sqlite.js";
|
|
24
|
+
import { isQueryAdapter } from "./types.js";
|
|
25
|
+
/** Separates a database from the session inside it, in a reference id. */
|
|
26
|
+
export const SESSION_SEPARATOR = '#';
|
|
27
|
+
/** Directory depth walked below each root. */
|
|
28
|
+
const MAX_DEPTH = 6;
|
|
29
|
+
/** Probe windows, in bytes, tried in order until enough lines are recovered. */
|
|
30
|
+
const PROBE_STEPS = [32 * 1024, 256 * 1024, 2 * 1024 * 1024];
|
|
31
|
+
/** Chunk size for walking a large transcript backwards from its end. */
|
|
32
|
+
const TAIL_CHUNK_BYTES = 1024 * 1024;
|
|
33
|
+
/**
|
|
34
|
+
* Find every transcript under the given roots.
|
|
35
|
+
*
|
|
36
|
+
* Reads directory entries and stats only — no file contents — because this runs
|
|
37
|
+
* behind a keystroke debounce. A root that does not exist is an ordinary state
|
|
38
|
+
* (that agent is not installed) and is skipped rather than raised.
|
|
39
|
+
* @param roots - directories to search, each tagged with its agent.
|
|
40
|
+
* @param adapters - adapters by kind, used to test filenames.
|
|
41
|
+
* @param maxTranscripts - most descriptors to return, newest first.
|
|
42
|
+
* @param signal - cancellation from the caller.
|
|
43
|
+
* @returns descriptors sorted newest-modified first, at most `maxTranscripts`.
|
|
44
|
+
*/
|
|
45
|
+
export async function listTranscripts(roots, adapters, maxTranscripts, signal) {
|
|
46
|
+
const found = [];
|
|
47
|
+
for (const root of roots) {
|
|
48
|
+
const adapter = adapters.get(root.kind);
|
|
49
|
+
if (adapter === undefined)
|
|
50
|
+
continue;
|
|
51
|
+
await walk(root, root.path, 0, adapter, found, signal, maxTranscripts);
|
|
52
|
+
}
|
|
53
|
+
found.sort((left, right) => right.mtimeMs - left.mtimeMs);
|
|
54
|
+
return found.slice(0, maxTranscripts);
|
|
55
|
+
}
|
|
56
|
+
async function walk(root, dir, depth, adapter, found, signal, maxSessions = 200) {
|
|
57
|
+
signal?.throwIfAborted();
|
|
58
|
+
if (depth > MAX_DEPTH)
|
|
59
|
+
return;
|
|
60
|
+
// An unreadable directory is one nothing could be referenced from anyway;
|
|
61
|
+
// the remaining roots still list.
|
|
62
|
+
const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
63
|
+
for (const entry of entries) {
|
|
64
|
+
const path = join(dir, entry.name);
|
|
65
|
+
if (entry.isDirectory()) {
|
|
66
|
+
await walk(root, path, depth + 1, adapter, found, signal, maxSessions);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (!entry.isFile())
|
|
70
|
+
continue;
|
|
71
|
+
const relPath = relative(root.path, path);
|
|
72
|
+
if (!adapter.matches(relPath))
|
|
73
|
+
continue;
|
|
74
|
+
const stats = await stat(path).catch(() => undefined);
|
|
75
|
+
if (stats === undefined)
|
|
76
|
+
continue;
|
|
77
|
+
if (isQueryAdapter(adapter)) {
|
|
78
|
+
found.push(...await expandSessions(root.kind, path, relPath, stats.mtimeMs, stats.size, adapter, maxSessions));
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
found.push({ kind: root.kind, path, relPath, mtimeMs: stats.mtimeMs, size: stats.size });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Turn one database into a descriptor per conversation inside it.
|
|
86
|
+
*
|
|
87
|
+
* This is the only place discovery reads content rather than directory
|
|
88
|
+
* metadata, and it is affordable for the same reason the bookmark table exists:
|
|
89
|
+
* the session list is a single indexed query over a few hundred small rows,
|
|
90
|
+
* where the file-backed formats would need a byte probe per transcript.
|
|
91
|
+
* @param kind - the format this root holds.
|
|
92
|
+
* @param path - absolute path to the database.
|
|
93
|
+
* @param relPath - the database's path relative to its root.
|
|
94
|
+
* @param mtimeMs - the database file's modification time, used when a session has none.
|
|
95
|
+
* @param size - the database file's size, which is what a cursor is validated against.
|
|
96
|
+
* @param adapter - the adapter that knows this schema.
|
|
97
|
+
* @param limit - most sessions to take from this database.
|
|
98
|
+
* @returns one descriptor per session, or none when the database was unreadable.
|
|
99
|
+
*/
|
|
100
|
+
async function expandSessions(kind, path, relPath, mtimeMs, size, adapter, limit) {
|
|
101
|
+
const sessions = await withDatabase(path, db => adapter.sessions(db, limit));
|
|
102
|
+
if (sessions === undefined)
|
|
103
|
+
return [];
|
|
104
|
+
return sessions.map(session => ({
|
|
105
|
+
kind,
|
|
106
|
+
path,
|
|
107
|
+
relPath: `${relPath}${SESSION_SEPARATOR}${session.id}`,
|
|
108
|
+
// The session's own recency, so the menu orders conversations rather than
|
|
109
|
+
// giving every session in one database the same position.
|
|
110
|
+
mtimeMs: session.updatedAt > 0 ? session.updatedAt : mtimeMs,
|
|
111
|
+
size,
|
|
112
|
+
session,
|
|
113
|
+
}));
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Look up the row that describes one conversation.
|
|
117
|
+
*
|
|
118
|
+
* A read may arrive without a listing having preceded it — the model can hold a
|
|
119
|
+
* reference id across turns, and `locate()` resolves a path rather than
|
|
120
|
+
* replaying discovery — so this is what gives a database-backed snapshot its
|
|
121
|
+
* title, working directory, and times.
|
|
122
|
+
* @param path - absolute path to the database.
|
|
123
|
+
* @param adapter - the adapter that knows this schema.
|
|
124
|
+
* @param sessionId - which conversation to describe.
|
|
125
|
+
* @param signal - cancellation from the caller.
|
|
126
|
+
* @returns the session row, or undefined when the database or the row is unreadable.
|
|
127
|
+
*/
|
|
128
|
+
export async function readSessionInfo(path, adapter, sessionId, signal) {
|
|
129
|
+
signal?.throwIfAborted();
|
|
130
|
+
return await withDatabase(path, db => adapter.session(db, sessionId));
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Read one conversation out of a database.
|
|
134
|
+
* @param path - absolute path to the database.
|
|
135
|
+
* @param adapter - the adapter that knows this schema.
|
|
136
|
+
* @param sessionId - which conversation to fold.
|
|
137
|
+
* @param options - projection settings.
|
|
138
|
+
* @param maxRecords - most message rows to materialize, counted from the end.
|
|
139
|
+
* @param signal - cancellation from the caller.
|
|
140
|
+
* @returns the folded turns, oldest first.
|
|
141
|
+
*/
|
|
142
|
+
export async function readSessionTurns(path, adapter, sessionId, options, maxRecords, signal) {
|
|
143
|
+
signal?.throwIfAborted();
|
|
144
|
+
const page = await withDatabase(path, db => adapter.turns(db, sessionId, options, maxRecords));
|
|
145
|
+
if (page === undefined) {
|
|
146
|
+
throw new ReferenceAnythingError(`could not read ${JSON.stringify(adapter.displayName)} database ${JSON.stringify(path)}; it is missing, locked, or in a shape this adapter does not understand`, 'REFERENCE_READ_FAILED');
|
|
147
|
+
}
|
|
148
|
+
return page;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Recover complete records from both ends of a transcript.
|
|
152
|
+
*
|
|
153
|
+
* Escalates through {@link PROBE_STEPS} until enough complete lines are in hand
|
|
154
|
+
* or the byte ceiling is reached, because a fixed window cannot serve a corpus
|
|
155
|
+
* whose p99 line is 1.4 MB. Giving up returns whatever was recovered rather
|
|
156
|
+
* than throwing: a transcript with no readable metadata still lists, under a
|
|
157
|
+
* fallback label.
|
|
158
|
+
* @param path - absolute path to the transcript.
|
|
159
|
+
* @param size - its size in bytes, from a prior stat.
|
|
160
|
+
* @param limits - how many lines are wanted and how far to read for them.
|
|
161
|
+
* @param signal - cancellation from the caller.
|
|
162
|
+
* @returns the complete records found at each end.
|
|
163
|
+
*/
|
|
164
|
+
export async function probeEnds(path, size, limits, signal) {
|
|
165
|
+
signal?.throwIfAborted();
|
|
166
|
+
// Nothing to recover, but the absence is conclusive: an empty file holds no
|
|
167
|
+
// conversation, and saying so beats making the caller guess.
|
|
168
|
+
if (size === 0)
|
|
169
|
+
return { headLines: [], tailLines: [], complete: true };
|
|
170
|
+
const handle = await open(path, 'r');
|
|
171
|
+
try {
|
|
172
|
+
const head = await probeEnd(handle, size, limits.headLines, limits.maxProbeBytes, 'head', signal);
|
|
173
|
+
const tail = await probeEnd(handle, size, limits.tailLines, limits.maxProbeBytes, 'tail', signal);
|
|
174
|
+
return { headLines: head.lines, tailLines: tail.lines, complete: head.covered };
|
|
175
|
+
}
|
|
176
|
+
finally {
|
|
177
|
+
await handle.close();
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
async function probeEnd(handle, size, wanted, maxProbeBytes, end, signal) {
|
|
181
|
+
if (wanted <= 0)
|
|
182
|
+
return { lines: [], covered: false };
|
|
183
|
+
let recovered = [];
|
|
184
|
+
let covered = false;
|
|
185
|
+
for (const step of PROBE_STEPS) {
|
|
186
|
+
signal?.throwIfAborted();
|
|
187
|
+
const span = Math.min(step, maxProbeBytes, size);
|
|
188
|
+
const offset = end === 'head' ? 0 : size - span;
|
|
189
|
+
const buffer = Buffer.alloc(span);
|
|
190
|
+
const { bytesRead } = await handle.read(buffer, 0, span, offset);
|
|
191
|
+
// A partial line sits at the window's inner edge unless the window reached
|
|
192
|
+
// that end of the file, in which case the line really does terminate there.
|
|
193
|
+
recovered = splitComplete(buffer.subarray(0, bytesRead).toString('utf8'), {
|
|
194
|
+
dropFirst: end === 'tail' && offset > 0,
|
|
195
|
+
dropLast: end === 'head' && span < size,
|
|
196
|
+
});
|
|
197
|
+
covered = span >= size;
|
|
198
|
+
if (recovered.length >= wanted || covered || span >= maxProbeBytes)
|
|
199
|
+
break;
|
|
200
|
+
}
|
|
201
|
+
// A window that swallowed the file is returned whole rather than clipped to
|
|
202
|
+
// `wanted`. The extra lines cost nothing to fold, and they are what makes a
|
|
203
|
+
// "no turns here" answer trustworthy — and they sharpen titles besides, since
|
|
204
|
+
// a short session's opening prompt can sit past the line the clip would keep.
|
|
205
|
+
if (end === 'head')
|
|
206
|
+
return { lines: covered ? recovered : recovered.slice(0, wanted), covered };
|
|
207
|
+
return { lines: recovered.slice(-wanted), covered };
|
|
208
|
+
}
|
|
209
|
+
/** Split a window into the lines it holds whole. */
|
|
210
|
+
function splitComplete(text, drop) {
|
|
211
|
+
const lines = text.split('\n');
|
|
212
|
+
if (drop.dropLast)
|
|
213
|
+
lines.pop();
|
|
214
|
+
if (drop.dropFirst)
|
|
215
|
+
lines.shift();
|
|
216
|
+
return lines.filter(line => line.trim() !== '');
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Read one window of turns from a transcript.
|
|
220
|
+
*
|
|
221
|
+
* Takes one of two branches. Under `maxScanBytes` it streams the whole file
|
|
222
|
+
* once, so `startIndex` and `totalTurns` are exact. Over it — the 192 MB
|
|
223
|
+
* rollout — it walks backwards from the end far enough to fill the window,
|
|
224
|
+
* which is honest but cannot see the beginning: `totalTurns` is then undefined
|
|
225
|
+
* and the caller must report the snapshot as partial.
|
|
226
|
+
* @param path - absolute path to the transcript.
|
|
227
|
+
* @param size - its size in bytes, from a prior stat.
|
|
228
|
+
* @param adapter - the format's adapter.
|
|
229
|
+
* @param options - projection settings.
|
|
230
|
+
* @param request - which turns to return.
|
|
231
|
+
* @param limits - how much of the file this read may touch.
|
|
232
|
+
* @param signal - cancellation from the caller.
|
|
233
|
+
* @returns the turns and their position in the transcript.
|
|
234
|
+
*/
|
|
235
|
+
export async function readTurns(path, size, adapter, options, request, limits, signal) {
|
|
236
|
+
// A document adapter folds nothing per line and emits everything from
|
|
237
|
+
// `flush()`, so the tail-anchored path is not available to it: a region of a
|
|
238
|
+
// document is not a smaller document — half a JSON object parses as nothing,
|
|
239
|
+
// and half a branch tree has no leaf to walk back from. It would fold to zero
|
|
240
|
+
// turns and read as "this session was empty" rather than as "this file is too
|
|
241
|
+
// large to read". Refusing says which. The byte cursor is likewise moot here,
|
|
242
|
+
// since only `readFromTail` ever mints one.
|
|
243
|
+
if (adapter.document === true) {
|
|
244
|
+
if (size > limits.maxScanBytes) {
|
|
245
|
+
throw new ReferenceAnythingError(`this transcript is ${size} bytes and its format has to be read whole; raise maxScanBytes past that to read it`, 'REFERENCE_TRANSCRIPT_TOO_LARGE');
|
|
246
|
+
}
|
|
247
|
+
return await readWholeFile(path, adapter, options, request, limits, signal);
|
|
248
|
+
}
|
|
249
|
+
if (size <= limits.maxScanBytes && request.beforeOffset === undefined) {
|
|
250
|
+
return await readWholeFile(path, adapter, options, request, limits, signal);
|
|
251
|
+
}
|
|
252
|
+
return await readFromTail(path, size, adapter, options, request, limits, signal);
|
|
253
|
+
}
|
|
254
|
+
/** One sequential pass: exact indices, memory bounded by the window size. */
|
|
255
|
+
async function readWholeFile(path, adapter, options, request, limits, signal) {
|
|
256
|
+
const state = adapter.createState();
|
|
257
|
+
const window = new TurnRing(request.limit);
|
|
258
|
+
let total = 0;
|
|
259
|
+
const collect = (turns) => {
|
|
260
|
+
for (const turn of turns) {
|
|
261
|
+
if (request.before === undefined || total < request.before)
|
|
262
|
+
window.push(turn, total);
|
|
263
|
+
total += 1;
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
for await (const line of iterateLines(path, limits.maxLineBytes, signal)) {
|
|
267
|
+
collect(adapter.step(line, state, options));
|
|
268
|
+
}
|
|
269
|
+
collect(adapter.flush(state, options));
|
|
270
|
+
const items = window.drain();
|
|
271
|
+
const startIndex = window.firstIndex ?? Math.min(request.before ?? total, total);
|
|
272
|
+
return {
|
|
273
|
+
items,
|
|
274
|
+
startIndex,
|
|
275
|
+
totalTurns: total,
|
|
276
|
+
hasOlder: startIndex > 0,
|
|
277
|
+
compacted: state.compacted,
|
|
278
|
+
anchored: false,
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Walk backwards from the end until the window is full.
|
|
283
|
+
*
|
|
284
|
+
* Each chunk's leading partial line is discarded — except at offset 0, where
|
|
285
|
+
* the line genuinely starts — and the recovered region is then parsed forward,
|
|
286
|
+
* because an adapter folds records in order and cannot run in reverse.
|
|
287
|
+
*/
|
|
288
|
+
async function readFromTail(path, size, adapter, options, request, limits, signal) {
|
|
289
|
+
const end = Math.min(request.beforeOffset ?? size, size);
|
|
290
|
+
const handle = await open(path, 'r');
|
|
291
|
+
try {
|
|
292
|
+
let regionStart = end;
|
|
293
|
+
let attempt = {
|
|
294
|
+
turns: [], offsets: [], compacted: false,
|
|
295
|
+
};
|
|
296
|
+
while (regionStart > 0 && attempt.turns.length < request.limit && end - regionStart < limits.maxScanBytes) {
|
|
297
|
+
signal?.throwIfAborted();
|
|
298
|
+
regionStart = Math.max(0, regionStart - TAIL_CHUNK_BYTES);
|
|
299
|
+
const span = end - regionStart;
|
|
300
|
+
const buffer = Buffer.alloc(span);
|
|
301
|
+
const { bytesRead } = await handle.read(buffer, 0, span, regionStart);
|
|
302
|
+
attempt = foldRegion(buffer.subarray(0, bytesRead), regionStart, regionStart > 0, adapter, options, limits.maxLineBytes);
|
|
303
|
+
}
|
|
304
|
+
const kept = attempt.turns.slice(-request.limit);
|
|
305
|
+
const keptOffsets = attempt.offsets.slice(-request.limit);
|
|
306
|
+
const startOffset = keptOffsets[0] ?? regionStart;
|
|
307
|
+
// Reaching offset 0 with everything folded means this really is the whole
|
|
308
|
+
// transcript, so the exact answers are available after all.
|
|
309
|
+
const sawStart = regionStart === 0 && attempt.turns.length <= request.limit;
|
|
310
|
+
return {
|
|
311
|
+
items: kept,
|
|
312
|
+
startIndex: 0,
|
|
313
|
+
...sawStart ? { totalTurns: attempt.turns.length } : {},
|
|
314
|
+
hasOlder: !sawStart,
|
|
315
|
+
compacted: attempt.compacted,
|
|
316
|
+
anchored: !sawStart,
|
|
317
|
+
...sawStart ? {} : { startOffset },
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
finally {
|
|
321
|
+
await handle.close();
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Fold one recovered byte region forward, remembering where each turn began.
|
|
326
|
+
*
|
|
327
|
+
* "Began" is the load-bearing word. An assistant run is emitted only when the
|
|
328
|
+
* *next* user record closes it, so attributing it to the line that flushed it
|
|
329
|
+
* would place it after records it actually precedes — and the next page back,
|
|
330
|
+
* reading everything below that offset, would emit it a second time. Each turn
|
|
331
|
+
* is therefore stamped with the offset of the first line that contributed to
|
|
332
|
+
* it, which makes the pages abut exactly.
|
|
333
|
+
*/
|
|
334
|
+
function foldRegion(region, regionStart, dropFirstLine, adapter, options, maxLineBytes) {
|
|
335
|
+
const state = adapter.createState();
|
|
336
|
+
const turns = [];
|
|
337
|
+
const offsets = [];
|
|
338
|
+
let runStart;
|
|
339
|
+
let cursor = 0;
|
|
340
|
+
let first = true;
|
|
341
|
+
while (cursor < region.length) {
|
|
342
|
+
const start = cursor;
|
|
343
|
+
const breakAt = region.indexOf(0x0a, start);
|
|
344
|
+
const stop = breakAt === -1 ? region.length : breakAt;
|
|
345
|
+
const offset = regionStart + start;
|
|
346
|
+
const skip = (first && dropFirstLine) || stop - start > maxLineBytes;
|
|
347
|
+
first = false;
|
|
348
|
+
cursor = stop + 1;
|
|
349
|
+
if (skip || stop === start)
|
|
350
|
+
continue;
|
|
351
|
+
for (const turn of adapter.step(region.subarray(start, stop).toString('utf8'), state, options)) {
|
|
352
|
+
turns.push(turn);
|
|
353
|
+
if (turn.role === 'assistant') {
|
|
354
|
+
offsets.push(runStart ?? offset);
|
|
355
|
+
runStart = undefined;
|
|
356
|
+
}
|
|
357
|
+
else {
|
|
358
|
+
offsets.push(offset);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
if (state.pending.length === 0)
|
|
362
|
+
runStart = undefined;
|
|
363
|
+
else
|
|
364
|
+
runStart ??= offset;
|
|
365
|
+
}
|
|
366
|
+
for (const turn of adapter.flush(state, options)) {
|
|
367
|
+
turns.push(turn);
|
|
368
|
+
offsets.push(runStart ?? regionStart + region.length);
|
|
369
|
+
}
|
|
370
|
+
return { turns, offsets, compacted: state.compacted };
|
|
371
|
+
}
|
|
372
|
+
/**
|
|
373
|
+
* Yield a file's lines without ever holding more than one in memory.
|
|
374
|
+
*
|
|
375
|
+
* Splits on `\n` over raw bytes, so a multi-byte character straddling a chunk
|
|
376
|
+
* boundary cannot be corrupted: `0x0a` never appears inside a UTF-8 sequence.
|
|
377
|
+
* A line longer than `maxLineBytes` is dropped rather than materialized —
|
|
378
|
+
* the corpus contains 4 MB records, and the memory bound is the point.
|
|
379
|
+
*/
|
|
380
|
+
async function* iterateLines(path, maxLineBytes, signal) {
|
|
381
|
+
const stream = createReadStream(path);
|
|
382
|
+
let held = [];
|
|
383
|
+
let heldBytes = 0;
|
|
384
|
+
let skipping = false;
|
|
385
|
+
try {
|
|
386
|
+
for await (const chunk of stream) {
|
|
387
|
+
signal?.throwIfAborted();
|
|
388
|
+
let cursor = 0;
|
|
389
|
+
while (cursor < chunk.length) {
|
|
390
|
+
const breakAt = chunk.indexOf(0x0a, cursor);
|
|
391
|
+
if (breakAt === -1) {
|
|
392
|
+
const rest = chunk.subarray(cursor);
|
|
393
|
+
if (!skipping) {
|
|
394
|
+
held.push(rest);
|
|
395
|
+
heldBytes += rest.length;
|
|
396
|
+
if (heldBytes > maxLineBytes) {
|
|
397
|
+
skipping = true;
|
|
398
|
+
held = [];
|
|
399
|
+
heldBytes = 0;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
break;
|
|
403
|
+
}
|
|
404
|
+
if (!skipping) {
|
|
405
|
+
held.push(chunk.subarray(cursor, breakAt));
|
|
406
|
+
heldBytes += breakAt - cursor;
|
|
407
|
+
if (heldBytes <= maxLineBytes) {
|
|
408
|
+
const line = Buffer.concat(held).toString('utf8');
|
|
409
|
+
if (line.trim() !== '')
|
|
410
|
+
yield line;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
held = [];
|
|
414
|
+
heldBytes = 0;
|
|
415
|
+
skipping = false;
|
|
416
|
+
cursor = breakAt + 1;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
if (!skipping && heldBytes > 0 && heldBytes <= maxLineBytes) {
|
|
420
|
+
const line = Buffer.concat(held).toString('utf8');
|
|
421
|
+
if (line.trim() !== '')
|
|
422
|
+
yield line;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
finally {
|
|
426
|
+
stream.destroy();
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
/**
|
|
430
|
+
* The newest `capacity` turns seen so far, and where the oldest of them sits.
|
|
431
|
+
*
|
|
432
|
+
* A transcript can hold thousands of turns and a window asks for tens, so the
|
|
433
|
+
* pass keeps only what it will return.
|
|
434
|
+
*/
|
|
435
|
+
class TurnRing {
|
|
436
|
+
capacity;
|
|
437
|
+
slots;
|
|
438
|
+
indices;
|
|
439
|
+
filled = 0;
|
|
440
|
+
next = 0;
|
|
441
|
+
constructor(capacity) {
|
|
442
|
+
this.capacity = capacity;
|
|
443
|
+
this.slots = new Array(Math.max(capacity, 0));
|
|
444
|
+
this.indices = new Array(Math.max(capacity, 0));
|
|
445
|
+
}
|
|
446
|
+
push(turn, index) {
|
|
447
|
+
if (this.capacity <= 0)
|
|
448
|
+
return;
|
|
449
|
+
this.slots[this.next] = turn;
|
|
450
|
+
this.indices[this.next] = index;
|
|
451
|
+
this.next = (this.next + 1) % this.capacity;
|
|
452
|
+
if (this.filled < this.capacity)
|
|
453
|
+
this.filled += 1;
|
|
454
|
+
}
|
|
455
|
+
/** Index of the oldest retained turn, or undefined when none was kept. */
|
|
456
|
+
get firstIndex() {
|
|
457
|
+
if (this.filled === 0)
|
|
458
|
+
return undefined;
|
|
459
|
+
return this.indices[(this.next - this.filled + this.capacity) % this.capacity];
|
|
460
|
+
}
|
|
461
|
+
/** The retained turns in chronological order. */
|
|
462
|
+
drain() {
|
|
463
|
+
const out = [];
|
|
464
|
+
for (let step = 0; step < this.filled; step += 1) {
|
|
465
|
+
const slot = (this.next - this.filled + step + this.capacity) % this.capacity;
|
|
466
|
+
const turn = this.slots[slot];
|
|
467
|
+
if (turn !== undefined)
|
|
468
|
+
out.push(turn);
|
|
469
|
+
}
|
|
470
|
+
return out;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one place `node:sqlite` is touched.
|
|
3
|
+
*
|
|
4
|
+
* Three of the supported agents keep their history in SQLite rather than in a
|
|
5
|
+
* file per conversation, so reading them needs a database driver. Node has one
|
|
6
|
+
* built in — but only since 22.5, and it still prints an experimental warning
|
|
7
|
+
* on load. Both facts argue for the same shape: import it lazily, from here,
|
|
8
|
+
* and treat its absence as "that format is unreadable on this runtime" rather
|
|
9
|
+
* than as a failure of the plugin. A user with no opencode install never pays
|
|
10
|
+
* for it, because nothing below runs until a database file has actually been
|
|
11
|
+
* found on disk.
|
|
12
|
+
*
|
|
13
|
+
* @module dsh-reference-anything/local-agent/sqlite
|
|
14
|
+
*/
|
|
15
|
+
import type { SqliteReader } from './types.ts';
|
|
16
|
+
/** The slice of `node:sqlite`'s `DatabaseSync` used here. */
|
|
17
|
+
interface Database {
|
|
18
|
+
prepare(sql: string): {
|
|
19
|
+
all(...params: readonly (string | number)[]): unknown[];
|
|
20
|
+
};
|
|
21
|
+
close(): void;
|
|
22
|
+
}
|
|
23
|
+
type DatabaseConstructor = new (path: string, options?: {
|
|
24
|
+
readOnly?: boolean;
|
|
25
|
+
}) => Database;
|
|
26
|
+
/**
|
|
27
|
+
* Load Node's SQLite driver, once.
|
|
28
|
+
* @returns the constructor, or undefined on a runtime that has no `node:sqlite`.
|
|
29
|
+
*/
|
|
30
|
+
export declare function sqliteDriver(): Promise<DatabaseConstructor | undefined>;
|
|
31
|
+
/**
|
|
32
|
+
* Open a database read-only and hand its reader to `use`.
|
|
33
|
+
*
|
|
34
|
+
* The handle never escapes: every query a caller wants happens inside the
|
|
35
|
+
* callback, and the database is closed on the way out however that goes. A
|
|
36
|
+
* database that cannot be opened at all — missing, locked by a live agent
|
|
37
|
+
* holding an exclusive lock, or not SQLite — yields `undefined` rather than
|
|
38
|
+
* throwing, because an agent's own database being busy is an ordinary state and
|
|
39
|
+
* must not take the `@` menu down with it.
|
|
40
|
+
* @param path - absolute path to the database file.
|
|
41
|
+
* @param use - what to read from it.
|
|
42
|
+
* @returns whatever `use` returned, or undefined when the database was unusable.
|
|
43
|
+
*/
|
|
44
|
+
export declare function withDatabase<T>(path: string, use: (db: SqliteReader) => T): Promise<T | undefined>;
|
|
45
|
+
export {};
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one place `node:sqlite` is touched.
|
|
3
|
+
*
|
|
4
|
+
* Three of the supported agents keep their history in SQLite rather than in a
|
|
5
|
+
* file per conversation, so reading them needs a database driver. Node has one
|
|
6
|
+
* built in — but only since 22.5, and it still prints an experimental warning
|
|
7
|
+
* on load. Both facts argue for the same shape: import it lazily, from here,
|
|
8
|
+
* and treat its absence as "that format is unreadable on this runtime" rather
|
|
9
|
+
* than as a failure of the plugin. A user with no opencode install never pays
|
|
10
|
+
* for it, because nothing below runs until a database file has actually been
|
|
11
|
+
* found on disk.
|
|
12
|
+
*
|
|
13
|
+
* @module dsh-reference-anything/local-agent/sqlite
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Resolved driver, or `null` once the runtime has been found not to have one.
|
|
17
|
+
*
|
|
18
|
+
* Cached either way: a Node without `node:sqlite` will not grow one mid-process,
|
|
19
|
+
* and repeating the failed import per keystroke would be pure cost.
|
|
20
|
+
*/
|
|
21
|
+
let driver;
|
|
22
|
+
/**
|
|
23
|
+
* Load Node's SQLite driver, once.
|
|
24
|
+
* @returns the constructor, or undefined on a runtime that has no `node:sqlite`.
|
|
25
|
+
*/
|
|
26
|
+
export async function sqliteDriver() {
|
|
27
|
+
if (driver !== undefined)
|
|
28
|
+
return driver ?? undefined;
|
|
29
|
+
try {
|
|
30
|
+
const module = await import('node:sqlite');
|
|
31
|
+
driver = module.DatabaseSync ?? null;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
// Node older than 22.5, or a build compiled without SQLite. Neither is an
|
|
35
|
+
// error here — the three database-backed formats simply do not list.
|
|
36
|
+
driver = null;
|
|
37
|
+
}
|
|
38
|
+
return driver ?? undefined;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Open a database read-only and hand its reader to `use`.
|
|
42
|
+
*
|
|
43
|
+
* The handle never escapes: every query a caller wants happens inside the
|
|
44
|
+
* callback, and the database is closed on the way out however that goes. A
|
|
45
|
+
* database that cannot be opened at all — missing, locked by a live agent
|
|
46
|
+
* holding an exclusive lock, or not SQLite — yields `undefined` rather than
|
|
47
|
+
* throwing, because an agent's own database being busy is an ordinary state and
|
|
48
|
+
* must not take the `@` menu down with it.
|
|
49
|
+
* @param path - absolute path to the database file.
|
|
50
|
+
* @param use - what to read from it.
|
|
51
|
+
* @returns whatever `use` returned, or undefined when the database was unusable.
|
|
52
|
+
*/
|
|
53
|
+
export async function withDatabase(path, use) {
|
|
54
|
+
const DatabaseSync = await sqliteDriver();
|
|
55
|
+
if (DatabaseSync === undefined)
|
|
56
|
+
return undefined;
|
|
57
|
+
let handle;
|
|
58
|
+
try {
|
|
59
|
+
handle = new DatabaseSync(path, { readOnly: true });
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
try {
|
|
65
|
+
return use(readerFor(handle));
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
// A schema this build does not recognise reads as "no conversations here",
|
|
69
|
+
// which is what a user of a *different* agent's database would expect.
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
finally {
|
|
73
|
+
try {
|
|
74
|
+
handle.close();
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
// Closing a database that failed to open properly is not worth a report.
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/** Wrap a live handle in the read-only surface adapters are given. */
|
|
82
|
+
function readerFor(handle) {
|
|
83
|
+
const columnCache = new Map();
|
|
84
|
+
return {
|
|
85
|
+
all(sql, ...params) {
|
|
86
|
+
return handle.prepare(sql).all(...params).filter(isRow);
|
|
87
|
+
},
|
|
88
|
+
columns(table) {
|
|
89
|
+
const cached = columnCache.get(table);
|
|
90
|
+
if (cached !== undefined)
|
|
91
|
+
return cached;
|
|
92
|
+
let names;
|
|
93
|
+
try {
|
|
94
|
+
// The table name cannot be bound as a parameter in a PRAGMA, so it is
|
|
95
|
+
// constrained instead: only a plain identifier is ever passed, and
|
|
96
|
+
// anything else yields an empty set rather than reaching SQLite.
|
|
97
|
+
names = /^[A-Za-z_][A-Za-z0-9_]*$/u.test(table)
|
|
98
|
+
? new Set(handle.prepare(`PRAGMA table_info(${table})`).all()
|
|
99
|
+
.filter(isRow)
|
|
100
|
+
.map(row => String(row['name'])))
|
|
101
|
+
: new Set();
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
names = new Set();
|
|
105
|
+
}
|
|
106
|
+
columnCache.set(table, names);
|
|
107
|
+
return names;
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
function isRow(value) {
|
|
112
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
113
|
+
}
|