dsh-context-graph 0.6.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +563 -0
- package/README.md +252 -97
- package/README.zh.md +82 -32
- package/cordis.patch.yml +10 -6
- package/index.js +7 -432
- package/package.json +11 -12
- package/src/client/client.js +880 -0
- package/src/core/builder.js +210 -0
- package/src/core/chat.js +34 -0
- package/src/core/graph.js +134 -0
- package/src/core/index.js +14 -0
- package/src/core/ledger.js +108 -0
- package/src/core/match.js +377 -0
- package/src/core/render.js +195 -0
- package/src/core/session.js +446 -0
- package/src/core/text.js +220 -0
- package/src/core/time.js +147 -0
- package/src/core/tree.js +39 -0
- package/src/host/builder.js +416 -0
- package/src/host/command.js +155 -0
- package/src/host/config.js +50 -0
- package/src/host/memory.js +175 -0
- package/src/host/meter.js +75 -0
- package/src/host/page.js +186 -0
- package/src/host/plugin.js +46 -0
- package/src/host/recall.js +227 -0
- package/src/host/runtime.js +599 -0
- package/src/store/chats.js +236 -0
- package/src/store/files.js +181 -0
- package/src/store/ledger.js +65 -0
- package/src/store/messages.js +88 -0
- package/src/store/migrate.js +143 -0
- package/graph.js +0 -496
- package/lib/client.js +0 -126
- package/store.js +0 -81
- /package/{update.js → src/host/update.js} +0 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* On-disk layout under `<dir>` (default `<workspace>/.context-graph`), one folder per chat:
|
|
3
|
+
* chats/<sessionId>/meta.json { cid, title, startedAt, lastActiveAt, turns, builtSeq,
|
|
4
|
+
* memoryOff?, private?, parent?, forkSeq?, legacyRoot? }
|
|
5
|
+
* chats/<sessionId>/topics.json the chat's topic graph (src/core/graph.js, version 2)
|
|
6
|
+
* chats/<sessionId>/msgs.json the verbatim messages its topics cite, each once (messages.js)
|
|
7
|
+
* chats/<sessionId>/graph.json only in folders from v0.4-v0.6: read-only backup (migrate.js)
|
|
8
|
+
* chats/.cid/<n> empty claim files: chat number n is taken
|
|
9
|
+
* Nothing is merged or copied between chats; the list of chats is the list of folders.
|
|
10
|
+
* Writes to one chat folder are serialized in-process (files.js `locked`).
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'
|
|
14
|
+
import { join } from 'node:path'
|
|
15
|
+
|
|
16
|
+
import { GRAPH_VERSION, emptyGraph, normalizeGraph } from '../core/graph.js'
|
|
17
|
+
import { mergeGaps } from '../core/session.js'
|
|
18
|
+
import { locked, readJson, readJsonCached, readJsonDetailed, writeJson } from './files.js'
|
|
19
|
+
import { TOPICS, upgradeChat } from './migrate.js'
|
|
20
|
+
|
|
21
|
+
export const chatDir = (dir, sid) => join(dir, 'chats', String(sid))
|
|
22
|
+
|
|
23
|
+
const exists = (path) => stat(path).then(() => true, () => false)
|
|
24
|
+
|
|
25
|
+
const graphOf = (value) => (value?.version === GRAPH_VERSION ? normalizeGraph(value) : undefined)
|
|
26
|
+
|
|
27
|
+
const LEGACY_CHECK_MS = 60000
|
|
28
|
+
const legacyChecks = new Map()
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Bring a v0.4-v0.6 `graph.json` in this folder up to date, if there is one. Readers look at most
|
|
32
|
+
* once a minute per folder: that stat (and, while the old file is kept as a backup, a re-read of
|
|
33
|
+
* the topics) was a third of the per-message cost at 150 chats (#33). A reader inside the minute
|
|
34
|
+
* waits for the check that is still running, so a folder's first upgrade is never read half-done.
|
|
35
|
+
* The builder looks on every write, so topics an older plugin copy adds to the old file are merged
|
|
36
|
+
* before anything is saved. A failed check is not remembered.
|
|
37
|
+
*/
|
|
38
|
+
function legacyUpgrade(chat, writable) {
|
|
39
|
+
const last = legacyChecks.get(chat)
|
|
40
|
+
if (!writable && last && Date.now() - last.at < LEGACY_CHECK_MS) return last.done
|
|
41
|
+
const entry = { at: Date.now(), done: exists(join(chat, 'graph.json')).then((found) => (found ? upgradeChat(chat) : false)) }
|
|
42
|
+
legacyChecks.set(chat, entry)
|
|
43
|
+
entry.done.catch(() => { if (legacyChecks.get(chat) === entry) legacyChecks.delete(chat) })
|
|
44
|
+
return entry.done
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The chat's graph, upgraded from an older layout when one is present, repaired; empty when there
|
|
49
|
+
* is none. Read-only and shared through the read cache (#33) unless `writable` (the builder).
|
|
50
|
+
*/
|
|
51
|
+
export async function loadGraph(chat, { writable = false } = {}) {
|
|
52
|
+
await legacyUpgrade(chat, writable)
|
|
53
|
+
const file = join(chat, TOPICS)
|
|
54
|
+
const graph = writable ? graphOf(await readJson(file)) : await readJsonCached(file, graphOf)
|
|
55
|
+
return graph ?? emptyGraph()
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export const saveGraph = (chat, graph) => locked(chat, () => writeJson(join(chat, TOPICS), graph))
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* A chat's meta (read-only). One that exists but cannot be read, or is gone while a set-aside copy
|
|
62
|
+
* of it exists, reads as private with memory off (and its chat number, when that can be recovered):
|
|
63
|
+
* the lost file may have said so, and memory on is the default (#50). Nothing is written here;
|
|
64
|
+
* updateMeta writes the same fail-closed meta on the next change.
|
|
65
|
+
*/
|
|
66
|
+
export async function loadMeta(dir, sid) {
|
|
67
|
+
const folder = chatDir(dir, sid)
|
|
68
|
+
const file = join(folder, 'meta.json')
|
|
69
|
+
const meta = await readJsonCached(file)
|
|
70
|
+
if (meta && typeof meta === 'object') return meta
|
|
71
|
+
const lost = await setAside(folder, 'meta.json')
|
|
72
|
+
return lost === undefined ? undefined : Object.freeze({ ...recoverMeta(lost), private: true, memoryOff: true, recovered: true })
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The next free chat number. Claimed with an exclusive create, so two chats starting at once (or two
|
|
77
|
+
* dsh windows on one workspace) can never share one (#11). Existing chats keep their numbers.
|
|
78
|
+
*/
|
|
79
|
+
function claimCid(dir) {
|
|
80
|
+
return locked(`${dir}#cid`, async () => {
|
|
81
|
+
const chats = await readChats(dir)
|
|
82
|
+
let cid = chats.reduce((max, c) => Math.max(max, Number(c.meta.cid) || 0), 0) + 1
|
|
83
|
+
const claims = join(dir, 'chats', '.cid')
|
|
84
|
+
await mkdir(claims, { recursive: true })
|
|
85
|
+
for (; ; cid++) {
|
|
86
|
+
try {
|
|
87
|
+
await writeFile(join(claims, String(cid)), '', { flag: 'wx' })
|
|
88
|
+
return cid
|
|
89
|
+
} catch (error) {
|
|
90
|
+
if (error?.code !== 'EEXIST') throw error
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
})
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** The newest set-aside copy of `name` in `folder` (`<name>.corrupt-<time>`), as text; undefined when there is none. */
|
|
97
|
+
async function setAside(folder, name) {
|
|
98
|
+
let files = []
|
|
99
|
+
try { files = await readdir(folder) } catch { return undefined }
|
|
100
|
+
const newest = files.filter((f) => f.startsWith(`${name}.corrupt-`)).sort().at(-1)
|
|
101
|
+
return newest ? readFile(join(folder, newest), 'utf8') : undefined
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* What can still be read from an unreadable meta.json: its chat number, index and fork positions,
|
|
106
|
+
* the title and summary it had (made safely before the damage), and the stretches typed while
|
|
107
|
+
* memory was off (#59). A meta whose stretches cannot be read back may have had some, so every turn
|
|
108
|
+
* of that chat stays closed to other chats, and no title is taken from its log (review).
|
|
109
|
+
*/
|
|
110
|
+
function recoverMeta(raw) {
|
|
111
|
+
const text = String(raw ?? '')
|
|
112
|
+
const number = (name) => {
|
|
113
|
+
const m = new RegExp(`"${name}"\\s*:\\s*(\\d+)`).exec(text)
|
|
114
|
+
return m ? Number(m[1]) : undefined
|
|
115
|
+
}
|
|
116
|
+
const string = (name) => {
|
|
117
|
+
const m = new RegExp(`"${name}"\\s*:\\s*("(?:[^"\\\\]|\\\\.)*")`).exec(text)
|
|
118
|
+
try { return m ? JSON.parse(m[1]) : undefined } catch { return undefined }
|
|
119
|
+
}
|
|
120
|
+
const out = {}
|
|
121
|
+
for (const name of ['cid', 'builtSeq', 'offSeq', 'forkSeq', 'closedThrough']) if (number(name) !== undefined) out[name] = number(name)
|
|
122
|
+
for (const name of ['title', 'summary', 'firstKey']) if (typeof string(name) === 'string') out[name] = string(name)
|
|
123
|
+
const m = /"gaps"\s*:\s*(\[(?:\s*\[\s*-?\d+\s*,\s*-?\d+\s*\]\s*,?)*\s*\])/.exec(text)
|
|
124
|
+
let gaps
|
|
125
|
+
try { gaps = m ? JSON.parse(m[1]) : undefined } catch { gaps = undefined }
|
|
126
|
+
if (Array.isArray(gaps)) out.gaps = gaps
|
|
127
|
+
// A meta repaired before stays unsure where it was (a second damage must not reopen those turns).
|
|
128
|
+
if (!Array.isArray(gaps) || /"turnsClosed"\s*:\s*true/.test(text)) out.turnsClosed = true
|
|
129
|
+
return out
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Read-modify-write of one chat's meta under the chat's lock, so a flag set by a command and a
|
|
134
|
+
* build finishing at the same moment can no longer undo each other (#10). A chat seen for the first
|
|
135
|
+
* time gets `init()` plus a freshly claimed chat number. A meta.json that exists but cannot be read
|
|
136
|
+
* is never replaced by a fresh identity: its chat number and index position are recovered where
|
|
137
|
+
* possible, and the chat fails closed (private, memory off) because the lost file may have said so.
|
|
138
|
+
*/
|
|
139
|
+
export function updateMeta(dir, sid, change = (meta) => meta, init = () => ({})) {
|
|
140
|
+
return locked(chatDir(dir, sid), async () => {
|
|
141
|
+
const file = join(chatDir(dir, sid), 'meta.json')
|
|
142
|
+
const read = await readJsonDetailed(file)
|
|
143
|
+
let meta = read.value
|
|
144
|
+
if (!meta || typeof meta !== 'object') {
|
|
145
|
+
// An unreadable meta may already have been set aside by a reader (listChats): use that copy.
|
|
146
|
+
const lost = read.state === 'quarantined' ? read.raw : await setAside(chatDir(dir, sid), 'meta.json')
|
|
147
|
+
// A fresh meta's title comes from the log, which may hold an off stretch the lost file knew of:
|
|
148
|
+
// a recovered meta keeps only the title it can read back (review).
|
|
149
|
+
// A fork's fresh meta holds its ancestors' off stretches (offsInherited): they join the ones
|
|
150
|
+
// read back, never give way to them (second review of #69).
|
|
151
|
+
const fresh = await init()
|
|
152
|
+
const back = lost === undefined ? undefined : recoverMeta(lost)
|
|
153
|
+
meta = back === undefined ? fresh : { ...fresh, title: undefined, ...back, ...(Array.isArray(fresh?.gaps) && Array.isArray(back.gaps) ? { gaps: mergeGaps(fresh.gaps, back.gaps) } : {}) }
|
|
154
|
+
if (lost !== undefined) Object.assign(meta, { private: true, memoryOff: true, recovered: new Date().toISOString() })
|
|
155
|
+
if (!Number.isInteger(meta.cid)) meta.cid = await claimCid(dir)
|
|
156
|
+
}
|
|
157
|
+
const next = change({ ...meta }) ?? meta
|
|
158
|
+
await writeJson(file, next)
|
|
159
|
+
return next
|
|
160
|
+
})
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Every chat folder with a readable meta.json, unsorted. Folders starting with '.' are bookkeeping. */
|
|
164
|
+
async function readChats(dir) {
|
|
165
|
+
let names = []
|
|
166
|
+
try { names = await readdir(join(dir, 'chats')) } catch (error) { if (error?.code !== 'ENOENT') throw error }
|
|
167
|
+
const metas = await Promise.all(names.filter((n) => !n.startsWith('.')).map(async (sid) => ({
|
|
168
|
+
sid,
|
|
169
|
+
// One entry that cannot be read (a stray file, a locked folder) must not hide every other chat.
|
|
170
|
+
meta: await loadMeta(dir, sid).catch((error) => {
|
|
171
|
+
console.error(`context-graph: skipping chat ${sid} (${error?.code ?? error?.message})`)
|
|
172
|
+
return undefined
|
|
173
|
+
})
|
|
174
|
+
})))
|
|
175
|
+
return metas.filter((c) => c.meta && typeof c.meta === 'object')
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const migrations = new Map()
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Every chat with a readable meta.json, newest activity first. A meta.json that does not parse is
|
|
182
|
+
* skipped instead of failing the whole workspace (#10). The first call for a workspace also brings
|
|
183
|
+
* memory written by v0.1-v0.3 into the chat layout (#17); a migration that failed is retried by a
|
|
184
|
+
* later call, at most once a minute.
|
|
185
|
+
*/
|
|
186
|
+
export async function listChats(dir) {
|
|
187
|
+
const entry = migrations.get(dir)
|
|
188
|
+
if (!entry || (entry.failedAt && Date.now() - entry.failedAt > 60000)) {
|
|
189
|
+
const next = {}
|
|
190
|
+
next.done = migrateRootLayout(dir).catch((error) => {
|
|
191
|
+
next.failedAt = Date.now()
|
|
192
|
+
console.error('context-graph: memory from before v0.4 is not migrated yet; retrying later', error)
|
|
193
|
+
})
|
|
194
|
+
migrations.set(dir, next)
|
|
195
|
+
}
|
|
196
|
+
await migrations.get(dir).done
|
|
197
|
+
const chats = await readChats(dir)
|
|
198
|
+
return chats.sort((a, b) => String(b.meta.lastActiveAt ?? '').localeCompare(String(a.meta.lastActiveAt ?? '')))
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* v0.1-v0.3 kept one graph per workspace at `<dir>/graph.json` with sources in `<dir>/src/`; v0.4
|
|
203
|
+
* moved to one folder per chat and stopped reading it, so that memory became invisible (#17).
|
|
204
|
+
* The files are COPIED into a chat folder (the original session's id when every topic came from one
|
|
205
|
+
* session, else `legacy`), given a meta, and upgraded; the root copy is removed only after all of
|
|
206
|
+
* that succeeded, so an interrupted run simply repeats. A folder that belongs to a real chat is never
|
|
207
|
+
* used as the target.
|
|
208
|
+
*/
|
|
209
|
+
export function migrateRootLayout(dir) {
|
|
210
|
+
return locked(`${dir}#root`, async () => {
|
|
211
|
+
const root = join(dir, 'graph.json')
|
|
212
|
+
const graph = await readJson(root)
|
|
213
|
+
if (!graph || typeof graph.nodes !== 'object' || graph.nodes === null) return undefined
|
|
214
|
+
const nodes = Object.values(graph.nodes)
|
|
215
|
+
const sessions = new Set(nodes.flatMap((n) => (Array.isArray(n.src) ? n.src : []).map((s) => s?.session)).filter(Boolean))
|
|
216
|
+
const candidates = [...(sessions.size === 1 ? [String([...sessions][0])] : []), 'legacy', ...Array.from({ length: 20 }, (_, i) => `legacy-${i + 2}`)]
|
|
217
|
+
let sid
|
|
218
|
+
for (const name of candidates) {
|
|
219
|
+
// Free, half-copied by an interrupted run (no meta.json yet), or already ours: never a real chat.
|
|
220
|
+
const meta = join(chatDir(dir, name), 'meta.json')
|
|
221
|
+
if (!(await exists(chatDir(dir, name))) || !(await exists(meta)) || (await loadMeta(dir, name))?.legacyRoot === true) { sid = name; break }
|
|
222
|
+
}
|
|
223
|
+
if (!sid) throw new Error('no free folder name for the migrated memory')
|
|
224
|
+
const chat = chatDir(dir, sid)
|
|
225
|
+
await mkdir(chat, { recursive: true })
|
|
226
|
+
await cp(root, join(chat, 'graph.json'))
|
|
227
|
+
if (await exists(join(dir, 'src'))) await cp(join(dir, 'src'), join(chat, 'src'), { recursive: true })
|
|
228
|
+
const times = nodes.flatMap((n) => [n.updated, ...(Array.isArray(n.src) ? n.src : []).map((s) => s?.at)]).filter(Boolean).sort()
|
|
229
|
+
const now = new Date().toISOString()
|
|
230
|
+
await updateMeta(dir, sid, (meta) => ({ ...meta, legacyRoot: true }), () => ({ title: 'Memory from before v0.4', startedAt: times[0] ?? now, lastActiveAt: times.at(-1) ?? now, turns: 0 }))
|
|
231
|
+
await upgradeChat(chat)
|
|
232
|
+
await rm(root, { force: true })
|
|
233
|
+
await rm(join(dir, 'src'), { recursive: true, force: true })
|
|
234
|
+
return sid
|
|
235
|
+
})
|
|
236
|
+
}
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JSON files on disk, written so that concurrent writers and crashes cannot leave a half-written
|
|
3
|
+
* file, and read so that one damaged file can neither take the rest of the memory down nor be
|
|
4
|
+
* silently overwritten.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { randomUUID } from 'node:crypto'
|
|
8
|
+
import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
9
|
+
import { dirname } from 'node:path'
|
|
10
|
+
|
|
11
|
+
/** Index just past the first complete top-level JSON value in `raw` (string-aware), or 0. */
|
|
12
|
+
function firstValueEnd(raw) {
|
|
13
|
+
let depth = 0
|
|
14
|
+
let inString = false
|
|
15
|
+
for (let i = 0; i < raw.length; i++) {
|
|
16
|
+
const ch = raw[i]
|
|
17
|
+
if (inString) {
|
|
18
|
+
if (ch === '\\') i++
|
|
19
|
+
else if (ch === '"') inString = false
|
|
20
|
+
} else if (ch === '"') inString = true
|
|
21
|
+
else if (ch === '{' || ch === '[') depth++
|
|
22
|
+
else if ((ch === '}' || ch === ']') && --depth === 0) return i + 1
|
|
23
|
+
}
|
|
24
|
+
return 0
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The JSON value at the start of `raw` when only junk follows it: what v0.6's interleaved writes
|
|
29
|
+
* left behind. Cut where the parser says the junk starts, else after the first complete value; both
|
|
30
|
+
* are one pass, however long the junk (#54).
|
|
31
|
+
*/
|
|
32
|
+
function salvage(raw, error) {
|
|
33
|
+
const at = /at position (\d+)/.exec(String(error?.message))
|
|
34
|
+
for (const end of [at ? Number(at[1]) : 0, firstValueEnd(raw)]) {
|
|
35
|
+
if (end <= 0) continue
|
|
36
|
+
try { return JSON.parse(raw.slice(0, end)) } catch { /* try the next cut */ }
|
|
37
|
+
}
|
|
38
|
+
return undefined
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Read a JSON file and say what was found:
|
|
43
|
+
* { state: 'ok', value } parsed
|
|
44
|
+
* { state: 'missing' } no such file
|
|
45
|
+
* { state: 'salvaged', value } trailing junk cut off (v0.6 could leave it); the next write repairs the file
|
|
46
|
+
* { state: 'quarantined', raw } unreadable; moved aside to `<file>.corrupt-<time>` so nothing overwrites it
|
|
47
|
+
* Errors other than "no such file" are thrown: the caller must not treat them as missing data (#10).
|
|
48
|
+
* So is an unreadable file that cannot be moved aside (#79).
|
|
49
|
+
*/
|
|
50
|
+
export async function readJsonDetailed(path) {
|
|
51
|
+
let raw
|
|
52
|
+
try {
|
|
53
|
+
raw = await readFile(path, 'utf8')
|
|
54
|
+
} catch (error) {
|
|
55
|
+
if (error?.code === 'ENOENT') return { state: 'missing' }
|
|
56
|
+
throw error
|
|
57
|
+
}
|
|
58
|
+
try {
|
|
59
|
+
return { state: 'ok', value: JSON.parse(raw) }
|
|
60
|
+
} catch (error) {
|
|
61
|
+
const repaired = salvage(raw, error)
|
|
62
|
+
if (repaired !== undefined) {
|
|
63
|
+
console.error(`context-graph: repaired ${path} (${error.message}); it is rewritten on the next save`)
|
|
64
|
+
return { state: 'salvaged', value: repaired }
|
|
65
|
+
}
|
|
66
|
+
const aside = `${path}.corrupt-${Date.now()}`
|
|
67
|
+
// Gone: another reader moved it first. Any other failure (on Windows a sync tool or a scan that
|
|
68
|
+
// holds the file: EBUSY) leaves it in place, and reading it as missing let a memory-off chat
|
|
69
|
+
// read as on and a writer replace the file with no copy kept (#79).
|
|
70
|
+
try { await rename(path, aside) } catch (error) { if (error?.code !== 'ENOENT') throw error }
|
|
71
|
+
console.error(`context-graph: ${path} is unreadable (${error.message}); kept as ${aside}`)
|
|
72
|
+
return { state: 'quarantined', raw }
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Parsed JSON, or undefined when the file is missing or unreadable (see readJsonDetailed). */
|
|
77
|
+
export async function readJson(path) {
|
|
78
|
+
return (await readJsonDetailed(path)).value
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const cache = new Map()
|
|
82
|
+
// Two or three entries per chat are read on every message, so the cap sits far above any real
|
|
83
|
+
// workspace: at 2,000 a workspace of 1,500 chats evicted what the same pass needed next and took
|
|
84
|
+
// about 1 s per message (#63). ponytail: entry count, not bytes; size by bytes if memory matters.
|
|
85
|
+
const CACHE_MAX = 20000
|
|
86
|
+
|
|
87
|
+
function deepFreeze(value) {
|
|
88
|
+
if (value && typeof value === 'object' && !Object.isFrozen(value)) {
|
|
89
|
+
Object.freeze(value)
|
|
90
|
+
for (const v of Object.values(value)) deepFreeze(v)
|
|
91
|
+
}
|
|
92
|
+
return value
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Parsed JSON for READ-ONLY use, cached by the file's stat (#33): while a file is unchanged it is
|
|
97
|
+
* not read or parsed again, which took the per-message cost at 150 chats from 37 ms of reads to a
|
|
98
|
+
* few stats. `prepare(value)` runs once per change (to normalize a graph). The result is frozen and
|
|
99
|
+
* shared, so code that tries to change it fails loudly instead of corrupting every later read.
|
|
100
|
+
* Writers use readJson / readJsonDetailed.
|
|
101
|
+
*/
|
|
102
|
+
export async function readJsonCached(path, prepare = (value) => value) {
|
|
103
|
+
let st
|
|
104
|
+
try {
|
|
105
|
+
st = await stat(path)
|
|
106
|
+
} catch (error) {
|
|
107
|
+
if (error?.code !== 'ENOENT') throw error
|
|
108
|
+
cache.delete(path)
|
|
109
|
+
return undefined
|
|
110
|
+
}
|
|
111
|
+
const key = `${st.mtimeMs}:${st.size}:${st.ino}`
|
|
112
|
+
const hit = cache.get(path)
|
|
113
|
+
if (hit?.key === key) {
|
|
114
|
+
cache.delete(path)
|
|
115
|
+
cache.set(path, hit)
|
|
116
|
+
return hit.value
|
|
117
|
+
}
|
|
118
|
+
const read = await readJsonDetailed(path)
|
|
119
|
+
const value = read.value === undefined ? undefined : deepFreeze(prepare(read.value))
|
|
120
|
+
// A repaired value is cached too, so the repair (and its log line) happens once per change of the
|
|
121
|
+
// file, not on every message until the next save rewrites it.
|
|
122
|
+
if (read.state === 'ok' || read.state === 'salvaged') {
|
|
123
|
+
cache.set(path, { key, value })
|
|
124
|
+
if (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value)
|
|
125
|
+
}
|
|
126
|
+
return value
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Rename with a retry: on Windows a rename over a file fails while anyone has that file open, even
|
|
131
|
+
* for a moment (another chat reading it, a sync tool, a virus scan, which can take seconds on a
|
|
132
|
+
* busy machine). Backing off for up to about 5 s outlasts such holds; the old 250 ms lost a meta
|
|
133
|
+
* write now and then under load (review). Writes are off the critical path, so waiting costs nothing.
|
|
134
|
+
*/
|
|
135
|
+
export async function renameRetry(from, to) {
|
|
136
|
+
for (let attempt = 0; ; attempt++) {
|
|
137
|
+
try {
|
|
138
|
+
await rename(from, to)
|
|
139
|
+
return
|
|
140
|
+
} catch (error) {
|
|
141
|
+
if (attempt < 11 && (error?.code === 'EPERM' || error?.code === 'EBUSY' || error?.code === 'EACCES')) {
|
|
142
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(800, 25 * 2 ** attempt)))
|
|
143
|
+
continue
|
|
144
|
+
}
|
|
145
|
+
throw error
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Atomic write: a uniquely named temp file, then rename over the target. A shared temp name let two
|
|
152
|
+
* concurrent writes interleave and corrupt the file (#10).
|
|
153
|
+
*/
|
|
154
|
+
export async function writeJson(path, value) {
|
|
155
|
+
await mkdir(dirname(path), { recursive: true })
|
|
156
|
+
const tmp = `${path}.${process.pid}.${randomUUID().slice(0, 8)}.tmp`
|
|
157
|
+
await writeFile(tmp, JSON.stringify(value), 'utf8')
|
|
158
|
+
try {
|
|
159
|
+
await renameRetry(tmp, path)
|
|
160
|
+
} catch (error) {
|
|
161
|
+
await rm(tmp, { force: true })
|
|
162
|
+
throw error
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const chains = new Map()
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Run `fn` after every earlier call with the same key has settled, so read-modify-write sequences on
|
|
170
|
+
* one chat folder never interleave inside this process. Never call `locked` with a key from inside
|
|
171
|
+
* a callback holding the same key: it would wait for itself. ponytail: in-process only; two dsh
|
|
172
|
+
* processes writing one workspace still race (chat numbers are protected separately, see chats.js).
|
|
173
|
+
*/
|
|
174
|
+
export function locked(key, fn) {
|
|
175
|
+
const previous = chains.get(key) ?? Promise.resolve()
|
|
176
|
+
const run = previous.then(fn, fn)
|
|
177
|
+
const tail = run.then(() => {}, () => {})
|
|
178
|
+
chains.set(key, tail)
|
|
179
|
+
tail.then(() => { if (chains.get(key) === tail) chains.delete(key) })
|
|
180
|
+
return run
|
|
181
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The savings ledger of a workspace (#35): one small file per chat, `<dir>/ledger/<sessionId>.json`,
|
|
3
|
+
* so the estimate survives restarts and a turn end writes only its own chat's entry, not the whole
|
|
4
|
+
* workspace's (#83). Written at most once per turn per chat, off the critical path; a chat needs no
|
|
5
|
+
* folder of its own to have an entry.
|
|
6
|
+
* Format: `{ c: counters, used: [topic ids], msgs: ["<cid>:<seq>"] }`, zero counters and empty lists
|
|
7
|
+
* left out (#78): `msgs` are the source messages already counted as stood in for, so each counts
|
|
8
|
+
* once per chat. `<dir>/ledger.json`, the one file 0.9.0 kept (`{ v: 1, chats: { "<sessionId>":
|
|
9
|
+
* entry } }`), is only read: a chat's entry there is where its own file starts.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { readdir } from 'node:fs/promises'
|
|
13
|
+
import { join } from 'node:path'
|
|
14
|
+
|
|
15
|
+
import { addCounters, counters, emptyCounters } from '../core/ledger.js'
|
|
16
|
+
import { locked, readJson, readJsonCached, writeJson } from './files.js'
|
|
17
|
+
|
|
18
|
+
const folder = (dir) => join(dir, 'ledger')
|
|
19
|
+
const file = (dir, sid) => join(folder(dir), `${sid}.json`)
|
|
20
|
+
|
|
21
|
+
/** Topic ids remembered per chat as used, so each counts once; the newest are kept. */
|
|
22
|
+
const USED_MAX = 500
|
|
23
|
+
const MSGS_MAX = 1000
|
|
24
|
+
|
|
25
|
+
/** The entries 0.9.0 kept in one file, read-only (cached: nothing writes that file any more). */
|
|
26
|
+
async function oldEntries(dir) {
|
|
27
|
+
const value = await readJsonCached(join(dir, 'ledger.json'))
|
|
28
|
+
return value && typeof value.chats === 'object' && value.chats !== null ? value.chats : {}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Every chat's entry, read-only (cached per file): `{ v: 1, chats: { "<sessionId>": entry } }`. */
|
|
32
|
+
export async function loadLedger(dir) {
|
|
33
|
+
const chats = { ...(await oldEntries(dir)) }
|
|
34
|
+
let names = []
|
|
35
|
+
try { names = (await readdir(folder(dir))).filter((name) => name.endsWith('.json')) } catch (error) { if (error?.code !== 'ENOENT') throw error }
|
|
36
|
+
const entries = await Promise.all(names.map((name) => readJsonCached(join(folder(dir), name))))
|
|
37
|
+
names.forEach((name, i) => { if (entries[i]) chats[name.slice(0, -'.json'.length)] = entries[i] })
|
|
38
|
+
return { v: 1, chats }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** One chat's counters, used topic ids and credited source messages. */
|
|
42
|
+
export async function chatEntry(dir, sid) {
|
|
43
|
+
const entry = (await readJsonCached(file(dir, sid))) ?? (await oldEntries(dir))[String(sid)]
|
|
44
|
+
return { c: counters(entry?.c), used: Array.isArray(entry?.used) ? entry.used : [], msgs: Array.isArray(entry?.msgs) ? entry.msgs : [] }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The workspace total. */
|
|
48
|
+
export async function workspaceTotal(dir) {
|
|
49
|
+
return Object.values((await loadLedger(dir)).chats).reduce((sum, entry) => addCounters(sum, counters(entry?.c)), emptyCounters())
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Add `delta` to a chat's counters and remember newly used topic ids. Locked read-modify-write of
|
|
54
|
+
* the chat's own file, so two writes for one chat (a slow turn end, then the next) cannot undo each other.
|
|
55
|
+
*/
|
|
56
|
+
export function addToLedger(dir, sid, delta, usedIds = [], msgKeys = []) {
|
|
57
|
+
const path = file(dir, sid)
|
|
58
|
+
return locked(path, async () => {
|
|
59
|
+
const prev = (await readJson(path)) ?? (await oldEntries(dir))[String(sid)] ?? {}
|
|
60
|
+
const used = [...new Set([...(Array.isArray(prev.used) ? prev.used : []), ...usedIds])].slice(-USED_MAX)
|
|
61
|
+
const msgs = [...new Set([...(Array.isArray(prev.msgs) ? prev.msgs : []), ...msgKeys])].slice(-MSGS_MAX)
|
|
62
|
+
const c = Object.fromEntries(Object.entries(addCounters(counters(prev.c), delta)).filter(([, n]) => n !== 0))
|
|
63
|
+
await writeJson(path, { c, ...(used.length ? { used } : {}), ...(msgs.length ? { msgs } : {}) })
|
|
64
|
+
})
|
|
65
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The deepest stored layer: the verbatim messages topics were drawn from, one file per chat
|
|
3
|
+
* (`chats/<sid>/msgs.json`), each message stored ONCE under its seq however many topics cite it.
|
|
4
|
+
* Up to v0.6 every topic kept its own copy (3x on real data) and grew a section per rebuild (#12).
|
|
5
|
+
* Format: `{ v: 2, m: { "<seq>": { r: "user"|"assistant", t: turn, a: time, x: text } } }`, `t`
|
|
6
|
+
* being dsh's own turn number (#71); a `v: 1` store (up to 0.9.0) counted typed prompts instead.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { join } from 'node:path'
|
|
10
|
+
|
|
11
|
+
import { capText } from '../core/text.js'
|
|
12
|
+
import { locked, readJson, readJsonCached, writeJson } from './files.js'
|
|
13
|
+
|
|
14
|
+
/** Long messages keep their head and tail (core/text.js); the full text stays in dsh's own session log. */
|
|
15
|
+
export { capText }
|
|
16
|
+
|
|
17
|
+
const file = (chat) => join(chat, 'msgs.json')
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* The store for READ-ONLY use (cached, #33). A file that cannot be read now (a damaged one another
|
|
21
|
+
* program holds, #79) reads as nothing stored: readers show no source instead of failing the whole
|
|
22
|
+
* block or recall, as the page and the savings estimate already did (review of #79). Writers use readJson.
|
|
23
|
+
*/
|
|
24
|
+
const readStore = (chat) => readJsonCached(file(chat)).catch(() => undefined)
|
|
25
|
+
|
|
26
|
+
/** `{ [seq]: { r, t, a, x } }` for one chat, read-only; empty when nothing is stored (or readable) yet. */
|
|
27
|
+
export async function loadMessages(chat) {
|
|
28
|
+
const store = await readStore(chat)
|
|
29
|
+
return store && typeof store.m === 'object' && store.m !== null ? store.m : {}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Whether a chat's stored turn ids still count typed prompts (a store from 0.9.0 or older): such an
|
|
34
|
+
* id names another turn than dsh's, so it is not shown until relabelTurns converts it (review of #71).
|
|
35
|
+
*/
|
|
36
|
+
export async function oldTurns(chat) {
|
|
37
|
+
const store = await readStore(chat)
|
|
38
|
+
return store?.v !== 2 && Object.values(store?.m ?? {}).some((m) => m?.t !== undefined)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Add messages (`[{ seq, role, text, turn, time }]`) that are not stored yet. Idempotent: writing
|
|
43
|
+
* the same seq twice is a no-op, so a build that is retried after a crash cannot duplicate text.
|
|
44
|
+
*/
|
|
45
|
+
export function saveMessages(chat, messages) {
|
|
46
|
+
return locked(`${chat}#msgs`, async () => {
|
|
47
|
+
const store = (await readJson(file(chat))) ?? { v: 2, m: {} }
|
|
48
|
+
if (typeof store.m !== 'object' || store.m === null) store.m = {}
|
|
49
|
+
let changed = false
|
|
50
|
+
for (const m of messages) {
|
|
51
|
+
const key = String(m.seq)
|
|
52
|
+
if (!Number.isInteger(m.seq) || Object.hasOwn(store.m, key)) continue
|
|
53
|
+
store.m[key] = { r: m.role, ...(m.turn ? { t: m.turn } : {}), ...(m.time ? { a: m.time } : {}), x: capText(m.text) }
|
|
54
|
+
changed = true
|
|
55
|
+
}
|
|
56
|
+
if (changed) await writeJson(file(chat), store)
|
|
57
|
+
})
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Give stored messages the turn `turnAt(seq)` says (#71: turn ids follow dsh's own turns now) and
|
|
62
|
+
* mark the store as numbered so (v 2). Under the lock saveMessages takes; when nothing changes,
|
|
63
|
+
* nothing is written.
|
|
64
|
+
*/
|
|
65
|
+
export function relabelTurns(chat, turnAt) {
|
|
66
|
+
return locked(`${chat}#msgs`, async () => {
|
|
67
|
+
const store = await readJson(file(chat))
|
|
68
|
+
if (typeof store?.m !== 'object' || store.m === null) return
|
|
69
|
+
let changed = store.v !== 2
|
|
70
|
+
for (const [seq, m] of Object.entries(store.m)) {
|
|
71
|
+
const t = turnAt(Number(seq))
|
|
72
|
+
if (t === undefined || !m || typeof m !== 'object' || m.t === t) continue
|
|
73
|
+
m.t = t
|
|
74
|
+
changed = true
|
|
75
|
+
}
|
|
76
|
+
if (changed) await writeJson(file(chat), { ...store, v: 2 })
|
|
77
|
+
})
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The stored messages a topic cites, oldest first: `[{ seq, role, turn, text }]`. */
|
|
81
|
+
export function sourceOf(messages, node) {
|
|
82
|
+
return (node?.refs ?? [])
|
|
83
|
+
.filter((seq) => Object.hasOwn(messages, String(seq)))
|
|
84
|
+
.map((seq) => {
|
|
85
|
+
const m = messages[String(seq)]
|
|
86
|
+
return { seq, role: m.r, turn: m.t, text: m.x }
|
|
87
|
+
})
|
|
88
|
+
}
|