dsh-context-graph 0.6.2 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,236 @@
1
+ /**
2
+ * On-disk layout under `<dir>` (default `<workspace>/.context-graph`), one folder per chat:
3
+ * chats/<sessionId>/meta.json { cid, title, startedAt, lastActiveAt, turns, builtSeq,
4
+ * memoryOff?, private?, parent?, forkSeq?, legacyRoot? }
5
+ * chats/<sessionId>/topics.json the chat's topic graph (src/core/graph.js, version 2)
6
+ * chats/<sessionId>/msgs.json the verbatim messages its topics cite, each once (messages.js)
7
+ * chats/<sessionId>/graph.json only in folders from v0.4-v0.6: read-only backup (migrate.js)
8
+ * chats/.cid/<n> empty claim files: chat number n is taken
9
+ * Nothing is merged or copied between chats; the list of chats is the list of folders.
10
+ * Writes to one chat folder are serialized in-process (files.js `locked`).
11
+ */
12
+
13
+ import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'
14
+ import { join } from 'node:path'
15
+
16
+ import { GRAPH_VERSION, emptyGraph, normalizeGraph } from '../core/graph.js'
17
+ import { mergeGaps } from '../core/session.js'
18
+ import { locked, readJson, readJsonCached, readJsonDetailed, writeJson } from './files.js'
19
+ import { TOPICS, upgradeChat } from './migrate.js'
20
+
21
+ export const chatDir = (dir, sid) => join(dir, 'chats', String(sid))
22
+
23
+ const exists = (path) => stat(path).then(() => true, () => false)
24
+
25
+ const graphOf = (value) => (value?.version === GRAPH_VERSION ? normalizeGraph(value) : undefined)
26
+
27
+ const LEGACY_CHECK_MS = 60000
28
+ const legacyChecks = new Map()
29
+
30
+ /**
31
+ * Bring a v0.4-v0.6 `graph.json` in this folder up to date, if there is one. Readers look at most
32
+ * once a minute per folder: that stat (and, while the old file is kept as a backup, a re-read of
33
+ * the topics) was a third of the per-message cost at 150 chats (#33). A reader inside the minute
34
+ * waits for the check that is still running, so a folder's first upgrade is never read half-done.
35
+ * The builder looks on every write, so topics an older plugin copy adds to the old file are merged
36
+ * before anything is saved. A failed check is not remembered.
37
+ */
38
+ function legacyUpgrade(chat, writable) {
39
+ const last = legacyChecks.get(chat)
40
+ if (!writable && last && Date.now() - last.at < LEGACY_CHECK_MS) return last.done
41
+ const entry = { at: Date.now(), done: exists(join(chat, 'graph.json')).then((found) => (found ? upgradeChat(chat) : false)) }
42
+ legacyChecks.set(chat, entry)
43
+ entry.done.catch(() => { if (legacyChecks.get(chat) === entry) legacyChecks.delete(chat) })
44
+ return entry.done
45
+ }
46
+
47
+ /**
48
+ * The chat's graph, upgraded from an older layout when one is present, repaired; empty when there
49
+ * is none. Read-only and shared through the read cache (#33) unless `writable` (the builder).
50
+ */
51
+ export async function loadGraph(chat, { writable = false } = {}) {
52
+ await legacyUpgrade(chat, writable)
53
+ const file = join(chat, TOPICS)
54
+ const graph = writable ? graphOf(await readJson(file)) : await readJsonCached(file, graphOf)
55
+ return graph ?? emptyGraph()
56
+ }
57
+
58
+ export const saveGraph = (chat, graph) => locked(chat, () => writeJson(join(chat, TOPICS), graph))
59
+
60
+ /**
61
+ * A chat's meta (read-only). One that exists but cannot be read, or is gone while a set-aside copy
62
+ * of it exists, reads as private with memory off (and its chat number, when that can be recovered):
63
+ * the lost file may have said so, and memory on is the default (#50). Nothing is written here;
64
+ * updateMeta writes the same fail-closed meta on the next change.
65
+ */
66
+ export async function loadMeta(dir, sid) {
67
+ const folder = chatDir(dir, sid)
68
+ const file = join(folder, 'meta.json')
69
+ const meta = await readJsonCached(file)
70
+ if (meta && typeof meta === 'object') return meta
71
+ const lost = await setAside(folder, 'meta.json')
72
+ return lost === undefined ? undefined : Object.freeze({ ...recoverMeta(lost), private: true, memoryOff: true, recovered: true })
73
+ }
74
+
75
+ /**
76
+ * The next free chat number. Claimed with an exclusive create, so two chats starting at once (or two
77
+ * dsh windows on one workspace) can never share one (#11). Existing chats keep their numbers.
78
+ */
79
+ function claimCid(dir) {
80
+ return locked(`${dir}#cid`, async () => {
81
+ const chats = await readChats(dir)
82
+ let cid = chats.reduce((max, c) => Math.max(max, Number(c.meta.cid) || 0), 0) + 1
83
+ const claims = join(dir, 'chats', '.cid')
84
+ await mkdir(claims, { recursive: true })
85
+ for (; ; cid++) {
86
+ try {
87
+ await writeFile(join(claims, String(cid)), '', { flag: 'wx' })
88
+ return cid
89
+ } catch (error) {
90
+ if (error?.code !== 'EEXIST') throw error
91
+ }
92
+ }
93
+ })
94
+ }
95
+
96
+ /** The newest set-aside copy of `name` in `folder` (`<name>.corrupt-<time>`), as text; undefined when there is none. */
97
+ async function setAside(folder, name) {
98
+ let files = []
99
+ try { files = await readdir(folder) } catch { return undefined }
100
+ const newest = files.filter((f) => f.startsWith(`${name}.corrupt-`)).sort().at(-1)
101
+ return newest ? readFile(join(folder, newest), 'utf8') : undefined
102
+ }
103
+
104
+ /**
105
+ * What can still be read from an unreadable meta.json: its chat number, index and fork positions,
106
+ * the title and summary it had (made safely before the damage), and the stretches typed while
107
+ * memory was off (#59). A meta whose stretches cannot be read back may have had some, so every turn
108
+ * of that chat stays closed to other chats, and no title is taken from its log (review).
109
+ */
110
+ function recoverMeta(raw) {
111
+ const text = String(raw ?? '')
112
+ const number = (name) => {
113
+ const m = new RegExp(`"${name}"\\s*:\\s*(\\d+)`).exec(text)
114
+ return m ? Number(m[1]) : undefined
115
+ }
116
+ const string = (name) => {
117
+ const m = new RegExp(`"${name}"\\s*:\\s*("(?:[^"\\\\]|\\\\.)*")`).exec(text)
118
+ try { return m ? JSON.parse(m[1]) : undefined } catch { return undefined }
119
+ }
120
+ const out = {}
121
+ for (const name of ['cid', 'builtSeq', 'offSeq', 'forkSeq', 'closedThrough']) if (number(name) !== undefined) out[name] = number(name)
122
+ for (const name of ['title', 'summary', 'firstKey']) if (typeof string(name) === 'string') out[name] = string(name)
123
+ const m = /"gaps"\s*:\s*(\[(?:\s*\[\s*-?\d+\s*,\s*-?\d+\s*\]\s*,?)*\s*\])/.exec(text)
124
+ let gaps
125
+ try { gaps = m ? JSON.parse(m[1]) : undefined } catch { gaps = undefined }
126
+ if (Array.isArray(gaps)) out.gaps = gaps
127
+ // A meta repaired before stays unsure where it was (a second damage must not reopen those turns).
128
+ if (!Array.isArray(gaps) || /"turnsClosed"\s*:\s*true/.test(text)) out.turnsClosed = true
129
+ return out
130
+ }
131
+
132
+ /**
133
+ * Read-modify-write of one chat's meta under the chat's lock, so a flag set by a command and a
134
+ * build finishing at the same moment can no longer undo each other (#10). A chat seen for the first
135
+ * time gets `init()` plus a freshly claimed chat number. A meta.json that exists but cannot be read
136
+ * is never replaced by a fresh identity: its chat number and index position are recovered where
137
+ * possible, and the chat fails closed (private, memory off) because the lost file may have said so.
138
+ */
139
+ export function updateMeta(dir, sid, change = (meta) => meta, init = () => ({})) {
140
+ return locked(chatDir(dir, sid), async () => {
141
+ const file = join(chatDir(dir, sid), 'meta.json')
142
+ const read = await readJsonDetailed(file)
143
+ let meta = read.value
144
+ if (!meta || typeof meta !== 'object') {
145
+ // An unreadable meta may already have been set aside by a reader (listChats): use that copy.
146
+ const lost = read.state === 'quarantined' ? read.raw : await setAside(chatDir(dir, sid), 'meta.json')
147
+ // A fresh meta's title comes from the log, which may hold an off stretch the lost file knew of:
148
+ // a recovered meta keeps only the title it can read back (review).
149
+ // A fork's fresh meta holds its ancestors' off stretches (offsInherited): they join the ones
150
+ // read back, never give way to them (second review of #69).
151
+ const fresh = await init()
152
+ const back = lost === undefined ? undefined : recoverMeta(lost)
153
+ meta = back === undefined ? fresh : { ...fresh, title: undefined, ...back, ...(Array.isArray(fresh?.gaps) && Array.isArray(back.gaps) ? { gaps: mergeGaps(fresh.gaps, back.gaps) } : {}) }
154
+ if (lost !== undefined) Object.assign(meta, { private: true, memoryOff: true, recovered: new Date().toISOString() })
155
+ if (!Number.isInteger(meta.cid)) meta.cid = await claimCid(dir)
156
+ }
157
+ const next = change({ ...meta }) ?? meta
158
+ await writeJson(file, next)
159
+ return next
160
+ })
161
+ }
162
+
163
+ /** Every chat folder with a readable meta.json, unsorted. Folders starting with '.' are bookkeeping. */
164
+ async function readChats(dir) {
165
+ let names = []
166
+ try { names = await readdir(join(dir, 'chats')) } catch (error) { if (error?.code !== 'ENOENT') throw error }
167
+ const metas = await Promise.all(names.filter((n) => !n.startsWith('.')).map(async (sid) => ({
168
+ sid,
169
+ // One entry that cannot be read (a stray file, a locked folder) must not hide every other chat.
170
+ meta: await loadMeta(dir, sid).catch((error) => {
171
+ console.error(`context-graph: skipping chat ${sid} (${error?.code ?? error?.message})`)
172
+ return undefined
173
+ })
174
+ })))
175
+ return metas.filter((c) => c.meta && typeof c.meta === 'object')
176
+ }
177
+
178
+ const migrations = new Map()
179
+
180
+ /**
181
+ * Every chat with a readable meta.json, newest activity first. A meta.json that does not parse is
182
+ * skipped instead of failing the whole workspace (#10). The first call for a workspace also brings
183
+ * memory written by v0.1-v0.3 into the chat layout (#17); a migration that failed is retried by a
184
+ * later call, at most once a minute.
185
+ */
186
+ export async function listChats(dir) {
187
+ const entry = migrations.get(dir)
188
+ if (!entry || (entry.failedAt && Date.now() - entry.failedAt > 60000)) {
189
+ const next = {}
190
+ next.done = migrateRootLayout(dir).catch((error) => {
191
+ next.failedAt = Date.now()
192
+ console.error('context-graph: memory from before v0.4 is not migrated yet; retrying later', error)
193
+ })
194
+ migrations.set(dir, next)
195
+ }
196
+ await migrations.get(dir).done
197
+ const chats = await readChats(dir)
198
+ return chats.sort((a, b) => String(b.meta.lastActiveAt ?? '').localeCompare(String(a.meta.lastActiveAt ?? '')))
199
+ }
200
+
201
+ /**
202
+ * v0.1-v0.3 kept one graph per workspace at `<dir>/graph.json` with sources in `<dir>/src/`; v0.4
203
+ * moved to one folder per chat and stopped reading it, so that memory became invisible (#17).
204
+ * The files are COPIED into a chat folder (the original session's id when every topic came from one
205
+ * session, else `legacy`), given a meta, and upgraded; the root copy is removed only after all of
206
+ * that succeeded, so an interrupted run simply repeats. A folder that belongs to a real chat is never
207
+ * used as the target.
208
+ */
209
+ export function migrateRootLayout(dir) {
210
+ return locked(`${dir}#root`, async () => {
211
+ const root = join(dir, 'graph.json')
212
+ const graph = await readJson(root)
213
+ if (!graph || typeof graph.nodes !== 'object' || graph.nodes === null) return undefined
214
+ const nodes = Object.values(graph.nodes)
215
+ const sessions = new Set(nodes.flatMap((n) => (Array.isArray(n.src) ? n.src : []).map((s) => s?.session)).filter(Boolean))
216
+ const candidates = [...(sessions.size === 1 ? [String([...sessions][0])] : []), 'legacy', ...Array.from({ length: 20 }, (_, i) => `legacy-${i + 2}`)]
217
+ let sid
218
+ for (const name of candidates) {
219
+ // Free, half-copied by an interrupted run (no meta.json yet), or already ours: never a real chat.
220
+ const meta = join(chatDir(dir, name), 'meta.json')
221
+ if (!(await exists(chatDir(dir, name))) || !(await exists(meta)) || (await loadMeta(dir, name))?.legacyRoot === true) { sid = name; break }
222
+ }
223
+ if (!sid) throw new Error('no free folder name for the migrated memory')
224
+ const chat = chatDir(dir, sid)
225
+ await mkdir(chat, { recursive: true })
226
+ await cp(root, join(chat, 'graph.json'))
227
+ if (await exists(join(dir, 'src'))) await cp(join(dir, 'src'), join(chat, 'src'), { recursive: true })
228
+ const times = nodes.flatMap((n) => [n.updated, ...(Array.isArray(n.src) ? n.src : []).map((s) => s?.at)]).filter(Boolean).sort()
229
+ const now = new Date().toISOString()
230
+ await updateMeta(dir, sid, (meta) => ({ ...meta, legacyRoot: true }), () => ({ title: 'Memory from before v0.4', startedAt: times[0] ?? now, lastActiveAt: times.at(-1) ?? now, turns: 0 }))
231
+ await upgradeChat(chat)
232
+ await rm(root, { force: true })
233
+ await rm(join(dir, 'src'), { recursive: true, force: true })
234
+ return sid
235
+ })
236
+ }
@@ -0,0 +1,181 @@
1
+ /**
2
+ * JSON files on disk, written so that concurrent writers and crashes cannot leave a half-written
3
+ * file, and read so that one damaged file can neither take the rest of the memory down nor be
4
+ * silently overwritten.
5
+ */
6
+
7
+ import { randomUUID } from 'node:crypto'
8
+ import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'
9
+ import { dirname } from 'node:path'
10
+
11
+ /** Index just past the first complete top-level JSON value in `raw` (string-aware), or 0. */
12
+ function firstValueEnd(raw) {
13
+ let depth = 0
14
+ let inString = false
15
+ for (let i = 0; i < raw.length; i++) {
16
+ const ch = raw[i]
17
+ if (inString) {
18
+ if (ch === '\\') i++
19
+ else if (ch === '"') inString = false
20
+ } else if (ch === '"') inString = true
21
+ else if (ch === '{' || ch === '[') depth++
22
+ else if ((ch === '}' || ch === ']') && --depth === 0) return i + 1
23
+ }
24
+ return 0
25
+ }
26
+
27
+ /**
28
+ * The JSON value at the start of `raw` when only junk follows it: what v0.6's interleaved writes
29
+ * left behind. Cut where the parser says the junk starts, else after the first complete value; both
30
+ * are one pass, however long the junk (#54).
31
+ */
32
+ function salvage(raw, error) {
33
+ const at = /at position (\d+)/.exec(String(error?.message))
34
+ for (const end of [at ? Number(at[1]) : 0, firstValueEnd(raw)]) {
35
+ if (end <= 0) continue
36
+ try { return JSON.parse(raw.slice(0, end)) } catch { /* try the next cut */ }
37
+ }
38
+ return undefined
39
+ }
40
+
41
+ /**
42
+ * Read a JSON file and say what was found:
43
+ * { state: 'ok', value } parsed
44
+ * { state: 'missing' } no such file
45
+ * { state: 'salvaged', value } trailing junk cut off (v0.6 could leave it); the next write repairs the file
46
+ * { state: 'quarantined', raw } unreadable; moved aside to `<file>.corrupt-<time>` so nothing overwrites it
47
+ * Errors other than "no such file" are thrown: the caller must not treat them as missing data (#10).
48
+ * So is an unreadable file that cannot be moved aside (#79).
49
+ */
50
+ export async function readJsonDetailed(path) {
51
+ let raw
52
+ try {
53
+ raw = await readFile(path, 'utf8')
54
+ } catch (error) {
55
+ if (error?.code === 'ENOENT') return { state: 'missing' }
56
+ throw error
57
+ }
58
+ try {
59
+ return { state: 'ok', value: JSON.parse(raw) }
60
+ } catch (error) {
61
+ const repaired = salvage(raw, error)
62
+ if (repaired !== undefined) {
63
+ console.error(`context-graph: repaired ${path} (${error.message}); it is rewritten on the next save`)
64
+ return { state: 'salvaged', value: repaired }
65
+ }
66
+ const aside = `${path}.corrupt-${Date.now()}`
67
+ // Gone: another reader moved it first. Any other failure (on Windows a sync tool or a scan that
68
+ // holds the file: EBUSY) leaves it in place, and reading it as missing let a memory-off chat
69
+ // read as on and a writer replace the file with no copy kept (#79).
70
+ try { await rename(path, aside) } catch (error) { if (error?.code !== 'ENOENT') throw error }
71
+ console.error(`context-graph: ${path} is unreadable (${error.message}); kept as ${aside}`)
72
+ return { state: 'quarantined', raw }
73
+ }
74
+ }
75
+
76
+ /** Parsed JSON, or undefined when the file is missing or unreadable (see readJsonDetailed). */
77
+ export async function readJson(path) {
78
+ return (await readJsonDetailed(path)).value
79
+ }
80
+
81
+ const cache = new Map()
82
+ // Two or three entries per chat are read on every message, so the cap sits far above any real
83
+ // workspace: at 2,000 a workspace of 1,500 chats evicted what the same pass needed next and took
84
+ // about 1 s per message (#63). ponytail: entry count, not bytes; size by bytes if memory matters.
85
+ const CACHE_MAX = 20000
86
+
87
+ function deepFreeze(value) {
88
+ if (value && typeof value === 'object' && !Object.isFrozen(value)) {
89
+ Object.freeze(value)
90
+ for (const v of Object.values(value)) deepFreeze(v)
91
+ }
92
+ return value
93
+ }
94
+
95
+ /**
96
+ * Parsed JSON for READ-ONLY use, cached by the file's stat (#33): while a file is unchanged it is
97
+ * not read or parsed again, which took the per-message cost at 150 chats from 37 ms of reads to a
98
+ * few stats. `prepare(value)` runs once per change (to normalize a graph). The result is frozen and
99
+ * shared, so code that tries to change it fails loudly instead of corrupting every later read.
100
+ * Writers use readJson / readJsonDetailed.
101
+ */
102
+ export async function readJsonCached(path, prepare = (value) => value) {
103
+ let st
104
+ try {
105
+ st = await stat(path)
106
+ } catch (error) {
107
+ if (error?.code !== 'ENOENT') throw error
108
+ cache.delete(path)
109
+ return undefined
110
+ }
111
+ const key = `${st.mtimeMs}:${st.size}:${st.ino}`
112
+ const hit = cache.get(path)
113
+ if (hit?.key === key) {
114
+ cache.delete(path)
115
+ cache.set(path, hit)
116
+ return hit.value
117
+ }
118
+ const read = await readJsonDetailed(path)
119
+ const value = read.value === undefined ? undefined : deepFreeze(prepare(read.value))
120
+ // A repaired value is cached too, so the repair (and its log line) happens once per change of the
121
+ // file, not on every message until the next save rewrites it.
122
+ if (read.state === 'ok' || read.state === 'salvaged') {
123
+ cache.set(path, { key, value })
124
+ if (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value)
125
+ }
126
+ return value
127
+ }
128
+
129
+ /**
130
+ * Rename with a retry: on Windows a rename over a file fails while anyone has that file open, even
131
+ * for a moment (another chat reading it, a sync tool, a virus scan, which can take seconds on a
132
+ * busy machine). Backing off for up to about 5 s outlasts such holds; the old 250 ms lost a meta
133
+ * write now and then under load (review). Writes are off the critical path, so waiting costs nothing.
134
+ */
135
+ export async function renameRetry(from, to) {
136
+ for (let attempt = 0; ; attempt++) {
137
+ try {
138
+ await rename(from, to)
139
+ return
140
+ } catch (error) {
141
+ if (attempt < 11 && (error?.code === 'EPERM' || error?.code === 'EBUSY' || error?.code === 'EACCES')) {
142
+ await new Promise((resolve) => setTimeout(resolve, Math.min(800, 25 * 2 ** attempt)))
143
+ continue
144
+ }
145
+ throw error
146
+ }
147
+ }
148
+ }
149
+
150
+ /**
151
+ * Atomic write: a uniquely named temp file, then rename over the target. A shared temp name let two
152
+ * concurrent writes interleave and corrupt the file (#10).
153
+ */
154
+ export async function writeJson(path, value) {
155
+ await mkdir(dirname(path), { recursive: true })
156
+ const tmp = `${path}.${process.pid}.${randomUUID().slice(0, 8)}.tmp`
157
+ await writeFile(tmp, JSON.stringify(value), 'utf8')
158
+ try {
159
+ await renameRetry(tmp, path)
160
+ } catch (error) {
161
+ await rm(tmp, { force: true })
162
+ throw error
163
+ }
164
+ }
165
+
166
+ const chains = new Map()
167
+
168
+ /**
169
+ * Run `fn` after every earlier call with the same key has settled, so read-modify-write sequences on
170
+ * one chat folder never interleave inside this process. Never call `locked` with a key from inside
171
+ * a callback holding the same key: it would wait for itself. ponytail: in-process only; two dsh
172
+ * processes writing one workspace still race (chat numbers are protected separately, see chats.js).
173
+ */
174
+ export function locked(key, fn) {
175
+ const previous = chains.get(key) ?? Promise.resolve()
176
+ const run = previous.then(fn, fn)
177
+ const tail = run.then(() => {}, () => {})
178
+ chains.set(key, tail)
179
+ tail.then(() => { if (chains.get(key) === tail) chains.delete(key) })
180
+ return run
181
+ }
@@ -0,0 +1,65 @@
1
+ /**
2
+ * The savings ledger of a workspace (#35): one small file per chat, `<dir>/ledger/<sessionId>.json`,
3
+ * so the estimate survives restarts and a turn end writes only its own chat's entry, not the whole
4
+ * workspace's (#83). Written at most once per turn per chat, off the critical path; a chat needs no
5
+ * folder of its own to have an entry.
6
+ * Format: `{ c: counters, used: [topic ids], msgs: ["<cid>:<seq>"] }`, zero counters and empty lists
7
+ * left out (#78): `msgs` are the source messages already counted as stood in for, so each counts
8
+ * once per chat. `<dir>/ledger.json`, the one file 0.9.0 kept (`{ v: 1, chats: { "<sessionId>":
9
+ * entry } }`), is only read: a chat's entry there is where its own file starts.
10
+ */
11
+
12
+ import { readdir } from 'node:fs/promises'
13
+ import { join } from 'node:path'
14
+
15
+ import { addCounters, counters, emptyCounters } from '../core/ledger.js'
16
+ import { locked, readJson, readJsonCached, writeJson } from './files.js'
17
+
18
+ const folder = (dir) => join(dir, 'ledger')
19
+ const file = (dir, sid) => join(folder(dir), `${sid}.json`)
20
+
21
+ /** Topic ids remembered per chat as used, so each counts once; the newest are kept. */
22
+ const USED_MAX = 500
23
+ const MSGS_MAX = 1000
24
+
25
+ /** The entries 0.9.0 kept in one file, read-only (cached: nothing writes that file any more). */
26
+ async function oldEntries(dir) {
27
+ const value = await readJsonCached(join(dir, 'ledger.json'))
28
+ return value && typeof value.chats === 'object' && value.chats !== null ? value.chats : {}
29
+ }
30
+
31
+ /** Every chat's entry, read-only (cached per file): `{ v: 1, chats: { "<sessionId>": entry } }`. */
32
+ export async function loadLedger(dir) {
33
+ const chats = { ...(await oldEntries(dir)) }
34
+ let names = []
35
+ try { names = (await readdir(folder(dir))).filter((name) => name.endsWith('.json')) } catch (error) { if (error?.code !== 'ENOENT') throw error }
36
+ const entries = await Promise.all(names.map((name) => readJsonCached(join(folder(dir), name))))
37
+ names.forEach((name, i) => { if (entries[i]) chats[name.slice(0, -'.json'.length)] = entries[i] })
38
+ return { v: 1, chats }
39
+ }
40
+
41
+ /** One chat's counters, used topic ids and credited source messages. */
42
+ export async function chatEntry(dir, sid) {
43
+ const entry = (await readJsonCached(file(dir, sid))) ?? (await oldEntries(dir))[String(sid)]
44
+ return { c: counters(entry?.c), used: Array.isArray(entry?.used) ? entry.used : [], msgs: Array.isArray(entry?.msgs) ? entry.msgs : [] }
45
+ }
46
+
47
+ /** The workspace total. */
48
+ export async function workspaceTotal(dir) {
49
+ return Object.values((await loadLedger(dir)).chats).reduce((sum, entry) => addCounters(sum, counters(entry?.c)), emptyCounters())
50
+ }
51
+
52
+ /**
53
+ * Add `delta` to a chat's counters and remember newly used topic ids. Locked read-modify-write of
54
+ * the chat's own file, so two writes for one chat (a slow turn end, then the next) cannot undo each other.
55
+ */
56
+ export function addToLedger(dir, sid, delta, usedIds = [], msgKeys = []) {
57
+ const path = file(dir, sid)
58
+ return locked(path, async () => {
59
+ const prev = (await readJson(path)) ?? (await oldEntries(dir))[String(sid)] ?? {}
60
+ const used = [...new Set([...(Array.isArray(prev.used) ? prev.used : []), ...usedIds])].slice(-USED_MAX)
61
+ const msgs = [...new Set([...(Array.isArray(prev.msgs) ? prev.msgs : []), ...msgKeys])].slice(-MSGS_MAX)
62
+ const c = Object.fromEntries(Object.entries(addCounters(counters(prev.c), delta)).filter(([, n]) => n !== 0))
63
+ await writeJson(path, { c, ...(used.length ? { used } : {}), ...(msgs.length ? { msgs } : {}) })
64
+ })
65
+ }
@@ -0,0 +1,88 @@
1
+ /**
2
+ * The deepest stored layer: the verbatim messages topics were drawn from, one file per chat
3
+ * (`chats/<sid>/msgs.json`), each message stored ONCE under its seq however many topics cite it.
4
+ * Up to v0.6 every topic kept its own copy (3x on real data) and grew a section per rebuild (#12).
5
+ * Format: `{ v: 2, m: { "<seq>": { r: "user"|"assistant", t: turn, a: time, x: text } } }`, `t`
6
+ * being dsh's own turn number (#71); a `v: 1` store (up to 0.9.0) counted typed prompts instead.
7
+ */
8
+
9
+ import { join } from 'node:path'
10
+
11
+ import { capText } from '../core/text.js'
12
+ import { locked, readJson, readJsonCached, writeJson } from './files.js'
13
+
14
+ /** Long messages keep their head and tail (core/text.js); the full text stays in dsh's own session log. */
15
+ export { capText }
16
+
17
+ const file = (chat) => join(chat, 'msgs.json')
18
+
19
+ /**
20
+ * The store for READ-ONLY use (cached, #33). A file that cannot be read now (a damaged one another
21
+ * program holds, #79) reads as nothing stored: readers show no source instead of failing the whole
22
+ * block or recall, as the page and the savings estimate already did (review of #79). Writers use readJson.
23
+ */
24
+ const readStore = (chat) => readJsonCached(file(chat)).catch(() => undefined)
25
+
26
+ /** `{ [seq]: { r, t, a, x } }` for one chat, read-only; empty when nothing is stored (or readable) yet. */
27
+ export async function loadMessages(chat) {
28
+ const store = await readStore(chat)
29
+ return store && typeof store.m === 'object' && store.m !== null ? store.m : {}
30
+ }
31
+
32
+ /**
33
+ * Whether a chat's stored turn ids still count typed prompts (a store from 0.9.0 or older): such an
34
+ * id names another turn than dsh's, so it is not shown until relabelTurns converts it (review of #71).
35
+ */
36
+ export async function oldTurns(chat) {
37
+ const store = await readStore(chat)
38
+ return store?.v !== 2 && Object.values(store?.m ?? {}).some((m) => m?.t !== undefined)
39
+ }
40
+
41
+ /**
42
+ * Add messages (`[{ seq, role, text, turn, time }]`) that are not stored yet. Idempotent: writing
43
+ * the same seq twice is a no-op, so a build that is retried after a crash cannot duplicate text.
44
+ */
45
+ export function saveMessages(chat, messages) {
46
+ return locked(`${chat}#msgs`, async () => {
47
+ const store = (await readJson(file(chat))) ?? { v: 2, m: {} }
48
+ if (typeof store.m !== 'object' || store.m === null) store.m = {}
49
+ let changed = false
50
+ for (const m of messages) {
51
+ const key = String(m.seq)
52
+ if (!Number.isInteger(m.seq) || Object.hasOwn(store.m, key)) continue
53
+ store.m[key] = { r: m.role, ...(m.turn ? { t: m.turn } : {}), ...(m.time ? { a: m.time } : {}), x: capText(m.text) }
54
+ changed = true
55
+ }
56
+ if (changed) await writeJson(file(chat), store)
57
+ })
58
+ }
59
+
60
+ /**
61
+ * Give stored messages the turn `turnAt(seq)` says (#71: turn ids follow dsh's own turns now) and
62
+ * mark the store as numbered so (v 2). Under the lock saveMessages takes; when nothing changes,
63
+ * nothing is written.
64
+ */
65
+ export function relabelTurns(chat, turnAt) {
66
+ return locked(`${chat}#msgs`, async () => {
67
+ const store = await readJson(file(chat))
68
+ if (typeof store?.m !== 'object' || store.m === null) return
69
+ let changed = store.v !== 2
70
+ for (const [seq, m] of Object.entries(store.m)) {
71
+ const t = turnAt(Number(seq))
72
+ if (t === undefined || !m || typeof m !== 'object' || m.t === t) continue
73
+ m.t = t
74
+ changed = true
75
+ }
76
+ if (changed) await writeJson(file(chat), { ...store, v: 2 })
77
+ })
78
+ }
79
+
80
+ /** The stored messages a topic cites, oldest first: `[{ seq, role, turn, text }]`. */
81
+ export function sourceOf(messages, node) {
82
+ return (node?.refs ?? [])
83
+ .filter((seq) => Object.hasOwn(messages, String(seq)))
84
+ .map((seq) => {
85
+ const m = messages[String(seq)]
86
+ return { seq, role: m.r, turn: m.t, text: m.x }
87
+ })
88
+ }