dsh-context-graph 0.6.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +563 -0
- package/README.md +252 -97
- package/README.zh.md +82 -32
- package/cordis.patch.yml +10 -6
- package/index.js +7 -432
- package/package.json +11 -12
- package/src/client/client.js +880 -0
- package/src/core/builder.js +210 -0
- package/src/core/chat.js +34 -0
- package/src/core/graph.js +134 -0
- package/src/core/index.js +14 -0
- package/src/core/ledger.js +108 -0
- package/src/core/match.js +377 -0
- package/src/core/render.js +195 -0
- package/src/core/session.js +446 -0
- package/src/core/text.js +220 -0
- package/src/core/time.js +147 -0
- package/src/core/tree.js +39 -0
- package/src/host/builder.js +416 -0
- package/src/host/command.js +155 -0
- package/src/host/config.js +50 -0
- package/src/host/memory.js +175 -0
- package/src/host/meter.js +75 -0
- package/src/host/page.js +186 -0
- package/src/host/plugin.js +46 -0
- package/src/host/recall.js +227 -0
- package/src/host/runtime.js +599 -0
- package/src/store/chats.js +236 -0
- package/src/store/files.js +181 -0
- package/src/store/ledger.js +65 -0
- package/src/store/messages.js +88 -0
- package/src/store/migrate.js +143 -0
- package/graph.js +0 -496
- package/lib/client.js +0 -126
- package/store.js +0 -81
- /package/{update.js → src/host/update.js} +0 -0
|
@@ -0,0 +1,599 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-plugin runtime shared by the host modules: configuration, per-chat state, and the helpers
|
|
3
|
+
* that map a dsh agent to its memory folder, its meta and the views of its workspace.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { createHash } from 'node:crypto'
|
|
7
|
+
import { basename, isAbsolute, join } from 'node:path'
|
|
8
|
+
|
|
9
|
+
import { closedAt, memoryOn, offPoint, sharedOn, walkGaps } from '../core/chat.js'
|
|
10
|
+
import { emptyCounters, standIn } from '../core/ledger.js'
|
|
11
|
+
import { collectSpan, eventText, inGap, isPrompt, mergeGaps, promptText, shownContext, titleOf } from '../core/session.js'
|
|
12
|
+
import { chatDir, listChats, loadGraph, loadMeta, updateMeta } from '../store/chats.js'
|
|
13
|
+
import { chatEntry } from '../store/ledger.js'
|
|
14
|
+
import { capText, loadMessages } from '../store/messages.js'
|
|
15
|
+
|
|
16
|
+
const ownParts = new WeakMap()
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* A chat's first prompt, kept only as a fingerprint: forks are matched to their ancestors by it,
|
|
20
|
+
* and the text, which may have been typed while memory was off, is never stored (#59, review).
|
|
21
|
+
*/
|
|
22
|
+
const firstKey = (text) => (text === undefined ? undefined : createHash('sha1').update(text).digest('hex').slice(0, 16))
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* What a chat's fingerprint is made from: its first typed prompt (titleOf), or while it has none,
|
|
26
|
+
* its first prompt of any kind, such as a /goal chat's objective, so that its meta too says it is
|
|
27
|
+
* no 0.4-0.7 meta (forkUnknown; review of #69).
|
|
28
|
+
*/
|
|
29
|
+
const keyPrompt = (eventAt) => titleOf(eventAt) ?? firstPromptOf(eventAt)
|
|
30
|
+
|
|
31
|
+
/** A chat's first prompt of any kind (typed, a /goal round's objective, a reminder). */
|
|
32
|
+
function firstPromptOf(eventAt) {
|
|
33
|
+
for (let seq = 0; ; seq++) {
|
|
34
|
+
const event = eventAt(seq)
|
|
35
|
+
if (!event) return undefined
|
|
36
|
+
if (isPrompt(event)) return promptText(event.data)
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* What an ancestor's fingerprint can be made from, as a fork's log shows it (keyPrompt, when the
|
|
42
|
+
* ancestor's meta was written): its first typed prompt, or its first prompt of any kind for a chat
|
|
43
|
+
* begun with /goal (second review of #52).
|
|
44
|
+
*/
|
|
45
|
+
const ancestorKeys = (eventAt) => [...new Set([titleOf(eventAt), firstPromptOf(eventAt)].filter((p) => p !== undefined))]
|
|
46
|
+
|
|
47
|
+
/** Whether a chat's first prompt is `first`; metas written before 0.9.0 keep it as text, or only a title. */
|
|
48
|
+
const startsWith = (meta, first) => first !== undefined && (meta?.firstKey !== undefined ? meta.firstKey === firstKey(first) : (meta?.firstPrompt ?? meta?.title) === first)
|
|
49
|
+
/**
|
|
50
|
+
* Whether a chat may be an ancestor of `session`'s chat (same first prompt). Its keys are read once
|
|
51
|
+
* per session: the inherited part never changes, and a pasted first prompt is long (third check).
|
|
52
|
+
*/
|
|
53
|
+
const keyCache = new WeakMap()
|
|
54
|
+
function ancestorOf(meta, session) {
|
|
55
|
+
let keys = keyCache.get(session)
|
|
56
|
+
if (!keys) keyCache.set(session, (keys = ancestorKeys((seq) => session.eventAt(seq)).map((text) => ({ text, key: firstKey(text) }))))
|
|
57
|
+
return keys.some(({ text, key }) => (meta?.firstKey !== undefined ? meta.firstKey === key : (meta?.firstPrompt ?? meta?.title) === text))
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Whether a chat's meta cannot say if the chat is a fork: written by 0.4-0.7 and not since (#58).
|
|
62
|
+
* From 0.8.0 on, the write that stores the first prompt (firstPrompt, then its fingerprint firstKey)
|
|
63
|
+
* also stores forkSeq when the chat is a fork, so a meta with either says (#72).
|
|
64
|
+
*/
|
|
65
|
+
/** How long a failed listing of dsh's sessions, or a fork point not learned, waits to be tried again. */
|
|
66
|
+
const RETRY_MS = 30000
|
|
67
|
+
|
|
68
|
+
const forkUnknown = (meta) => !Number.isInteger(meta?.forkSeq) && meta?.firstKey === undefined && meta?.firstPrompt === undefined
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* A fork's graph without the topics it inherited: topics whose every cited message lies before the
|
|
72
|
+
* fork point are dropped, mixed ones keep only their own sources. Memoized per loaded graph, so the
|
|
73
|
+
* same objects come back while the file is unchanged (the token memo in match.js relies on it).
|
|
74
|
+
*/
|
|
75
|
+
function ownPart(graph, forkSeq) {
|
|
76
|
+
if (!forkSeq) return graph
|
|
77
|
+
const cached = ownParts.get(graph)
|
|
78
|
+
if (cached?.forkSeq === forkSeq) return cached.graph
|
|
79
|
+
const nodes = {}
|
|
80
|
+
let changed = false
|
|
81
|
+
for (const [id, node] of Object.entries(graph.nodes)) {
|
|
82
|
+
const refs = node.refs ?? []
|
|
83
|
+
const own = refs.filter((seq) => seq >= forkSeq)
|
|
84
|
+
if (refs.length > 0 && own.length === 0) { changed = true; continue }
|
|
85
|
+
if (own.length !== refs.length) { changed = true; nodes[id] = { ...node, refs: own } } else nodes[id] = node
|
|
86
|
+
}
|
|
87
|
+
const result = changed ? { ...graph, nodes, edges: graph.edges.filter(([a, b]) => Object.hasOwn(nodes, a) && Object.hasOwn(nodes, b)) } : graph
|
|
88
|
+
ownParts.set(graph, { forkSeq, graph: result })
|
|
89
|
+
return result
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** What an unresolvable owner reads as: nothing shared, memory off. */
|
|
93
|
+
const CLOSED = Object.freeze({ private: true, memoryOff: true })
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* @param agentById the live agent of a session id (dsh's `agents` service), used to find the chat
|
|
97
|
+
* behind a nested subagent.
|
|
98
|
+
*/
|
|
99
|
+
export function createRuntime(cfg, { agentById = () => undefined, sessionQuery = () => undefined } = {}) {
|
|
100
|
+
const sessions = new Map()
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* In-memory state of one chat for this app run:
|
|
104
|
+
* building an indexing call is running
|
|
105
|
+
* announce /context-graph on asked for the chat list again
|
|
106
|
+
* listed the chat list was already sent in this run (see memory.js)
|
|
107
|
+
* retryAt after a failed build, wait for this many unindexed human turns
|
|
108
|
+
* pin after a failed build, the last seq the retry may read (#45)
|
|
109
|
+
* failedAnswers unusable answers in a row for the pinned span
|
|
110
|
+
* and for the savings estimate (#35):
|
|
111
|
+
* chat a human turn or a build of this chat was seen in this run (meter.js charges it)
|
|
112
|
+
* delta counters not yet written to the ledger
|
|
113
|
+
* inContext tokens of the plugin's blocks and recall results still in the chat's context
|
|
114
|
+
* justSent tokens added since the last request (counted as sent, not re-sent)
|
|
115
|
+
* pending topics shown this turn, to check against the answer at turn end
|
|
116
|
+
* newlyUsed topic ids used since the last write; usedIds: every id used (loaded once)
|
|
117
|
+
* newlyCredited source messages counted since the last write; credited: all of them (loaded once)
|
|
118
|
+
* toCredit topics judged used (or recalled) and not credited yet: kept until they are
|
|
119
|
+
* generation the surface's replaceGeneration when inContext was last read from it
|
|
120
|
+
*/
|
|
121
|
+
const state = (sid) => {
|
|
122
|
+
let st = sessions.get(sid)
|
|
123
|
+
if (!st) {
|
|
124
|
+
st = { building: false, announce: false, listed: false, retryAt: 0, chat: false, delta: emptyCounters(), inContext: 0, justSent: 0, pending: [], newlyUsed: [], usedIds: undefined, newlyCredited: [], credited: undefined, toCredit: [], generation: undefined }
|
|
125
|
+
sessions.set(sid, st)
|
|
126
|
+
}
|
|
127
|
+
return st
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** The state of a chat the savings estimate follows, or undefined. */
|
|
131
|
+
const tracked = (sid) => {
|
|
132
|
+
const st = sessions.get(sid)
|
|
133
|
+
return st?.chat ? st : undefined
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const sidOf = (agent) => String(agent.session.id)
|
|
137
|
+
const isSubagent = (agent) => agent.options?.origin === 'subagent' || agent.session.header?.origin === 'subagent'
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Input the user typed that dsh holds and has not logged yet (queued turns, steering, a late
|
|
141
|
+
* answer to a question, #70); dsh's own notices do not count.
|
|
142
|
+
*/
|
|
143
|
+
const pendingInput = (agent) => [...(agent.inbox?.nextTurn ?? []), ...(agent.inbox?.nextStep ?? [])].some((m) => m?.source?.kind === 'user' || m?.source?.kind === 'user-question-reply')
|
|
144
|
+
|
|
145
|
+
function dirFor(agent) {
|
|
146
|
+
const base = agent.options?.cwd ?? agent.session.header?.cwd ?? process.cwd()
|
|
147
|
+
return isAbsolute(cfg.dir) ? cfg.dir : join(base, cfg.dir)
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* This agent's meta. A fork seen for the first time gets its meta at once, so the flags it
|
|
152
|
+
* inherits (private, memory off) hold from its first message, not only from its first index
|
|
153
|
+
* (#60, review); other chats get a meta only when they are first indexed (#38).
|
|
154
|
+
*/
|
|
155
|
+
async function metaFor(agent) {
|
|
156
|
+
const meta = await metaOf(dirFor(agent), sidOf(agent))
|
|
157
|
+
if (meta || isSubagent(agent) || !forkSeqOf(agent.session)) return meta
|
|
158
|
+
return withPending(sidOf(agent), await updateFor(agent, (m) => m))
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* A chat's meta as every memory check must see it: an `off` whose write still waits for the file
|
|
163
|
+
* (command.js, up to ~5 s while another program holds it) already holds, for this chat and for
|
|
164
|
+
* anything that asks about it (a fork made now, a subagent, a waiting `on`) (final review).
|
|
165
|
+
*/
|
|
166
|
+
function withPending(sid, meta) {
|
|
167
|
+
const from = sessions.get(sid)?.offFrom
|
|
168
|
+
if (from === undefined) return meta
|
|
169
|
+
return { ...(meta ?? {}), memoryOff: true, offSeq: memoryOn(meta) || !Number.isInteger(meta?.offSeq) ? from : meta.offSeq }
|
|
170
|
+
}
|
|
171
|
+
const metaOf = async (dir, sid) => withPending(String(sid), await loadMeta(dir, String(sid)))
|
|
172
|
+
|
|
173
|
+
/** Memory folders used in this app run, for the plugin's page (with dsh's registered workspaces). */
|
|
174
|
+
const seenDirs = new Map()
|
|
175
|
+
function noteDir(agent) {
|
|
176
|
+
const dir = dirFor(agent)
|
|
177
|
+
if (seenDirs.has(dir)) return
|
|
178
|
+
const path = agent.options?.cwd ?? agent.session.header?.cwd ?? process.cwd()
|
|
179
|
+
seenDirs.set(dir, { title: basename(path) || path, path })
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Where a forked chat's own history starts. dsh copies the parent's events into the fork with the
|
|
184
|
+
* same seqs and reports how many in `inheritedEventCount`; `header.isSeeded` marks the copy. The
|
|
185
|
+
* parent's id (`header.parentSession`) is optional: some dsh builds omit it for ordinary forks.
|
|
186
|
+
* @param session a live session, or dsh's observation of one (the same two fields).
|
|
187
|
+
* @returns the first seq that is the fork's own, or undefined for a chat that is not a fork.
|
|
188
|
+
*/
|
|
189
|
+
function forkSeqOf(session) {
|
|
190
|
+
const inherited = session.inheritedEventCount
|
|
191
|
+
return session.header?.isSeeded === true && Number.isInteger(inherited) && inherited > 0 ? inherited : undefined
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** The last seq already indexed: from the meta, else just before a fork's own history (#16, #55). */
|
|
195
|
+
const indexStart = (agent, meta) => (Number.isInteger(meta?.builtSeq) ? meta.builtSeq : (forkSeqOf(agent.session) ?? 0) - 1)
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Count a topic this chat used (#35): once per chat, standing in for its newest messages, each of
|
|
199
|
+
* which also counts once per chat (review). `stored` is the topic's chat's msgs.json.
|
|
200
|
+
*/
|
|
201
|
+
async function creditUse(agent, { id, node, cid, stored }) {
|
|
202
|
+
const sid = sidOf(agent)
|
|
203
|
+
const st = state(sid)
|
|
204
|
+
if (!st.usedIds || !st.credited) {
|
|
205
|
+
const entry = await chatEntry(dirFor(agent), sid)
|
|
206
|
+
st.usedIds ??= new Set(entry.used)
|
|
207
|
+
st.credited ??= new Set(entry.msgs)
|
|
208
|
+
}
|
|
209
|
+
if (st.usedIds.has(id)) return
|
|
210
|
+
const { tokens, keys } = standIn(node, stored, cid, st.credited, cfg.sourceMaxMessages)
|
|
211
|
+
st.delta.stoodIn += tokens
|
|
212
|
+
st.delta.used++
|
|
213
|
+
st.usedIds.add(id)
|
|
214
|
+
st.newlyUsed.push(id)
|
|
215
|
+
for (const key of keys) st.credited.add(key)
|
|
216
|
+
st.newlyCredited.push(...keys)
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** dsh's observation of a chat's session (header, events), or undefined when it cannot be read here. */
|
|
220
|
+
async function observe(sid, signal) {
|
|
221
|
+
const query = sessionQuery()
|
|
222
|
+
if (typeof query?.observeSession !== 'function') return undefined
|
|
223
|
+
try {
|
|
224
|
+
return await query.observeSession(sid, { ...(signal ? { signal } : {}), projectionMode: 'none' })
|
|
225
|
+
} catch {
|
|
226
|
+
return undefined
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Whether `stored`, a message an ancestor stored at `seq`, is word for word this fork's own event
|
|
232
|
+
* there. Compared with what the builder read at that seq (a goal round's objective, an approved
|
|
233
|
+
* plan, the user's answers), not with the raw event, as that is what was stored (second review of
|
|
234
|
+
* #52); a fork's inherited part never changes, so it is read once per session.
|
|
235
|
+
*/
|
|
236
|
+
const readCache = new WeakMap()
|
|
237
|
+
function sameText(stored, agent, forkSeq, seq) {
|
|
238
|
+
const session = agent.session
|
|
239
|
+
let read = readCache.get(session)
|
|
240
|
+
if (!read) {
|
|
241
|
+
read = new Map()
|
|
242
|
+
for (const m of collectSpan((s) => session.eventAt(s)).messages) if (m.seq < forkSeq) read.set(m.seq, m.text)
|
|
243
|
+
readCache.set(session, read)
|
|
244
|
+
}
|
|
245
|
+
return [read.get(seq), eventText(session.eventAt(seq))].some((own) => own && (stored === own || stored === capText(own)))
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* The meta of the chat a fork was made from, for the flags it carries over. dsh names the parent;
|
|
250
|
+
* a named parent with no meta is an ordinary chat with no flags to carry over, unless dsh says it
|
|
251
|
+
* is itself a fork (one never seen here), whose own parent is asked in turn (review of #60). Only
|
|
252
|
+
* where no parent is named, or dsh cannot tell, a chat whose log this one starts with is looked
|
|
253
|
+
* for (same first prompt, its stored messages word for word in this log); a candidate with
|
|
254
|
+
* nothing stored to compare counts (fail closed).
|
|
255
|
+
*/
|
|
256
|
+
async function parentMetaOf(agent, forkSeq) {
|
|
257
|
+
const dir = dirFor(agent)
|
|
258
|
+
let named = agent.session.header?.parentSession
|
|
259
|
+
for (let hops = 0; named && hops < 16; hops++) {
|
|
260
|
+
const meta = await metaOf(dir, named)
|
|
261
|
+
if (meta) return meta
|
|
262
|
+
const observation = await observe(String(named))
|
|
263
|
+
if (!observation) break
|
|
264
|
+
let header
|
|
265
|
+
try { header = observation.header } finally { observation[Symbol.dispose]?.() }
|
|
266
|
+
if (header?.isSeeded !== true) return undefined
|
|
267
|
+
named = header.parentSession
|
|
268
|
+
}
|
|
269
|
+
const eventAt = (seq) => agent.session.eventAt(seq)
|
|
270
|
+
for (const chat of await listChats(dir)) {
|
|
271
|
+
if (chat.sid === sidOf(agent) || !ancestorOf(chat.meta, agent.session)) continue
|
|
272
|
+
if (chat.meta.private !== true && chat.meta.memoryOff !== true) continue
|
|
273
|
+
const stored = await loadMessages(chatDir(dir, chat.sid))
|
|
274
|
+
const inherited = Object.keys(stored).map(Number).filter((seq) => seq < forkSeq)
|
|
275
|
+
const same = inherited.every((seq) => sameText(stored[String(seq)].x, agent, forkSeq, seq))
|
|
276
|
+
if (same) return chat.meta
|
|
277
|
+
}
|
|
278
|
+
return undefined
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* What a chat's meta starts with. A fork starts its index after the inherited events, so its first
|
|
283
|
+
* build does not re-index the parent's history (#16); it is titled from its own first prompt. A
|
|
284
|
+
* fork of a private chat starts private (#60). A fork made where its parent's memory was off (the
|
|
285
|
+
* parent was off then, or the fork point lies in one of its off stretches) starts memory-off and
|
|
286
|
+
* has no title until `on`, its own prompts so far being typed while off (review). The inherited
|
|
287
|
+
* first prompt is kept as `firstKey`, the fingerprint ancestors are matched by. Its ancestors'
|
|
288
|
+
* off stretches become its first gaps: a goal set in one and resumed in the fork stays private
|
|
289
|
+
* (#69, review); the fork never indexes or serves its inherited part anyway.
|
|
290
|
+
*/
|
|
291
|
+
async function initialMeta(agent) {
|
|
292
|
+
const session = agent.session
|
|
293
|
+
const eventAt = (seq) => session.eventAt(seq)
|
|
294
|
+
const now = new Date().toISOString()
|
|
295
|
+
const forkSeq = forkSeqOf(session)
|
|
296
|
+
const parent = session.header?.parentSession
|
|
297
|
+
const first = titleOf(eventAt)
|
|
298
|
+
const base = { title: first, firstKey: firstKey(first), startedAt: now, lastActiveAt: now }
|
|
299
|
+
if (!forkSeq) return base
|
|
300
|
+
const from = await parentMetaOf(agent, forkSeq)
|
|
301
|
+
const point = forkSeq - 1
|
|
302
|
+
const offAtFork = await offAt(dirFor(agent), from, point)
|
|
303
|
+
const inherited = await offUpTo(dirFor(agent), from, point)
|
|
304
|
+
return {
|
|
305
|
+
...base,
|
|
306
|
+
title: offAtFork ? undefined : titleOf(eventAt, 60, forkSeq),
|
|
307
|
+
builtSeq: point,
|
|
308
|
+
forkSeq,
|
|
309
|
+
...(parent ? { parent: String(parent) } : {}),
|
|
310
|
+
...(from?.private === true ? { private: true } : {}),
|
|
311
|
+
...(inherited.length > 0 ? { gaps: inherited } : {}),
|
|
312
|
+
offsInherited: true,
|
|
313
|
+
// Off from its first message: `on` turns all of the fork's own part so far into a gap (#59).
|
|
314
|
+
...(offAtFork ? { memoryOff: true } : {})
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Whether memory was off at `point` of the chat whose meta is `meta`: the point lies in one of its
|
|
320
|
+
* off stretches, or it is off now and the point is past where it went off. The log does not show
|
|
321
|
+
* whether a fork made right where memory went off came before or after the `off`, and a meta
|
|
322
|
+
* whose stretches were lost cannot tell up to where memory was turned on again (turnsClosed,
|
|
323
|
+
* closedThrough): both count as off (fail closed). A point in the chat's own inherited part is
|
|
324
|
+
* its parent's to answer, and so on up (third and fourth reviews).
|
|
325
|
+
*/
|
|
326
|
+
async function offAt(dir, meta, point) {
|
|
327
|
+
for (let hops = 0; meta && hops < 16; hops++) {
|
|
328
|
+
if (closedAt(meta, point) || inGap(point, meta.gaps) || inGap(point + 1, meta.gaps) || (!memoryOn(meta) && point >= offPoint(meta))) return true
|
|
329
|
+
if (!(Number.isInteger(meta.forkSeq) && point < meta.forkSeq && meta.parent)) return false
|
|
330
|
+
meta = await metaOf(dir, meta.parent)
|
|
331
|
+
}
|
|
332
|
+
return false
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* The off stretches up to `point` of the chat whose meta is `meta` and of its ancestors, as gaps,
|
|
337
|
+
* by the same rules as offAt: its gaps; while it is off, all after where it went off; for a meta
|
|
338
|
+
* whose stretches were lost, all up to where memory was turned on again (review of #69).
|
|
339
|
+
*/
|
|
340
|
+
async function offUpTo(dir, meta, point) {
|
|
341
|
+
const out = []
|
|
342
|
+
for (let hops = 0; meta && point >= 0 && hops < 16; hops++) {
|
|
343
|
+
for (const g of Array.isArray(meta.gaps) ? meta.gaps : []) if (Array.isArray(g) && g[0] <= point) out.push([g[0], Math.min(g[1], point)])
|
|
344
|
+
if (!memoryOn(meta) && offPoint(meta) < point) out.push([offPoint(meta) + 1, point])
|
|
345
|
+
if (meta.turnsClosed === true) out.push([0, Number.isInteger(meta.closedThrough) ? Math.min(meta.closedThrough, point) : point])
|
|
346
|
+
if (!(Number.isInteger(meta.forkSeq) && meta.parent)) break
|
|
347
|
+
point = Math.min(point, meta.forkSeq - 1)
|
|
348
|
+
meta = await metaOf(dir, meta.parent)
|
|
349
|
+
}
|
|
350
|
+
return mergeGaps([], out)
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* The off stretches another chat's log is judged by (turn recall): its gaps, and for a fork whose
|
|
355
|
+
* meta came before forks took their ancestors' (offsInherited), theirs too, read-only (second
|
|
356
|
+
* review of #69).
|
|
357
|
+
*/
|
|
358
|
+
async function offsOf(dir, meta) {
|
|
359
|
+
const own = walkGaps(meta)
|
|
360
|
+
if (!Number.isInteger(meta?.forkSeq) || !meta.parent || meta.offsInherited === true) return own
|
|
361
|
+
return mergeGaps(own, await offUpTo(dir, await metaOf(dir, meta.parent), meta.forkSeq - 1))
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* A fork whose meta was written before forks took their ancestors' off stretches (up to 0.9.0, or
|
|
366
|
+
* a meta repaired after damage) gets them once, before it is next indexed: a goal set in one of
|
|
367
|
+
* them and resumed in the fork stays private there too (second review of #69). Fails closed: a
|
|
368
|
+
* write that cannot land stops the build.
|
|
369
|
+
*/
|
|
370
|
+
async function inheritOffs(agent, meta) {
|
|
371
|
+
const forkSeq = forkSeqOf(agent.session)
|
|
372
|
+
if (!meta || !forkSeq || meta.offsInherited === true) return meta
|
|
373
|
+
const from = await parentMetaOf(agent, forkSeq)
|
|
374
|
+
const inherited = await offUpTo(dirFor(agent), from, forkSeq - 1)
|
|
375
|
+
const updated = await updateFor(agent, (m) => {
|
|
376
|
+
const gaps = mergeGaps(Array.isArray(m.gaps) ? m.gaps : [], inherited)
|
|
377
|
+
return { ...m, ...(gaps.length > 0 ? { gaps } : {}), offsInherited: true }
|
|
378
|
+
})
|
|
379
|
+
return withPending(sidOf(agent), updated)
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* A fork whose meta was written before forks had a fork point (v0.4-v0.6) gets one the next time
|
|
384
|
+
* it is written, from dsh's own count, so its inherited topics can be held back (second review).
|
|
385
|
+
*/
|
|
386
|
+
function withForkPoint(agent, meta) {
|
|
387
|
+
const forkSeq = forkSeqOf(agent.session)
|
|
388
|
+
if (!meta || !forkSeq || Number.isInteger(meta.forkSeq)) return meta
|
|
389
|
+
const parent = agent.session.header?.parentSession
|
|
390
|
+
return { ...meta, forkSeq, ...(parent && !meta.parent ? { parent: String(parent) } : {}) }
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* A meta written before 0.9.0 (no fingerprint yet: 0.8.0 kept the first prompt as text, older
|
|
395
|
+
* versions not at all) is brought up to date the next time it is written: the first prompt becomes
|
|
396
|
+
* its fingerprint, and a fork drops the title it got then (often its parent's: dsh's "<title> (1)",
|
|
397
|
+
* or the inherited first prompt), so its next index titles it from its own prompts (reviews).
|
|
398
|
+
* A new meta passes here too: one of a chat with no typed prompt gets its fingerprint here.
|
|
399
|
+
*/
|
|
400
|
+
function upToDate(agent, meta) {
|
|
401
|
+
if (!meta) return meta
|
|
402
|
+
const eventAt = (seq) => agent.session.eventAt(seq)
|
|
403
|
+
// An absolute local path that nothing reads, in a folder users may sync: dropped (#78).
|
|
404
|
+
const { workspace, ...current } = meta
|
|
405
|
+
// Kept once made: forks are matched by either key a chat can have (ancestorKeys), and a fork
|
|
406
|
+
// made before a /goal chat's first typed prompt has only its objective (second review of #69).
|
|
407
|
+
if (current.firstKey !== undefined) return current
|
|
408
|
+
const { firstPrompt, ...rest } = current
|
|
409
|
+
const first = firstPrompt ?? keyPrompt(eventAt)
|
|
410
|
+
return { ...rest, ...(first !== undefined ? { firstKey: firstKey(first) } : {}), ...(Number.isInteger(rest.forkSeq) ? { title: undefined } : {}) }
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/**
|
|
414
|
+
* Locked read-modify-write of this agent's meta; creates it (with a chat number) on first use. An
|
|
415
|
+
* older meta is brought up to date before `change` sees it, so a title the change sets is kept.
|
|
416
|
+
*/
|
|
417
|
+
const updateFor = (agent, change) => updateMeta(dirFor(agent), sidOf(agent), (meta) => withForkPoint(agent, change(upToDate(agent, withForkPoint(agent, meta)))), () => initialMeta(agent))
|
|
418
|
+
|
|
419
|
+
/** Set flags such as `memoryOff` or `private`, creating the chat's meta if needed. */
|
|
420
|
+
const setFlags = (agent, flags) => updateFor(agent, (meta) => ({ ...meta, ...flags, lastActiveAt: new Date().toISOString() }))
|
|
421
|
+
|
|
422
|
+
/** Fork points learned from dsh for chats whose meta has none, for this app run (0: not a fork). */
|
|
423
|
+
const forkPoints = new Map()
|
|
424
|
+
const forkPointOf = (sid, meta) => Math.max(Number(meta?.forkSeq) || 0, forkPoints.get(sid) ?? 0)
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Learn from dsh where a chat's own history starts when its meta cannot say (forkUnknown): one
|
|
428
|
+
* read of its log, remembered for this app run. Resolves to whether the chat is a fork.
|
|
429
|
+
*/
|
|
430
|
+
async function learnForkPoint(sid, meta, signal) {
|
|
431
|
+
if (!forkUnknown(meta) || forkPoints.has(sid)) return false
|
|
432
|
+
const observation = await observe(sid, signal)
|
|
433
|
+
if (!observation) return false
|
|
434
|
+
try {
|
|
435
|
+
forkPoints.set(sid, forkSeqOf(observation) ?? 0)
|
|
436
|
+
} finally {
|
|
437
|
+
observation[Symbol.dispose]?.()
|
|
438
|
+
}
|
|
439
|
+
return forkPoints.get(sid) > 0
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* Ids of the dsh sessions that are forks (header.isSeeded), from one listing of session headers
|
|
444
|
+
* per app run (#58); undefined while dsh cannot list them here, null when the listing failed. A
|
|
445
|
+
* listing that failed is not tried again in this run: every message would wait for it (review of #58).
|
|
446
|
+
*/
|
|
447
|
+
let seeded
|
|
448
|
+
function seededIds() {
|
|
449
|
+
const query = sessionQuery()
|
|
450
|
+
if (typeof query?.listSessions !== 'function') return undefined
|
|
451
|
+
seeded ??= Promise.resolve()
|
|
452
|
+
.then(() => query.listSessions())
|
|
453
|
+
.then((records) => new Set(records.filter((r) => r?.header?.isSeeded === true).map((r) => String(r.header.id))))
|
|
454
|
+
.catch(() => {
|
|
455
|
+
// Tried again later (third check): until then the unsure chats stay held back.
|
|
456
|
+
setTimeout(() => { seeded = undefined }, RETRY_MS).unref?.()
|
|
457
|
+
return null
|
|
458
|
+
})
|
|
459
|
+
return seeded
|
|
460
|
+
}
|
|
461
|
+
const learning = new Map()
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* A fork indexed by 0.4-0.6 records no fork point, so its inherited topics would be matched as its
|
|
465
|
+
* own until it is opened or recalled (#58). dsh's listing, which the first message of a run that
|
|
466
|
+
* meets such a meta waits for, names the forks: one whose fork point is not known yet is left out
|
|
467
|
+
* (fail closed) while it is learned in the background, once per run (a log that cannot be read
|
|
468
|
+
* keeps it out); a chat dsh does not list as a fork is not one.
|
|
469
|
+
*/
|
|
470
|
+
async function withoutUnknownForks(sid, chats) {
|
|
471
|
+
const unsure = chats.filter((c) => c.sid !== sid && forkUnknown(c.meta) && !forkPoints.has(c.sid))
|
|
472
|
+
const forks = unsure.length > 0 ? await seededIds() : undefined
|
|
473
|
+
if (forks === undefined) return chats
|
|
474
|
+
const waiting = new Set()
|
|
475
|
+
for (const c of unsure) {
|
|
476
|
+
// A listing that failed tells nothing: each such chat is learned from its own log and held
|
|
477
|
+
// back meanwhile (fail closed, second review of #58).
|
|
478
|
+
if (forks && !forks.has(c.sid)) { forkPoints.set(c.sid, 0); continue }
|
|
479
|
+
waiting.add(c.sid)
|
|
480
|
+
if (!learning.has(c.sid)) learning.set(c.sid, learnForkPoint(c.sid, c.meta).catch(() => false).then((learned) => {
|
|
481
|
+
if (!forkPoints.has(c.sid)) setTimeout(() => learning.delete(c.sid), RETRY_MS).unref?.()
|
|
482
|
+
return learned
|
|
483
|
+
}))
|
|
484
|
+
}
|
|
485
|
+
return chats.filter((c) => !waiting.has(c.sid))
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
const viewFrom = (dir, sid, { sid: other, meta }) => loadGraph(chatDir(dir, other)).then((graph) => ({
|
|
489
|
+
sid: other,
|
|
490
|
+
cid: meta.cid,
|
|
491
|
+
meta,
|
|
492
|
+
own: other === sid,
|
|
493
|
+
// Another chat that is a fork shows only what is its own: topics drawn only from inherited
|
|
494
|
+
// messages (which only v0.4-v0.6 indexed) belong to the chat it was forked from.
|
|
495
|
+
graph: other === sid ? graph : ownPart(graph, forkPointOf(other, meta))
|
|
496
|
+
}))
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* Own chat plus every shared chat of the workspace (`all`: private ones too, for the map). Reads
|
|
500
|
+
* go through the stat-keyed cache, so an unchanged workspace costs a few stats per message (#33).
|
|
501
|
+
*/
|
|
502
|
+
async function loadViews(agent, { all = false } = {}) {
|
|
503
|
+
const dir = dirFor(agent)
|
|
504
|
+
const sid = sidOf(agent)
|
|
505
|
+
const listed = (await listChats(dir)).filter((c) => all || c.sid === sid || sharedOn(c.meta))
|
|
506
|
+
// The map (all) is the user's own view of every chat; nothing in it reaches a model.
|
|
507
|
+
const chats = all ? listed : await withoutUnknownForks(sid, listed)
|
|
508
|
+
// One chat whose topics cannot be read (a file locked by a sync tool, no permission) is
|
|
509
|
+
// skipped, like a bad meta, instead of failing the whole workspace (#62).
|
|
510
|
+
const views = await Promise.all(chats.map((c) => viewFrom(dir, sid, c).catch((error) => {
|
|
511
|
+
console.error(`context-graph: skipping chat ${c.sid} (${error?.code ?? error?.message})`)
|
|
512
|
+
return undefined
|
|
513
|
+
})))
|
|
514
|
+
return views.filter(Boolean)
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* One chat by its number, loading only that chat's topics (#49); undefined unless it is this chat
|
|
519
|
+
* or shared. Without a number: this chat (its meta may not exist yet).
|
|
520
|
+
*/
|
|
521
|
+
async function viewOf(agent, cid) {
|
|
522
|
+
const dir = dirFor(agent)
|
|
523
|
+
const sid = sidOf(agent)
|
|
524
|
+
if (cid === undefined) {
|
|
525
|
+
const meta = (await metaFor(agent)) ?? {}
|
|
526
|
+
return { sid, cid: meta.cid, meta, own: true, graph: await loadGraph(chatDir(dir, sid)) }
|
|
527
|
+
}
|
|
528
|
+
// Visible chats only: a private chat that shares a number with a shared one (possible in data
|
|
529
|
+
// from before v0.7) must not hide it. Blocks never name this chat, so another chat comes first.
|
|
530
|
+
const matches = (await listChats(dir)).filter((c) => String(c.meta.cid) === String(cid) && (c.sid === sid || sharedOn(c.meta)))
|
|
531
|
+
// A copy whose topics cannot be read is left out, as in loadViews (fourth review).
|
|
532
|
+
const views = (await Promise.all(matches.map((c) => viewFrom(dir, sid, c).catch(() => undefined)))).filter(Boolean)
|
|
533
|
+
const others = views.filter((v) => !v.own)
|
|
534
|
+
// Another chat holds the number but cannot be read: never this chat in its place (final review).
|
|
535
|
+
if (others.length === 0 && matches.some((c) => c.sid !== sid)) return undefined
|
|
536
|
+
// A repeated number (a copied folder) goes to the chat with topics, as on the page (page.js drawn).
|
|
537
|
+
return others.find((v) => Object.keys(v.graph.nodes).length > 0) ?? others[0] ?? views[0]
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/** The stored source messages of a view's chat (read-only, cached). */
|
|
541
|
+
const messagesOf = (agent, view) => loadMessages(chatDir(dirFor(agent), view.sid))
|
|
542
|
+
|
|
543
|
+
function resolveTarget(agent) {
|
|
544
|
+
if (cfg.provider && cfg.model) return { provider: cfg.provider, model: cfg.model }
|
|
545
|
+
const routed = agent.session.requestHeader()?.config
|
|
546
|
+
if (routed?.provider && routed?.model) return { provider: routed.provider, model: routed.model }
|
|
547
|
+
const { provider, model } = agent.options ?? {}
|
|
548
|
+
return provider && model ? { provider, model } : undefined
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* The meta whose memory flag governs this agent: its own, or for a subagent the meta of the chat
|
|
553
|
+
* that started the task (a subagent works for that chat and has no memory folder of its own).
|
|
554
|
+
* dsh records only a subagent's direct parent, which for a nested subagent is another subagent,
|
|
555
|
+
* so the chain is walked up through the live agents. A chain that cannot be followed fails
|
|
556
|
+
* closed: memory off for the chat that started it must hold at every depth.
|
|
557
|
+
*/
|
|
558
|
+
async function ownerMeta(agent) {
|
|
559
|
+
if (!isSubagent(agent)) return metaFor(agent)
|
|
560
|
+
let header = agent.session.header
|
|
561
|
+
for (let hops = 0; hops < 32; hops++) {
|
|
562
|
+
const parentId = header?.parentSession
|
|
563
|
+
if (!parentId) return CLOSED
|
|
564
|
+
const parentHeader = agentById(String(parentId))?.session?.header
|
|
565
|
+
const parentIsChat = parentHeader ? parentHeader.origin !== 'subagent' : (Number(header.delegationDepth) || 1) <= 1
|
|
566
|
+
if (parentIsChat) return metaOf(dirFor(agent), parentId)
|
|
567
|
+
// An unknown parent ends the walk below: no header, no parentSession, closed.
|
|
568
|
+
header = parentHeader
|
|
569
|
+
}
|
|
570
|
+
return CLOSED
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* For a subagent forked from a chat (dsh seeds it with the chat's log, blocks included, and gives
|
|
575
|
+
* it the chat's tools): the state of the chat that started it, which its requests are charged to,
|
|
576
|
+
* and the block tokens it carries, read again after a compaction (review of #35). Else undefined.
|
|
577
|
+
*/
|
|
578
|
+
const carried = new WeakMap()
|
|
579
|
+
function forkedFrom(sid) {
|
|
580
|
+
const agent = agentById(sid)
|
|
581
|
+
if (!agent || !isSubagent(agent) || !forkSeqOf(agent.session)) return undefined
|
|
582
|
+
let header = agent.session.header
|
|
583
|
+
for (let hops = 0; hops < 32 && header?.parentSession; hops++) {
|
|
584
|
+
const parentId = String(header.parentSession)
|
|
585
|
+
const parentHeader = agentById(parentId)?.session?.header
|
|
586
|
+
if (parentHeader?.origin === 'subagent') { header = parentHeader; continue }
|
|
587
|
+
const st = tracked(parentId)
|
|
588
|
+
if (!st) return undefined
|
|
589
|
+
const generation = agent.session.surface?.replaceGeneration
|
|
590
|
+
let seen = carried.get(agent)
|
|
591
|
+
if (!seen || seen.generation !== generation) carried.set(agent, (seen = { generation, tokens: shownContext(agent.session.surface.nodes, (seq) => agent.session.eventAt(seq)).tokens }))
|
|
592
|
+
return { st, tokens: seen.tokens }
|
|
593
|
+
}
|
|
594
|
+
return undefined
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
// toolTokens: what the recall tool's definition costs per request (set by recall.js).
|
|
598
|
+
return { cfg, state, tracked, seenDirs, noteDir, sidOf, isSubagent, pendingInput, dirFor, metaFor, inheritOffs, offsOf, ancestorOf, sameText, ownerMeta, forkedFrom, updateFor, setFlags, loadViews, viewOf, messagesOf, resolveTarget, forkSeqOf, indexStart, forkPoints, learnForkPoint, observe, creditUse, toolTokens: 0 }
|
|
599
|
+
}
|