dsh-context-graph 0.6.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +563 -0
- package/README.md +252 -97
- package/README.zh.md +82 -32
- package/cordis.patch.yml +10 -6
- package/index.js +7 -432
- package/package.json +11 -12
- package/src/client/client.js +880 -0
- package/src/core/builder.js +210 -0
- package/src/core/chat.js +34 -0
- package/src/core/graph.js +134 -0
- package/src/core/index.js +14 -0
- package/src/core/ledger.js +108 -0
- package/src/core/match.js +377 -0
- package/src/core/render.js +195 -0
- package/src/core/session.js +446 -0
- package/src/core/text.js +220 -0
- package/src/core/time.js +147 -0
- package/src/core/tree.js +39 -0
- package/src/host/builder.js +416 -0
- package/src/host/command.js +155 -0
- package/src/host/config.js +50 -0
- package/src/host/memory.js +175 -0
- package/src/host/meter.js +75 -0
- package/src/host/page.js +186 -0
- package/src/host/plugin.js +46 -0
- package/src/host/recall.js +227 -0
- package/src/host/runtime.js +599 -0
- package/src/store/chats.js +236 -0
- package/src/store/files.js +181 -0
- package/src/store/ledger.js +65 -0
- package/src/store/messages.js +88 -0
- package/src/store/migrate.js +143 -0
- package/graph.js +0 -496
- package/lib/client.js +0 -126
- package/store.js +0 -81
- /package/{update.js → src/host/update.js} +0 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `/context-graph` command. Nested plugin: it starts only where a human-command surface exists
|
|
3
|
+
* (the Web and desktop apps), not in headless or ACP runs.
|
|
4
|
+
*
|
|
5
|
+
* No subcommand takes the agent's maintenance slot: that held the user's next message for the whole
|
|
6
|
+
* indexing call and failed outright while a turn was running (#19). Builds read the session and
|
|
7
|
+
* write only this plugin's files, and the per-chat `building` flag already prevents overlap.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { memoryOn, offPoint, sharedOn } from '../core/chat.js'
|
|
11
|
+
import { addCounters, savingsDetail, savingsLine } from '../core/ledger.js'
|
|
12
|
+
import { SCORE, matchAcross } from '../core/match.js'
|
|
13
|
+
import { renderMap } from '../core/render.js'
|
|
14
|
+
import { collectSpan } from '../core/session.js'
|
|
15
|
+
import { parseTimeRange } from '../core/time.js'
|
|
16
|
+
import { chatDir, loadGraph } from '../store/chats.js'
|
|
17
|
+
import { chatEntry, workspaceTotal } from '../store/ledger.js'
|
|
18
|
+
import { USAGE, version } from './config.js'
|
|
19
|
+
|
|
20
|
+
const plural = (n, word) => `${n} ${word}${n === 1 ? '' : 's'}`
|
|
21
|
+
|
|
22
|
+
/** Chats `/context-graph map` lists: about 40 KB at most, whatever the workspace's size (#82). */
|
|
23
|
+
const MAP_MAX_CHATS = 40
|
|
24
|
+
|
|
25
|
+
export function registerCommand(ctx, rt, { build, turnOn, onAtIdle, meter, updates }) {
|
|
26
|
+
const { cfg } = rt
|
|
27
|
+
ctx.plugin({
|
|
28
|
+
name: 'context-graph-command',
|
|
29
|
+
inject: ['commands'],
|
|
30
|
+
apply(cmdCtx) {
|
|
31
|
+
cmdCtx.effect(function* () {
|
|
32
|
+
yield cmdCtx.commands.register({
|
|
33
|
+
name: 'context-graph',
|
|
34
|
+
description: `Chat memory graph: /context-graph ${USAGE}`,
|
|
35
|
+
// Without `input`, dsh clients run the bare name and send any typed arguments to the model as chat.
|
|
36
|
+
input: { hint: USAGE },
|
|
37
|
+
async handler(invocation) {
|
|
38
|
+
const [sub = 'stats', ...rest] = invocation.rawInput.trim().split(/\s+/).filter(Boolean)
|
|
39
|
+
const agent = invocation.agent
|
|
40
|
+
const sid = rt.sidOf(agent)
|
|
41
|
+
if (sub === 'on') {
|
|
42
|
+
// Input typed while memory was off can still be on its way into the log: the rest of a
|
|
43
|
+
// running turn, or messages the user typed that dsh holds in the agent's inbox until it
|
|
44
|
+
// takes them (its own notices do not count). Memory then comes back when the chat is
|
|
45
|
+
// idle, so all of it stays in the off stretch (review of #59).
|
|
46
|
+
const busy = collectSpan((seq) => agent.session.eventAt(seq), -1, { completeOnly: true }).open || rt.pendingInput(agent)
|
|
47
|
+
let meta
|
|
48
|
+
if (busy && !memoryOn(await rt.metaFor(agent))) {
|
|
49
|
+
// builder.js turns it on once the chat is idle. The turn may have ended while the
|
|
50
|
+
// flag was written, with no turn end left to come: checked once more (third review).
|
|
51
|
+
await rt.updateFor(agent, (m) => ({ ...m, onWhenIdle: true }))
|
|
52
|
+
if (!(await onAtIdle(agent))) return { kind: 'success', text: 'Memory on when this turn ends · what was typed or queued while it was off stays private\n/context-graph stats shows when it is on.' }
|
|
53
|
+
meta = await rt.metaFor(agent)
|
|
54
|
+
} else {
|
|
55
|
+
// The flag is the real effect; indexing continues in the background.
|
|
56
|
+
meta = await turnOn(agent)
|
|
57
|
+
}
|
|
58
|
+
const shared = (await rt.loadViews(agent)).filter((v) => !v.own && Object.keys(v.graph.nodes).length > 0).length
|
|
59
|
+
// Says whether other chats can use this one: a private chat stays private after on (#64).
|
|
60
|
+
const reach = sharedOn(meta) ? 'other chats in this workspace can use it; /context-graph unshare keeps it private' : 'it stays private; /context-graph share lets other chats use it'
|
|
61
|
+
return { kind: 'success', text: `Memory on · indexing this chat now · ${plural(shared, 'shared chat')} to draw on\nThis chat indexes itself every ${cfg.buildEveryTurns} turns and ${reach}. /context-graph stats shows progress; /context-graph off stops it.` }
|
|
62
|
+
}
|
|
63
|
+
if (sub === 'off') {
|
|
64
|
+
// Where memory went off: from here until on, nothing is indexed or shared (#59). It holds
|
|
65
|
+
// in this app run at once: the write can wait seconds for a file another program holds,
|
|
66
|
+
// and a turn ending meanwhile must not be indexed past this point (fourth review).
|
|
67
|
+
const end = collectSpan((seq) => agent.session.eventAt(seq)).lastSeq
|
|
68
|
+
const st = rt.state(sid)
|
|
69
|
+
// Counted: a second off while the first still waits keeps it held after the first settles.
|
|
70
|
+
st.offFrom = Math.min(st.offFrom ?? end, end)
|
|
71
|
+
st.offsPending = (st.offsPending ?? 0) + 1
|
|
72
|
+
try {
|
|
73
|
+
await rt.updateFor(agent, (m) => {
|
|
74
|
+
const { onWhenIdle, ...rest } = m
|
|
75
|
+
return { ...rest, memoryOff: true, offSeq: offPoint(m) ?? end, lastActiveAt: new Date().toISOString() }
|
|
76
|
+
})
|
|
77
|
+
} finally {
|
|
78
|
+
if (--st.offsPending === 0) delete st.offFrom
|
|
79
|
+
}
|
|
80
|
+
return { kind: 'success', text: 'Memory off for this chat · no memory blocks are added and recall cannot read other chats\nThe graph stays; /context-graph on resumes. To turn memory off in every chat, switch dsh-context-graph off on the Plugins page.' }
|
|
81
|
+
}
|
|
82
|
+
if (sub === 'share' || sub === 'unshare') {
|
|
83
|
+
const meta = await rt.setFlags(agent, { private: sub === 'unshare' })
|
|
84
|
+
const topics = Object.keys((await loadGraph(chatDir(rt.dirFor(agent), sid))).nodes).length
|
|
85
|
+
const hint = memoryOn(meta) ? '' : '\nMemory is off for this chat, so it has nothing new to share until /context-graph on.'
|
|
86
|
+
return { kind: 'success', text: sub === 'share' ? `Shared with the workspace · ${plural(topics, 'topic')}\nOther chats with memory on can now match and recall this chat.${hint}` : `No longer shared · ${plural(topics, 'topic')} stay private to this chat` }
|
|
87
|
+
}
|
|
88
|
+
if (sub === 'build') {
|
|
89
|
+
const result = await build(agent, { force: true, signal: invocation.signal, completeOnly: true })
|
|
90
|
+
if (result.status !== 'built') return { kind: 'error', text: `Build ${result.status}: ${result.reason ?? ''}`.trim() }
|
|
91
|
+
return { kind: 'success', text: `${plural(result.nodes, 'topic')} indexed (${result.newNodes} new) · this chat has ${plural(result.totalNodes, 'topic')}${result.more ? ' · older turns read; the rest follows at the next build' : ''}\nLinks: ${result.edges}\nFolder: ${result.dir}` }
|
|
92
|
+
}
|
|
93
|
+
if (sub === 'map') {
|
|
94
|
+
// Every chat of the workspace, private ones too: this is the user's own view. Loaded once (#49).
|
|
95
|
+
const all = await rt.loadViews(agent, { all: true })
|
|
96
|
+
// dsh keeps the card's whole text in the chat's session log, at every call: the newest
|
|
97
|
+
// chats only (loadViews keeps listChats' order, newest first), the page has the rest (#82).
|
|
98
|
+
const more = all.length > MAP_MAX_CHATS ? `\n… ${plural(all.length - MAP_MAX_CHATS, 'more chat')}; Memory tree in the sidebar has them all` : ''
|
|
99
|
+
return { kind: 'success', text: `${plural(all.length, 'chat')} in ${rt.dirFor(agent)}\n${renderMap(all.slice(0, MAP_MAX_CHATS))}${more}` }
|
|
100
|
+
}
|
|
101
|
+
const views = await rt.loadViews(agent)
|
|
102
|
+
if (sub === 'find') {
|
|
103
|
+
const query = rest.join(' ')
|
|
104
|
+
// Every match, each copy on its own line: a person searching wants to see all of them.
|
|
105
|
+
const { hits } = matchAcross(views, query, { max: 10, range: parseTimeRange(query), floor: SCORE.label, cover: false, collapse: false })
|
|
106
|
+
if (hits.length === 0) return { kind: 'success', text: 'No matching topics in shared chats' }
|
|
107
|
+
return { kind: 'success', text: [`${hits.length} matching ${hits.length === 1 ? 'topic' : 'topics'}`, ...hits.map(({ id, node }) => `${id} ${node.label}: ${node.gist}`)].join('\n') }
|
|
108
|
+
}
|
|
109
|
+
if (sub !== 'stats') return { kind: 'error', text: `Usage: /context-graph ${USAGE}` }
|
|
110
|
+
const own = views.find((v) => v.own)
|
|
111
|
+
// The chat's own flags, also when its topics could not be read (#62): never "on, shared" by default.
|
|
112
|
+
const meta = own?.meta ?? (await rt.metaFor(agent))
|
|
113
|
+
const m = meter.main
|
|
114
|
+
const cached = m.cacheRead + m.cacheWrite
|
|
115
|
+
const hitRate = m.input + cached > 0 ? Math.round((m.cacheRead / (m.input + cached)) * 100) : 0
|
|
116
|
+
const session = agent.session
|
|
117
|
+
// A new fork counts from its own first turn, not from the inherited ones (#55).
|
|
118
|
+
// While memory is off nothing after the off point waits to be indexed: on makes it a gap (#59).
|
|
119
|
+
const gaps = memoryOn(meta) ? meta?.gaps : [...(meta?.gaps ?? []), [offPoint(meta) + 1, Infinity]]
|
|
120
|
+
const pending = collectSpan((seq) => session.eventAt(seq), rt.indexStart(agent, meta), { gaps }).userTurns
|
|
121
|
+
const shared = views.filter((v) => !v.own).length
|
|
122
|
+
const o = meter.other
|
|
123
|
+
const last = rt.state(sid).lastBuild
|
|
124
|
+
// Counters not yet written (this turn) are included, so the card is current. A ledger
|
|
125
|
+
// that cannot be read now (a sync tool holding it) costs the savings lines, not the card.
|
|
126
|
+
const pendingDelta = rt.state(sid).delta
|
|
127
|
+
const ledger = await Promise.all([chatEntry(rt.dirFor(agent), sid), workspaceTotal(rt.dirFor(agent))]).catch(() => undefined)
|
|
128
|
+
const here = ledger && addCounters(ledger[0].c, pendingDelta)
|
|
129
|
+
const everywhere = ledger && addCounters(ledger[1], pendingDelta)
|
|
130
|
+
const lastLine = !last ? 'Last index: none since the app started'
|
|
131
|
+
: last.status === 'built' ? `Last index: ${plural(last.nodes, 'topic')} (${last.newNodes} new) at ${last.at.slice(11, 16)} UTC`
|
|
132
|
+
: `Last index ${last.status} at ${last.at.slice(11, 16)} UTC: ${last.reason}`
|
|
133
|
+
return {
|
|
134
|
+
kind: 'success',
|
|
135
|
+
text: [
|
|
136
|
+
`Memory ${memoryOn(meta) ? 'on' : meta?.onWhenIdle === true ? 'off until this turn ends' : 'off'} · ${own || !meta ? `${plural(Object.keys(own?.graph.nodes ?? {}).length, 'topic')} in this chat` : 'topics of this chat could not be read'} · ${sharedOn(meta) ? 'shared' : 'private'} · ${plural(shared, 'shared chat')} · ${plural(pending, 'turn')} waiting to be indexed${updates.current() ? ' · update available' : ''}`,
|
|
137
|
+
// What memory did for this chat and the workspace, with the estimate's terms (#35).
|
|
138
|
+
ledger ? savingsLine(here, everywhere) : 'Savings: unavailable for now (the savings ledger could not be read)',
|
|
139
|
+
lastLine,
|
|
140
|
+
`Version ${version}: ${updates.current() ?? 'no newer version found'}`,
|
|
141
|
+
`Graph folder: ${rt.dirFor(agent)}`,
|
|
142
|
+
...(ledger ? [savingsDetail(here)] : []),
|
|
143
|
+
'Counters below are since the app started:',
|
|
144
|
+
`Builds: ${meter.builds}; builder tokens in/out: ${meter.builder.input + meter.builder.cacheRead + meter.builder.cacheWrite}/${meter.builder.output}`,
|
|
145
|
+
`Injected blocks: ${meter.injected} (~${meter.injectedTokens} tokens); recalls: ${meter.recalls}`,
|
|
146
|
+
`Conversation requests: ${m.requests}; input ${m.input} uncached + ${m.cacheRead} cache-read + ${m.cacheWrite} cache-write; cache-hit ${hitRate}%`,
|
|
147
|
+
`Other model calls (compaction, titles): ${o.requests}; input ${o.input + o.cacheRead + o.cacheWrite}, output ${o.output}`
|
|
148
|
+
].join('\n')
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
})
|
|
152
|
+
}, 'context-graph command')
|
|
153
|
+
}
|
|
154
|
+
})
|
|
155
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plugin configuration defaults and fixed names. Every key here can be overridden in the
|
|
3
|
+
* `config:` block of the plugin's row in cordis.patch.yml.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { readFileSync } from 'node:fs'
|
|
7
|
+
|
|
8
|
+
export const PURPOSE = 'context-graph'
|
|
9
|
+
export const USAGE = '[on|off|share|unshare|stats|build|map|find <words>]'
|
|
10
|
+
export const { version } = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8'))
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Budgets are in tokens (estimateTokens), not characters: Chinese costs about 3.5x more per
|
|
14
|
+
* character, so a character cap let a block cost three times its English size (#39).
|
|
15
|
+
*/
|
|
16
|
+
export const DEFAULTS = {
|
|
17
|
+
dir: '.context-graph',
|
|
18
|
+
buildEveryTurns: 4,
|
|
19
|
+
buildMinChars: 400,
|
|
20
|
+
/** Topics per message at most; usually fewer, only relevant ones pass (#23). */
|
|
21
|
+
maxHits: 5,
|
|
22
|
+
/** A clearly leading hit has its source inlined only up to this size. */
|
|
23
|
+
inlineMaxTokens: 300,
|
|
24
|
+
/** The chat list a new chat gets once (#27). */
|
|
25
|
+
listMaxTokens: 200,
|
|
26
|
+
/** recall of a topic: its newest cited messages, sharing one token budget (#29). */
|
|
27
|
+
sourceMaxMessages: 3,
|
|
28
|
+
sourceMaxTokens: 1000,
|
|
29
|
+
/** recall of a turn (#47). */
|
|
30
|
+
turnMaxTokens: 2000,
|
|
31
|
+
builderMaxTokens: 8192,
|
|
32
|
+
builderMaxInputChars: 60000,
|
|
33
|
+
rangeMaxChats: 3,
|
|
34
|
+
rangeMaxTopics: 3,
|
|
35
|
+
provider: '',
|
|
36
|
+
model: ''
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Keys of an older config block (cordis.patch.yml asks users to restate the whole block) mapped to
|
|
41
|
+
* today's: `inlineMaxChars` (0.2-0.7) becomes `inlineMaxTokens` at about 4 characters a token, so
|
|
42
|
+
* `0` still turns inlining off; `surfaceMaxChats` becomes `listMaxChats`, a cap on the lines of the
|
|
43
|
+
* chat list. An explicit new key wins.
|
|
44
|
+
*/
|
|
45
|
+
export function upgradeConfig(config) {
|
|
46
|
+
const { inlineMaxChars, surfaceMaxChats, ...out } = config
|
|
47
|
+
if (inlineMaxChars !== undefined && out.inlineMaxTokens === undefined) out.inlineMaxTokens = Math.round((Number(inlineMaxChars) || 0) / 4)
|
|
48
|
+
if (surfaceMaxChats !== undefined && out.listMaxChats === undefined) out.listMaxChats = Number(surfaceMaxChats) || 0
|
|
49
|
+
return out
|
|
50
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory injection: on every human turn, match the prompt against the other shared chats of the
|
|
3
|
+
* workspace and append one short `[context-graph]` block at the END of the request, so the
|
|
4
|
+
* provider's prefix cache survives. A chat's first block also carries the top layer: one line per
|
|
5
|
+
* earlier chat. Unrelated messages get nothing (#23). Nothing else is ever added to a chat.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { createUserMessage } from '@deepseek-ai/dsh-llm'
|
|
9
|
+
|
|
10
|
+
import { SOURCE_KIND, memoryOn } from '../core/chat.js'
|
|
11
|
+
import { matchAcross } from '../core/match.js'
|
|
12
|
+
import { formatSource, renderChats, renderHits, renderRange } from '../core/render.js'
|
|
13
|
+
import { PROMPT_KINDS, promptText, shownContext } from '../core/session.js'
|
|
14
|
+
import { estimateTokens } from '../core/text.js'
|
|
15
|
+
import { asksAboutPast, parseTimeRange } from '../core/time.js'
|
|
16
|
+
import { sourceOf } from '../store/messages.js'
|
|
17
|
+
|
|
18
|
+
const TOPIC_LINE = /^\s*- (c\d+n\d+) /gm
|
|
19
|
+
|
|
20
|
+
/** A goal round of a revision that already had one on the surface: its objective was matched then (#69). */
|
|
21
|
+
const roundSeen = (agent, source) => source?.kind === 'goal' && agent.session.surface.nodes.some((seq) => {
|
|
22
|
+
const event = agent.session.eventAt(seq)
|
|
23
|
+
const s = event?.type === 'user/message' ? event.data?.source : undefined
|
|
24
|
+
return s?.kind === 'goal' && s.goalId === source.goalId && s.revision === source.revision
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
export function registerMemory(ctx, rt, meter) {
|
|
28
|
+
const { cfg } = rt
|
|
29
|
+
|
|
30
|
+
ctx.on('agent/pre-step', async (payload, next) => {
|
|
31
|
+
const agent = payload.agent
|
|
32
|
+
// The user's input however dsh drove it: typed, a /goal round (its objective, once per goal
|
|
33
|
+
// revision), a reminder, a late answer to a question (#69).
|
|
34
|
+
const prompt = payload.messages.filter((m) => PROMPT_KINDS.has(m.source?.kind) && !roundSeen(agent, m.source)).map(promptText).filter(Boolean).join('\n')
|
|
35
|
+
const decision = await next()
|
|
36
|
+
if (decision.kind !== 'enter') return decision
|
|
37
|
+
refreshContext(agent)
|
|
38
|
+
return addMemory(agent, prompt, decision)
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* After a compaction, blocks no longer on the surface stop being charged as re-sent (#35, review):
|
|
43
|
+
* checked on every step, memory on or off; read again only when dsh's replace count moved.
|
|
44
|
+
*/
|
|
45
|
+
function refreshContext(agent) {
|
|
46
|
+
const st = rt.isSubagent(agent) ? undefined : rt.tracked(rt.sidOf(agent))
|
|
47
|
+
const generation = agent.session.surface.replaceGeneration
|
|
48
|
+
if (!st || st.generation === generation) return
|
|
49
|
+
st.generation = generation
|
|
50
|
+
st.inContext = shownContext(agent.session.surface.nodes, (seq) => agent.session.eventAt(seq)).tokens
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Whether a fork already holds a topic (#16, #52). A fork carries its ancestors' events under the
|
|
55
|
+
* same seqs, so a topic of ANY chat (parent, grandparent ...) whose every cited message is word for
|
|
56
|
+
* word this chat's own event at that seq, before its fork point, is already in its context.
|
|
57
|
+
* Decided by content, so no parent has to be identified or remembered.
|
|
58
|
+
*/
|
|
59
|
+
function holds(agent, forkSeq, stored, node) {
|
|
60
|
+
const refs = node.refs ?? []
|
|
61
|
+
if (refs.length === 0 || !refs.every((seq) => seq < forkSeq)) return false
|
|
62
|
+
return refs.every((seq) => stored[String(seq)]?.x !== undefined && rt.sameText(stored[String(seq)].x, agent, forkSeq, seq))
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Every ancestor of a fork starts with the fork's own first prompt (dsh copies the log from seq 0),
|
|
67
|
+
* so only chats with that first prompt need their stored messages compared. This keeps a fork's
|
|
68
|
+
* first message from parsing every chat's msgs.json (review of #52).
|
|
69
|
+
*/
|
|
70
|
+
const ancestorLike = (view, agent) => rt.ancestorOf(view.meta, agent.session)
|
|
71
|
+
|
|
72
|
+
/** The hits a fork already holds. */
|
|
73
|
+
async function inherited(agent, forkSeq, hits) {
|
|
74
|
+
const out = new Set()
|
|
75
|
+
if (!forkSeq) return out
|
|
76
|
+
for (const hit of hits) {
|
|
77
|
+
if (ancestorLike(hit.view, agent) && holds(agent, forkSeq, await rt.messagesOf(agent, hit.view), hit.node)) out.add(hit)
|
|
78
|
+
}
|
|
79
|
+
return out
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Chats a fork holds entirely (its ancestors up to the fork point): not worth a line in its list. */
|
|
83
|
+
async function heldChats(agent, forkSeq, views) {
|
|
84
|
+
const out = new Set()
|
|
85
|
+
if (!forkSeq) return out
|
|
86
|
+
for (const view of views) {
|
|
87
|
+
const nodes = Object.values(view.graph.nodes)
|
|
88
|
+
if (view.own || nodes.length === 0 || !ancestorLike(view, agent) || !nodes.every((n) => (n.refs ?? []).every((seq) => seq < forkSeq))) continue
|
|
89
|
+
const stored = await rt.messagesOf(agent, view)
|
|
90
|
+
if (nodes.every((node) => holds(agent, forkSeq, stored, node))) out.add(view)
|
|
91
|
+
}
|
|
92
|
+
return out
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async function addMemory(agent, prompt, decision) {
|
|
96
|
+
try {
|
|
97
|
+
if (rt.isSubagent(agent) || !prompt) return decision
|
|
98
|
+
const st = rt.state(rt.sidOf(agent))
|
|
99
|
+
// From now on the savings estimate follows this chat (meter.js), memory on or off: the recall
|
|
100
|
+
// tool rides with its requests either way.
|
|
101
|
+
st.chat = true
|
|
102
|
+
rt.noteDir(agent)
|
|
103
|
+
// An off whose write is still waiting for the file already holds here (runtime.js withPending).
|
|
104
|
+
if (!memoryOn(await rt.metaFor(agent))) return decision
|
|
105
|
+
const views = await rt.loadViews(agent)
|
|
106
|
+
// What this chat already holds: blocks and recalls still on its surface.
|
|
107
|
+
const shown = shownContext(agent.session.surface.nodes, (seq) => agent.session.eventAt(seq))
|
|
108
|
+
// Read from the surface on every human turn, so a compaction that dropped old blocks is seen.
|
|
109
|
+
st.inContext = shown.tokens
|
|
110
|
+
st.generation = agent.session.surface.replaceGeneration
|
|
111
|
+
const forkSeq = rt.forkSeqOf(agent.session)
|
|
112
|
+
const parts = []
|
|
113
|
+
let layer
|
|
114
|
+
// Chats this fork holds word for word (its ancestors): listed nowhere, neither in the list nor in a day digest.
|
|
115
|
+
let heldViews
|
|
116
|
+
const held = async () => (heldViews ??= await heldChats(agent, forkSeq, views))
|
|
117
|
+
const range = parseTimeRange(prompt)
|
|
118
|
+
const { hits: found, lead, total } = matchAcross(views, prompt, { max: cfg.maxHits, range, skip: shown.ids })
|
|
119
|
+
const holds = await inherited(agent, forkSeq, found)
|
|
120
|
+
const hits = found.filter((hit) => !holds.has(hit))
|
|
121
|
+
if (total === 0 && range && asksAboutPast(prompt)) {
|
|
122
|
+
// A question about a day with no topic hit gets that day's digest, kept small; recall("YYYY-MM-DD") has the rest (#24).
|
|
123
|
+
const skipped = await held()
|
|
124
|
+
// Also bounded in tokens, like a day recall (#77): long gists cannot grow it past a recall's size.
|
|
125
|
+
const block = renderRange(views.filter((v) => !v.own && !skipped.has(v)), range, { maxChats: cfg.rangeMaxChats, maxTopics: cfg.rangeMaxTopics, maxTokens: cfg.sourceMaxTokens })
|
|
126
|
+
const ids = [...block.matchAll(TOPIC_LINE)].map((m) => m[1])
|
|
127
|
+
const seen = shown.texts.has(block) || (ids.length > 0 && ids.every((id) => shown.ids.has(id)))
|
|
128
|
+
if (!seen) parts.push(block)
|
|
129
|
+
}
|
|
130
|
+
let inline
|
|
131
|
+
if (hits.length > 0) {
|
|
132
|
+
// Only a hit that clearly answers the message has its source inlined, within a token budget (#39).
|
|
133
|
+
if (lead && hits[0] === lead) {
|
|
134
|
+
const text = formatSource(sourceOf(await rt.messagesOf(agent, lead.view), lead.node))
|
|
135
|
+
if (text && estimateTokens(text) <= cfg.inlineMaxTokens) inline = { id: lead.id, text }
|
|
136
|
+
}
|
|
137
|
+
parts.push(renderHits(hits, { inline }))
|
|
138
|
+
}
|
|
139
|
+
// The top layer, once per chat (#26). Evidence before tokens (#76): it rides with the chat's
|
|
140
|
+
// first block or a question about earlier work, on whichever turn that comes (a chat that
|
|
141
|
+
// opens with a greeting gets it later, review); a message about something else gets nothing,
|
|
142
|
+
// as the list would be re-sent with every later request. The first block is the one sent
|
|
143
|
+
// while no block is on the surface: an app restart, which leaves the blocks there, does not
|
|
144
|
+
// bring the list back, nor a compaction in the same run (listed: only once it is sent). A fork
|
|
145
|
+
// carries its parent's context: like any chat it gets the list once more after
|
|
146
|
+
// /context-graph on, not before. Never while a list is still on the surface.
|
|
147
|
+
const due = st.announce || (!forkSeq && shown.texts.size === 0)
|
|
148
|
+
if (due && !st.listed && !shown.list && (parts.length > 0 || asksAboutPast(prompt))) {
|
|
149
|
+
const skipped = await held()
|
|
150
|
+
const list = renderChats(views.filter((v) => !skipped.has(v)), { prompt, maxTokens: cfg.listMaxTokens, maxChats: cfg.listMaxChats })
|
|
151
|
+
if (list) {
|
|
152
|
+
parts.unshift(list)
|
|
153
|
+
layer = 'surface'
|
|
154
|
+
st.listed = true
|
|
155
|
+
st.announce = false
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
if (parts.length === 0) return decision
|
|
159
|
+
const text = parts.join('\n\n')
|
|
160
|
+
const tokens = estimateTokens(text)
|
|
161
|
+
meter.injected++
|
|
162
|
+
meter.injectedTokens += tokens
|
|
163
|
+
st.delta.sent += tokens
|
|
164
|
+
st.inContext += tokens
|
|
165
|
+
st.justSent += tokens
|
|
166
|
+
// Checked against the answer at turn end: a topic the answer uses counts as saved (#35).
|
|
167
|
+
for (const hit of hits) st.pending.push({ id: hit.id, sid: hit.view.sid, cid: hit.view.cid, node: hit.node, prompt })
|
|
168
|
+
const message = createUserMessage({ content: [{ type: 'text', text }], source: layer ? { kind: SOURCE_KIND, layer } : { kind: SOURCE_KIND } })
|
|
169
|
+
return { ...decision, messages: [...decision.messages, message] }
|
|
170
|
+
} catch (error) {
|
|
171
|
+
console.error('context-graph: matcher failed (non-fatal)', error)
|
|
172
|
+
return decision
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token meter: watches every `llm/stream` call and adds its usage to one bucket:
|
|
3
|
+
* main the conversation's own requests (dsh's isAgentLoopRequest)
|
|
4
|
+
* builder this plugin's indexing calls (purpose 'context-graph')
|
|
5
|
+
* other everything else, such as dsh compaction and session titles (#21)
|
|
6
|
+
* These counters live for the app run; /context-graph stats shows them.
|
|
7
|
+
*
|
|
8
|
+
* It also feeds the per-chat savings estimate (#35, src/core/ledger.js) of every chat that had a
|
|
9
|
+
* human turn in this run: each conversation request re-sends the plugin's blocks still in context
|
|
10
|
+
* and carries the recall tool, and each indexing call reports its real usage. All of it is counted
|
|
11
|
+
* in memory and written to the ledger at turn end (src/host/builder.js).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { isAgentLoopRequest } from '@deepseek-ai/dsh-llm'
|
|
15
|
+
|
|
16
|
+
import { PURPOSE } from './config.js'
|
|
17
|
+
|
|
18
|
+
const bucket = () => ({ requests: 0, input: 0, cacheRead: 0, cacheWrite: 0, output: 0 })
|
|
19
|
+
|
|
20
|
+
export function createMeter() {
|
|
21
|
+
return { main: bucket(), builder: bucket(), other: bucket(), builds: 0, injected: 0, injectedTokens: 0, recalls: 0 }
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Which bucket a request belongs to. `isAgentLoopRequest` checks object identity, so pass the options as received. */
|
|
25
|
+
export function bucketOf(meter, options) {
|
|
26
|
+
if (options?.purpose === PURPOSE) return meter.builder
|
|
27
|
+
return isAgentLoopRequest(options) ? meter.main : meter.other
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function registerMeter(ctx, meter, rt) {
|
|
31
|
+
ctx.on('llm/stream', (options, next) => {
|
|
32
|
+
const inner = next()
|
|
33
|
+
const target = bucketOf(meter, options)
|
|
34
|
+
// Only chats the memory hook has seen (a human turn in this run): subagents and other sessions are not charged.
|
|
35
|
+
const st = options?.sessionId === undefined ? undefined : rt?.tracked(String(options.sessionId))
|
|
36
|
+
if (st && target === meter.main) {
|
|
37
|
+
st.delta.requests++
|
|
38
|
+
// Blocks sent with this very request count as sent, not as re-sent.
|
|
39
|
+
st.delta.resent += Math.max(0, st.inContext - st.justSent)
|
|
40
|
+
st.delta.tool += rt.toolTokens
|
|
41
|
+
st.justSent = 0
|
|
42
|
+
} else if (!st && target === meter.main && options?.sessionId !== undefined) {
|
|
43
|
+
// A subagent forked from a chat re-sends that chat's blocks and carries the recall tool with
|
|
44
|
+
// every request it makes: charged to the chat that started it (review of #35).
|
|
45
|
+
const fork = rt?.forkedFrom?.(String(options.sessionId))
|
|
46
|
+
if (fork) {
|
|
47
|
+
fork.st.delta.requests++
|
|
48
|
+
fork.st.delta.resent += fork.tokens
|
|
49
|
+
fork.st.delta.tool += rt.toolTokens
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async function* tracked() {
|
|
53
|
+
try {
|
|
54
|
+
for await (const chunk of inner) {
|
|
55
|
+
if (chunk?.type === 'usage' && chunk.usage) {
|
|
56
|
+
const u = chunk.usage
|
|
57
|
+
target.input += u.inputTokens ?? 0
|
|
58
|
+
target.cacheRead += u.cacheReadTokens ?? 0
|
|
59
|
+
target.cacheWrite += u.cacheWriteTokens ?? 0
|
|
60
|
+
target.output += u.outputTokens ?? 0
|
|
61
|
+
if (st && target === meter.builder) {
|
|
62
|
+
st.delta.indexIn += (u.inputTokens ?? 0) + (u.cacheWriteTokens ?? 0)
|
|
63
|
+
st.delta.indexCached += u.cacheReadTokens ?? 0
|
|
64
|
+
st.delta.indexOut += u.outputTokens ?? 0
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
yield chunk
|
|
68
|
+
}
|
|
69
|
+
} finally {
|
|
70
|
+
target.requests++
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return tracked()
|
|
74
|
+
})
|
|
75
|
+
}
|
package/src/host/page.js
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-only routes for the client module: the savings on the plugin's page in Plugins (#35) and
|
|
3
|
+
* the Memory tree page (#85, #86).
|
|
4
|
+
* GET api/context-graph.overview workspaces with memory: chats, topics, savings
|
|
5
|
+
* GET api/context-graph.tree?w=<key> one workspace as a tree: chats, their topics, links (#86)
|
|
6
|
+
* GET api/context-graph.topic?w=<key>&id=c3n12 one topic: gist, aliases, links, its sources
|
|
7
|
+
*
|
|
8
|
+
* Nested plugin with `inject: ['connection']`: it starts only where dsh's connection exists, and
|
|
9
|
+
* dsh answers 403 to every principal but the owner on these routes. A workspace is named by a key
|
|
10
|
+
* the overview handed out, never by a path in the request, so nothing outside the known memory
|
|
11
|
+
* folders can be read. This is the owner's own view: private and memory-off chats are included
|
|
12
|
+
* and marked. Nothing here writes, and nothing reaches a chat's context.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { createHash } from 'node:crypto'
|
|
16
|
+
import { stat } from 'node:fs/promises'
|
|
17
|
+
import { isAbsolute, join } from 'node:path'
|
|
18
|
+
|
|
19
|
+
import { memoryOn, sharedOn } from '../core/chat.js'
|
|
20
|
+
import { has, neighbors } from '../core/graph.js'
|
|
21
|
+
import { memoryTree } from '../core/tree.js'
|
|
22
|
+
import { addCounters, balance, counters, emptyCounters } from '../core/ledger.js'
|
|
23
|
+
import { chatSummary } from '../core/render.js'
|
|
24
|
+
import { chatDir, listChats, loadGraph } from '../store/chats.js'
|
|
25
|
+
import { loadLedger } from '../store/ledger.js'
|
|
26
|
+
import { loadMessages, oldTurns, sourceOf } from '../store/messages.js'
|
|
27
|
+
|
|
28
|
+
const json = (value, status = 200) => new Response(JSON.stringify(value), { status, headers: { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' } })
|
|
29
|
+
const keyOf = (dir) => createHash('sha1').update(dir).digest('hex').slice(0, 12)
|
|
30
|
+
const exists = (path) => stat(path).then(() => true, () => false)
|
|
31
|
+
|
|
32
|
+
export function registerPage(ctx, rt) {
|
|
33
|
+
const { cfg } = rt
|
|
34
|
+
|
|
35
|
+
/** Every memory folder this app knows: dsh's registered workspaces, and folders used in this run. */
|
|
36
|
+
async function known() {
|
|
37
|
+
const dirs = new Map()
|
|
38
|
+
let workspaces = []
|
|
39
|
+
try { workspaces = ctx.get('workspaceRegistry')?.list?.() ?? [] } catch { /* not mounted here */ }
|
|
40
|
+
for (const w of workspaces) {
|
|
41
|
+
if (typeof w?.path !== 'string') continue
|
|
42
|
+
dirs.set(isAbsolute(cfg.dir) ? cfg.dir : join(w.path, cfg.dir), { title: w.title || w.path, path: w.path })
|
|
43
|
+
}
|
|
44
|
+
for (const [dir, info] of rt.seenDirs) if (!dirs.has(dir)) dirs.set(dir, info)
|
|
45
|
+
const out = []
|
|
46
|
+
for (const [dir, info] of dirs) if (await exists(join(dir, 'chats'))) out.push({ key: keyOf(dir), dir, ...info })
|
|
47
|
+
return out
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** The ledger, or none for this answer while it cannot be read (a sync tool holding it): chats and the tree still load. */
|
|
51
|
+
const ledgerOf = (dir) => loadLedger(dir).catch((error) => {
|
|
52
|
+
console.error(`context-graph: page reads no savings for ${dir} (${error?.code ?? error?.message})`)
|
|
53
|
+
return { v: 1, chats: {}, unreadable: true }
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
/** `savings: false`: the topic route needs no ledger, which is one file per chat (#83). */
|
|
57
|
+
async function chatsOf(dir, { savings = true } = {}) {
|
|
58
|
+
const ledger = savings ? await ledgerOf(dir) : { v: 1, chats: {} }
|
|
59
|
+
// One chat whose topics cannot be read is left out, as in the chats themselves (#62).
|
|
60
|
+
const read = async ({ sid, meta }) => ({ sid, cid: meta.cid, meta, graph: await loadGraph(chatDir(dir, sid)), c: counters(ledger.chats[sid]?.c), unreadable: ledger.unreadable === true })
|
|
61
|
+
const chats = await Promise.all((await listChats(dir)).map((chat) => read(chat).catch((error) => {
|
|
62
|
+
console.error(`context-graph: page skips chat ${chat.sid} (${error?.code ?? error?.message})`)
|
|
63
|
+
return undefined
|
|
64
|
+
})))
|
|
65
|
+
return chats.filter(Boolean)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function overview() {
|
|
69
|
+
const out = []
|
|
70
|
+
for (const w of await known()) {
|
|
71
|
+
const chats = await chatsOf(w.dir).catch((error) => {
|
|
72
|
+
console.error(`context-graph: page skips workspace ${w.title} (${error?.code ?? error?.message})`)
|
|
73
|
+
return undefined
|
|
74
|
+
})
|
|
75
|
+
if (!chats) continue
|
|
76
|
+
const ledger = await ledgerOf(w.dir)
|
|
77
|
+
const total = Object.values(ledger.chats).reduce((sum, e) => addCounters(sum, counters(e?.c)), emptyCounters())
|
|
78
|
+
out.push({
|
|
79
|
+
key: w.key,
|
|
80
|
+
title: w.title,
|
|
81
|
+
path: w.path,
|
|
82
|
+
chats: chats.length,
|
|
83
|
+
indexed: chats.filter((c) => Object.keys(c.graph.nodes).length > 0).length,
|
|
84
|
+
topics: chats.reduce((n, c) => n + Object.keys(c.graph.nodes).length, 0),
|
|
85
|
+
used: total.used,
|
|
86
|
+
// null while the ledger cannot be read: the page says so instead of showing zeros (third review).
|
|
87
|
+
savings: ledger.unreadable ? null : balance(total)
|
|
88
|
+
})
|
|
89
|
+
}
|
|
90
|
+
return { workspaces: out }
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* The chats the tree draws, which are the chats the topic route opens: each needs topics and its
|
|
95
|
+
* own number (a meta that could not be read has none until it is written again, and a copied
|
|
96
|
+
* folder can repeat one: the first chat with topics keeps it).
|
|
97
|
+
*/
|
|
98
|
+
async function drawn(dir, options) {
|
|
99
|
+
const seen = new Set()
|
|
100
|
+
return (await chatsOf(dir, options)).filter((c) => Number.isInteger(c.cid) && Object.keys(c.graph.nodes).length > 0 && !seen.has(c.cid) && seen.add(c.cid))
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function tree(w) {
|
|
104
|
+
const chats = await drawn(w.dir)
|
|
105
|
+
const byCid = new Map(chats.map((c) => [c.cid, c]))
|
|
106
|
+
return {
|
|
107
|
+
key: w.key,
|
|
108
|
+
title: w.title,
|
|
109
|
+
// Labels only: gists and sources are fetched when a topic is opened.
|
|
110
|
+
chats: memoryTree(chats.map((c) => ({ cid: c.cid, lastActiveAt: c.meta.lastActiveAt, graph: c.graph }))).map((branch) => {
|
|
111
|
+
const c = byCid.get(branch.cid)
|
|
112
|
+
return {
|
|
113
|
+
...branch,
|
|
114
|
+
title: c.meta.title ?? '',
|
|
115
|
+
summary: chatSummary({ meta: c.meta, graph: c.graph }),
|
|
116
|
+
shared: sharedOn(c.meta),
|
|
117
|
+
memory: memoryOn(c.meta),
|
|
118
|
+
startedAt: c.meta.startedAt,
|
|
119
|
+
lastActiveAt: c.meta.lastActiveAt,
|
|
120
|
+
savings: c.unreadable ? null : balance(c.c),
|
|
121
|
+
used: c.c.used
|
|
122
|
+
}
|
|
123
|
+
})
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async function topic(w, id) {
|
|
128
|
+
const m = /^c(\d+)(n\d+)$/.exec(String(id ?? ''))
|
|
129
|
+
if (!m) return undefined
|
|
130
|
+
const chat = (await drawn(w.dir, { savings: false })).find((c) => String(c.cid) === m[1])
|
|
131
|
+
if (!chat) return undefined
|
|
132
|
+
const graph = chat.graph
|
|
133
|
+
if (!has(graph, m[2])) return undefined
|
|
134
|
+
const node = graph.nodes[m[2]]
|
|
135
|
+
// Sources that cannot be read leave the topic itself readable (review).
|
|
136
|
+
const stored = await loadMessages(chatDir(w.dir, chat.sid)).catch(() => ({}))
|
|
137
|
+
// Turn numbers stored by 0.9.0 counted typed prompts, not dsh's turns: not shown until a
|
|
138
|
+
// recall or a build of that chat converts them; the page never writes (second review of #71).
|
|
139
|
+
const named = !(await oldTurns(chatDir(w.dir, chat.sid)))
|
|
140
|
+
const sources = sourceOf(stored, node).slice(-3).map((s) => ({ role: s.role, turn: named ? s.turn : undefined, text: String(s.text ?? '').slice(0, 600) }))
|
|
141
|
+
return {
|
|
142
|
+
id,
|
|
143
|
+
label: node.label,
|
|
144
|
+
gist: node.gist,
|
|
145
|
+
kind: node.kind,
|
|
146
|
+
aliases: node.aliases ?? [],
|
|
147
|
+
updated: node.updated,
|
|
148
|
+
chat: { cid: chat.meta.cid, title: chat.meta.title ?? '', summary: chatSummary({ meta: chat.meta, graph }) },
|
|
149
|
+
links: neighbors(graph, m[2]).map((l) => ({ id: `c${chat.meta.cid}${l.id}`, label: graph.nodes[l.id].label, relation: l.relation, direction: l.direction })),
|
|
150
|
+
sources
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const route = (name, handle) => ({
|
|
155
|
+
path: `/api/context-graph.${name}`,
|
|
156
|
+
methods: ['GET'],
|
|
157
|
+
requestBody: 'buffered',
|
|
158
|
+
fetch: async (request) => {
|
|
159
|
+
try {
|
|
160
|
+
const query = new URL(request.url).searchParams
|
|
161
|
+
if (name === 'overview') return json(await overview())
|
|
162
|
+
const w = (await known()).find((k) => k.key === query.get('w'))
|
|
163
|
+
if (!w) return json({ error: 'unknown workspace' }, 404)
|
|
164
|
+
const value = await handle(w, query)
|
|
165
|
+
return value === undefined ? json({ error: 'not found' }, 404) : json(value)
|
|
166
|
+
} catch (error) {
|
|
167
|
+
console.error('context-graph: page request failed (non-fatal)', error)
|
|
168
|
+
return json({ error: 'failed' }, 500)
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
ctx.plugin({
|
|
174
|
+
name: 'context-graph-page',
|
|
175
|
+
inject: ['connection'],
|
|
176
|
+
apply(pageCtx) {
|
|
177
|
+
for (const r of [route('overview'), route('tree', (w) => tree(w)), route('topic', (w, q) => topic(w, q.get('id')))]) {
|
|
178
|
+
pageCtx.effect(function* () {
|
|
179
|
+
yield pageCtx.connection.fetch.register(r)
|
|
180
|
+
}, `context-graph page: ${r.path}`)
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
return { overview, tree, topic, known }
|
|
186
|
+
}
|