dsh-context-graph 0.6.2 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,446 @@
1
+ /**
2
+ * Reading dsh session events: the human/assistant exchange, turns, titles, and what this plugin has
3
+ * already put into a chat. Pure: callers pass `eventAt(seq)` and surface seqs.
4
+ */
5
+
6
+ import { SOURCE_KIND } from './chat.js'
7
+ import { clipTokens, estimateTokens, sliceChars, textOf } from './text.js'
8
+
9
+ export const isHuman = (event) => event?.type === 'user/message' && event.data?.source?.kind === 'user'
10
+
11
+ /**
12
+ * `source.kind` of the user/message events that carry the user's input, however dsh drove it (#69,
13
+ * #70): typed (a steer too), a /goal round, a scheduled reminder, a late answer to a question.
14
+ */
15
+ export const PROMPT_KINDS = new Set(['user', 'goal', 'schedule', 'user-question-reply'])
16
+ export const isPrompt = (event) => event?.type === 'user/message' && PROMPT_KINDS.has(event.data?.source?.kind)
17
+
18
+ /** The text of a user or assistant event, as the builder stores it. */
19
+ export const eventText = (event) => textOf(event?.data?.content ?? event?.data?.message?.content)
20
+
21
+ const parse = (text) => { try { return JSON.parse(text) } catch { return undefined } }
22
+
23
+ // dsh writes these values as JSON on a line of their own: dsh-goal-round-driver and dsh-schedule.
24
+ const OBJECTIVE = /^Objective: (.+)$/m
25
+ const REMINDER = /^reminder_prompt_json: (.+)$/m
26
+ const REMINDERS = /^reminders_json: (.+)$/m
27
+ const jsonAt = (re, text) => parse(re.exec(text)?.[1])
28
+
29
+ /** A reminder's own prompts (dsh frames one, or a batch), trimmed as dsh stores them. */
30
+ function reminderPrompts(text) {
31
+ const one = jsonAt(REMINDER, text)
32
+ const batch = jsonAt(REMINDERS, text)
33
+ const prompts = typeof one === 'string' ? [one] : Array.isArray(batch) ? batch.map((r) => r?.reminder_prompt) : []
34
+ return prompts.filter((p) => typeof p === 'string').map((p) => p.trim())
35
+ }
36
+
37
+ /** The user's choices, each after its question when known: "Which database? → Postgres 16" (#70). */
38
+ function answersText(answers, questions) {
39
+ if (!Array.isArray(answers)) return ''
40
+ const asked = new Map((Array.isArray(questions) ? questions : []).map((q) => [q?.id, q?.question]))
41
+ return answers.map((a) => {
42
+ const chosen = [...(Array.isArray(a?.selected) ? a.selected : []), a?.custom].filter((s) => typeof s === 'string' && s.trim()).join('; ')
43
+ const question = asked.get(a?.id)
44
+ return chosen && (typeof question === 'string' ? `${question} → ${chosen}` : chosen)
45
+ }).filter(Boolean).join('\n')
46
+ }
47
+
48
+ /**
49
+ * A prompt's words as the builder and the matcher read them (#69, #70): a goal round's objective and
50
+ * a reminder's own prompt without dsh's framing around them, the choices of a late answer.
51
+ */
52
+ export function promptText(message) {
53
+ const text = textOf(message?.content)
54
+ switch (message?.source?.kind) {
55
+ case 'goal': {
56
+ const objective = jsonAt(OBJECTIVE, text)
57
+ return typeof objective === 'string' ? objective : text
58
+ }
59
+ case 'schedule': return reminderPrompts(text).join('\n') || text
60
+ case 'user-question-reply': {
61
+ const reply = parse(text)
62
+ return reply ? answersText(reply.answers, reply.questions) : text
63
+ }
64
+ default: return text
65
+ }
66
+ }
67
+
68
+ const argsOf = (block) => (typeof block?.arguments === 'string' ? parse(block.arguments) : block?.arguments) ?? {}
69
+
70
+ // How dsh-plan-mode's exit_plan_mode result words the user's review: approved, or sent back to
71
+ // planning with the user's own words (review of #70).
72
+ const APPROVED = /^Plan approved\b/
73
+ const SENT_BACK = /^(?:Error: )?The user chose to keep planning; their feedback: ([^]+)$/
74
+
75
+ const planText = (plan) => (typeof plan === 'string' && plan.trim() ? `Plan:\n${plan.trim()}` : '')
76
+
77
+ /**
78
+ * An assistant message's text, with each plan it handed to exit_plan_mode that the user approved
79
+ * (`approved(callId)`): dsh keeps the plan only as that call's argument (#70), and a plan the user
80
+ * sent back is not the plan (review).
81
+ */
82
+ function assistantText(blocks, approved) {
83
+ const plans = blocks.filter((b) => b?.type === 'tool-call' && b.name === 'exit_plan_mode' && approved(b.id)).map((b) => planText(argsOf(b).plan))
84
+ return [textOf(blocks), ...plans].filter(Boolean).join('\n\n')
85
+ }
86
+
87
+ /**
88
+ * The log event by event, with dsh's own turn numbers (#71) and what stays private (#59, #69).
89
+ * Turns: a message belongs to the turn its `turn/start` opened, so a steer stays in its turn and a
90
+ * goal round or reminder has one of its own; a log without numbered turns counts prompts instead.
91
+ * Private: a seq in `gaps`; a turn whose first prompt is private, as a whole; after a private
92
+ * prompt, the rest of its turn. A prompt is private when typed in a gap or driven from there: a
93
+ * round of a goal that /goal or a goal tool changed in a private stretch, a reminder scheduled
94
+ * there, the answer to a question asked there (one whose goal or question is unknown, too).
95
+ * Each step: { seq, event, turn, start, prompt, hidden, from, role, text }, `from` being where the
96
+ * private stretch of a hidden step starts (a turn made private by its prompt, from its turn/start).
97
+ * Tool calls count however the model made them: as blocks of its message, or under dsh's ptc
98
+ * preset from its run_code program, which dsh logs as tool/ptc-dispatch-start, then (settled, with
99
+ * the result) tool/ptc-dispatch (review of #69, #70). `limit`: the last seq a call's result is
100
+ * looked for at.
101
+ */
102
+ function* walk(eventAt, gaps = [], limit = Infinity) {
103
+ const goals = new Map()
104
+ const reminders = new Set()
105
+ const asks = new Map()
106
+ const plans = new Set()
107
+ const drivenFromGap = ({ source, content }) => {
108
+ if (source.kind === 'goal') return goals.get(source.goalId) ?? true
109
+ if (source.kind === 'schedule') return reminderPrompts(textOf(content)).some((p) => reminders.has(p))
110
+ if (source.kind === 'user-question-reply') return asks.get(source.callId)?.hidden ?? true
111
+ return false
112
+ }
113
+ // A tool call, `isHidden` when made in a private stretch.
114
+ const called = (name, args, id, isHidden) => {
115
+ if (name === 'ask_user_question') asks.set(id, { questions: args.questions, hidden: isHidden })
116
+ else if (name === 'exit_plan_mode') plans.add(id)
117
+ // A reminder scheduled in a private stretch stays private when it comes back (#69).
118
+ else if ((name === 'schedule_create' || name === 'schedule_update') && isHidden && typeof args.prompt === 'string') reminders.add(args.prompt.trim())
119
+ }
120
+ // What a call's result holds of the user's own: the answers to ask_user_question (#70), the
121
+ // words a plan was sent back with (review). The only tool output read (#47).
122
+ const answerIn = (id, content) => {
123
+ const ask = asks.get(id)
124
+ if (ask) return answersText(parse(textOf(content))?.answers, ask.questions)
125
+ return plans.has(id) ? SENT_BACK.exec(textOf(content))?.[1].trim() ?? '' : ''
126
+ }
127
+ // Whether the result of exit_plan_mode call `id` (made at `seq`) approves the plan: looked for in
128
+ // the call's turn, up to `limit` and outside the gaps (review of #70).
129
+ const approvedAfter = (seq, id) => {
130
+ for (let s = seq + 1; s <= limit; s++) {
131
+ const e = eventAt(s)
132
+ if (!e || e.type === 'turn/start' || e.type === 'turn/end') return false
133
+ if (e.type === 'tool/result' && e.data?.message?.toolCallId === id) return !inGap(s, gaps) && APPROVED.test(textOf(e.data.message.content))
134
+ }
135
+ return false
136
+ }
137
+ let numbered = false
138
+ let turn
139
+ let turnSeq = -1
140
+ let turnHidden = false
141
+ let hidden = false
142
+ let prompted = false
143
+ for (let seq = 0; ; seq++) {
144
+ const event = eventAt(seq)
145
+ if (!event) return
146
+ const gap = inGap(seq, gaps)
147
+ const prompt = isPrompt(event)
148
+ let start = false
149
+ if (event.type === 'turn/start' && Number.isInteger(event.data?.turn)) { numbered = true; turn = event.data.turn; start = true }
150
+ else if (prompt && !numbered) { turn = (turn ?? 0) + 1; start = true }
151
+ if (start) { turnSeq = seq; turnHidden = gap; hidden = gap; prompted = false }
152
+ let from = seq
153
+ let role
154
+ let text
155
+ if (prompt) {
156
+ const own = gap || drivenFromGap(event.data)
157
+ if (!prompted && own && !turnHidden) { turnHidden = true; from = turnSeq }
158
+ hidden = turnHidden || own
159
+ prompted = true
160
+ role = 'user'
161
+ text = promptText(event.data)
162
+ } else if (event.type === 'assistant/message') {
163
+ const blocks = Array.isArray(event.data?.message?.content) ? event.data.message.content : []
164
+ role = 'assistant'
165
+ text = assistantText(blocks, (id) => approvedAfter(seq, id))
166
+ for (const b of blocks) if (b?.type === 'tool-call') called(b.name, argsOf(b), b.id, gap || hidden)
167
+ } else if (event.type === 'tool/result') {
168
+ const id = event.data?.message?.toolCallId
169
+ text = id === undefined ? '' : answerIn(id, event.data.message.content)
170
+ if (text) role = 'user'
171
+ } else if (event.type === 'tool/ptc-dispatch-start') {
172
+ called(event.data?.name, argsOf(event.data), event.data?.subCallId, gap || hidden)
173
+ } else if (event.type === 'tool/ptc-dispatch') {
174
+ const { name, subCallId, content } = event.data ?? {}
175
+ if (name === 'exit_plan_mode' && APPROVED.test(textOf(content))) {
176
+ role = 'assistant'
177
+ text = planText(argsOf(event.data).plan)
178
+ } else {
179
+ text = answerIn(subCallId, content)
180
+ if (text) role = 'user'
181
+ }
182
+ } else if (event.type === 'goal/change') {
183
+ const id = event.data?.goal?.id ?? event.data?.cleared?.id
184
+ if (id !== undefined) goals.set(id, goals.get(id) === true || gap || hidden)
185
+ }
186
+ const step = { seq, event, turn, start, prompt, hidden: gap || hidden, from, role, text }
187
+ // What comes between turns (a /goal typed then) is judged by the gaps alone.
188
+ if (event.type === 'turn/end') hidden = false
189
+ yield step
190
+ }
191
+ }
192
+
193
+ /**
194
+ * The human/assistant exchange after `afterSeq`, read from the durable session log (seq 0, 1, 2 ...
195
+ * until `eventAt` returns nothing). Not from the surface: dsh compaction replaces old turns on the
196
+ * surface with one checkpoint, and turns that were still waiting to be indexed would be lost (#13).
197
+ * Within one session the log is never edited, so nothing outside the conversation is read.
198
+ * Each message carries its dsh turn number (see walk) and its time. Prompts are what the user
199
+ * typed or dsh drove (a goal round's objective once per goal revision, #69); assistant text
200
+ * includes the plans handed to exit_plan_mode that the user approved, and the user's answers to
201
+ * ask_user_question, or the words a plan was sent back with, are user text (#70, review).
202
+ * `completeOnly`: stop before a turn that is still running (a `turn/start` with no `turn/end` after
203
+ * it), so a build started by a command mid-turn never splits that turn's question from its answer.
204
+ * `gaps`: seq ranges `[from, to]` typed while memory was off; never read, nor what they drove (#59, #69).
205
+ * `titleFrom`: only dsh titles at or after this seq, and made only from prompts at or after it,
206
+ * count: a fork passes its first own prompt, so neither its parent's titles nor the rename dsh
207
+ * gives a new fork ("<parent's title> (1)") nor a title listing inherited prompts is used (#60, review).
208
+ * @returns { messages: [{ seq, role, text, turn, time }], lastSeq, userTurns, turns, title, open, hidden, turnAt }
209
+ * `userTurns`: turns of the span with a prompt that is not private; `turns`: turns with a prompt
210
+ * up to the end; `title`: the chat's latest dsh title (model-made, or the user's own rename), if
211
+ * any (#41); `open`: the span stopped before a running turn; `hidden`: the private stretches of
212
+ * the whole log, `[from, to]`, the gaps included; `turnAt(seq)`: the turn of any seq of the log.
213
+ */
214
+ export function collectSpan(eventAt, afterSeq = -1, { completeOnly = false, gaps = [], titleFrom = 0 } = {}) {
215
+ const all = []
216
+ const prompts = []
217
+ const starts = []
218
+ const hidden = []
219
+ const hiddenPrompts = new Set()
220
+ const objectives = new Set()
221
+ let end = -1
222
+ let openTurn = -1
223
+ let title
224
+ let run
225
+ for (const step of walk(eventAt, gaps)) {
226
+ const { seq, event } = step
227
+ end = seq
228
+ if (step.start) starts.push([seq, step.turn])
229
+ if (step.hidden) {
230
+ if (run && step.from <= run[1] + 1) run[1] = seq
231
+ else hidden.push(run = [step.from, seq])
232
+ }
233
+ if (step.prompt) {
234
+ prompts.push(step)
235
+ if (step.hidden) hiddenPrompts.add(seq)
236
+ }
237
+ if (event.type === 'turn/start') openTurn = seq
238
+ else if (event.type === 'turn/end') openTurn = -1
239
+ else if (event.type === 'session/title' && seq >= titleFrom && typeof event.data?.title === 'string' && event.data.title.trim() && !titleFromGap(event, seq, gaps, hiddenPrompts) && !madeBefore(event, titleFrom)) title = event.data.title.trim()
240
+ let text = step.text
241
+ const source = step.prompt && !step.hidden ? event.data.source : undefined
242
+ if (source?.kind === 'goal') {
243
+ // Every round repeats its goal's objective word for word: read once per goal revision (#69).
244
+ const key = `${source.goalId}:${source.revision}`
245
+ if (objectives.has(key)) text = ''
246
+ objectives.add(key)
247
+ }
248
+ if (text && seq > afterSeq && !step.hidden) all.push({ seq, role: step.role, text, turn: step.turn, time: event.time })
249
+ }
250
+ const limit = completeOnly && openTurn >= 0 ? openTurn - 1 : end
251
+ const turnsOf = (list) => new Set(list.filter((p) => p.seq <= limit).map((p) => p.turn)).size
252
+ return {
253
+ messages: all.filter((m) => m.seq <= limit),
254
+ lastSeq: Math.max(afterSeq, limit),
255
+ userTurns: turnsOf(prompts.filter((p) => !p.hidden && p.seq > afterSeq)),
256
+ turns: turnsOf(prompts),
257
+ title,
258
+ open: completeOnly && openTurn >= 0,
259
+ hidden,
260
+ // Seqs past the log (messages migrated from other sessions) have no turn here.
261
+ turnAt: (seq) => (seq <= end ? starts.findLast(([s]) => s <= seq)?.[1] : undefined)
262
+ }
263
+ }
264
+
265
+ /**
266
+ * A dsh title that comes from what stays private: written in an off stretch, or derived from a
267
+ * prompt typed there or driven from there (`messageSeqs` names the prompts a generated title was
268
+ * made from; #59, #69). Never shown to other chats.
269
+ */
270
+ const titleFromGap = (event, seq, gaps, hiddenPrompts) => inGap(seq, gaps) || (Array.isArray(event.data?.messageSeqs) && event.data.messageSeqs.some((s) => inGap(s, gaps) || hiddenPrompts.has(s)))
271
+
272
+ /** A dsh title made (in part) from prompts before `from`: in a fork, its parent's prompts. */
273
+ const madeBefore = (event, from) => Array.isArray(event.data?.messageSeqs) && event.data.messageSeqs.some((s) => s < from)
274
+
275
+ /** The seq of the first human prompt at or after `from`, or undefined (none yet). */
276
+ export function firstPromptSeq(eventAt, from = 0) {
277
+ for (let seq = from; ; seq++) {
278
+ const event = eventAt(seq)
279
+ if (!event) return undefined
280
+ if (isHuman(event)) return seq
281
+ }
282
+ }
283
+
284
+ /** Whether a seq lies in one of the `[from, to]` ranges typed while memory was off (#59). */
285
+ export const inGap = (seq, gaps = []) => (Array.isArray(gaps) ? gaps : []).some((g) => Array.isArray(g) && seq >= g[0] && seq <= g[1])
286
+
287
+ /** The first gap that starts after `seq`, or undefined. */
288
+ export const nextGap = (seq, gaps = []) => (Array.isArray(gaps) ? gaps : []).filter((g) => Array.isArray(g) && g[0] > seq).sort((a, b) => a[0] - b[0])[0]
289
+
290
+ /**
291
+ * `gaps` with `ranges` added, in order, touching ones joined. Past 50 the two oldest merge, also
292
+ * covering the stretch between them: a gap is never dropped (as `on` keeps them).
293
+ */
294
+ export function mergeGaps(gaps = [], ranges = []) {
295
+ const out = []
296
+ for (const [from, to] of [...(Array.isArray(gaps) ? gaps : []), ...ranges].filter(Array.isArray).sort((a, b) => a[0] - b[0])) {
297
+ const last = out.at(-1)
298
+ if (last && from <= last[1] + 1) last[1] = Math.max(last[1], to)
299
+ else out.push([from, to])
300
+ }
301
+ while (out.length > 50) out.splice(0, 2, [out[0][0], out[1][1]])
302
+ return out
303
+ }
304
+
305
+ const TOPIC_LINE = /^\s*- (c\d+n\d+) /gm
306
+ const RECALLED = /^(?:# |\s*- )(c\d+n\d+) /gm
307
+ const RECALL_RESULT = /^(?:# [ct]\d|\[context-graph\])/
308
+
309
+ /**
310
+ * What this chat already holds in context, read from the surface (so after compaction drops a
311
+ * block, its topics can be sent again):
312
+ * ids topics listed in the plugin's blocks, or fetched with recall (#31). Only topic lines
313
+ * count, never ids inside prose, so an example id cannot hide a real topic (#51).
314
+ * texts whole block texts (an identical day digest is not sent twice)
315
+ * list whether the chat list (the top layer) is on the surface (#26)
316
+ * tokens the plugin's blocks and recall results still in context: what every request of the
317
+ * chat re-sends as cached input (#35)
318
+ */
319
+ export function shownContext(nodes, eventAt) {
320
+ const ids = new Set()
321
+ const texts = new Set()
322
+ let list = false
323
+ let tokens = 0
324
+ for (const seq of nodes) {
325
+ const event = eventAt(seq)
326
+ if (event?.type === 'user/message' && event.data?.source?.kind === SOURCE_KIND) {
327
+ const text = textOf(event.data.content)
328
+ texts.add(text)
329
+ tokens += estimateTokens(text)
330
+ if (event.data.source.layer === 'surface') list = true
331
+ for (const m of text.matchAll(TOPIC_LINE)) ids.add(m[1])
332
+ } else if (event?.type === 'tool/result') {
333
+ const text = textOf(event.data?.message?.content)
334
+ // Only a recall result names what this chat holds: a file or command output with a
335
+ // "- c3n12 ..." line hides no topic, and is not scanned (#73).
336
+ if (!RECALL_RESULT.test(text)) continue
337
+ tokens += estimateTokens(text)
338
+ for (const m of text.matchAll(RECALLED)) ids.add(m[1])
339
+ }
340
+ }
341
+ return { ids, texts, list, tokens }
342
+ }
343
+
344
+ /**
345
+ * The answer of the newest turn: the assistant's text after the last prompt on the surface (a
346
+ * goal round's answer is its own, #69). Used once per turn to see which shown topics the answer
347
+ * actually used (#35).
348
+ */
349
+ export function lastAnswer(nodes, eventAt) {
350
+ const parts = []
351
+ for (let i = nodes.length - 1; i >= 0; i--) {
352
+ const event = eventAt(nodes[i])
353
+ if (isPrompt(event)) break
354
+ if (event?.type === 'assistant/message') parts.unshift(textOf(event.data?.message?.content))
355
+ }
356
+ return parts.filter(Boolean).join('\n')
357
+ }
358
+
359
+ /**
360
+ * First human prompt of the log (at or after `from`), shortened: the chat's title until dsh names
361
+ * it; from seq 0 it is also the key fork detection uses, since a fork starts with its parent's log.
362
+ */
363
+ export function titleOf(eventAt, width = 60, from = 0, gaps = []) {
364
+ for (let seq = from; ; seq++) {
365
+ const event = eventAt(seq)
366
+ if (!event) return undefined
367
+ if (isHuman(event) && !inGap(seq, gaps)) {
368
+ const text = textOf(event.data.content).replace(/\s+/g, ' ')
369
+ return text.length > width ? `${sliceChars(text, width - 1)}…` : text
370
+ }
371
+ }
372
+ }
373
+
374
+ /** "read_file ×3, bash": tool names in first-use order with counts. */
375
+ function toolLine(names) {
376
+ const counts = new Map()
377
+ for (const name of names) counts.set(name, (counts.get(name) ?? 0) + 1)
378
+ return `[tools: ${[...counts].map(([name, n]) => (n > 1 ? `${name} ×${n}` : name)).join(', ')}]`
379
+ }
380
+
381
+ /**
382
+ * Parts kept within `maxTokens`: the prompt and the final answer first (each clipped if huge), then
383
+ * the steps between them, newest first; the oldest steps that do not fit become one marker.
384
+ */
385
+ function fit(parts, maxTokens) {
386
+ const joined = (list) => list.map((p) => p.text).join('\n\n')
387
+ if (estimateTokens(joined(parts)) <= maxTokens) return joined(parts)
388
+ const lastAnswer = parts.findLastIndex((p, i) => i > 0 && p.kind === 'assistant')
389
+ const head = { ...parts[0], text: clipTokens(parts[0].text, Math.floor(maxTokens * 0.35)) }
390
+ const answer = lastAnswer > 0 ? { ...parts[lastAnswer], text: clipTokens(parts[lastAnswer].text, Math.floor(maxTokens * 0.45)) } : undefined
391
+ const after = lastAnswer > 0 ? parts.slice(lastAnswer + 1) : []
392
+ const middle = parts.slice(1, lastAnswer > 0 ? lastAnswer : parts.length)
393
+ let room = maxTokens - estimateTokens(joined([head, ...(answer ? [answer] : []), ...after])) - 16
394
+ const kept = []
395
+ for (let i = middle.length - 1; i >= 0; i--) {
396
+ const cost = estimateTokens(middle[i].text) + 1
397
+ if (cost > room) break
398
+ room -= cost
399
+ kept.unshift(middle[i])
400
+ }
401
+ const dropped = middle.length - kept.length
402
+ const marker = dropped > 0 ? [{ text: `…[${dropped} earlier step${dropped === 1 ? '' : 's'} left out]…` }] : []
403
+ return joined([head, ...marker, ...kept, ...(answer ? [answer] : []), ...after])
404
+ }
405
+
406
+ /** How a recalled turn names a prompt dsh drove rather than one the user typed. */
407
+ const DRIVEN = { goal: 'goal', schedule: 'reminder' }
408
+
409
+ /**
410
+ * Turn `n` as dsh numbers it (#71: from its turn/start to the next one), lean (#47): its prompts,
411
+ * the assistant's text and approved plans, the user's answers to its questions and to its plans
412
+ * (#70, review), and its tool use as one line; tool output is left out (it averaged 9,000 tokens
413
+ * on real logs). Capped at about `maxTokens`.
414
+ * @param limitSeq last seq that may be read: another chat's turn stops at its indexed position.
415
+ * @param gaps another chat's off stretches: the turn ends at its first private step, typed there or
416
+ * driven from there (a goal's round, a reminder, a late answer), judged from the log as the
417
+ * builder judges it (second review of #69).
418
+ * @returns { text, seq, endSeq } or undefined when the log has nothing of turn n at or before `limitSeq`.
419
+ */
420
+ export function turnTranscript(eventAt, n, { maxTokens = 2000, limitSeq = Infinity, gaps = [] } = {}) {
421
+ const parts = []
422
+ let tools = []
423
+ const flush = () => {
424
+ if (tools.length > 0) parts.push({ kind: 'tools', text: toolLine(tools) })
425
+ tools = []
426
+ }
427
+ let start = -1
428
+ let end = -1
429
+ // A plan's approval past `limitSeq` is not read either.
430
+ for (const step of walk(eventAt, gaps, limitSeq)) {
431
+ if (step.seq > limitSeq || (start >= 0 && step.start)) break
432
+ if (start < 0 && !(step.start && step.turn === n)) continue
433
+ if (step.hidden) break
434
+ if (start < 0) start = step.seq
435
+ end = step.seq
436
+ const { event } = step
437
+ if (step.role && (step.text || step.prompt)) {
438
+ flush()
439
+ parts.push({ kind: step.role, text: `${(step.prompt && DRIVEN[event.data.source.kind]) || step.role}:\n${step.text}` })
440
+ }
441
+ const blocks = event.type === 'assistant/message' && Array.isArray(event.data?.message?.content) ? event.data.message.content : []
442
+ for (const b of blocks) if (b?.type === 'tool-call' && b.name) tools.push(b.name)
443
+ }
444
+ flush()
445
+ return parts.length > 0 ? { text: fit(parts, maxTokens), seq: start, endSeq: end } : undefined
446
+ }
@@ -0,0 +1,220 @@
1
+ /**
2
+ * Text helpers shared by every layer: token estimates, search words and message text.
3
+ * Pure: no I/O and no dsh imports, so `node --test` covers it without a harness.
4
+ */
5
+
6
+ /** Hiragana, katakana, CJK ideographs and Hangul: scripts written without spaces between words. */
7
+ const CJK_RUN = /[぀-ヿ㐀-鿿가-힯]+/g
8
+ const CJK_COUNT = /[぀-ヿ㐀-鿿가-힯]/g
9
+ const HIRAGANA = /[぀-ゟ]/
10
+
11
+ export const STOPWORDS = new Set((
12
+ 'a an the and or but if then so of to in on at for with by from as is are was were be been being ' +
13
+ 'do does did done have has had can could should would will shall may might must not no yes ' +
14
+ 'i me my we our you your he she it its they them this that these those there here what which who ' +
15
+ 'how when where why also just like get got make made use used using want need let please ok okay ' +
16
+ 'now new one two about into over under again more most some any all each other than too very ' +
17
+ // Everyday words that say nothing about a topic (#23): "can you help, I got an error again".
18
+ 'many much before after still even only really thing things something anything way lot lots bit ' +
19
+ 'kind sort maybe sure thanks thank hi hello hey right good great best better bad try tried help ' +
20
+ 'know think say said tell go going see look show give take put keep keeps kept since until while ' +
21
+ 'because though however actually basically back first last next time day days week month year ' +
22
+ 'today tomorrow yesterday ago morning tonight ' +
23
+ '的 了 是 在 我 你 他 她 它 们 这 那 个 和 与 或 吗 呢 吧 啊 把 被 给 让 就 都 也 还 很 不 没 有 要 会 能 ' +
24
+ '可以 什么 怎么 这个 那个 一下 一个 ' +
25
+ // Chinese function words: in a prompt they glue the question together and match any topic (#23).
26
+ '之前 上次 最后 最近 现在 已经 还是 时候 我们 你们 他们 可能 应该 需要 觉得 知道 然后 因为 所以 ' +
27
+ '但是 如果 这样 那样 一些 问题 东西 办法 比较 有点 帮我 给我 请问 一直 怎样 如何 为什 哪个 哪里 ' +
28
+ '哪些 多少 今天 昨天 明天 前天 上周 本周 这周 上个 本月 上月'
29
+ ).split(/\s+/))
30
+
31
+ /**
32
+ * Characters that only glue a Chinese sentence together. A bigram made of two of them ("我的",
33
+ * "了吗", "是不") is never a search word; one next to a content character ("会话", "性能") still is.
34
+ */
35
+ const FUNCTION_CHARS = new Set([...'的了吗呢吧啊呀么嘛哦啦我你他她它们这那哪个些是在和与或把被给让就都也还很不没有要会能又再得地着过对从跟并而且之其此啥咋怎什'])
36
+
37
+ /** Rough token estimate for budgets; CJK is close to one token per character. */
38
+ export function estimateTokens(text) {
39
+ const s = String(text ?? '')
40
+ const cjk = s.match(CJK_COUNT)?.length ?? 0
41
+ return Math.ceil(cjk + (s.length - cjk) / 3.5)
42
+ }
43
+
44
+ const CJK_CHAR = /[぀-ヿ㐀-鿿가-힯]/
45
+ const charTokens = (ch) => (CJK_CHAR.test(ch) ? 1 : 1 / 3.5)
46
+
47
+ /**
48
+ * `text` cut to about `maxTokens`, keeping its head (70%) and tail (30%) around a marker that says
49
+ * how much was left out and where the rest is (`more`). The cut points are found by adding up each
50
+ * character's cost, so Chinese before English (or the reverse) cannot overshoot the budget.
51
+ */
52
+ export function clipTokens(text, maxTokens, more = '') {
53
+ const s = String(text ?? '')
54
+ if (estimateTokens(s) <= maxTokens) return s
55
+ const budget = Math.max(0, maxTokens - 24)
56
+ let used = 0
57
+ let head = 0
58
+ while (head < s.length && used + charTokens(s[head]) <= budget * 0.7) used += charTokens(s[head++])
59
+ used = 0
60
+ let tail = s.length
61
+ while (tail > head && used + charTokens(s[tail - 1]) <= budget * 0.3) used += charTokens(s[--tail])
62
+ head = cutBefore(s, head)
63
+ tail = Math.max(head, cutAfter(s, tail))
64
+ return `${s.slice(0, head)}\n…[${tail - head} chars cut${more ? `; ${more}` : ''}]…${tail < s.length ? `\n${s.slice(tail)}` : ''}`
65
+ }
66
+
67
+ const isHigh = (code) => code >= 0xd800 && code <= 0xdbff
68
+ const isLow = (code) => code >= 0xdc00 && code <= 0xdfff
69
+
70
+ /**
71
+ * A cut position that does not split a surrogate pair (an emoji, a rare CJK character): a lone
72
+ * half would make the text invalid, and dsh's own log writers refuse that. `cutBefore` moves an
73
+ * end position back, `cutAfter` moves a start position forward.
74
+ */
75
+ export const cutBefore = (s, i) => (i > 0 && i < s.length && isHigh(s.charCodeAt(i - 1)) && isLow(s.charCodeAt(i)) ? i - 1 : i)
76
+ export const cutAfter = (s, i) => (i > 0 && i < s.length && isLow(s.charCodeAt(i)) && isHigh(s.charCodeAt(i - 1)) ? i + 1 : i)
77
+
78
+ const HEAD = 4500
79
+ const TAIL = 1000
80
+
81
+ /**
82
+ * A long message as it is stored: its head and tail; the full text stays in dsh's own session log.
83
+ * Here in the core because the savings estimate counts every source as if it was stored this way.
84
+ */
85
+ export function capText(text) {
86
+ const s = String(text ?? '')
87
+ if (s.length <= HEAD + TAIL + 200) return s
88
+ const head = cutBefore(s, HEAD)
89
+ const tail = cutAfter(s, s.length - TAIL)
90
+ return `${s.slice(0, head)}\n…[${tail - head} chars omitted; the full turn is in the chat's dsh log]…\n${s.slice(tail)}`
91
+ }
92
+
93
+ /** The first `max` characters of `s`, never half a surrogate pair. */
94
+ export const sliceChars = (s, max) => s.slice(0, cutBefore(s, Math.min(max, s.length)))
95
+
96
+ const searchable = (a, b) => !(FUNCTION_CHARS.has(a) && FUNCTION_CHARS.has(b)) && !(HIRAGANA.test(a) && HIRAGANA.test(b))
97
+
98
+ // Katakana words (transliterations such as データベース) and Hangul words are written as one run
99
+ // of syllables: a run of either script is one word, however long.
100
+ const SINGLE_WORD_SCRIPT = /[゠-ヿ]+|[가-힯]+|[^゠-ヿ가-힯]+/g
101
+ const isSingleWordScript = (run) => /^[゠-ヿ]+$/.test(run) || /^[가-힯]+$/.test(run)
102
+
103
+ /**
104
+ * Whether a word is code: it has _ or . (identifiers, file names), or is a path of three or more
105
+ * parts, or has a slash and a digit (api/v1). "ci/cd" or "read/write" are plain words.
106
+ */
107
+ function isCode(word) {
108
+ const slashes = word.split('/').length - 1
109
+ return /[_.]/.test(word) || slashes >= 2 || (slashes === 1 && /\d/.test(word))
110
+ }
111
+
112
+ // Product names that read as plurals stay whole, whatever their case (review of #75): Windows is
113
+ // not window, Teams not team. A capital cannot tell them apart: every label starts with one
114
+ // ("Timestamps stored as timestamptz" must still find "timestamp"), and aliases are lowercase ("windows path").
115
+ const PLURAL_NAMES = new Set(['windows', 'teams', 'sheets', 'pages', 'actions'])
116
+
117
+ /**
118
+ * A plural and its singular as one search word (#75): tests and test, queries and query, branches
119
+ * and branch. Plain lowercase words of 4+ letters only, and never into a stopword ("times" stays).
120
+ * Singulars the rules would cut wrongly are folded the same way: cache = caches, cookie = cookies.
121
+ * No dictionary: irregular plurals (indices, aliases, statuses) still differ, as before.
122
+ */
123
+ function singular(word) {
124
+ if (word.length < 4 || !/^[a-z]+$/.test(word) || PLURAL_NAMES.has(word)) return word
125
+ let s = word
126
+ if (s.endsWith('s')) {
127
+ if (s.endsWith('ies') && s.length > 4) s = `${s.slice(0, -3)}y`
128
+ else if (/(?:ch|sh|ss|x)es$/.test(s)) s = s.slice(0, -2)
129
+ else if (!/[su]s$/.test(s)) s = s.slice(0, -1)
130
+ }
131
+ if (s.endsWith('che')) s = s.slice(0, -1)
132
+ else if (s.endsWith('ie') && s.length > 3) s = `${s.slice(0, -2)}y` // ties stays tie, like tie itself
133
+ return s !== word && STOPWORDS.has(s) ? word : s
134
+ }
135
+
136
+ /** A word's tokens: `word`, then each of its parts that is a search word (plurals folded), once. */
137
+ function withParts(word, parts) {
138
+ const tokens = [word]
139
+ for (const part of parts) if (part.length >= 2 && !STOPWORDS.has(part) && !tokens.includes(singular(part))) tokens.push(singular(part))
140
+ return tokens
141
+ }
142
+
143
+ /**
144
+ * A lowerCamel identifier (loadViews, isAgentLoopRequest) is code, as snake_case is (#74): the
145
+ * word counts double where a topic has it, its humps are its parts (unless `humps` is false). 7+
146
+ * characters with a first hump of 2+, so iPhone, eCommerce or macOS stay plain words; PascalCase
147
+ * (GitHub) does too. The word folds like a plain word: loadViews, LoadViews and loadviews are one.
148
+ * At most 40 characters and 10 humps: a pasted base64 blob or key is one plain word, not an
149
+ * identifier with a thousand parts that every topic would rescan (review of #74), while
150
+ * findAllByCustomerIdAndCreatedAtBetween still is one (second review); one with a digit, at most 6, as
151
+ * random ids of mixed case nearly always hold one (third check).
152
+ */
153
+ function camelUnit(word, humps) {
154
+ if (word.length < 7 || word.length > 40 || !/^[a-z][a-z\d]+[A-Z][A-Za-z\d]*$/.test(word)) return undefined
155
+ const parts = word.split(/(?<=[a-z\d])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])/)
156
+ if (parts.length > (/\d/.test(word) ? 6 : 10)) return undefined
157
+ return { kind: 'word', tokens: withParts(singular(word.toLowerCase()), humps ? parts.map((h) => h.toLowerCase()) : []), code: true }
158
+ }
159
+
160
+ /**
161
+ * A text's search words, grouped by the word they came from, so evidence can be counted per word:
162
+ * { kind: 'word', tokens, code } one word in any alphabet (NFKC, so full-width "API" is "api"),
163
+ * then its identifier parts; `code`: see isCode and camelUnit
164
+ * { kind: 'cjk', bigrams, one } one run of CJK text as its bigrams in order, null where a bigram
165
+ * is not a search word; `one` when the run is a single word
166
+ * (katakana or Hangul)
167
+ * Stopwords are dropped, and so are CJK bigrams made only of particles/pronouns or only of
168
+ * hiragana, which match everything (#23, #25). `humps: false` leaves out the humps of lowerCamel.
169
+ */
170
+ export function searchUnits(text, { humps = true } = {}) {
171
+ const units = []
172
+ const nfkc = String(text ?? '').normalize('NFKC')
173
+ const lower = nfkc.toLowerCase()
174
+ for (const cjk of lower.match(CJK_RUN) ?? []) {
175
+ for (const run of cjk.match(SINGLE_WORD_SCRIPT) ?? []) {
176
+ if (run.length === 1) {
177
+ if (!STOPWORDS.has(run) && !FUNCTION_CHARS.has(run) && !HIRAGANA.test(run)) units.push({ kind: 'cjk', bigrams: [run], one: false })
178
+ continue
179
+ }
180
+ const bigrams = []
181
+ for (let i = 0; i + 1 < run.length; i++) {
182
+ const bigram = run.slice(i, i + 2)
183
+ bigrams.push(!STOPWORDS.has(bigram) && searchable(run[i], run[i + 1]) ? bigram : null)
184
+ }
185
+ if (bigrams.some(Boolean)) units.push({ kind: 'cjk', bigrams, one: isSingleWordScript(run) })
186
+ }
187
+ }
188
+ // Words are cut before lowercasing, so a lowerCamel identifier can be told from a plain word (#74).
189
+ for (const raw of nfkc.replace(CJK_RUN, ' ').match(/[\p{L}\p{N}_./-]{2,}/gu) ?? []) {
190
+ const cased = raw.replace(/[./-]+$/, '')
191
+ const word = cased.toLowerCase()
192
+ // Most words have no separator: one plain word, no parts (#66).
193
+ if (!/[_./-]/.test(word)) {
194
+ if (word.length >= 2 && !STOPWORDS.has(word)) units.push(camelUnit(cased, humps) ?? { kind: 'word', tokens: [singular(word)], code: false })
195
+ continue
196
+ }
197
+ if (!/[\p{L}\p{N}]/u.test(word) || STOPWORDS.has(word)) continue
198
+ // The compound itself stays exact (user_ids is not user_id); only its parts fold plurals (#75).
199
+ units.push({ kind: 'word', tokens: withParts(word, word.split(/[_./-]+/)), code: isCode(word) })
200
+ }
201
+ return units
202
+ }
203
+
204
+ /**
205
+ * Every search word of a text, once (`options`: see searchUnits).
206
+ * "clear_conversation tool" -> ["clear_conversation", "clear", "conversation", "tool"]
207
+ * "matchAcross tests" -> ["matchacross", "match", "across", "test"]
208
+ * "清除上下文" -> ["清除", "除上", "上下", "下文"]
209
+ */
210
+ export function tokenize(text, options) {
211
+ const out = new Set()
212
+ for (const unit of searchUnits(text, options)) for (const t of unit.kind === 'cjk' ? unit.bigrams : unit.tokens) if (t) out.add(t)
213
+ return [...out]
214
+ }
215
+
216
+ /** The text blocks of a dsh message's content, joined. */
217
+ export function textOf(content) {
218
+ if (!Array.isArray(content)) return ''
219
+ return content.filter((b) => b?.type === 'text' && typeof b.text === 'string').map((b) => b.text).join('\n').trim()
220
+ }