@yeaft/webchat-agent 0.1.629 → 0.1.631
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/unify/config.js +9 -0
- package/unify/dream-v2/apply.js +271 -0
- package/unify/dream-v2/limits.js +46 -0
- package/unify/dream-v2/merge.js +87 -0
- package/unify/dream-v2/runner.js +267 -0
- package/unify/dream-v2/schedule.js +73 -0
- package/unify/dream-v2/segment.js +191 -0
- package/unify/dream-v2/snapshot.js +80 -0
- package/unify/dream-v2/state.js +177 -0
- package/unify/dream-v2/triage.js +287 -0
- package/unify/engine.js +38 -4
- package/unify/memory/recall-v2.js +258 -0
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dream-v2/runner.js — DESIGN-v2 §13.
|
|
3
|
+
*
|
|
4
|
+
* Orchestrates one full dream pass:
|
|
5
|
+
*
|
|
6
|
+
* trigger
|
|
7
|
+
* ↓
|
|
8
|
+
* enumerateGroups() via opts.listGroups()
|
|
9
|
+
* ↓
|
|
10
|
+
* for each group with newCount ≥ MIN_NEW_PER_GROUP (auto)
|
|
11
|
+
* or > 0 (manual):
|
|
12
|
+
* loadDiff() via opts.loadGroupDiff(groupId, sinceId)
|
|
13
|
+
* applyOverlap() via opts.loadOverlapPreamble(...)
|
|
14
|
+
* segment() segmentDiff(...)
|
|
15
|
+
* triageGroupSegments() → group-local actions[]
|
|
16
|
+
*
|
|
17
|
+
* mergeByTarget() → per-target actions
|
|
18
|
+
* for each merged target:
|
|
19
|
+
* applyMergedTarget() (snapshot + UPDATE/CREATE + atomic write)
|
|
20
|
+
*
|
|
21
|
+
* bookkeep:
|
|
22
|
+
* for each processed group:
|
|
23
|
+
* group .dream-state ←
|
|
24
|
+
* { lastDreamMessageId: tail of real diff,
|
|
25
|
+
* lastDreamAt: nowIso,
|
|
26
|
+
* messageCount: <after> }
|
|
27
|
+
*
|
|
28
|
+
* pruneOldSnapshots()
|
|
29
|
+
*
|
|
30
|
+
* onProgress emits dream_progress events that the web bridge forwards
|
|
31
|
+
* as `unify_output` messages so the debug panel can render live state
|
|
32
|
+
* (DESIGN-v2 §19.4). All events flow through the same channel; no new
|
|
33
|
+
* WebSocket message type is introduced.
|
|
34
|
+
*
|
|
35
|
+
* Everything that touches a shell of the system (LLM, message store,
|
|
36
|
+
* scope listing, topic tree) is injected. The default exports below
|
|
37
|
+
* are pure orchestration.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
import { existsSync } from 'fs';
|
|
41
|
+
import { join } from 'path';
|
|
42
|
+
|
|
43
|
+
import { listScopes, readSummary } from '../memory/store-v2.js';
|
|
44
|
+
import {
|
|
45
|
+
DEFAULT_LIMITS,
|
|
46
|
+
} from './limits.js';
|
|
47
|
+
import { readGroupState, writeGroupState } from './state.js';
|
|
48
|
+
import { segmentDiff, truncateMessage, estimateMessagesTokens } from './segment.js';
|
|
49
|
+
import { triageGroupSegments } from './triage.js';
|
|
50
|
+
import { mergeByTarget } from './merge.js';
|
|
51
|
+
import { applyMergedTarget } from './apply.js';
|
|
52
|
+
import { tsForBackup, pruneOldSnapshots } from './snapshot.js';
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* @typedef {Object} RunDreamOpts
|
|
56
|
+
* @property {string} root — memory root, e.g. ~/.yeaft/memory
|
|
57
|
+
* @property {boolean} [manual=false] — manual trigger overrides newCount<20 skip
|
|
58
|
+
* @property {string[]} [scopeFilter] — optional: only dream these targets (still respects newCount per group; '*' allowed)
|
|
59
|
+
* @property {(req: {pass:string, prompt:string, system:string}) => Promise<string>} llm
|
|
60
|
+
* @property {() => Promise<Array<string>>} listGroups — return all group ids (incl. '_no-group')
|
|
61
|
+
* @property {(groupId: string) => Promise<number>} countMessages — total message count for a group
|
|
62
|
+
* @property {(groupId: string, sinceMessageId: string|null) => Promise<Array<object>>} loadGroupDiff
|
|
63
|
+
* @property {(groupId: string, beforeMessageId: string|null, count: number) => Promise<Array<object>>} loadOverlapPreamble
|
|
64
|
+
* @property {() => Promise<Array<{path:string, summary:string}>>} [listTopicSummaries]
|
|
65
|
+
* @property {(target: string) => Promise<Array<{path:string, summary:string}>>} [siblingTopicsFor]
|
|
66
|
+
* @property {(event: object) => void} [onProgress]
|
|
67
|
+
* @property {object} [limits] — override DEFAULT_LIMITS
|
|
68
|
+
* @property {() => string} [nowIso]
|
|
69
|
+
*/
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Run one dream pass. Returns a structured report consumed by the
|
|
73
|
+
* debug panel (and tests).
|
|
74
|
+
*
|
|
75
|
+
* @param {RunDreamOpts} opts
|
|
76
|
+
*/
|
|
77
|
+
export async function runDream(opts) {
|
|
78
|
+
if (!opts || !opts.root) throw new Error('runDream: opts.root required');
|
|
79
|
+
if (!opts.llm) throw new Error('runDream: opts.llm required');
|
|
80
|
+
const limits = { ...DEFAULT_LIMITS, ...(opts.limits || {}) };
|
|
81
|
+
const onProgress = typeof opts.onProgress === 'function' ? opts.onProgress : () => {};
|
|
82
|
+
const nowIso = (opts.nowIso ? opts.nowIso() : new Date().toISOString());
|
|
83
|
+
const ts = tsForBackup(new Date(nowIso));
|
|
84
|
+
const startedAt = Date.now();
|
|
85
|
+
|
|
86
|
+
onProgress({ phase: 'start', manual: !!opts.manual, ts });
|
|
87
|
+
|
|
88
|
+
// 1. enumerate groups
|
|
89
|
+
const groupIds = await safeCall(opts.listGroups, []);
|
|
90
|
+
const filter = Array.isArray(opts.scopeFilter) ? new Set(opts.scopeFilter) : null;
|
|
91
|
+
const groupsReport = [];
|
|
92
|
+
const groupTriages = [];
|
|
93
|
+
const processedGroups = [];
|
|
94
|
+
|
|
95
|
+
// 2. per-group: skip / segment / triage
|
|
96
|
+
const topicSummaries = opts.listTopicSummaries
|
|
97
|
+
? await safeCall(opts.listTopicSummaries, [])
|
|
98
|
+
: await defaultListTopicSummaries(opts.root).catch(() => []);
|
|
99
|
+
|
|
100
|
+
for (const groupId of groupIds) {
|
|
101
|
+
// NOTE: scopeFilter is applied at the merge/apply stage, not here.
|
|
102
|
+
// A filter like ['user'] still requires triaging every group so their
|
|
103
|
+
// hard-rule actions can contribute to the user target.
|
|
104
|
+
const state = await readGroupState(opts.root, groupId);
|
|
105
|
+
const beforeCount = await safeCall(() => opts.countMessages(groupId), 0);
|
|
106
|
+
const newCount = Math.max(0, beforeCount - (state.messageCount || 0));
|
|
107
|
+
|
|
108
|
+
if (newCount === 0) {
|
|
109
|
+
groupsReport.push({ groupId, new: 0, status: 'skipped', reason: 'no-new-messages' });
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (!opts.manual && newCount < limits.MIN_NEW_PER_GROUP) {
|
|
113
|
+
groupsReport.push({ groupId, new: newCount, status: 'skipped', reason: 'below-threshold' });
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
onProgress({ phase: 'load-diff', groupId });
|
|
118
|
+
const diffNew = await safeCall(() => opts.loadGroupDiff(groupId, state.lastDreamMessageId), []);
|
|
119
|
+
if (!diffNew || diffNew.length === 0) {
|
|
120
|
+
groupsReport.push({ groupId, new: newCount, status: 'skipped', reason: 'empty-diff' });
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const overlapMessages = state.lastDreamMessageId
|
|
124
|
+
? await safeCall(
|
|
125
|
+
() => opts.loadOverlapPreamble
|
|
126
|
+
? opts.loadOverlapPreamble(groupId, state.lastDreamMessageId, limits.DREAM_OVERLAP)
|
|
127
|
+
: [],
|
|
128
|
+
[],
|
|
129
|
+
)
|
|
130
|
+
: [];
|
|
131
|
+
const taggedOverlap = overlapMessages.map(m => ({ ...m, kind: 'overlap', body: truncateMessage(m.body || '') }));
|
|
132
|
+
const taggedNew = diffNew.map(m => ({ ...m, kind: 'new', body: truncateMessage(m.body || '') }));
|
|
133
|
+
const fullDiff = [...taggedOverlap, ...taggedNew];
|
|
134
|
+
|
|
135
|
+
const segments = segmentDiff(fullDiff, limits.MAX_DIFF_TOKENS_PER_TRIAGE, limits.DREAM_OVERLAP);
|
|
136
|
+
onProgress({ phase: 'triage', groupId, status: 'running', segments: segments.length });
|
|
137
|
+
|
|
138
|
+
let actions;
|
|
139
|
+
try {
|
|
140
|
+
actions = await triageGroupSegments({
|
|
141
|
+
groupId,
|
|
142
|
+
segments,
|
|
143
|
+
topicSummaries,
|
|
144
|
+
llm: opts.llm,
|
|
145
|
+
onProgress,
|
|
146
|
+
});
|
|
147
|
+
} catch (err) {
|
|
148
|
+
groupsReport.push({ groupId, new: newCount, status: 'error', error: err.message });
|
|
149
|
+
onProgress({ phase: 'triage', groupId, status: 'error', error: err.message });
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
onProgress({ phase: 'triage', groupId, status: 'done', actions: actions.length });
|
|
154
|
+
groupTriages.push({ groupId, diff: fullDiff, actions });
|
|
155
|
+
|
|
156
|
+
const tailId = lastMessageId(diffNew);
|
|
157
|
+
processedGroups.push({ groupId, tailId, beforeCount, newCount, segments: segments.length, actions: actions.length });
|
|
158
|
+
groupsReport.push({ groupId, new: newCount, segments: segments.length, actions: actions.length, status: 'triaged' });
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// 3. merge
|
|
162
|
+
const mergedTargets = mergeByTarget(groupTriages);
|
|
163
|
+
const targetsToApply = filter && filter.size > 0 && !filter.has('*')
|
|
164
|
+
? mergedTargets.filter(t => filter.has(t.target))
|
|
165
|
+
: mergedTargets;
|
|
166
|
+
|
|
167
|
+
onProgress({ phase: 'merge', targets: targetsToApply.length });
|
|
168
|
+
|
|
169
|
+
// 4. apply
|
|
170
|
+
const targetsReport = [];
|
|
171
|
+
for (const merged of targetsToApply) {
|
|
172
|
+
try {
|
|
173
|
+
const r = await applyMergedTarget(merged, {
|
|
174
|
+
root: opts.root,
|
|
175
|
+
ts,
|
|
176
|
+
llm: opts.llm,
|
|
177
|
+
limits,
|
|
178
|
+
nowIso: opts.nowIso || (() => nowIso),
|
|
179
|
+
onProgress,
|
|
180
|
+
siblingTopicsFor: opts.siblingTopicsFor,
|
|
181
|
+
});
|
|
182
|
+
targetsReport.push({ ...r, sources: merged.sources.length, status: 'done' });
|
|
183
|
+
} catch (err) {
|
|
184
|
+
targetsReport.push({
|
|
185
|
+
target: merged.target,
|
|
186
|
+
kind: merged.kind,
|
|
187
|
+
sources: merged.sources.length,
|
|
188
|
+
status: 'error',
|
|
189
|
+
error: err.message,
|
|
190
|
+
});
|
|
191
|
+
onProgress({ phase: 'apply', target: merged.target, status: 'error', error: err.message });
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// 5. bookkeep — only when at least one apply for this group's actions
|
|
196
|
+
// succeeded. We use a permissive policy: if ANY merged-target apply
|
|
197
|
+
// succeeded for a group's contributed actions, advance that group's
|
|
198
|
+
// cursor. (If everything errored, we keep the cursor so next run
|
|
199
|
+
// retries.)
|
|
200
|
+
const successfulTargets = new Set(targetsReport.filter(r => r.status === 'done').map(r => r.target));
|
|
201
|
+
for (const pg of processedGroups) {
|
|
202
|
+
const contributed = (groupTriages.find(g => g.groupId === pg.groupId) || { actions: [] })
|
|
203
|
+
.actions.map(a => a.scope);
|
|
204
|
+
const anySuccess = contributed.some(t => successfulTargets.has(t));
|
|
205
|
+
if (!anySuccess) continue;
|
|
206
|
+
if (pg.tailId) {
|
|
207
|
+
await writeGroupState(opts.root, pg.groupId, {
|
|
208
|
+
lastDreamMessageId: pg.tailId,
|
|
209
|
+
lastDreamAt: nowIso,
|
|
210
|
+
messageCount: pg.beforeCount,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// 6. prune backups
|
|
216
|
+
const pruned = await pruneOldSnapshots(opts.root, limits.DREAM_BACKUP_KEEP);
|
|
217
|
+
|
|
218
|
+
const duration = Date.now() - startedAt;
|
|
219
|
+
onProgress({
|
|
220
|
+
phase: 'done',
|
|
221
|
+
groups: processedGroups.length,
|
|
222
|
+
targets: targetsReport.length,
|
|
223
|
+
duration,
|
|
224
|
+
backupsKept: pruned.kept.length,
|
|
225
|
+
backupsRemoved: pruned.removed.length,
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
return {
|
|
229
|
+
startedAt: nowIso,
|
|
230
|
+
durationMs: duration,
|
|
231
|
+
groups: groupsReport,
|
|
232
|
+
targets: targetsReport,
|
|
233
|
+
backups: pruned,
|
|
234
|
+
ts,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// ─── helpers ──────────────────────────────────────────────────
|
|
239
|
+
|
|
240
|
+
function lastMessageId(messages) {
|
|
241
|
+
if (!Array.isArray(messages) || messages.length === 0) return null;
|
|
242
|
+
for (let i = messages.length - 1; i >= 0; i -= 1) {
|
|
243
|
+
if (messages[i] && messages[i].id) return messages[i].id;
|
|
244
|
+
}
|
|
245
|
+
return null;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async function safeCall(fn, fallback) {
|
|
249
|
+
try {
|
|
250
|
+
if (typeof fn !== 'function') return fallback;
|
|
251
|
+
const v = await fn();
|
|
252
|
+
return v == null ? fallback : v;
|
|
253
|
+
} catch {
|
|
254
|
+
return fallback;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async function defaultListTopicSummaries(root) {
|
|
259
|
+
const all = await listScopes({ root });
|
|
260
|
+
const out = [];
|
|
261
|
+
for (const sc of all) {
|
|
262
|
+
if (sc.kind !== 'topic') continue;
|
|
263
|
+
const summary = await readSummary(sc, { root });
|
|
264
|
+
out.push({ path: sc.path.join('/'), summary });
|
|
265
|
+
}
|
|
266
|
+
return out;
|
|
267
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dream-v2/schedule.js — DESIGN-v2 §10.2.
|
|
3
|
+
*
|
|
4
|
+
* Two trigger paths:
|
|
5
|
+
*
|
|
6
|
+
* 1. 12-hour interval timer.
|
|
7
|
+
* 2. Manual trigger (UI button or `/dream` command), routed in via
|
|
8
|
+
* `triggerNow()` — sets `manual: true` so the per-group threshold
|
|
9
|
+
* is bypassed.
|
|
10
|
+
*
|
|
11
|
+
* The scheduler is a thin wrapper around `runDream()` that prevents
|
|
12
|
+
* concurrent passes (a second tick while the previous is still running
|
|
13
|
+
* is dropped, not queued — DESIGN-v2 §10.1: "slow is OK, doesn't
|
|
14
|
+
* compete with user latency").
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { DREAM_INTERVAL_HOURS } from './limits.js';
|
|
18
|
+
|
|
19
|
+
export const DEFAULT_INTERVAL_MS = DREAM_INTERVAL_HOURS * 60 * 60 * 1000;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Build a scheduler around `runDream()`. The runner closure captures
|
|
23
|
+
* everything `runDream` needs (memory root, llm, message-store hooks,
|
|
24
|
+
* onProgress sink); the scheduler only knows how to call it.
|
|
25
|
+
*
|
|
26
|
+
* @param {{
|
|
27
|
+
* run: (opts: { manual: boolean, scopeFilter?: string[] }) => Promise<object>,
|
|
28
|
+
* intervalMs?: number,
|
|
29
|
+
* logger?: { info?: (...a:any) => void, warn?: (...a:any) => void, error?: (...a:any) => void },
|
|
30
|
+
* }} args
|
|
31
|
+
*/
|
|
32
|
+
export function createDreamScheduler({ run, intervalMs = DEFAULT_INTERVAL_MS, logger }) {
|
|
33
|
+
if (typeof run !== 'function') throw new Error('createDreamScheduler: run callable required');
|
|
34
|
+
const log = logger || {};
|
|
35
|
+
let timer = null;
|
|
36
|
+
let inflight = null;
|
|
37
|
+
|
|
38
|
+
async function fire(opts) {
|
|
39
|
+
if (inflight) {
|
|
40
|
+
log.warn?.('[dream] tick dropped — previous run still in progress');
|
|
41
|
+
return inflight;
|
|
42
|
+
}
|
|
43
|
+
inflight = (async () => {
|
|
44
|
+
try {
|
|
45
|
+
return await run(opts);
|
|
46
|
+
} catch (err) {
|
|
47
|
+
log.error?.('[dream] run failed:', err && err.message ? err.message : err);
|
|
48
|
+
return { error: err && err.message ? err.message : String(err) };
|
|
49
|
+
} finally {
|
|
50
|
+
inflight = null;
|
|
51
|
+
}
|
|
52
|
+
})();
|
|
53
|
+
return inflight;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
return {
|
|
57
|
+
start() {
|
|
58
|
+
if (timer) return;
|
|
59
|
+
timer = setInterval(() => { fire({ manual: false }).catch(() => {}); }, intervalMs);
|
|
60
|
+
// Don't keep the event loop alive solely for the dream ticker.
|
|
61
|
+
if (typeof timer.unref === 'function') timer.unref();
|
|
62
|
+
},
|
|
63
|
+
stop() {
|
|
64
|
+
if (timer) { clearInterval(timer); timer = null; }
|
|
65
|
+
},
|
|
66
|
+
triggerNow(scopeFilter) {
|
|
67
|
+
return fire({ manual: true, scopeFilter });
|
|
68
|
+
},
|
|
69
|
+
isRunning() { return !!inflight; },
|
|
70
|
+
/** Test hook: fires once without scheduling a timer. */
|
|
71
|
+
_fire: fire,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dream-v2/segment.js — DESIGN-v2 §17.
|
|
3
|
+
*
|
|
4
|
+
* Three independent length-control concerns, kept pure so they can be
|
|
5
|
+
* unit-tested without touching disk or any LLM:
|
|
6
|
+
*
|
|
7
|
+
* 1. truncateMessage — clamp a single message body to
|
|
8
|
+
* MAX_SINGLE_MESSAGE_CHARS, appending a clear notice. The full
|
|
9
|
+
* body is still preserved in the conversation log; this only
|
|
10
|
+
* affects what dream sees. (§17.3)
|
|
11
|
+
*
|
|
12
|
+
* 2. estimateTokens — rough chars-to-tokens approximation (we use 4
|
|
13
|
+
* chars/token, a stable industry approximation that doesn't drag
|
|
14
|
+
* a tokenizer into this layer; precise counts aren't required for
|
|
15
|
+
* "should we segment?" decisions and a small over-count is the
|
|
16
|
+
* safe direction).
|
|
17
|
+
*
|
|
18
|
+
* 3. segmentDiff — split a long per-group diff into K consecutive
|
|
19
|
+
* slices, each ≤ MAX_DIFF_TOKENS_PER_TRIAGE, with a 3-message
|
|
20
|
+
* overlap between adjacent slices for context continuity. (§17.1)
|
|
21
|
+
*
|
|
22
|
+
* 4. needsBatchedApply / batchSourcesForApply — when an Apply target's
|
|
23
|
+
* memory + summary + sources cumulatively exceed MAX_APPLY_TOKENS,
|
|
24
|
+
* split the sources (one source = one group's contribution) into
|
|
25
|
+
* batches; the LLM is then called once per batch, threading the
|
|
26
|
+
* written-back memory.md as input to the next batch. (§17.2)
|
|
27
|
+
*
|
|
28
|
+
* No side-effects. All functions are deterministic given their inputs.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import {
|
|
32
|
+
MAX_SINGLE_MESSAGE_CHARS,
|
|
33
|
+
MAX_DIFF_TOKENS_PER_TRIAGE,
|
|
34
|
+
MAX_APPLY_TOKENS,
|
|
35
|
+
DREAM_OVERLAP,
|
|
36
|
+
} from './limits.js';
|
|
37
|
+
|
|
38
|
+
const TRUNCATION_NOTICE = '\n\n[message truncated for dream, original preserved in conversation log]';
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Truncate a single message body if it exceeds the per-message char cap.
|
|
42
|
+
* Idempotent: passing in an already-truncated body returns it unchanged.
|
|
43
|
+
*
|
|
44
|
+
* @param {string} body
|
|
45
|
+
* @returns {string}
|
|
46
|
+
*/
|
|
47
|
+
export function truncateMessage(body) {
|
|
48
|
+
const s = String(body || '');
|
|
49
|
+
if (s.length <= MAX_SINGLE_MESSAGE_CHARS) return s;
|
|
50
|
+
if (s.endsWith(TRUNCATION_NOTICE)) return s;
|
|
51
|
+
// Reserve room for the notice without overflowing the cap.
|
|
52
|
+
const room = Math.max(0, MAX_SINGLE_MESSAGE_CHARS - TRUNCATION_NOTICE.length);
|
|
53
|
+
return s.slice(0, room) + TRUNCATION_NOTICE;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Conservative chars-to-tokens approximation. We over-count slightly
|
|
58
|
+
* (1 token ≈ 4 chars) to make MAX_*_TOKENS act as a true upper bound.
|
|
59
|
+
*
|
|
60
|
+
* @param {string} text
|
|
61
|
+
* @returns {number}
|
|
62
|
+
*/
|
|
63
|
+
export function estimateTokens(text) {
|
|
64
|
+
if (!text) return 0;
|
|
65
|
+
return Math.ceil(String(text).length / 4);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Estimate the token cost of an array of messages (header + body for each).
|
|
70
|
+
* @param {Array<{id?: string, role?: string, body?: string}>} msgs
|
|
71
|
+
*/
|
|
72
|
+
export function estimateMessagesTokens(msgs) {
|
|
73
|
+
if (!Array.isArray(msgs)) return 0;
|
|
74
|
+
let n = 0;
|
|
75
|
+
for (const m of msgs) {
|
|
76
|
+
n += estimateTokens(m.role || '');
|
|
77
|
+
n += estimateTokens(m.body || '');
|
|
78
|
+
n += 2; // separator overhead
|
|
79
|
+
}
|
|
80
|
+
return n;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Split a contiguous group diff into ≤MAX-token segments, with a
|
|
85
|
+
* DREAM_OVERLAP-message tail/head overlap between consecutive segments.
|
|
86
|
+
*
|
|
87
|
+
* Returns segments in temporal order. Each segment is `{ messages, kind }`
|
|
88
|
+
* where `kind` is 'overlap' for messages that exist only as continuity
|
|
89
|
+
* preamble (because they appeared in a prior segment), and 'new' for
|
|
90
|
+
* the rest. The first segment has no overlap header.
|
|
91
|
+
*
|
|
92
|
+
* Properties:
|
|
93
|
+
* - The union of `kind: 'new'` messages across all segments equals
|
|
94
|
+
* the input diff exactly, in order, with no duplicates.
|
|
95
|
+
* - Each segment's total token estimate ≤ MAX_DIFF_TOKENS_PER_TRIAGE
|
|
96
|
+
* unless a single message alone exceeds the cap, in which case
|
|
97
|
+
* that message gets its own segment (we never split a message).
|
|
98
|
+
*
|
|
99
|
+
* @param {Array<{id?: string, role?: string, body?: string}>} diff
|
|
100
|
+
* @param {number} [maxTokens=MAX_DIFF_TOKENS_PER_TRIAGE]
|
|
101
|
+
* @param {number} [overlap=DREAM_OVERLAP]
|
|
102
|
+
* @returns {Array<{ messages: Array<object>, overlapCount: number, newCount: number }>}
|
|
103
|
+
*/
|
|
104
|
+
export function segmentDiff(diff, maxTokens = MAX_DIFF_TOKENS_PER_TRIAGE, overlap = DREAM_OVERLAP) {
|
|
105
|
+
const msgs = Array.isArray(diff) ? diff : [];
|
|
106
|
+
if (msgs.length === 0) return [];
|
|
107
|
+
|
|
108
|
+
// Fast path: whole diff fits in one segment.
|
|
109
|
+
if (estimateMessagesTokens(msgs) <= maxTokens) {
|
|
110
|
+
return [{ messages: msgs, overlapCount: 0, newCount: msgs.length }];
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const segments = [];
|
|
114
|
+
let cursor = 0;
|
|
115
|
+
while (cursor < msgs.length) {
|
|
116
|
+
const overlapHead = segments.length > 0
|
|
117
|
+
? msgs.slice(Math.max(0, cursor - overlap), cursor)
|
|
118
|
+
: [];
|
|
119
|
+
let used = estimateMessagesTokens(overlapHead);
|
|
120
|
+
let end = cursor;
|
|
121
|
+
while (end < msgs.length) {
|
|
122
|
+
const cost = estimateTokens(msgs[end].body || '') + estimateTokens(msgs[end].role || '') + 2;
|
|
123
|
+
if (used + cost > maxTokens && end > cursor) break;
|
|
124
|
+
used += cost;
|
|
125
|
+
end += 1;
|
|
126
|
+
}
|
|
127
|
+
// If we made no progress (single oversized message), advance by 1.
|
|
128
|
+
if (end === cursor) end = cursor + 1;
|
|
129
|
+
segments.push({
|
|
130
|
+
messages: [...overlapHead, ...msgs.slice(cursor, end)],
|
|
131
|
+
overlapCount: overlapHead.length,
|
|
132
|
+
newCount: end - cursor,
|
|
133
|
+
});
|
|
134
|
+
cursor = end;
|
|
135
|
+
}
|
|
136
|
+
return segments;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// ─── apply batching ───────────────────────────────────────────
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Decide whether a merged apply target needs to be split into batches.
|
|
143
|
+
*
|
|
144
|
+
* @param {{ memoryMd?: string, summaryMd?: string, sources: Array<{ groupId: string, diff: any }> }} merged
|
|
145
|
+
* @param {number} [maxTokens=MAX_APPLY_TOKENS]
|
|
146
|
+
*/
|
|
147
|
+
export function needsBatchedApply(merged, maxTokens = MAX_APPLY_TOKENS) {
|
|
148
|
+
return totalApplyTokens(merged) > maxTokens;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function totalApplyTokens(merged) {
|
|
152
|
+
let n = estimateTokens(merged.memoryMd || '') + estimateTokens(merged.summaryMd || '');
|
|
153
|
+
for (const src of merged.sources || []) n += estimateMessagesTokens(src.diff || []);
|
|
154
|
+
return n;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Pack `merged.sources` into ordered batches such that each batch's
|
|
159
|
+
* (memoryMd + summaryMd + that batch's sources) ≤ maxTokens. The first
|
|
160
|
+
* batch uses the original memoryMd; subsequent batches assume the LLM's
|
|
161
|
+
* previous-batch output replaces memoryMd, so we account for the same
|
|
162
|
+
* baseline cost in each batch.
|
|
163
|
+
*
|
|
164
|
+
* If a single source (one group's diff) alone would overflow, it still
|
|
165
|
+
* goes into its own batch — we never split a source diff here (segment
|
|
166
|
+
* happens earlier, in triage).
|
|
167
|
+
*
|
|
168
|
+
* @param {{ memoryMd?: string, summaryMd?: string, sources: Array<{ groupId: string, diff: any }> }} merged
|
|
169
|
+
* @param {number} [maxTokens=MAX_APPLY_TOKENS]
|
|
170
|
+
* @returns {Array<{ groupId: string, diff: any }[]>}
|
|
171
|
+
*/
|
|
172
|
+
export function batchSourcesForApply(merged, maxTokens = MAX_APPLY_TOKENS) {
|
|
173
|
+
const sources = Array.isArray(merged.sources) ? merged.sources : [];
|
|
174
|
+
if (sources.length === 0) return [];
|
|
175
|
+
const baseline = estimateTokens(merged.memoryMd || '') + estimateTokens(merged.summaryMd || '');
|
|
176
|
+
const batches = [];
|
|
177
|
+
let cur = [];
|
|
178
|
+
let used = baseline;
|
|
179
|
+
for (const src of sources) {
|
|
180
|
+
const cost = estimateMessagesTokens(src.diff || []);
|
|
181
|
+
if (cur.length > 0 && used + cost > maxTokens) {
|
|
182
|
+
batches.push(cur);
|
|
183
|
+
cur = [];
|
|
184
|
+
used = baseline;
|
|
185
|
+
}
|
|
186
|
+
cur.push(src);
|
|
187
|
+
used += cost;
|
|
188
|
+
}
|
|
189
|
+
if (cur.length > 0) batches.push(cur);
|
|
190
|
+
return batches;
|
|
191
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dream-v2/snapshot.js — DESIGN-v2 §16.3 + §10.
|
|
3
|
+
*
|
|
4
|
+
* Pre-Apply backup of memory.md + summary.md to
|
|
5
|
+
* `~/.yeaft/memory/.dream-bak/<ts>/<scope-path>/`. The runner takes a
|
|
6
|
+
* snapshot once per merged target before Apply mutates it; that snapshot
|
|
7
|
+
* is the unit of rollback in case of LLM error or write failure.
|
|
8
|
+
*
|
|
9
|
+
* `pruneOldSnapshots()` keeps the most recent DREAM_BACKUP_KEEP
|
|
10
|
+
* timestamp directories under `.dream-bak/` and rm-rf's the rest.
|
|
11
|
+
*
|
|
12
|
+
* Pure I/O. No LLM. No control-flow.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { promises as fsp, existsSync } from 'fs';
|
|
16
|
+
import { join, dirname } from 'path';
|
|
17
|
+
|
|
18
|
+
import { DREAM_BACKUP_KEEP } from './limits.js';
|
|
19
|
+
|
|
20
|
+
/** Folder name where snapshots live, relative to memory root. */
|
|
21
|
+
export const BACKUP_DIRNAME = '.dream-bak';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Build a stable filesystem-safe ISO timestamp string. Same shape that
|
|
25
|
+
* the migration script uses (`migrate-r6-to-v2.js`).
|
|
26
|
+
*/
|
|
27
|
+
export function tsForBackup(d = new Date()) {
|
|
28
|
+
return d.toISOString().replace(/[:.]/g, '-');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Snapshot a single scope's memory.md + summary.md into
|
|
33
|
+
* `<root>/.dream-bak/<ts>/<scopeRelDir>/`. Missing source files are
|
|
34
|
+
* skipped silently; the destination dir is always created so that an
|
|
35
|
+
* absent snapshot is still distinguishable from "didn't run".
|
|
36
|
+
*
|
|
37
|
+
* @param {string} root — memory root
|
|
38
|
+
* @param {string} ts — timestamp folder name (re-use across all
|
|
39
|
+
* scopes in one dream pass)
|
|
40
|
+
* @param {string} scopeRelDir — e.g. 'user', 'group/g-eng', 'topic/sci/phys'
|
|
41
|
+
* @returns {Promise<{ backupDir: string, copied: string[] }>}
|
|
42
|
+
*/
|
|
43
|
+
export async function snapshotScope(root, ts, scopeRelDir) {
|
|
44
|
+
const srcDir = join(root, scopeRelDir);
|
|
45
|
+
const dstDir = join(root, BACKUP_DIRNAME, ts, scopeRelDir);
|
|
46
|
+
await fsp.mkdir(dstDir, { recursive: true });
|
|
47
|
+
const copied = [];
|
|
48
|
+
for (const name of ['memory.md', 'summary.md']) {
|
|
49
|
+
const s = join(srcDir, name);
|
|
50
|
+
if (!existsSync(s)) continue;
|
|
51
|
+
const d = join(dstDir, name);
|
|
52
|
+
await fsp.copyFile(s, d);
|
|
53
|
+
copied.push(name);
|
|
54
|
+
}
|
|
55
|
+
return { backupDir: dstDir, copied };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Keep the `keep` newest snapshot timestamps, rm-rf the rest.
|
|
60
|
+
*
|
|
61
|
+
* @param {string} root
|
|
62
|
+
* @param {number} [keep=DREAM_BACKUP_KEEP]
|
|
63
|
+
* @returns {Promise<{ kept: string[], removed: string[] }>}
|
|
64
|
+
*/
|
|
65
|
+
export async function pruneOldSnapshots(root, keep = DREAM_BACKUP_KEEP) {
|
|
66
|
+
const baseDir = join(root, BACKUP_DIRNAME);
|
|
67
|
+
if (!existsSync(baseDir)) return { kept: [], removed: [] };
|
|
68
|
+
let entries;
|
|
69
|
+
try { entries = await fsp.readdir(baseDir, { withFileTypes: true }); }
|
|
70
|
+
catch (err) { if (err && err.code === 'ENOENT') return { kept: [], removed: [] }; throw err; }
|
|
71
|
+
const dirs = entries.filter(e => e.isDirectory()).map(e => e.name).sort();
|
|
72
|
+
// sort() of ISO-with-dashes timestamps is chronological.
|
|
73
|
+
const cutoff = Math.max(0, dirs.length - keep);
|
|
74
|
+
const removed = dirs.slice(0, cutoff);
|
|
75
|
+
const kept = dirs.slice(cutoff);
|
|
76
|
+
for (const name of removed) {
|
|
77
|
+
await fsp.rm(join(baseDir, name), { recursive: true, force: true }).catch(() => {});
|
|
78
|
+
}
|
|
79
|
+
return { kept, removed };
|
|
80
|
+
}
|