@yeaft/webchat-agent 1.0.440 → 1.0.442
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/local-runtime/version.json +1 -1
- package/local-runtime/web/app.bundle.js +38 -36
- package/local-runtime/web/app.bundle.js.gz +0 -0
- package/local-runtime/web/index.html +1 -1
- package/package.json +1 -1
- package/yeaft/config-api.js +7 -3
- package/yeaft/config.js +10 -1
- package/yeaft/dream/apply.js +90 -37
- package/yeaft/dream/limits.js +5 -0
- package/yeaft/dream/runner.js +61 -33
- package/yeaft/dream/segment-extract.js +9 -3
- package/yeaft/dream/segment.js +104 -11
- package/yeaft/dream/session-wiring.js +13 -4
- package/yeaft/dream/topic-consolidation.js +5 -4
- package/yeaft/dream/triage.js +13 -9
- package/yeaft/engine.js +2 -2
|
Binary file
|
package/package.json
CHANGED
package/yeaft/config-api.js
CHANGED
|
@@ -196,7 +196,7 @@ export function updateLlmConfig(update, dir) {
|
|
|
196
196
|
* stable shape — `normaliseYeaftSection` guarantees that.
|
|
197
197
|
*
|
|
198
198
|
* @param {string} [dir] — Yeaft data directory
|
|
199
|
-
* @returns {{ maxConcurrentThreads: number, autoArchiveIdleDays: number, recentTurnsLimit: number } | { error: string }}
|
|
199
|
+
* @returns {{ maxConcurrentThreads: number, autoArchiveIdleDays: number, recentTurnsLimit: number, dream: object } | { error: string }}
|
|
200
200
|
*/
|
|
201
201
|
export function getYeaftSettings(dir) {
|
|
202
202
|
const root = dir || process.env.YEAFT_DIR || DEFAULT_YEAFT_DIR;
|
|
@@ -215,14 +215,15 @@ export function getYeaftSettings(dir) {
|
|
|
215
215
|
* Update the Yeaft-section of config.json. Merges into existing config
|
|
216
216
|
* (LLM provider / model fields are untouched) and validates each field:
|
|
217
217
|
* `maxConcurrentThreads` must be 1..50, `autoArchiveIdleDays` must be
|
|
218
|
-
* 1..3650, `recentTurnsLimit` must be 1..500.
|
|
218
|
+
* 1..3650, `recentTurnsLimit` must be 1..500. Dream limits are read-only
|
|
219
|
+
* runtime defaults here; invalid values are rejected
|
|
219
220
|
* outright so the UI sees an error rather than silently reverting — a
|
|
220
221
|
* silent revert would make "I set it to 100 and nothing happened"
|
|
221
222
|
* impossible to debug.
|
|
222
223
|
*
|
|
223
224
|
* @param {{ maxConcurrentThreads?: number, autoArchiveIdleDays?: number, recentTurnsLimit?: number }} update
|
|
224
225
|
* @param {string} [dir]
|
|
225
|
-
* @returns {{ maxConcurrentThreads: number, autoArchiveIdleDays: number, recentTurnsLimit: number } | { error: string }}
|
|
226
|
+
* @returns {{ maxConcurrentThreads: number, autoArchiveIdleDays: number, recentTurnsLimit: number, dream: object } | { error: string }}
|
|
226
227
|
*/
|
|
227
228
|
export function updateYeaftSettings(update, dir) {
|
|
228
229
|
const root = dir || process.env.YEAFT_DIR || DEFAULT_YEAFT_DIR;
|
|
@@ -266,6 +267,9 @@ export function updateYeaftSettings(update, dir) {
|
|
|
266
267
|
? Math.floor(Number(update.recentTurnsLimit))
|
|
267
268
|
: prev.recentTurnsLimit,
|
|
268
269
|
};
|
|
270
|
+
if (existing.yeaft?.dream && typeof existing.yeaft.dream === 'object') {
|
|
271
|
+
merged.dream = existing.yeaft.dream;
|
|
272
|
+
}
|
|
269
273
|
existing.yeaft = merged;
|
|
270
274
|
return merged;
|
|
271
275
|
});
|
package/yeaft/config.js
CHANGED
|
@@ -29,6 +29,7 @@ import { normalizeKnownProviderForRuntime } from './llm/known-providers.js';
|
|
|
29
29
|
import { createDenyAllPluginConfig, normalizePluginConfig } from './plugins.js';
|
|
30
30
|
import { normaliseBrowserRuntimeSection } from '../browser-runtime/config.js';
|
|
31
31
|
import { readWorkspaceFile } from './workspace-file.js';
|
|
32
|
+
import { DEFAULT_LIMITS } from './dream/limits.js';
|
|
32
33
|
|
|
33
34
|
/** Default configuration values. */
|
|
34
35
|
const DEFAULTS = {
|
|
@@ -254,13 +255,14 @@ function isTruthy(val) {
|
|
|
254
255
|
* bounds via `clampYeaftField`.
|
|
255
256
|
*
|
|
256
257
|
* @param {any} raw — jsonConfig.yeaft (may be undefined / malformed)
|
|
257
|
-
* @returns {{ maxConcurrentThreads: number, autoArchiveIdleDays: number }}
|
|
258
|
+
* @returns {{ maxConcurrentThreads: number, autoArchiveIdleDays: number, recentTurnsLimit: number, dream: object }}
|
|
258
259
|
*/
|
|
259
260
|
export function normaliseYeaftSection(raw) {
|
|
260
261
|
const out = {
|
|
261
262
|
maxConcurrentThreads: DEFAULTS.yeaftMaxConcurrentThreads,
|
|
262
263
|
autoArchiveIdleDays: DEFAULTS.yeaftAutoArchiveIdleDays,
|
|
263
264
|
recentTurnsLimit: DEFAULTS.yeaftRecentTurnsLimit,
|
|
265
|
+
dream: { ...DEFAULT_LIMITS },
|
|
264
266
|
};
|
|
265
267
|
if (!raw || typeof raw !== 'object') return out;
|
|
266
268
|
const mc = clampYeaftField(raw.maxConcurrentThreads, 'maxConcurrentThreads');
|
|
@@ -269,6 +271,13 @@ export function normaliseYeaftSection(raw) {
|
|
|
269
271
|
if (ad !== null) out.autoArchiveIdleDays = ad;
|
|
270
272
|
const rt = clampYeaftField(raw.recentTurnsLimit, 'recentTurnsLimit');
|
|
271
273
|
if (rt !== null) out.recentTurnsLimit = rt;
|
|
274
|
+
const dream = raw.dream;
|
|
275
|
+
if (dream && typeof dream === 'object' && !Array.isArray(dream)) {
|
|
276
|
+
for (const key of Object.keys(DEFAULT_LIMITS)) {
|
|
277
|
+
const value = Number(dream[key]);
|
|
278
|
+
if (Number.isFinite(value) && value > 0) out.dream[key] = Math.floor(value);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
272
281
|
return out;
|
|
273
282
|
}
|
|
274
283
|
|
package/yeaft/dream/apply.js
CHANGED
|
@@ -26,10 +26,16 @@ import { inspect } from 'util';
|
|
|
26
26
|
import { writeContent, writeSummary, readContent, readMemory, readSummary } from '../memory/store.js';
|
|
27
27
|
import { parseSegments } from '../memory/segment.js';
|
|
28
28
|
import { syncScope } from '../memory/segment-sync.js';
|
|
29
|
-
import {
|
|
29
|
+
import {
|
|
30
|
+
batchSourcesForApply,
|
|
31
|
+
boundDreamPrompt,
|
|
32
|
+
needsBatchedApply,
|
|
33
|
+
truncateMessage,
|
|
34
|
+
} from './segment.js';
|
|
30
35
|
import { snapshotScope } from './snapshot.js';
|
|
31
36
|
import { parseJsonSafe } from './triage.js';
|
|
32
37
|
import { render } from './prompts/index.js';
|
|
38
|
+
import { MAX_DREAM_PROMPT_CHARS } from './limits.js';
|
|
33
39
|
|
|
34
40
|
function malformedJsonError(message, raw) {
|
|
35
41
|
const err = new Error(message);
|
|
@@ -239,7 +245,7 @@ export function targetToScope(target) {
|
|
|
239
245
|
* root: string,
|
|
240
246
|
* ts: string, // shared timestamp folder
|
|
241
247
|
* llm: (req: { pass: string, prompt: string, system: string }) => Promise<string>,
|
|
242
|
-
* limits?: { MAX_APPLY_TOKENS?: number },
|
|
248
|
+
* limits?: { MAX_APPLY_TOKENS?: number, MAX_DREAM_PROMPT_CHARS?: number },
|
|
243
249
|
* snapshot?: typeof snapshotScope,
|
|
244
250
|
* nowIso?: () => string,
|
|
245
251
|
* onProgress?: (event: object) => void,
|
|
@@ -272,55 +278,81 @@ export async function applyMergedTarget(merged, opts) {
|
|
|
272
278
|
}
|
|
273
279
|
|
|
274
280
|
let batchesUsed = 0;
|
|
275
|
-
const maxApply = (opts.limits && opts.limits.MAX_APPLY_TOKENS) ||
|
|
281
|
+
const maxApply = (opts.limits && opts.limits.MAX_APPLY_TOKENS) || 80000;
|
|
282
|
+
// Leave room for the prompt template and JSON framing. The bounded context
|
|
283
|
+
// and source chunks are prompt projections; canonical files remain complete.
|
|
284
|
+
const maxPromptChars = (opts.limits && opts.limits.MAX_DREAM_PROMPT_CHARS) || MAX_DREAM_PROMPT_CHARS;
|
|
285
|
+
const providerPromptBudget = Math.floor(maxPromptChars / 4) - 2048;
|
|
286
|
+
const promptBudget = Math.max(1024, Math.min(Math.floor(maxApply) - 2048, providerPromptBudget));
|
|
287
|
+
const initialContext = boundApplyContext(contentMd, summaryMd, promptBudget);
|
|
276
288
|
|
|
277
289
|
if (merged.kind === 'create') {
|
|
278
290
|
const siblings = opts.siblingTopicsFor ? await opts.siblingTopicsFor(merged.target) : [];
|
|
279
|
-
const
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
const
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
291
|
+
const sourceBatches = needsBatchedApply(
|
|
292
|
+
{ memoryMd: '', summaryMd: '', sources: merged.sources },
|
|
293
|
+
promptBudget,
|
|
294
|
+
)
|
|
295
|
+
? batchSourcesForApply({ memoryMd: '', summaryMd: '', sources: merged.sources }, promptBudget)
|
|
296
|
+
: [merged.sources];
|
|
297
|
+
|
|
298
|
+
let i = 0;
|
|
299
|
+
for (const batch of sourceBatches) {
|
|
300
|
+
i += 1;
|
|
301
|
+
const prompt = boundDreamPrompt(i === 1
|
|
302
|
+
? buildCreatePrompt({
|
|
303
|
+
target: merged.target,
|
|
304
|
+
sources: batch,
|
|
305
|
+
siblingTopics: siblings,
|
|
306
|
+
language: opts.language,
|
|
307
|
+
})
|
|
308
|
+
: buildUpdatePrompt({
|
|
309
|
+
target: merged.target,
|
|
310
|
+
contentMd: boundApplyContext(contentMd, summaryMd, promptBudget).contentMd,
|
|
311
|
+
summaryMd: boundApplyContext(contentMd, summaryMd, promptBudget).summaryMd,
|
|
312
|
+
sources: batch,
|
|
313
|
+
batchInfo: { index: i, total: sourceBatches.length },
|
|
314
|
+
language: opts.language,
|
|
315
|
+
}), maxPromptChars);
|
|
316
|
+
if (opts.onProgress) opts.onProgress({ phase: 'apply', target: merged.target, status: 'llm', batch: i, of: sourceBatches.length });
|
|
317
|
+
const raw = await opts.llm({ pass: i === 1 ? 'create' : 'update', prompt, system: applySystem(opts.language) });
|
|
318
|
+
const parsed = parseJsonSafe(raw);
|
|
319
|
+
const parsedContent = parsed && typeof parsed.content_md === 'string'
|
|
320
|
+
? parsed.content_md
|
|
321
|
+
: (parsed && typeof parsed.memory_md === 'string' ? parsed.memory_md : null);
|
|
322
|
+
if (parsedContent === null) {
|
|
323
|
+
throw malformedJsonError(`apply: ${i === 1 ? 'CREATE' : 'UPDATE'} batch ${i} returned malformed JSON for ${merged.target}`, raw);
|
|
324
|
+
}
|
|
325
|
+
const ensured = ensurePrimarySessionOutput({
|
|
326
|
+
target: merged.target,
|
|
327
|
+
memoryMd: parsedContent,
|
|
328
|
+
summaryMd: typeof parsed.summary_md === 'string' ? parsed.summary_md : summaryMd,
|
|
329
|
+
sources: batch,
|
|
330
|
+
language: opts.language,
|
|
331
|
+
});
|
|
332
|
+
contentMd = ensured.memoryMd;
|
|
333
|
+
summaryMd = ensured.summaryMd;
|
|
293
334
|
}
|
|
294
|
-
|
|
295
|
-
target: merged.target,
|
|
296
|
-
memoryMd: parsedContent,
|
|
297
|
-
summaryMd: typeof parsed.summary_md === 'string' ? parsed.summary_md : '',
|
|
298
|
-
sources: merged.sources,
|
|
299
|
-
language: opts.language,
|
|
300
|
-
});
|
|
301
|
-
contentMd = ensured.memoryMd;
|
|
302
|
-
summaryMd = ensured.summaryMd;
|
|
303
|
-
batchesUsed = 1;
|
|
335
|
+
batchesUsed = sourceBatches.length;
|
|
304
336
|
} else {
|
|
305
|
-
// UPDATE — possibly batched.
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
? batchSourcesForApply({ memoryMd: contentMd, summaryMd, sources: merged.sources }, maxApply)
|
|
337
|
+
// UPDATE — possibly batched. A single Session source may be split into
|
|
338
|
+
// many bounded batches; old overlap is never present in merged.sources.
|
|
339
|
+
const input = { ...initialContext, sources: merged.sources };
|
|
340
|
+
const batches = needsBatchedApply(input, promptBudget)
|
|
341
|
+
? batchSourcesForApply(input, promptBudget)
|
|
311
342
|
: [merged.sources];
|
|
312
343
|
|
|
313
344
|
let i = 0;
|
|
314
345
|
for (const batch of batches) {
|
|
315
346
|
i += 1;
|
|
316
|
-
const
|
|
347
|
+
const context = boundApplyContext(contentMd, summaryMd, promptBudget);
|
|
348
|
+
const prompt = boundDreamPrompt(buildUpdatePrompt({
|
|
317
349
|
target: merged.target,
|
|
318
|
-
contentMd,
|
|
319
|
-
summaryMd,
|
|
350
|
+
contentMd: context.contentMd,
|
|
351
|
+
summaryMd: context.summaryMd,
|
|
320
352
|
sources: batch,
|
|
321
353
|
batchInfo: { index: i, total: batches.length },
|
|
322
354
|
language: opts.language,
|
|
323
|
-
});
|
|
355
|
+
}), maxPromptChars);
|
|
324
356
|
if (opts.onProgress) opts.onProgress({ phase: 'apply', target: merged.target, status: 'llm', batch: i, of: batches.length });
|
|
325
357
|
const raw = await opts.llm({ pass: 'update', prompt, system: applySystem(opts.language) });
|
|
326
358
|
const parsed = parseJsonSafe(raw);
|
|
@@ -392,6 +424,27 @@ function scopeRelDir(scope) {
|
|
|
392
424
|
|
|
393
425
|
function oneLine(s) { return String(s || '').replace(/\s+/g, ' ').trim().slice(0, 200); }
|
|
394
426
|
|
|
427
|
+
function boundApplyContext(contentMd, summaryMd, budget) {
|
|
428
|
+
const maxChars = Math.max(4096, Math.floor(Math.max(1, budget) * 4));
|
|
429
|
+
const summary = truncatePromptText(summaryMd, Math.min(maxChars, 8_000));
|
|
430
|
+
const contentBudget = Math.max(0, maxChars - summary.length);
|
|
431
|
+
return {
|
|
432
|
+
contentMd: truncatePromptText(contentMd, contentBudget),
|
|
433
|
+
summaryMd: summary,
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function truncatePromptText(value, maxChars) {
|
|
438
|
+
const text = String(value || '');
|
|
439
|
+
if (text.length <= maxChars) return text;
|
|
440
|
+
const marker = '\n\n[canonical memory clipped for Dream apply; durable source remains on disk]\n\n';
|
|
441
|
+
if (maxChars <= marker.length) return text.slice(0, Math.max(0, maxChars));
|
|
442
|
+
const room = maxChars - marker.length;
|
|
443
|
+
const head = Math.ceil(room * 0.62);
|
|
444
|
+
const tail = room - head;
|
|
445
|
+
return text.slice(0, head) + marker + (tail > 0 ? text.slice(-tail) : '');
|
|
446
|
+
}
|
|
447
|
+
|
|
395
448
|
function isLegacyCanonicalMemory(memoryMd, scope) {
|
|
396
449
|
const raw = String(memoryMd || '').trim();
|
|
397
450
|
if (!raw) return false;
|
package/yeaft/dream/limits.js
CHANGED
|
@@ -14,6 +14,10 @@ export const DREAM_INTERVAL_HOURS = 1;
|
|
|
14
14
|
export const DREAM_OVERLAP = 3;
|
|
15
15
|
export const MIN_NEW_PER_GROUP = 20;
|
|
16
16
|
export const MAX_SINGLE_MESSAGE_CHARS = 8000;
|
|
17
|
+
// Provider tokenizers differ substantially for CJK and serialized tool text.
|
|
18
|
+
// Keep an independent character ceiling so the approximate token budget cannot
|
|
19
|
+
// turn into a multi-million-token wire request.
|
|
20
|
+
export const MAX_DREAM_PROMPT_CHARS = 96000;
|
|
17
21
|
export const MAX_DIFF_TOKENS_PER_TRIAGE = 60000;
|
|
18
22
|
export const MAX_APPLY_TOKENS = 80000;
|
|
19
23
|
export const DREAM_BACKUP_KEEP = 7;
|
|
@@ -28,6 +32,7 @@ export const DEFAULT_LIMITS = Object.freeze({
|
|
|
28
32
|
DREAM_OVERLAP,
|
|
29
33
|
MIN_NEW_PER_GROUP,
|
|
30
34
|
MAX_SINGLE_MESSAGE_CHARS,
|
|
35
|
+
MAX_DREAM_PROMPT_CHARS,
|
|
31
36
|
MAX_DIFF_TOKENS_PER_TRIAGE,
|
|
32
37
|
MAX_APPLY_TOKENS,
|
|
33
38
|
DREAM_BACKUP_KEEP,
|
package/yeaft/dream/runner.js
CHANGED
|
@@ -8,8 +8,7 @@
|
|
|
8
8
|
* enumerateSessions() via opts.listSessions()
|
|
9
9
|
* ↓
|
|
10
10
|
* for each session with newCount ≥ MIN_NEW_PER_GROUP (auto)
|
|
11
|
-
* or > 0 (manual)
|
|
12
|
-
* or prior messages in a scoped manual session rerun:
|
|
11
|
+
* or > 0 (manual):
|
|
13
12
|
* loadDiff() via opts.loadSessionDiff(sessionId, sinceId)
|
|
14
13
|
* applyOverlap() via opts.loadOverlapPreamble(...)
|
|
15
14
|
* segment() segmentDiff(...)
|
|
@@ -46,7 +45,11 @@ import {
|
|
|
46
45
|
DEFAULT_LIMITS,
|
|
47
46
|
} from './limits.js';
|
|
48
47
|
import { clearDreamError, readSessionState, writeSessionState, writeDreamError } from './state.js';
|
|
49
|
-
import {
|
|
48
|
+
import {
|
|
49
|
+
segmentDiff,
|
|
50
|
+
compressDreamMessages,
|
|
51
|
+
selectDreamNewMessages,
|
|
52
|
+
} from './segment.js';
|
|
50
53
|
import { triageGroupSegments } from './triage.js';
|
|
51
54
|
import { mergeByTarget } from './merge.js';
|
|
52
55
|
import { applyMergedTarget } from './apply.js';
|
|
@@ -58,7 +61,7 @@ import { tsForBackup, pruneOldSnapshots } from './snapshot.js';
|
|
|
58
61
|
* @typedef {Object} RunDreamOpts
|
|
59
62
|
* @property {string} root — memory root, e.g. ~/.yeaft/memory
|
|
60
63
|
* @property {boolean} [manual=false] — manual trigger overrides newCount<20 skip
|
|
61
|
-
* @property {string[]} [scopeFilter] — optional: only dream these targets
|
|
64
|
+
* @property {string[]} [scopeFilter] — optional: only dream these targets ('*' allowed)
|
|
62
65
|
* @property {(req: {pass:string, prompt:string, system:string}) => Promise<string>} llm
|
|
63
66
|
* @property {() => Promise<Array<string>>} listSessions — return all session ids (incl. '_no-session')
|
|
64
67
|
* @property {(sessionId: string) => Promise<number>} countMessages — total message count for a session
|
|
@@ -109,11 +112,9 @@ export async function runDream(opts) {
|
|
|
109
112
|
};
|
|
110
113
|
|
|
111
114
|
for (const sessionId of sessionIds) {
|
|
112
|
-
//
|
|
113
|
-
//
|
|
114
|
-
//
|
|
115
|
-
// sessions/<id>". Pure target filters such as ['user'] still triage every
|
|
116
|
-
// session so their hard-rule actions can contribute to the requested scope.
|
|
115
|
+
// A Session filter constrains enumeration; a pure target filter such as
|
|
116
|
+
// ['user'] still triages every Session so hard-rule actions can contribute
|
|
117
|
+
// to the requested scope.
|
|
117
118
|
if (sessionFilter && !sessionFilter.has(sessionId)) {
|
|
118
119
|
sessionsReport.push({ sessionId, new: 0, status: 'skipped', reason: 'scope-filtered' });
|
|
119
120
|
continue;
|
|
@@ -122,13 +123,7 @@ export async function runDream(opts) {
|
|
|
122
123
|
const beforeCount = await safeCall(() => opts.countMessages(sessionId), 0);
|
|
123
124
|
const newCount = Math.max(0, beforeCount - (state.messageCount || 0));
|
|
124
125
|
|
|
125
|
-
|
|
126
|
-
&& sessionFilter
|
|
127
|
-
&& sessionFilter.has(sessionId)
|
|
128
|
-
&& newCount === 0
|
|
129
|
-
&& beforeCount > 0;
|
|
130
|
-
|
|
131
|
-
if (newCount === 0 && !rerunScopedManual) {
|
|
126
|
+
if (newCount === 0) {
|
|
132
127
|
const topics = await resolveTopicSummaries(sessionId);
|
|
133
128
|
if (opts.manual && topics.length >= 2) {
|
|
134
129
|
try {
|
|
@@ -140,6 +135,7 @@ export async function runDream(opts) {
|
|
|
140
135
|
language: opts.language,
|
|
141
136
|
ts,
|
|
142
137
|
segmentIndex: opts.segmentIndex || null,
|
|
138
|
+
maxPromptChars: limits.MAX_DREAM_PROMPT_CHARS,
|
|
143
139
|
});
|
|
144
140
|
sessionsReport.push({ sessionId, new: 0, status: 'consolidated', ...result });
|
|
145
141
|
} catch (err) {
|
|
@@ -156,14 +152,14 @@ export async function runDream(opts) {
|
|
|
156
152
|
}
|
|
157
153
|
|
|
158
154
|
onProgress({ phase: 'load-diff', sessionId });
|
|
159
|
-
const diffCursor =
|
|
155
|
+
const diffCursor = state.lastDreamMessageId;
|
|
160
156
|
const loadDiff = opts.loadSessionDiff || opts.loadGroupDiff;
|
|
161
157
|
const diffNew = await safeCall(() => loadDiff(sessionId, diffCursor), []);
|
|
162
158
|
if (!diffNew || diffNew.length === 0) {
|
|
163
159
|
sessionsReport.push({ sessionId, new: newCount, status: 'skipped', reason: 'empty-diff' });
|
|
164
160
|
continue;
|
|
165
161
|
}
|
|
166
|
-
const overlapMessages = state.lastDreamMessageId
|
|
162
|
+
const overlapMessages = state.lastDreamMessageId
|
|
167
163
|
? await safeCall(
|
|
168
164
|
() => opts.loadOverlapPreamble
|
|
169
165
|
? opts.loadOverlapPreamble(sessionId, state.lastDreamMessageId, limits.DREAM_OVERLAP)
|
|
@@ -171,11 +167,33 @@ export async function runDream(opts) {
|
|
|
171
167
|
[],
|
|
172
168
|
)
|
|
173
169
|
: [];
|
|
174
|
-
const taggedOverlap = overlapMessages
|
|
175
|
-
|
|
170
|
+
const taggedOverlap = compressDreamMessages(overlapMessages)
|
|
171
|
+
.map(m => ({ ...m, kind: 'overlap' }));
|
|
172
|
+
const taggedNew = compressDreamMessages(diffNew)
|
|
173
|
+
.map(m => ({ ...m, kind: 'new' }));
|
|
176
174
|
const fullDiff = [...taggedOverlap, ...taggedNew];
|
|
175
|
+
if (taggedNew.length === 0) {
|
|
176
|
+
// Tool-only traffic is intentionally not sent to Dream, but it still
|
|
177
|
+
// belongs to the processed transcript range. Advance the cursor so a
|
|
178
|
+
// failed/empty pass cannot replay the same execution history forever.
|
|
179
|
+
const tailId = lastMessageId(diffNew);
|
|
180
|
+
if (tailId) {
|
|
181
|
+
await writeSessionState(opts.root, sessionId, {
|
|
182
|
+
lastDreamMessageId: tailId,
|
|
183
|
+
lastDreamAt: nowIso,
|
|
184
|
+
messageCount: beforeCount,
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
sessionsReport.push({ sessionId, new: newCount, status: 'skipped', reason: 'no-durable-new-messages' });
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
177
190
|
|
|
178
|
-
const segments = segmentDiff(
|
|
191
|
+
const segments = segmentDiff(
|
|
192
|
+
fullDiff,
|
|
193
|
+
limits.MAX_DIFF_TOKENS_PER_TRIAGE,
|
|
194
|
+
limits.DREAM_OVERLAP,
|
|
195
|
+
limits.MAX_DREAM_PROMPT_CHARS,
|
|
196
|
+
);
|
|
179
197
|
onProgress({ phase: 'triage', sessionId, status: 'running', segments: segments.length });
|
|
180
198
|
|
|
181
199
|
let actions;
|
|
@@ -186,6 +204,7 @@ export async function runDream(opts) {
|
|
|
186
204
|
sessionId,
|
|
187
205
|
segments,
|
|
188
206
|
topicSummaries,
|
|
207
|
+
maxPromptChars: limits.MAX_DREAM_PROMPT_CHARS,
|
|
189
208
|
llm: dreamLlmForSession(opts.llm, sessionId),
|
|
190
209
|
onProgress,
|
|
191
210
|
language: opts.language,
|
|
@@ -206,11 +225,15 @@ export async function runDream(opts) {
|
|
|
206
225
|
}
|
|
207
226
|
|
|
208
227
|
onProgress({ phase: 'triage', sessionId, status: 'done', actions: actions.length });
|
|
209
|
-
sessionTriages.push({
|
|
228
|
+
sessionTriages.push({
|
|
229
|
+
sessionId,
|
|
230
|
+
diff: selectDreamNewMessages(taggedNew),
|
|
231
|
+
actions,
|
|
232
|
+
});
|
|
210
233
|
|
|
211
234
|
const tailId = lastMessageId(diffNew);
|
|
212
235
|
processedSessions.push({ sessionId, tailId, beforeCount, newCount, segments: segments.length, actions: actions.length });
|
|
213
|
-
sessionsReport.push({ sessionId, new: newCount, segments: segments.length, actions: actions.length, status: 'triaged'
|
|
236
|
+
sessionsReport.push({ sessionId, new: newCount, segments: segments.length, actions: actions.length, status: 'triaged' });
|
|
214
237
|
}
|
|
215
238
|
|
|
216
239
|
// 3. merge
|
|
@@ -277,6 +300,7 @@ export async function runDream(opts) {
|
|
|
277
300
|
targets,
|
|
278
301
|
llm: dreamLlmForSession(opts.llm, triage.sessionId),
|
|
279
302
|
language: opts.language,
|
|
303
|
+
limits,
|
|
280
304
|
nowIso: opts.nowIso || (() => nowIso),
|
|
281
305
|
segmentIndex: opts.segmentIndex || null,
|
|
282
306
|
});
|
|
@@ -310,6 +334,7 @@ export async function runDream(opts) {
|
|
|
310
334
|
language: opts.language,
|
|
311
335
|
ts,
|
|
312
336
|
segmentIndex: opts.segmentIndex || null,
|
|
337
|
+
maxPromptChars: limits.MAX_DREAM_PROMPT_CHARS,
|
|
313
338
|
});
|
|
314
339
|
topicConsolidation.push({ sessionId, status: 'done', ...result });
|
|
315
340
|
onProgress({ phase: 'topic-consolidation', sessionId, status: 'done', ...result });
|
|
@@ -324,17 +349,20 @@ export async function runDream(opts) {
|
|
|
324
349
|
}
|
|
325
350
|
}
|
|
326
351
|
|
|
327
|
-
// 7. bookkeep —
|
|
328
|
-
//
|
|
329
|
-
//
|
|
330
|
-
//
|
|
331
|
-
//
|
|
332
|
-
|
|
352
|
+
// 7. bookkeep — advance a Session cursor only after every Apply target
|
|
353
|
+
// selected for that Session in this run completed successfully. Apply keeps
|
|
354
|
+
// all batch output in memory until the target's final canonical write, so a
|
|
355
|
+
// failed target has no durable progress that can safely move the cursor.
|
|
356
|
+
// Keeping the cursor on any target error lets the next run retry the failed
|
|
357
|
+
// target together with the same source diff.
|
|
358
|
+
const targetStatus = new Map(targetsReport.map(report => [report.target, report.status]));
|
|
333
359
|
for (const pg of processedSessions) {
|
|
334
|
-
const
|
|
335
|
-
.
|
|
336
|
-
|
|
337
|
-
if (!
|
|
360
|
+
const sessionTargets = targetsToApply
|
|
361
|
+
.filter(target => target.sources.some(source => source.sessionId === pg.sessionId))
|
|
362
|
+
.map(target => target.target);
|
|
363
|
+
if (sessionTargets.length === 0 || !sessionTargets.every(target => targetStatus.get(target) === 'done')) {
|
|
364
|
+
continue;
|
|
365
|
+
}
|
|
338
366
|
if (pg.tailId) {
|
|
339
367
|
await writeSessionState(opts.root, pg.sessionId, {
|
|
340
368
|
lastDreamMessageId: pg.tailId,
|
|
@@ -10,6 +10,7 @@ import { readScope, writeScope } from '../memory/segment-store.js';
|
|
|
10
10
|
import { syncScope } from '../memory/segment-sync.js';
|
|
11
11
|
import { makeSegment } from '../memory/segment.js';
|
|
12
12
|
import { render, extractTemplateForScope } from './prompts/index.js';
|
|
13
|
+
import { boundDreamPrompt } from './segment.js';
|
|
13
14
|
import { parseJsonSafe } from './triage.js';
|
|
14
15
|
|
|
15
16
|
const MAX_MESSAGES = 80;
|
|
@@ -40,6 +41,7 @@ const VALID_KINDS = new Set([
|
|
|
40
41
|
* language?: string,
|
|
41
42
|
* nowIso?: Function,
|
|
42
43
|
* segmentIndex?: import('../memory/index-db.js').SegmentIndex|null,
|
|
44
|
+
* limits?: { MAX_DREAM_PROMPT_CHARS?: number },
|
|
43
45
|
* }} opts
|
|
44
46
|
*/
|
|
45
47
|
export async function extractAndWriteMemorySegments(opts) {
|
|
@@ -68,6 +70,7 @@ export async function extractAndWriteMemorySegments(opts) {
|
|
|
68
70
|
language: opts.language,
|
|
69
71
|
now,
|
|
70
72
|
allowedSourceIds,
|
|
73
|
+
maxPromptChars: opts.limits?.MAX_DREAM_PROMPT_CHARS,
|
|
71
74
|
});
|
|
72
75
|
} catch (err) {
|
|
73
76
|
errors.push({ scope, error: err.message, rawSnippet: err.rawSnippet || '' });
|
|
@@ -92,10 +95,13 @@ export async function extractAndWriteMemorySegments(opts) {
|
|
|
92
95
|
return { scopes: scopeCount, segments: segmentCount, errors };
|
|
93
96
|
}
|
|
94
97
|
|
|
95
|
-
async function extractScopeSegments({ scope, sessionId, messages, llm, language, now, allowedSourceIds }) {
|
|
98
|
+
async function extractScopeSegments({ scope, sessionId, messages, llm, language, now, allowedSourceIds, maxPromptChars }) {
|
|
96
99
|
const template = extractTemplateForScope(scope);
|
|
97
100
|
const base = render(template, templateVarsForScope(scope, sessionId), { language });
|
|
98
|
-
const prompt =
|
|
101
|
+
const prompt = boundDreamPrompt(
|
|
102
|
+
`${base}\n\nTarget scope: ${scope}\n\nConversation diff, oldest first:\n${renderMessages(messages)}\n\nReturn only the JSON array. Do not wrap it in Markdown.`,
|
|
103
|
+
maxPromptChars,
|
|
104
|
+
);
|
|
99
105
|
const firstRaw = await llm({ pass: 'extract-segments', prompt, system: extractSystem(language) });
|
|
100
106
|
const firstParsed = parseJsonSafe(firstRaw);
|
|
101
107
|
if (Array.isArray(firstParsed)) {
|
|
@@ -104,7 +110,7 @@ async function extractScopeSegments({ scope, sessionId, messages, llm, language,
|
|
|
104
110
|
.filter(Boolean);
|
|
105
111
|
}
|
|
106
112
|
|
|
107
|
-
const retryPrompt = `${prompt}\n\nYour previous output was malformed JSON. Previous output snippet:\n${rawSnippet(firstRaw)}\n\nRetry now. Return only a strict JSON array
|
|
113
|
+
const retryPrompt = boundDreamPrompt(`${prompt}\n\nYour previous output was malformed JSON. Previous output snippet:\n${rawSnippet(firstRaw)}\n\nRetry now. Return only a strict JSON array.`, maxPromptChars);
|
|
108
114
|
const retryRaw = await llm({ pass: 'extract-segments-retry', prompt: retryPrompt, system: extractSystem(language) });
|
|
109
115
|
const retryParsed = parseJsonSafe(retryRaw);
|
|
110
116
|
if (!Array.isArray(retryParsed)) {
|