@qearlyao/familiar 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -14
- package/config.example.toml +1 -1
- package/dist/added-models.js +6 -15
- package/dist/agent-events.js +1 -3
- package/dist/agent.js +3 -4
- package/dist/browser-tools.js +15 -11
- package/dist/chat-log.js +3 -2
- package/dist/cli.js +2 -2
- package/dist/config-overrides.js +5 -14
- package/dist/config-registry.js +1 -4
- package/dist/config.js +45 -113
- package/dist/contact-note.js +2 -12
- package/dist/data-retention.js +1 -3
- package/dist/discord.js +72 -19
- package/dist/generated-media.js +3 -2
- package/dist/hot-reload.js +1 -3
- package/dist/image-gen.js +12 -51
- package/dist/inbound-attachments.js +64 -23
- package/dist/memory/diary/ambient-injector.js +1 -3
- package/dist/memory/diary/ambient.js +1 -3
- package/dist/memory/diary/chunks.js +1 -3
- package/dist/memory/diary/indexer.js +1 -3
- package/dist/memory/doctor.js +3 -8
- package/dist/memory/index/chunk-indexer.js +27 -6
- package/dist/memory/index/retrieval.js +1 -3
- package/dist/memory/index/store.js +47 -19
- package/dist/memory/lcm/backfill.js +19 -16
- package/dist/memory/lcm/context-transformer.js +17 -29
- package/dist/memory/lcm/context.js +10 -4
- package/dist/memory/lcm/eviction-score.js +25 -13
- package/dist/memory/lcm/indexer.js +1 -5
- package/dist/memory/lcm/normalize.js +22 -1
- package/dist/memory/lcm/store.js +27 -24
- package/dist/memory/operator.js +3 -31
- package/dist/memory/service.js +1 -3
- package/dist/memory/tools.js +0 -4
- package/dist/memory/util.js +6 -0
- package/dist/models.js +3 -0
- package/dist/persona.js +3 -15
- package/dist/runtime.js +12 -23
- package/dist/scheduler.js +15 -49
- package/dist/service.js +39 -27
- package/dist/settings.js +7 -32
- package/dist/silent-marker.js +64 -0
- package/dist/tts.js +0 -6
- package/dist/util/fs.js +41 -0
- package/dist/util/guards.js +8 -0
- package/dist/util/image-mime.js +31 -0
- package/dist/util/time.js +29 -0
- package/dist/web-auth.js +4 -1
- package/dist/web-static.js +36 -1
- package/dist/web-tools.js +8 -5
- package/dist/web.js +253 -69
- package/npm-shrinkwrap.json +5139 -0
- package/package.json +5 -4
- package/web/dist/assets/index-B23WT77N.js +63 -0
- package/web/dist/assets/index-D3MotFzN.css +2 -0
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-BPZQbZh5.js +0 -61
- package/web/dist/assets/index-CcQ13VAY.css +0 -2
|
@@ -15,8 +15,11 @@ export class ChunkIndexer {
|
|
|
15
15
|
async replaceSource(corpus, sourceId, inputs, signal) {
|
|
16
16
|
const prepared = this.prepare(inputs.map((input) => ({ ...input, corpus, sourceId })));
|
|
17
17
|
const keepMappings = prepared.map((item) => ({ contentHash: item.contentHash, chunkIndex: item.chunkIndex }));
|
|
18
|
-
this.
|
|
19
|
-
|
|
18
|
+
const result = await this.insertPrepared(prepared, inputs.length - prepared.length, signal, {
|
|
19
|
+
corpus,
|
|
20
|
+
sourceId,
|
|
21
|
+
keepMappings,
|
|
22
|
+
});
|
|
20
23
|
return result;
|
|
21
24
|
}
|
|
22
25
|
prepare(inputs) {
|
|
@@ -46,10 +49,14 @@ export class ChunkIndexer {
|
|
|
46
49
|
}
|
|
47
50
|
return prepared;
|
|
48
51
|
}
|
|
49
|
-
async insertPrepared(prepared, skipped, signal) {
|
|
52
|
+
async insertPrepared(prepared, skipped, signal, replaceSource) {
|
|
50
53
|
const startedAt = Date.now();
|
|
51
|
-
if (prepared.length === 0)
|
|
54
|
+
if (prepared.length === 0) {
|
|
55
|
+
if (replaceSource) {
|
|
56
|
+
this.store.deleteBySourceExceptMappings(replaceSource.corpus, replaceSource.sourceId, []);
|
|
57
|
+
}
|
|
52
58
|
return { ids: [], embedded: 0, reused: 0, skipped };
|
|
59
|
+
}
|
|
53
60
|
const present = this.store.whichHashesPresent(prepared.map((item) => item.contentHash));
|
|
54
61
|
for (const item of prepared)
|
|
55
62
|
item.existingId = present.get(item.contentHash) ?? null;
|
|
@@ -92,10 +99,13 @@ export class ChunkIndexer {
|
|
|
92
99
|
const toInsert = [];
|
|
93
100
|
const insertPositions = [];
|
|
94
101
|
const existingMappings = [];
|
|
102
|
+
const existingMappingIds = new Map();
|
|
103
|
+
const insertKnownMissing = new Set();
|
|
95
104
|
for (let resultIndex = 0; resultIndex < prepared.length; resultIndex++) {
|
|
96
105
|
const item = prepared[resultIndex];
|
|
97
106
|
if (item.existingId !== null) {
|
|
98
107
|
ids[resultIndex] = item.existingId;
|
|
108
|
+
existingMappingIds.set(item.contentHash, item.existingId);
|
|
99
109
|
existingMappings.push({
|
|
100
110
|
corpus: item.input.corpus,
|
|
101
111
|
sourceId: item.sourceId,
|
|
@@ -113,6 +123,7 @@ export class ChunkIndexer {
|
|
|
113
123
|
if (!embedding)
|
|
114
124
|
throw new Error("Missing embedding for memory chunk");
|
|
115
125
|
insertPositions.push(resultIndex);
|
|
126
|
+
insertKnownMissing.add(item.contentHash);
|
|
116
127
|
toInsert.push({
|
|
117
128
|
corpus: item.input.corpus,
|
|
118
129
|
sourceId: item.sourceId,
|
|
@@ -125,8 +136,18 @@ export class ChunkIndexer {
|
|
|
125
136
|
embedding,
|
|
126
137
|
});
|
|
127
138
|
}
|
|
128
|
-
|
|
129
|
-
const
|
|
139
|
+
let insertedIds = [];
|
|
140
|
+
const writeChunks = () => {
|
|
141
|
+
if (replaceSource) {
|
|
142
|
+
this.store.deleteBySourceExceptMappings(replaceSource.corpus, replaceSource.sourceId, replaceSource.keepMappings);
|
|
143
|
+
}
|
|
144
|
+
this.store.recordSourceMappings(existingMappings, existingMappingIds);
|
|
145
|
+
insertedIds = this.store.insertChunks(toInsert, undefined, insertKnownMissing);
|
|
146
|
+
};
|
|
147
|
+
if (replaceSource && !this.store.db.inTransaction)
|
|
148
|
+
this.store.db.transaction(writeChunks).immediate();
|
|
149
|
+
else
|
|
150
|
+
writeChunks();
|
|
130
151
|
for (let index = 0; index < insertPositions.length; index++) {
|
|
131
152
|
ids[insertPositions[index]] = insertedIds[index];
|
|
132
153
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { positiveIntegerOrDefault } from "../util.js";
|
|
1
2
|
const DEFAULT_LIMIT = 8;
|
|
2
3
|
const DEFAULT_CANDIDATE_MULTIPLIER = 4;
|
|
3
4
|
const RRF_K = 60;
|
|
@@ -241,6 +242,3 @@ function chunkSources(chunk) {
|
|
|
241
242
|
function uniqueStrings(values) {
|
|
242
243
|
return Array.from(new Set(values?.filter((value) => value.trim()) ?? []));
|
|
243
244
|
}
|
|
244
|
-
function positiveIntegerOrDefault(value, fallback) {
|
|
245
|
-
return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
|
|
246
|
-
}
|
|
@@ -55,25 +55,31 @@ export class MemoryIndexStore {
|
|
|
55
55
|
insertChunk(input) {
|
|
56
56
|
return this.insertChunks([input])[0];
|
|
57
57
|
}
|
|
58
|
-
insertChunks(inputs) {
|
|
58
|
+
insertChunks(inputs, preloadedIds, knownMissingHashes) {
|
|
59
59
|
if (inputs.length === 0)
|
|
60
60
|
return [];
|
|
61
61
|
const rows = inputs.map((input) => this.normalizeInput(input));
|
|
62
|
+
const knownIds = new Map(preloadedIds);
|
|
62
63
|
const out = [];
|
|
63
64
|
const insert = this.db.transaction((items) => {
|
|
64
65
|
for (const item of items)
|
|
65
|
-
out.push(this.insertNormalized(item));
|
|
66
|
+
out.push(this.insertNormalized(item, knownIds, knownMissingHashes));
|
|
66
67
|
});
|
|
67
68
|
insert.immediate(rows);
|
|
68
69
|
return out;
|
|
69
70
|
}
|
|
70
|
-
recordSourceMappings(inputs) {
|
|
71
|
+
recordSourceMappings(inputs, preloadedIds) {
|
|
71
72
|
if (inputs.length === 0)
|
|
72
73
|
return;
|
|
73
74
|
const rows = inputs.map((input) => this.normalizeInput(input));
|
|
74
75
|
this.db
|
|
75
76
|
.transaction((items) => {
|
|
76
77
|
for (const item of items) {
|
|
78
|
+
const preloadedId = preloadedIds?.get(item.contentHash);
|
|
79
|
+
if (preloadedId !== undefined) {
|
|
80
|
+
this.insertSourceMapping(preloadedId, item);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
77
83
|
const existing = this.db
|
|
78
84
|
.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?")
|
|
79
85
|
.get(item.contentHash);
|
|
@@ -88,8 +94,9 @@ export class MemoryIndexStore {
|
|
|
88
94
|
const out = [];
|
|
89
95
|
const replace = this.db.transaction(() => {
|
|
90
96
|
this.deleteBySourceInternal(corpus, sourceId);
|
|
97
|
+
const knownIds = new Map();
|
|
91
98
|
for (const item of rows)
|
|
92
|
-
out.push(this.insertNormalized(item));
|
|
99
|
+
out.push(this.insertNormalized(item, knownIds));
|
|
93
100
|
});
|
|
94
101
|
replace.immediate();
|
|
95
102
|
return out;
|
|
@@ -168,19 +175,19 @@ export class MemoryIndexStore {
|
|
|
168
175
|
return rows.map((row) => ({ id: row.id, score: row.score, chunk: rowToChunk(row) }));
|
|
169
176
|
}
|
|
170
177
|
searchSemanticLinear(query, normalized) {
|
|
171
|
-
const
|
|
172
|
-
.prepare(normalized.corpus
|
|
178
|
+
const stmt = this.db.prepare(normalized.corpus
|
|
173
179
|
? `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c WHERE c.corpus = ?`
|
|
174
|
-
: `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c`)
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
180
|
+
: `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c`);
|
|
181
|
+
const best = [];
|
|
182
|
+
for (const row of stmt.iterate(...(normalized.corpus ? [normalized.corpus] : []))) {
|
|
183
|
+
const hit = {
|
|
184
|
+
id: row.id,
|
|
185
|
+
score: cosineDistance(query, decodeVector(row.embedding, row.embedding_dimensions)),
|
|
186
|
+
chunk: rowToChunk(row),
|
|
187
|
+
};
|
|
188
|
+
insertBoundedHit(best, hit, normalized.limit);
|
|
189
|
+
}
|
|
190
|
+
return best;
|
|
184
191
|
}
|
|
185
192
|
deleteChunk(id) {
|
|
186
193
|
const remove = this.db.transaction(() => {
|
|
@@ -243,6 +250,7 @@ export class MemoryIndexStore {
|
|
|
243
250
|
this.deleteBySource(corpus, sourceId);
|
|
244
251
|
return;
|
|
245
252
|
}
|
|
253
|
+
const keptHashes = new Set(kept.map((item) => item.contentHash));
|
|
246
254
|
this.db
|
|
247
255
|
.transaction(() => {
|
|
248
256
|
const rows = this.db
|
|
@@ -259,7 +267,8 @@ export class MemoryIndexStore {
|
|
|
259
267
|
this.db
|
|
260
268
|
.prepare("DELETE FROM memory_index_sources WHERE corpus = ? AND source_id = ? AND chunk_index = ?")
|
|
261
269
|
.run(corpus, sourceId, row.chunk_index);
|
|
262
|
-
|
|
270
|
+
if (!keptHashes.has(row.content_hash))
|
|
271
|
+
this.deleteOrphanChunk(row.id);
|
|
263
272
|
}
|
|
264
273
|
})
|
|
265
274
|
.immediate();
|
|
@@ -362,9 +371,15 @@ export class MemoryIndexStore {
|
|
|
362
371
|
}),
|
|
363
372
|
};
|
|
364
373
|
}
|
|
365
|
-
insertNormalized(item) {
|
|
366
|
-
const
|
|
374
|
+
insertNormalized(item, knownIds, knownMissingHashes) {
|
|
375
|
+
const knownId = knownIds.get(item.contentHash);
|
|
376
|
+
const existing = knownId !== undefined
|
|
377
|
+
? { id: knownId }
|
|
378
|
+
: knownMissingHashes?.has(item.contentHash)
|
|
379
|
+
? undefined
|
|
380
|
+
: this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?").get(item.contentHash);
|
|
367
381
|
if (existing) {
|
|
382
|
+
knownIds.set(item.contentHash, existing.id);
|
|
368
383
|
this.insertSourceMapping(existing.id, item);
|
|
369
384
|
return existing.id;
|
|
370
385
|
}
|
|
@@ -383,6 +398,7 @@ export class MemoryIndexStore {
|
|
|
383
398
|
.prepare("INSERT INTO memory_vec(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)")
|
|
384
399
|
.run(id, encodeVector(item.embedding));
|
|
385
400
|
}
|
|
401
|
+
knownIds.set(item.contentHash, id);
|
|
386
402
|
this.insertSourceMapping(id, item);
|
|
387
403
|
return id;
|
|
388
404
|
}
|
|
@@ -434,6 +450,18 @@ function normalizeSearchOptions(options) {
|
|
|
434
450
|
corpus: options.corpus,
|
|
435
451
|
};
|
|
436
452
|
}
|
|
453
|
+
function insertBoundedHit(best, hit, limit) {
|
|
454
|
+
if (limit <= 0)
|
|
455
|
+
return;
|
|
456
|
+
let index = best.findIndex((candidate) => hit.score < candidate.score);
|
|
457
|
+
if (index < 0)
|
|
458
|
+
index = best.length;
|
|
459
|
+
if (index >= limit)
|
|
460
|
+
return;
|
|
461
|
+
best.splice(index, 0, hit);
|
|
462
|
+
if (best.length > limit)
|
|
463
|
+
best.length = limit;
|
|
464
|
+
}
|
|
437
465
|
function sourcesJsonSelect(chunkIdExpr) {
|
|
438
466
|
return `(SELECT json_group_array(json_object(
|
|
439
467
|
'corpus', s.corpus,
|
|
@@ -2,7 +2,6 @@ import { readdir, readFile } from "node:fs/promises";
|
|
|
2
2
|
import { relative, resolve } from "node:path";
|
|
3
3
|
import { indexLcmRecords } from "./indexer.js";
|
|
4
4
|
import { normalizeChatRecords } from "./normalize.js";
|
|
5
|
-
import { computeLcmRecordKey } from "./store.js";
|
|
6
5
|
const DEFAULT_YIELD_EVERY_N = 1024;
|
|
7
6
|
const INDEX_BATCH_SIZE = 32;
|
|
8
7
|
export async function backfillFromChatLogs(deps, options) {
|
|
@@ -66,15 +65,12 @@ export async function backfillFromChatLogs(deps, options) {
|
|
|
66
65
|
}
|
|
67
66
|
const inserted = [];
|
|
68
67
|
for (const record of batch.records) {
|
|
69
|
-
|
|
68
|
+
const result = deps.lcmStore.insertRecordReturningStored(record);
|
|
69
|
+
if (!result.inserted) {
|
|
70
70
|
report.recordsSkippedDuplicate += 1;
|
|
71
71
|
continue;
|
|
72
72
|
}
|
|
73
|
-
|
|
74
|
-
const stored = deps.lcmStore.getRecord(id);
|
|
75
|
-
if (!stored)
|
|
76
|
-
throw new Error(`Failed to read backfilled LCM record: ${id}`);
|
|
77
|
-
inserted.push(stored);
|
|
73
|
+
inserted.push(result.record);
|
|
78
74
|
report.recordsInserted += 1;
|
|
79
75
|
if (inserted.length >= INDEX_BATCH_SIZE) {
|
|
80
76
|
report.indexedChunks += (await indexLcmRecords({ indexer: deps.indexer, records: inserted, signal: options.signal })).ids.length;
|
|
@@ -225,17 +221,24 @@ function countMissingSegments(lcmStore, segmentIds) {
|
|
|
225
221
|
return missing;
|
|
226
222
|
}
|
|
227
223
|
function countExistingRecords(lcmStore, records) {
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
224
|
+
if (records.length === 0)
|
|
225
|
+
return 0;
|
|
226
|
+
const keys = records.map((record) => lcmStore.computeRecordKey(record));
|
|
227
|
+
const existingKeys = new Set();
|
|
228
|
+
for (const chunk of chunks([...new Set(keys)], 256)) {
|
|
229
|
+
const rows = lcmStore.db
|
|
230
|
+
.prepare(`SELECT record_key FROM lcm_records WHERE record_key IN (${chunk.map(() => "?").join(",")})`)
|
|
231
|
+
.all(...chunk);
|
|
232
|
+
for (const row of rows)
|
|
233
|
+
existingKeys.add(row.record_key);
|
|
232
234
|
}
|
|
233
|
-
return
|
|
235
|
+
return keys.reduce((total, key) => total + (existingKeys.has(key) ? 1 : 0), 0);
|
|
234
236
|
}
|
|
235
|
-
function
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
.
|
|
237
|
+
function chunks(items, size) {
|
|
238
|
+
const out = [];
|
|
239
|
+
for (let index = 0; index < items.length; index += size)
|
|
240
|
+
out.push(items.slice(index, index + size));
|
|
241
|
+
return out;
|
|
239
242
|
}
|
|
240
243
|
function errorCode(error) {
|
|
241
244
|
return error && typeof error === "object" && "code" in error
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { condense } from "./condense.js";
|
|
2
|
-
import { createRawContextItems, estimateAgentMessageTokens,
|
|
2
|
+
import { createRawContextItems, estimateAgentMessageTokens, renderLcmRecordPartsForSummary, resolveFreshTailStartIndex, selectLcmCompactionCandidatePromptAware, } from "./context.js";
|
|
3
3
|
import { indexLcmSummaries } from "./indexer.js";
|
|
4
4
|
import { createSyntheticLcmSummaryMessage } from "./summarizer.js";
|
|
5
5
|
const LCM_SUMMARY_OPEN_TAG = "<from_earlier>";
|
|
@@ -48,6 +48,7 @@ export class LcmContextTransformer {
|
|
|
48
48
|
signal,
|
|
49
49
|
model: options.model,
|
|
50
50
|
promptText,
|
|
51
|
+
initialPressure: pressure,
|
|
51
52
|
});
|
|
52
53
|
}
|
|
53
54
|
}
|
|
@@ -76,7 +77,9 @@ export class LcmContextTransformer {
|
|
|
76
77
|
}
|
|
77
78
|
async serviceCompactionDebtForState(input) {
|
|
78
79
|
for (let round = 0; input.state.compactionDebt > 0 && round < this.settings.maxRounds; round += 1) {
|
|
79
|
-
const pressure =
|
|
80
|
+
const pressure = round === 0 && input.initialPressure
|
|
81
|
+
? input.initialPressure
|
|
82
|
+
: this.evaluateCompactionPressure(input.state, input.model, input.promptText ?? "");
|
|
80
83
|
if (!pressure.candidate.shouldCompact) {
|
|
81
84
|
if (pressure.thresholdOverflowTokens > 0) {
|
|
82
85
|
const condensed = await this.condenseRuntimeSummaries({
|
|
@@ -149,19 +152,15 @@ export class LcmContextTransformer {
|
|
|
149
152
|
mode: candidate.reasons.includes("context_threshold") ? "aggressive" : "normal",
|
|
150
153
|
previousSummary,
|
|
151
154
|
}, input.signal);
|
|
152
|
-
const summaryId = `${input.sessionKey}:summary-${++state.summaryCounter}`;
|
|
153
155
|
const message = createSyntheticLcmSummaryMessage(renderLcmSummaryMessage(summaryText), this.now());
|
|
154
156
|
const summaryItem = {
|
|
155
157
|
type: "summary",
|
|
156
|
-
id:
|
|
158
|
+
id: "",
|
|
157
159
|
sourceIds: chunkItems.map((item) => item.id),
|
|
158
160
|
depth: 1,
|
|
159
161
|
message,
|
|
160
162
|
tokens: estimateAgentMessageTokens(message),
|
|
161
163
|
};
|
|
162
|
-
state.items.splice(startIndex, removeCount, summaryItem);
|
|
163
|
-
compacted = true;
|
|
164
|
-
tokensSaved = Math.max(0, candidate.chunkTokens - summaryItem.tokens);
|
|
165
164
|
const persisted = await this.persistRuntimeSummary({
|
|
166
165
|
text: summaryText,
|
|
167
166
|
sourceItems: chunkItems,
|
|
@@ -169,8 +168,12 @@ export class LcmContextTransformer {
|
|
|
169
168
|
sessionId: input.sessionId,
|
|
170
169
|
signal: input.signal,
|
|
171
170
|
});
|
|
171
|
+
summaryItem.id = `${input.sessionKey}:summary-${++state.summaryCounter}`;
|
|
172
172
|
if (persisted?.summaryId !== undefined)
|
|
173
173
|
summaryItem.persistedSummaryId = persisted.summaryId;
|
|
174
|
+
state.items.splice(startIndex, removeCount, summaryItem);
|
|
175
|
+
compacted = true;
|
|
176
|
+
tokensSaved = Math.max(0, candidate.chunkTokens - summaryItem.tokens);
|
|
174
177
|
await this.condenseRuntimeSummaries({ state, sessionKey: input.sessionKey, signal: input.signal });
|
|
175
178
|
};
|
|
176
179
|
input.state.compactionQueue = input.state.compactionQueue.then(run, run);
|
|
@@ -262,8 +265,9 @@ export class LcmContextTransformer {
|
|
|
262
265
|
this.lcmStore.db
|
|
263
266
|
.transaction(() => {
|
|
264
267
|
for (const insert of inserts) {
|
|
265
|
-
|
|
266
|
-
insert.item.
|
|
268
|
+
const result = this.lcmStore.insertRecordReturningStored(insert.input);
|
|
269
|
+
insert.item.recordId = result.record.id;
|
|
270
|
+
insert.item.record = result.record;
|
|
267
271
|
}
|
|
268
272
|
})
|
|
269
273
|
.immediate();
|
|
@@ -275,20 +279,7 @@ export class LcmContextTransformer {
|
|
|
275
279
|
const items = [];
|
|
276
280
|
for (const row of rows) {
|
|
277
281
|
if (row.type === "raw") {
|
|
278
|
-
|
|
279
|
-
if (!record) {
|
|
280
|
-
console.error(`memory LCM context item dropped because record ${row.recordId} is missing`);
|
|
281
|
-
continue;
|
|
282
|
-
}
|
|
283
|
-
const message = lcmRecordToAgentMessage(record);
|
|
284
|
-
items.push({
|
|
285
|
-
type: "raw",
|
|
286
|
-
id: row.fingerprint,
|
|
287
|
-
recordId: record.id,
|
|
288
|
-
record,
|
|
289
|
-
message,
|
|
290
|
-
tokens: estimateAgentMessageTokens(message),
|
|
291
|
-
});
|
|
282
|
+
// Raw history is owned by transcripts; legacy raw context rows must not replay it.
|
|
292
283
|
continue;
|
|
293
284
|
}
|
|
294
285
|
const summary = this.lcmStore.getSummary(row.summaryId);
|
|
@@ -359,8 +350,6 @@ function syncContextState(state, messages) {
|
|
|
359
350
|
const replacement = rawById.get(item.id);
|
|
360
351
|
if (replacement && !covered.has(item.id))
|
|
361
352
|
next.push(replacement);
|
|
362
|
-
else if (item.recordId !== null && !covered.has(item.id))
|
|
363
|
-
next.push(item);
|
|
364
353
|
}
|
|
365
354
|
for (const item of rawItems) {
|
|
366
355
|
if (!covered.has(item.id) && !next.some((existing) => existing.type === "raw" && existing.id === item.id)) {
|
|
@@ -390,8 +379,6 @@ function contextItemsForStorage(items) {
|
|
|
390
379
|
const timestamp = item.message.timestamp;
|
|
391
380
|
const happenedAt = typeof timestamp === "number" && Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
|
|
392
381
|
if (item.type === "raw") {
|
|
393
|
-
if (item.recordId !== null)
|
|
394
|
-
stored.push({ type: "raw", recordId: item.recordId, fingerprint: item.id, happenedAt });
|
|
395
382
|
continue;
|
|
396
383
|
}
|
|
397
384
|
if (item.persistedSummaryId !== undefined) {
|
|
@@ -522,9 +509,10 @@ function assembleWithinBudget(state, settings, model) {
|
|
|
522
509
|
const freshTail = state.items.slice(resolveFreshTailStartIndexForState(state.items, settings));
|
|
523
510
|
const selected = new Set(freshTail);
|
|
524
511
|
let tokens = sumItemTokens(freshTail);
|
|
512
|
+
const originalIndexes = new Map(state.items.map((item, index) => [item, index]));
|
|
525
513
|
const summaries = state.items
|
|
526
514
|
.filter((item) => item.type === "summary" && !selected.has(item))
|
|
527
|
-
.sort((a, b) => b.depth - a.depth ||
|
|
515
|
+
.sort((a, b) => b.depth - a.depth || (originalIndexes.get(b) ?? 0) - (originalIndexes.get(a) ?? 0));
|
|
528
516
|
for (const item of summaries) {
|
|
529
517
|
if (tokens + item.tokens > budget && selected.size > 0)
|
|
530
518
|
continue;
|
|
@@ -587,7 +575,7 @@ function lcmRecordPartsFromAgentMessage(message) {
|
|
|
587
575
|
kind: "tool_result",
|
|
588
576
|
toolCallId: toolResult.toolCallId ?? "",
|
|
589
577
|
toolName: toolResult.toolName ?? "tool",
|
|
590
|
-
output:
|
|
578
|
+
output: textFromContent(toolResult.content),
|
|
591
579
|
...(toolResult.isError ? { isError: true } : {}),
|
|
592
580
|
},
|
|
593
581
|
];
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { scoreEvictable, tokenBag } from "./eviction-score.js";
|
|
2
|
+
import { buildEvictionScoreContext, scoreEvictable, tokenBag } from "./eviction-score.js";
|
|
3
3
|
const MESSAGE_OVERHEAD_TOKENS = 6;
|
|
4
4
|
const RECORD_OVERHEAD_TOKENS = 4;
|
|
5
5
|
const IMAGE_TOKEN_ESTIMATE = 1200;
|
|
@@ -167,9 +167,12 @@ function selectLeafChunk(items, leafChunkTokens, promptText, config) {
|
|
|
167
167
|
const records = items.map((item) => item.record).filter((record) => !!record);
|
|
168
168
|
if (records.length === 0)
|
|
169
169
|
return selectOldestLeafChunk(items, leafChunkTokens);
|
|
170
|
+
const scoreContext = buildEvictionScoreContext(promptText, records);
|
|
171
|
+
if (!scoreContext)
|
|
172
|
+
return selectOldestLeafChunk(items, leafChunkTokens);
|
|
170
173
|
const scored = targetRanges.map((range) => ({
|
|
171
174
|
...range,
|
|
172
|
-
score: range.items.reduce((total, item) => total + (item.record ? scoreEvictable(item.record, promptText, records) : 0), 0),
|
|
175
|
+
score: range.items.reduce((total, item) => total + (item.record ? scoreEvictable(item.record, promptText, records, scoreContext) : 0), 0),
|
|
173
176
|
}));
|
|
174
177
|
scored.sort((a, b) => a.score - b.score || a.startIndex - b.startIndex);
|
|
175
178
|
return scored[0]?.items ?? [];
|
|
@@ -179,7 +182,7 @@ function createValidLeafRanges(items, leafChunkTokens) {
|
|
|
179
182
|
for (let startIndex = 0; startIndex < items.length; startIndex += 1) {
|
|
180
183
|
if (isToolResultContinuingPreviousToolCall(items, startIndex))
|
|
181
184
|
continue;
|
|
182
|
-
const chunk =
|
|
185
|
+
const chunk = selectOldestLeafChunkFromIndex(items, startIndex, leafChunkTokens);
|
|
183
186
|
if (chunk.length === 0)
|
|
184
187
|
continue;
|
|
185
188
|
const chunkTokens = chunk.reduce((total, item) => total + item.tokens, 0);
|
|
@@ -194,9 +197,12 @@ function createValidLeafRanges(items, leafChunkTokens) {
|
|
|
194
197
|
return ranges;
|
|
195
198
|
}
|
|
196
199
|
function selectOldestLeafChunk(items, leafChunkTokens) {
|
|
200
|
+
return selectOldestLeafChunkFromIndex(items, 0, leafChunkTokens);
|
|
201
|
+
}
|
|
202
|
+
function selectOldestLeafChunkFromIndex(items, startIndex, leafChunkTokens) {
|
|
197
203
|
const chunk = [];
|
|
198
204
|
let tokens = 0;
|
|
199
|
-
for (let index =
|
|
205
|
+
for (let index = startIndex; index < items.length; index += 1) {
|
|
200
206
|
const item = items[index];
|
|
201
207
|
if (!item)
|
|
202
208
|
continue;
|
|
@@ -4,34 +4,46 @@ export function tokenBag(text) {
|
|
|
4
4
|
.split(/[^a-z0-9]+/)
|
|
5
5
|
.filter((token) => token.length >= 2);
|
|
6
6
|
}
|
|
7
|
-
export function
|
|
7
|
+
export function buildEvictionScoreContext(prompt, allRecords) {
|
|
8
8
|
const promptTerms = tokenBag(prompt);
|
|
9
9
|
if (promptTerms.length === 0)
|
|
10
|
-
return
|
|
11
|
-
const recordTerms = tokenBag(record.text);
|
|
12
|
-
if (recordTerms.length === 0)
|
|
13
|
-
return 0;
|
|
14
|
-
const recordFreq = new Map();
|
|
15
|
-
for (const term of recordTerms)
|
|
16
|
-
recordFreq.set(term, (recordFreq.get(term) ?? 0) + 1);
|
|
10
|
+
return null;
|
|
17
11
|
const promptUniqueTerms = new Set(promptTerms);
|
|
18
12
|
const documentFrequencies = new Map();
|
|
13
|
+
const recordTerms = new Map();
|
|
19
14
|
for (const candidate of allRecords) {
|
|
20
|
-
const
|
|
15
|
+
const terms = tokenBag(candidate.text);
|
|
16
|
+
recordTerms.set(candidate, terms);
|
|
17
|
+
const candidateTerms = new Set(terms);
|
|
21
18
|
for (const term of promptUniqueTerms) {
|
|
22
19
|
if (candidateTerms.has(term))
|
|
23
20
|
documentFrequencies.set(term, (documentFrequencies.get(term) ?? 0) + 1);
|
|
24
21
|
}
|
|
25
22
|
}
|
|
26
|
-
|
|
23
|
+
return {
|
|
24
|
+
promptUniqueTerms,
|
|
25
|
+
documentFrequencies,
|
|
26
|
+
recordTerms,
|
|
27
|
+
candidateCount: Math.max(0, allRecords.length),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export function scoreEvictable(record, prompt, allRecords, context = buildEvictionScoreContext(prompt, allRecords)) {
|
|
31
|
+
if (!context)
|
|
32
|
+
return 0;
|
|
33
|
+
const recordTerms = context.recordTerms.get(record) ?? tokenBag(record.text);
|
|
34
|
+
if (recordTerms.length === 0)
|
|
35
|
+
return 0;
|
|
36
|
+
const recordFreq = new Map();
|
|
37
|
+
for (const term of recordTerms)
|
|
38
|
+
recordFreq.set(term, (recordFreq.get(term) ?? 0) + 1);
|
|
27
39
|
let score = 0;
|
|
28
|
-
for (const term of promptUniqueTerms) {
|
|
40
|
+
for (const term of context.promptUniqueTerms) {
|
|
29
41
|
const tf = recordFreq.get(term) ?? 0;
|
|
30
42
|
if (tf <= 0)
|
|
31
43
|
continue;
|
|
32
44
|
const normalizedTf = tf / recordTerms.length;
|
|
33
|
-
const df = documentFrequencies.get(term) ?? 0;
|
|
34
|
-
const idf = Math.log((candidateCount + 1) / (df + 1) + 1);
|
|
45
|
+
const df = context.documentFrequencies.get(term) ?? 0;
|
|
46
|
+
const idf = Math.log((context.candidateCount + 1) / (df + 1) + 1);
|
|
35
47
|
score += normalizedTf * idf;
|
|
36
48
|
}
|
|
37
49
|
return score;
|
|
@@ -8,11 +8,7 @@ export async function projectNormalizedLcmBatch(options) {
|
|
|
8
8
|
}
|
|
9
9
|
const storedRecords = [];
|
|
10
10
|
for (const record of options.batch.records) {
|
|
11
|
-
|
|
12
|
-
const stored = options.lcmStore.getRecord(id);
|
|
13
|
-
if (!stored)
|
|
14
|
-
throw new Error(`Failed to read projected LCM record: ${id}`);
|
|
15
|
-
storedRecords.push(stored);
|
|
11
|
+
storedRecords.push(options.lcmStore.insertRecordReturningStored(record).record);
|
|
16
12
|
}
|
|
17
13
|
return {
|
|
18
14
|
segmentIds,
|
|
@@ -59,7 +59,7 @@ export function normalizeChatRecords(chatRecords, options) {
|
|
|
59
59
|
kind: "tool_result",
|
|
60
60
|
toolCallId: record.event.toolCallId,
|
|
61
61
|
toolName: record.event.toolName,
|
|
62
|
-
output: record.event.result,
|
|
62
|
+
output: toolResultVisibleOutput(record.event.result),
|
|
63
63
|
...(record.event.isError ? { isError: true } : {}),
|
|
64
64
|
},
|
|
65
65
|
];
|
|
@@ -191,6 +191,27 @@ function briefJson(value, maxLength = 160) {
|
|
|
191
191
|
const compact = text.replace(/\s+/g, " ").trim();
|
|
192
192
|
return compact.length > maxLength ? `${compact.slice(0, maxLength - 1)}...` : compact;
|
|
193
193
|
}
|
|
194
|
+
function toolResultVisibleOutput(result) {
|
|
195
|
+
if (!result || typeof result !== "object" || !("content" in result))
|
|
196
|
+
return result;
|
|
197
|
+
const content = result.content;
|
|
198
|
+
if (!Array.isArray(content))
|
|
199
|
+
return result;
|
|
200
|
+
return textFromToolResultContent(content);
|
|
201
|
+
}
|
|
202
|
+
function textFromToolResultContent(content) {
|
|
203
|
+
if (!Array.isArray(content))
|
|
204
|
+
return content;
|
|
205
|
+
return content
|
|
206
|
+
.map((item) => {
|
|
207
|
+
if (item && typeof item === "object" && item.type === "text") {
|
|
208
|
+
return item.text;
|
|
209
|
+
}
|
|
210
|
+
return "";
|
|
211
|
+
})
|
|
212
|
+
.filter((item) => typeof item === "string" && item.length > 0)
|
|
213
|
+
.join("\n");
|
|
214
|
+
}
|
|
194
215
|
function chatRecordSource(record, sourcePath) {
|
|
195
216
|
const messageId = "messageId" in record && typeof record.messageId === "string"
|
|
196
217
|
? record.messageId
|