@qearlyao/familiar 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +6 -14
  2. package/config.example.toml +1 -1
  3. package/dist/added-models.js +6 -15
  4. package/dist/agent-events.js +1 -3
  5. package/dist/agent.js +3 -4
  6. package/dist/browser-tools.js +15 -11
  7. package/dist/chat-log.js +3 -2
  8. package/dist/cli.js +2 -2
  9. package/dist/config-overrides.js +5 -14
  10. package/dist/config-registry.js +1 -4
  11. package/dist/config.js +45 -113
  12. package/dist/contact-note.js +2 -12
  13. package/dist/data-retention.js +1 -3
  14. package/dist/discord.js +72 -19
  15. package/dist/generated-media.js +3 -2
  16. package/dist/hot-reload.js +1 -3
  17. package/dist/image-gen.js +12 -51
  18. package/dist/inbound-attachments.js +64 -23
  19. package/dist/memory/diary/ambient-injector.js +1 -3
  20. package/dist/memory/diary/ambient.js +1 -3
  21. package/dist/memory/diary/chunks.js +1 -3
  22. package/dist/memory/diary/indexer.js +1 -3
  23. package/dist/memory/doctor.js +3 -8
  24. package/dist/memory/index/chunk-indexer.js +27 -6
  25. package/dist/memory/index/retrieval.js +1 -3
  26. package/dist/memory/index/store.js +47 -19
  27. package/dist/memory/lcm/backfill.js +19 -16
  28. package/dist/memory/lcm/context-transformer.js +17 -29
  29. package/dist/memory/lcm/context.js +10 -4
  30. package/dist/memory/lcm/eviction-score.js +25 -13
  31. package/dist/memory/lcm/indexer.js +1 -5
  32. package/dist/memory/lcm/normalize.js +22 -1
  33. package/dist/memory/lcm/store.js +27 -24
  34. package/dist/memory/operator.js +3 -31
  35. package/dist/memory/service.js +1 -3
  36. package/dist/memory/tools.js +0 -4
  37. package/dist/memory/util.js +6 -0
  38. package/dist/models.js +3 -0
  39. package/dist/persona.js +3 -15
  40. package/dist/runtime.js +12 -23
  41. package/dist/scheduler.js +15 -49
  42. package/dist/service.js +39 -27
  43. package/dist/settings.js +7 -32
  44. package/dist/silent-marker.js +64 -0
  45. package/dist/tts.js +0 -6
  46. package/dist/util/fs.js +41 -0
  47. package/dist/util/guards.js +8 -0
  48. package/dist/util/image-mime.js +31 -0
  49. package/dist/util/time.js +29 -0
  50. package/dist/web-auth.js +4 -1
  51. package/dist/web-static.js +36 -1
  52. package/dist/web-tools.js +8 -5
  53. package/dist/web.js +253 -69
  54. package/npm-shrinkwrap.json +5139 -0
  55. package/package.json +5 -4
  56. package/web/dist/assets/index-B23WT77N.js +63 -0
  57. package/web/dist/assets/index-D3MotFzN.css +2 -0
  58. package/web/dist/index.html +2 -2
  59. package/web/dist/assets/index-BPZQbZh5.js +0 -61
  60. package/web/dist/assets/index-CcQ13VAY.css +0 -2
@@ -15,8 +15,11 @@ export class ChunkIndexer {
15
15
  async replaceSource(corpus, sourceId, inputs, signal) {
16
16
  const prepared = this.prepare(inputs.map((input) => ({ ...input, corpus, sourceId })));
17
17
  const keepMappings = prepared.map((item) => ({ contentHash: item.contentHash, chunkIndex: item.chunkIndex }));
18
- this.store.deleteBySourceExceptMappings(corpus, sourceId, keepMappings);
19
- const result = await this.insertPrepared(prepared, inputs.length - prepared.length, signal);
18
+ const result = await this.insertPrepared(prepared, inputs.length - prepared.length, signal, {
19
+ corpus,
20
+ sourceId,
21
+ keepMappings,
22
+ });
20
23
  return result;
21
24
  }
22
25
  prepare(inputs) {
@@ -46,10 +49,14 @@ export class ChunkIndexer {
46
49
  }
47
50
  return prepared;
48
51
  }
49
- async insertPrepared(prepared, skipped, signal) {
52
+ async insertPrepared(prepared, skipped, signal, replaceSource) {
50
53
  const startedAt = Date.now();
51
- if (prepared.length === 0)
54
+ if (prepared.length === 0) {
55
+ if (replaceSource) {
56
+ this.store.deleteBySourceExceptMappings(replaceSource.corpus, replaceSource.sourceId, []);
57
+ }
52
58
  return { ids: [], embedded: 0, reused: 0, skipped };
59
+ }
53
60
  const present = this.store.whichHashesPresent(prepared.map((item) => item.contentHash));
54
61
  for (const item of prepared)
55
62
  item.existingId = present.get(item.contentHash) ?? null;
@@ -92,10 +99,13 @@ export class ChunkIndexer {
92
99
  const toInsert = [];
93
100
  const insertPositions = [];
94
101
  const existingMappings = [];
102
+ const existingMappingIds = new Map();
103
+ const insertKnownMissing = new Set();
95
104
  for (let resultIndex = 0; resultIndex < prepared.length; resultIndex++) {
96
105
  const item = prepared[resultIndex];
97
106
  if (item.existingId !== null) {
98
107
  ids[resultIndex] = item.existingId;
108
+ existingMappingIds.set(item.contentHash, item.existingId);
99
109
  existingMappings.push({
100
110
  corpus: item.input.corpus,
101
111
  sourceId: item.sourceId,
@@ -113,6 +123,7 @@ export class ChunkIndexer {
113
123
  if (!embedding)
114
124
  throw new Error("Missing embedding for memory chunk");
115
125
  insertPositions.push(resultIndex);
126
+ insertKnownMissing.add(item.contentHash);
116
127
  toInsert.push({
117
128
  corpus: item.input.corpus,
118
129
  sourceId: item.sourceId,
@@ -125,8 +136,18 @@ export class ChunkIndexer {
125
136
  embedding,
126
137
  });
127
138
  }
128
- this.store.recordSourceMappings(existingMappings);
129
- const insertedIds = this.store.insertChunks(toInsert);
139
+ let insertedIds = [];
140
+ const writeChunks = () => {
141
+ if (replaceSource) {
142
+ this.store.deleteBySourceExceptMappings(replaceSource.corpus, replaceSource.sourceId, replaceSource.keepMappings);
143
+ }
144
+ this.store.recordSourceMappings(existingMappings, existingMappingIds);
145
+ insertedIds = this.store.insertChunks(toInsert, undefined, insertKnownMissing);
146
+ };
147
+ if (replaceSource && !this.store.db.inTransaction)
148
+ this.store.db.transaction(writeChunks).immediate();
149
+ else
150
+ writeChunks();
130
151
  for (let index = 0; index < insertPositions.length; index++) {
131
152
  ids[insertPositions[index]] = insertedIds[index];
132
153
  }
@@ -1,3 +1,4 @@
1
+ import { positiveIntegerOrDefault } from "../util.js";
1
2
  const DEFAULT_LIMIT = 8;
2
3
  const DEFAULT_CANDIDATE_MULTIPLIER = 4;
3
4
  const RRF_K = 60;
@@ -241,6 +242,3 @@ function chunkSources(chunk) {
241
242
  function uniqueStrings(values) {
242
243
  return Array.from(new Set(values?.filter((value) => value.trim()) ?? []));
243
244
  }
244
- function positiveIntegerOrDefault(value, fallback) {
245
- return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
246
- }
@@ -55,25 +55,31 @@ export class MemoryIndexStore {
55
55
  insertChunk(input) {
56
56
  return this.insertChunks([input])[0];
57
57
  }
58
- insertChunks(inputs) {
58
+ insertChunks(inputs, preloadedIds, knownMissingHashes) {
59
59
  if (inputs.length === 0)
60
60
  return [];
61
61
  const rows = inputs.map((input) => this.normalizeInput(input));
62
+ const knownIds = new Map(preloadedIds);
62
63
  const out = [];
63
64
  const insert = this.db.transaction((items) => {
64
65
  for (const item of items)
65
- out.push(this.insertNormalized(item));
66
+ out.push(this.insertNormalized(item, knownIds, knownMissingHashes));
66
67
  });
67
68
  insert.immediate(rows);
68
69
  return out;
69
70
  }
70
- recordSourceMappings(inputs) {
71
+ recordSourceMappings(inputs, preloadedIds) {
71
72
  if (inputs.length === 0)
72
73
  return;
73
74
  const rows = inputs.map((input) => this.normalizeInput(input));
74
75
  this.db
75
76
  .transaction((items) => {
76
77
  for (const item of items) {
78
+ const preloadedId = preloadedIds?.get(item.contentHash);
79
+ if (preloadedId !== undefined) {
80
+ this.insertSourceMapping(preloadedId, item);
81
+ continue;
82
+ }
77
83
  const existing = this.db
78
84
  .prepare("SELECT id FROM memory_chunks WHERE content_hash = ?")
79
85
  .get(item.contentHash);
@@ -88,8 +94,9 @@ export class MemoryIndexStore {
88
94
  const out = [];
89
95
  const replace = this.db.transaction(() => {
90
96
  this.deleteBySourceInternal(corpus, sourceId);
97
+ const knownIds = new Map();
91
98
  for (const item of rows)
92
- out.push(this.insertNormalized(item));
99
+ out.push(this.insertNormalized(item, knownIds));
93
100
  });
94
101
  replace.immediate();
95
102
  return out;
@@ -168,19 +175,19 @@ export class MemoryIndexStore {
168
175
  return rows.map((row) => ({ id: row.id, score: row.score, chunk: rowToChunk(row) }));
169
176
  }
170
177
  searchSemanticLinear(query, normalized) {
171
- const rows = this.db
172
- .prepare(normalized.corpus
178
+ const stmt = this.db.prepare(normalized.corpus
173
179
  ? `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c WHERE c.corpus = ?`
174
- : `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c`)
175
- .all(...(normalized.corpus ? [normalized.corpus] : []));
176
- return rows
177
- .map((row) => ({
178
- id: row.id,
179
- score: cosineDistance(query, decodeVector(row.embedding, row.embedding_dimensions)),
180
- chunk: rowToChunk(row),
181
- }))
182
- .sort((a, b) => a.score - b.score)
183
- .slice(0, normalized.limit);
180
+ : `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c`);
181
+ const best = [];
182
+ for (const row of stmt.iterate(...(normalized.corpus ? [normalized.corpus] : []))) {
183
+ const hit = {
184
+ id: row.id,
185
+ score: cosineDistance(query, decodeVector(row.embedding, row.embedding_dimensions)),
186
+ chunk: rowToChunk(row),
187
+ };
188
+ insertBoundedHit(best, hit, normalized.limit);
189
+ }
190
+ return best;
184
191
  }
185
192
  deleteChunk(id) {
186
193
  const remove = this.db.transaction(() => {
@@ -243,6 +250,7 @@ export class MemoryIndexStore {
243
250
  this.deleteBySource(corpus, sourceId);
244
251
  return;
245
252
  }
253
+ const keptHashes = new Set(kept.map((item) => item.contentHash));
246
254
  this.db
247
255
  .transaction(() => {
248
256
  const rows = this.db
@@ -259,7 +267,8 @@ export class MemoryIndexStore {
259
267
  this.db
260
268
  .prepare("DELETE FROM memory_index_sources WHERE corpus = ? AND source_id = ? AND chunk_index = ?")
261
269
  .run(corpus, sourceId, row.chunk_index);
262
- this.deleteOrphanChunk(row.id);
270
+ if (!keptHashes.has(row.content_hash))
271
+ this.deleteOrphanChunk(row.id);
263
272
  }
264
273
  })
265
274
  .immediate();
@@ -362,9 +371,15 @@ export class MemoryIndexStore {
362
371
  }),
363
372
  };
364
373
  }
365
- insertNormalized(item) {
366
- const existing = this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?").get(item.contentHash);
374
+ insertNormalized(item, knownIds, knownMissingHashes) {
375
+ const knownId = knownIds.get(item.contentHash);
376
+ const existing = knownId !== undefined
377
+ ? { id: knownId }
378
+ : knownMissingHashes?.has(item.contentHash)
379
+ ? undefined
380
+ : this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?").get(item.contentHash);
367
381
  if (existing) {
382
+ knownIds.set(item.contentHash, existing.id);
368
383
  this.insertSourceMapping(existing.id, item);
369
384
  return existing.id;
370
385
  }
@@ -383,6 +398,7 @@ export class MemoryIndexStore {
383
398
  .prepare("INSERT INTO memory_vec(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)")
384
399
  .run(id, encodeVector(item.embedding));
385
400
  }
401
+ knownIds.set(item.contentHash, id);
386
402
  this.insertSourceMapping(id, item);
387
403
  return id;
388
404
  }
@@ -434,6 +450,18 @@ function normalizeSearchOptions(options) {
434
450
  corpus: options.corpus,
435
451
  };
436
452
  }
453
+ function insertBoundedHit(best, hit, limit) {
454
+ if (limit <= 0)
455
+ return;
456
+ let index = best.findIndex((candidate) => hit.score < candidate.score);
457
+ if (index < 0)
458
+ index = best.length;
459
+ if (index >= limit)
460
+ return;
461
+ best.splice(index, 0, hit);
462
+ if (best.length > limit)
463
+ best.length = limit;
464
+ }
437
465
  function sourcesJsonSelect(chunkIdExpr) {
438
466
  return `(SELECT json_group_array(json_object(
439
467
  'corpus', s.corpus,
@@ -2,7 +2,6 @@ import { readdir, readFile } from "node:fs/promises";
2
2
  import { relative, resolve } from "node:path";
3
3
  import { indexLcmRecords } from "./indexer.js";
4
4
  import { normalizeChatRecords } from "./normalize.js";
5
- import { computeLcmRecordKey } from "./store.js";
6
5
  const DEFAULT_YIELD_EVERY_N = 1024;
7
6
  const INDEX_BATCH_SIZE = 32;
8
7
  export async function backfillFromChatLogs(deps, options) {
@@ -66,15 +65,12 @@ export async function backfillFromChatLogs(deps, options) {
66
65
  }
67
66
  const inserted = [];
68
67
  for (const record of batch.records) {
69
- if (recordExists(deps.lcmStore, record)) {
68
+ const result = deps.lcmStore.insertRecordReturningStored(record);
69
+ if (!result.inserted) {
70
70
  report.recordsSkippedDuplicate += 1;
71
71
  continue;
72
72
  }
73
- const id = deps.lcmStore.insertRecord(record);
74
- const stored = deps.lcmStore.getRecord(id);
75
- if (!stored)
76
- throw new Error(`Failed to read backfilled LCM record: ${id}`);
77
- inserted.push(stored);
73
+ inserted.push(result.record);
78
74
  report.recordsInserted += 1;
79
75
  if (inserted.length >= INDEX_BATCH_SIZE) {
80
76
  report.indexedChunks += (await indexLcmRecords({ indexer: deps.indexer, records: inserted, signal: options.signal })).ids.length;
@@ -225,17 +221,24 @@ function countMissingSegments(lcmStore, segmentIds) {
225
221
  return missing;
226
222
  }
227
223
  function countExistingRecords(lcmStore, records) {
228
- let existing = 0;
229
- for (const record of records) {
230
- if (recordExists(lcmStore, record))
231
- existing += 1;
224
+ if (records.length === 0)
225
+ return 0;
226
+ const keys = records.map((record) => lcmStore.computeRecordKey(record));
227
+ const existingKeys = new Set();
228
+ for (const chunk of chunks([...new Set(keys)], 256)) {
229
+ const rows = lcmStore.db
230
+ .prepare(`SELECT record_key FROM lcm_records WHERE record_key IN (${chunk.map(() => "?").join(",")})`)
231
+ .all(...chunk);
232
+ for (const row of rows)
233
+ existingKeys.add(row.record_key);
232
234
  }
233
- return existing;
235
+ return keys.reduce((total, key) => total + (existingKeys.has(key) ? 1 : 0), 0);
234
236
  }
235
- function recordExists(lcmStore, record) {
236
- return !!lcmStore.db
237
- .prepare("SELECT 1 FROM lcm_records WHERE record_key = ? LIMIT 1")
238
- .get(computeLcmRecordKey(record));
237
+ function chunks(items, size) {
238
+ const out = [];
239
+ for (let index = 0; index < items.length; index += size)
240
+ out.push(items.slice(index, index + size));
241
+ return out;
239
242
  }
240
243
  function errorCode(error) {
241
244
  return error && typeof error === "object" && "code" in error
@@ -1,5 +1,5 @@
1
1
  import { condense } from "./condense.js";
2
- import { createRawContextItems, estimateAgentMessageTokens, lcmRecordToAgentMessage, renderLcmRecordPartsForSummary, resolveFreshTailStartIndex, selectLcmCompactionCandidatePromptAware, } from "./context.js";
2
+ import { createRawContextItems, estimateAgentMessageTokens, renderLcmRecordPartsForSummary, resolveFreshTailStartIndex, selectLcmCompactionCandidatePromptAware, } from "./context.js";
3
3
  import { indexLcmSummaries } from "./indexer.js";
4
4
  import { createSyntheticLcmSummaryMessage } from "./summarizer.js";
5
5
  const LCM_SUMMARY_OPEN_TAG = "<from_earlier>";
@@ -48,6 +48,7 @@ export class LcmContextTransformer {
48
48
  signal,
49
49
  model: options.model,
50
50
  promptText,
51
+ initialPressure: pressure,
51
52
  });
52
53
  }
53
54
  }
@@ -76,7 +77,9 @@ export class LcmContextTransformer {
76
77
  }
77
78
  async serviceCompactionDebtForState(input) {
78
79
  for (let round = 0; input.state.compactionDebt > 0 && round < this.settings.maxRounds; round += 1) {
79
- const pressure = this.evaluateCompactionPressure(input.state, input.model, input.promptText ?? "");
80
+ const pressure = round === 0 && input.initialPressure
81
+ ? input.initialPressure
82
+ : this.evaluateCompactionPressure(input.state, input.model, input.promptText ?? "");
80
83
  if (!pressure.candidate.shouldCompact) {
81
84
  if (pressure.thresholdOverflowTokens > 0) {
82
85
  const condensed = await this.condenseRuntimeSummaries({
@@ -149,19 +152,15 @@ export class LcmContextTransformer {
149
152
  mode: candidate.reasons.includes("context_threshold") ? "aggressive" : "normal",
150
153
  previousSummary,
151
154
  }, input.signal);
152
- const summaryId = `${input.sessionKey}:summary-${++state.summaryCounter}`;
153
155
  const message = createSyntheticLcmSummaryMessage(renderLcmSummaryMessage(summaryText), this.now());
154
156
  const summaryItem = {
155
157
  type: "summary",
156
- id: summaryId,
158
+ id: "",
157
159
  sourceIds: chunkItems.map((item) => item.id),
158
160
  depth: 1,
159
161
  message,
160
162
  tokens: estimateAgentMessageTokens(message),
161
163
  };
162
- state.items.splice(startIndex, removeCount, summaryItem);
163
- compacted = true;
164
- tokensSaved = Math.max(0, candidate.chunkTokens - summaryItem.tokens);
165
164
  const persisted = await this.persistRuntimeSummary({
166
165
  text: summaryText,
167
166
  sourceItems: chunkItems,
@@ -169,8 +168,12 @@ export class LcmContextTransformer {
169
168
  sessionId: input.sessionId,
170
169
  signal: input.signal,
171
170
  });
171
+ summaryItem.id = `${input.sessionKey}:summary-${++state.summaryCounter}`;
172
172
  if (persisted?.summaryId !== undefined)
173
173
  summaryItem.persistedSummaryId = persisted.summaryId;
174
+ state.items.splice(startIndex, removeCount, summaryItem);
175
+ compacted = true;
176
+ tokensSaved = Math.max(0, candidate.chunkTokens - summaryItem.tokens);
174
177
  await this.condenseRuntimeSummaries({ state, sessionKey: input.sessionKey, signal: input.signal });
175
178
  };
176
179
  input.state.compactionQueue = input.state.compactionQueue.then(run, run);
@@ -262,8 +265,9 @@ export class LcmContextTransformer {
262
265
  this.lcmStore.db
263
266
  .transaction(() => {
264
267
  for (const insert of inserts) {
265
- insert.item.recordId = this.lcmStore.insertRecord(insert.input);
266
- insert.item.record = this.lcmStore.getRecord(insert.item.recordId);
268
+ const result = this.lcmStore.insertRecordReturningStored(insert.input);
269
+ insert.item.recordId = result.record.id;
270
+ insert.item.record = result.record;
267
271
  }
268
272
  })
269
273
  .immediate();
@@ -275,20 +279,7 @@ export class LcmContextTransformer {
275
279
  const items = [];
276
280
  for (const row of rows) {
277
281
  if (row.type === "raw") {
278
- const record = this.lcmStore.getRecord(row.recordId);
279
- if (!record) {
280
- console.error(`memory LCM context item dropped because record ${row.recordId} is missing`);
281
- continue;
282
- }
283
- const message = lcmRecordToAgentMessage(record);
284
- items.push({
285
- type: "raw",
286
- id: row.fingerprint,
287
- recordId: record.id,
288
- record,
289
- message,
290
- tokens: estimateAgentMessageTokens(message),
291
- });
282
+ // Raw history is owned by transcripts; legacy raw context rows must not replay it.
292
283
  continue;
293
284
  }
294
285
  const summary = this.lcmStore.getSummary(row.summaryId);
@@ -359,8 +350,6 @@ function syncContextState(state, messages) {
359
350
  const replacement = rawById.get(item.id);
360
351
  if (replacement && !covered.has(item.id))
361
352
  next.push(replacement);
362
- else if (item.recordId !== null && !covered.has(item.id))
363
- next.push(item);
364
353
  }
365
354
  for (const item of rawItems) {
366
355
  if (!covered.has(item.id) && !next.some((existing) => existing.type === "raw" && existing.id === item.id)) {
@@ -390,8 +379,6 @@ function contextItemsForStorage(items) {
390
379
  const timestamp = item.message.timestamp;
391
380
  const happenedAt = typeof timestamp === "number" && Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
392
381
  if (item.type === "raw") {
393
- if (item.recordId !== null)
394
- stored.push({ type: "raw", recordId: item.recordId, fingerprint: item.id, happenedAt });
395
382
  continue;
396
383
  }
397
384
  if (item.persistedSummaryId !== undefined) {
@@ -522,9 +509,10 @@ function assembleWithinBudget(state, settings, model) {
522
509
  const freshTail = state.items.slice(resolveFreshTailStartIndexForState(state.items, settings));
523
510
  const selected = new Set(freshTail);
524
511
  let tokens = sumItemTokens(freshTail);
512
+ const originalIndexes = new Map(state.items.map((item, index) => [item, index]));
525
513
  const summaries = state.items
526
514
  .filter((item) => item.type === "summary" && !selected.has(item))
527
- .sort((a, b) => b.depth - a.depth || state.items.indexOf(b) - state.items.indexOf(a));
515
+ .sort((a, b) => b.depth - a.depth || (originalIndexes.get(b) ?? 0) - (originalIndexes.get(a) ?? 0));
528
516
  for (const item of summaries) {
529
517
  if (tokens + item.tokens > budget && selected.size > 0)
530
518
  continue;
@@ -587,7 +575,7 @@ function lcmRecordPartsFromAgentMessage(message) {
587
575
  kind: "tool_result",
588
576
  toolCallId: toolResult.toolCallId ?? "",
589
577
  toolName: toolResult.toolName ?? "tool",
590
- output: toolResult.details ?? textFromContent(toolResult.content),
578
+ output: textFromContent(toolResult.content),
591
579
  ...(toolResult.isError ? { isError: true } : {}),
592
580
  },
593
581
  ];
@@ -1,5 +1,5 @@
1
1
  import { createHash } from "node:crypto";
2
- import { scoreEvictable, tokenBag } from "./eviction-score.js";
2
+ import { buildEvictionScoreContext, scoreEvictable, tokenBag } from "./eviction-score.js";
3
3
  const MESSAGE_OVERHEAD_TOKENS = 6;
4
4
  const RECORD_OVERHEAD_TOKENS = 4;
5
5
  const IMAGE_TOKEN_ESTIMATE = 1200;
@@ -167,9 +167,12 @@ function selectLeafChunk(items, leafChunkTokens, promptText, config) {
167
167
  const records = items.map((item) => item.record).filter((record) => !!record);
168
168
  if (records.length === 0)
169
169
  return selectOldestLeafChunk(items, leafChunkTokens);
170
+ const scoreContext = buildEvictionScoreContext(promptText, records);
171
+ if (!scoreContext)
172
+ return selectOldestLeafChunk(items, leafChunkTokens);
170
173
  const scored = targetRanges.map((range) => ({
171
174
  ...range,
172
- score: range.items.reduce((total, item) => total + (item.record ? scoreEvictable(item.record, promptText, records) : 0), 0),
175
+ score: range.items.reduce((total, item) => total + (item.record ? scoreEvictable(item.record, promptText, records, scoreContext) : 0), 0),
173
176
  }));
174
177
  scored.sort((a, b) => a.score - b.score || a.startIndex - b.startIndex);
175
178
  return scored[0]?.items ?? [];
@@ -179,7 +182,7 @@ function createValidLeafRanges(items, leafChunkTokens) {
179
182
  for (let startIndex = 0; startIndex < items.length; startIndex += 1) {
180
183
  if (isToolResultContinuingPreviousToolCall(items, startIndex))
181
184
  continue;
182
- const chunk = selectOldestLeafChunk(items.slice(startIndex), leafChunkTokens);
185
+ const chunk = selectOldestLeafChunkFromIndex(items, startIndex, leafChunkTokens);
183
186
  if (chunk.length === 0)
184
187
  continue;
185
188
  const chunkTokens = chunk.reduce((total, item) => total + item.tokens, 0);
@@ -194,9 +197,12 @@ function createValidLeafRanges(items, leafChunkTokens) {
194
197
  return ranges;
195
198
  }
196
199
  function selectOldestLeafChunk(items, leafChunkTokens) {
200
+ return selectOldestLeafChunkFromIndex(items, 0, leafChunkTokens);
201
+ }
202
+ function selectOldestLeafChunkFromIndex(items, startIndex, leafChunkTokens) {
197
203
  const chunk = [];
198
204
  let tokens = 0;
199
- for (let index = 0; index < items.length; index += 1) {
205
+ for (let index = startIndex; index < items.length; index += 1) {
200
206
  const item = items[index];
201
207
  if (!item)
202
208
  continue;
@@ -4,34 +4,46 @@ export function tokenBag(text) {
4
4
  .split(/[^a-z0-9]+/)
5
5
  .filter((token) => token.length >= 2);
6
6
  }
7
- export function scoreEvictable(record, prompt, allRecords) {
7
+ export function buildEvictionScoreContext(prompt, allRecords) {
8
8
  const promptTerms = tokenBag(prompt);
9
9
  if (promptTerms.length === 0)
10
- return 0;
11
- const recordTerms = tokenBag(record.text);
12
- if (recordTerms.length === 0)
13
- return 0;
14
- const recordFreq = new Map();
15
- for (const term of recordTerms)
16
- recordFreq.set(term, (recordFreq.get(term) ?? 0) + 1);
10
+ return null;
17
11
  const promptUniqueTerms = new Set(promptTerms);
18
12
  const documentFrequencies = new Map();
13
+ const recordTerms = new Map();
19
14
  for (const candidate of allRecords) {
20
- const candidateTerms = new Set(tokenBag(candidate.text));
15
+ const terms = tokenBag(candidate.text);
16
+ recordTerms.set(candidate, terms);
17
+ const candidateTerms = new Set(terms);
21
18
  for (const term of promptUniqueTerms) {
22
19
  if (candidateTerms.has(term))
23
20
  documentFrequencies.set(term, (documentFrequencies.get(term) ?? 0) + 1);
24
21
  }
25
22
  }
26
- const candidateCount = Math.max(0, allRecords.length);
23
+ return {
24
+ promptUniqueTerms,
25
+ documentFrequencies,
26
+ recordTerms,
27
+ candidateCount: Math.max(0, allRecords.length),
28
+ };
29
+ }
30
+ export function scoreEvictable(record, prompt, allRecords, context = buildEvictionScoreContext(prompt, allRecords)) {
31
+ if (!context)
32
+ return 0;
33
+ const recordTerms = context.recordTerms.get(record) ?? tokenBag(record.text);
34
+ if (recordTerms.length === 0)
35
+ return 0;
36
+ const recordFreq = new Map();
37
+ for (const term of recordTerms)
38
+ recordFreq.set(term, (recordFreq.get(term) ?? 0) + 1);
27
39
  let score = 0;
28
- for (const term of promptUniqueTerms) {
40
+ for (const term of context.promptUniqueTerms) {
29
41
  const tf = recordFreq.get(term) ?? 0;
30
42
  if (tf <= 0)
31
43
  continue;
32
44
  const normalizedTf = tf / recordTerms.length;
33
- const df = documentFrequencies.get(term) ?? 0;
34
- const idf = Math.log((candidateCount + 1) / (df + 1) + 1);
45
+ const df = context.documentFrequencies.get(term) ?? 0;
46
+ const idf = Math.log((context.candidateCount + 1) / (df + 1) + 1);
35
47
  score += normalizedTf * idf;
36
48
  }
37
49
  return score;
@@ -8,11 +8,7 @@ export async function projectNormalizedLcmBatch(options) {
8
8
  }
9
9
  const storedRecords = [];
10
10
  for (const record of options.batch.records) {
11
- const id = options.lcmStore.insertRecord(record);
12
- const stored = options.lcmStore.getRecord(id);
13
- if (!stored)
14
- throw new Error(`Failed to read projected LCM record: ${id}`);
15
- storedRecords.push(stored);
11
+ storedRecords.push(options.lcmStore.insertRecordReturningStored(record).record);
16
12
  }
17
13
  return {
18
14
  segmentIds,
@@ -59,7 +59,7 @@ export function normalizeChatRecords(chatRecords, options) {
59
59
  kind: "tool_result",
60
60
  toolCallId: record.event.toolCallId,
61
61
  toolName: record.event.toolName,
62
- output: record.event.result,
62
+ output: toolResultVisibleOutput(record.event.result),
63
63
  ...(record.event.isError ? { isError: true } : {}),
64
64
  },
65
65
  ];
@@ -191,6 +191,27 @@ function briefJson(value, maxLength = 160) {
191
191
  const compact = text.replace(/\s+/g, " ").trim();
192
192
  return compact.length > maxLength ? `${compact.slice(0, maxLength - 1)}...` : compact;
193
193
  }
194
+ function toolResultVisibleOutput(result) {
195
+ if (!result || typeof result !== "object" || !("content" in result))
196
+ return result;
197
+ const content = result.content;
198
+ if (!Array.isArray(content))
199
+ return result;
200
+ return textFromToolResultContent(content);
201
+ }
202
+ function textFromToolResultContent(content) {
203
+ if (!Array.isArray(content))
204
+ return content;
205
+ return content
206
+ .map((item) => {
207
+ if (item && typeof item === "object" && item.type === "text") {
208
+ return item.text;
209
+ }
210
+ return "";
211
+ })
212
+ .filter((item) => typeof item === "string" && item.length > 0)
213
+ .join("\n");
214
+ }
194
215
  function chatRecordSource(record, sourcePath) {
195
216
  const messageId = "messageId" in record && typeof record.messageId === "string"
196
217
  ? record.messageId