@qearlyao/familiar 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +31 -0
- package/HEARTBEAT.md +23 -0
- package/LICENSE +21 -0
- package/MEMORY.md +1 -0
- package/README.md +245 -0
- package/SOUL.md +13 -0
- package/USER.md +13 -0
- package/config.example.toml +221 -0
- package/dist/agent-events.js +167 -0
- package/dist/agent.js +590 -0
- package/dist/browser-tools.js +638 -0
- package/dist/chat-log.js +130 -0
- package/dist/cli.js +168 -0
- package/dist/config.js +804 -0
- package/dist/data-retention.js +54 -0
- package/dist/discord.js +1203 -0
- package/dist/generated-media.js +86 -0
- package/dist/image-derivatives.js +102 -0
- package/dist/image-gen.js +440 -0
- package/dist/inbound-attachments.js +266 -0
- package/dist/index.js +10 -0
- package/dist/media-understanding.js +120 -0
- package/dist/memory/diary/ambient-injector.js +180 -0
- package/dist/memory/diary/ambient.js +124 -0
- package/dist/memory/diary/chunks.js +231 -0
- package/dist/memory/diary/index.js +3 -0
- package/dist/memory/diary/indexer.js +93 -0
- package/dist/memory/doctor.js +250 -0
- package/dist/memory/index/chunk-indexer.js +151 -0
- package/dist/memory/index/embedding-provider.js +119 -0
- package/dist/memory/index/fts-query.js +18 -0
- package/dist/memory/index/retrieval.js +246 -0
- package/dist/memory/index/schema.js +157 -0
- package/dist/memory/index/store.js +513 -0
- package/dist/memory/index/vec.js +72 -0
- package/dist/memory/index/vector-codec.js +27 -0
- package/dist/memory/lcm/backfill.js +247 -0
- package/dist/memory/lcm/condense.js +146 -0
- package/dist/memory/lcm/context-transformer.js +662 -0
- package/dist/memory/lcm/context.js +421 -0
- package/dist/memory/lcm/eviction-score.js +38 -0
- package/dist/memory/lcm/index.js +6 -0
- package/dist/memory/lcm/indexer.js +200 -0
- package/dist/memory/lcm/normalize.js +235 -0
- package/dist/memory/lcm/schema.js +188 -0
- package/dist/memory/lcm/segment-manager.js +136 -0
- package/dist/memory/lcm/store.js +722 -0
- package/dist/memory/lcm/summarizer.js +258 -0
- package/dist/memory/lcm/types.js +1 -0
- package/dist/memory/operator.js +477 -0
- package/dist/memory/service.js +202 -0
- package/dist/memory/tools.js +205 -0
- package/dist/models.js +165 -0
- package/dist/persona.js +54 -0
- package/dist/runtime.js +493 -0
- package/dist/scheduler.js +200 -0
- package/dist/settings.js +116 -0
- package/dist/skills.js +38 -0
- package/dist/tts.js +143 -0
- package/dist/web-auth.js +105 -0
- package/dist/web-events.js +114 -0
- package/dist/web-http.js +29 -0
- package/dist/web-static.js +106 -0
- package/dist/web-tools.js +940 -0
- package/dist/web-types.js +2 -0
- package/dist/web.js +844 -0
- package/package.json +60 -0
- package/web/dist/assets/index-ClgkMgaq.css +2 -0
- package/web/dist/assets/index-Cu2QquuR.js +59 -0
- package/web/dist/favicon.svg +1 -0
- package/web/dist/icons.svg +24 -0
- package/web/dist/index.html +20 -0
|
@@ -0,0 +1,662 @@
|
|
|
1
|
+
import { condense } from "./condense.js";
|
|
2
|
+
import { createRawContextItems, estimateAgentMessageTokens, lcmRecordToAgentMessage, renderLcmRecordPartsForSummary, resolveFreshTailStartIndex, selectLcmCompactionCandidatePromptAware, } from "./context.js";
|
|
3
|
+
import { indexLcmSummaries } from "./indexer.js";
|
|
4
|
+
import { createSyntheticLcmSummaryMessage } from "./summarizer.js";
|
|
5
|
+
const LCM_SUMMARY_OPEN_TAG = "<from_earlier>";
|
|
6
|
+
const LCM_SUMMARY_CLOSE_TAG = "</from_earlier>";
|
|
7
|
+
export class LcmContextTransformer {
|
|
8
|
+
settings;
|
|
9
|
+
lcmStore;
|
|
10
|
+
indexer;
|
|
11
|
+
summarizer;
|
|
12
|
+
segmentManager;
|
|
13
|
+
now;
|
|
14
|
+
contextStates = new Map();
|
|
15
|
+
constructor(options) {
|
|
16
|
+
this.settings = options.settings;
|
|
17
|
+
this.lcmStore = options.lcmStore;
|
|
18
|
+
this.indexer = options.indexer;
|
|
19
|
+
this.summarizer = options.summarizer;
|
|
20
|
+
this.segmentManager = options.segmentManager;
|
|
21
|
+
this.now = options.now ?? Date.now;
|
|
22
|
+
}
|
|
23
|
+
async transformLcmContext(messages, signal, options) {
|
|
24
|
+
const settings = this.settings;
|
|
25
|
+
if (!settings.enabled)
|
|
26
|
+
return messages;
|
|
27
|
+
const promptText = lastUserText(messages);
|
|
28
|
+
const sessionKey = options.sessionKey ?? options.sessionId ?? "default";
|
|
29
|
+
const state = this.contextState(sessionKey);
|
|
30
|
+
const now = this.now();
|
|
31
|
+
const previousCacheTouchedAt = state.cacheTouchedAt;
|
|
32
|
+
state.cacheTouchedAt = now;
|
|
33
|
+
syncContextState(state, messages);
|
|
34
|
+
this.projectContextState(sessionKey, options.sessionId, state);
|
|
35
|
+
try {
|
|
36
|
+
const pressure = this.evaluateCompactionPressure(state, options.model, promptText);
|
|
37
|
+
state.compactionDebt += pressure.pressureScore;
|
|
38
|
+
if (shouldServiceCompactionDebt({
|
|
39
|
+
settings,
|
|
40
|
+
now,
|
|
41
|
+
previousCacheTouchedAt,
|
|
42
|
+
pressureScore: state.compactionDebt,
|
|
43
|
+
})) {
|
|
44
|
+
await this.serviceCompactionDebtForState({
|
|
45
|
+
state,
|
|
46
|
+
sessionKey,
|
|
47
|
+
sessionId: options.sessionId,
|
|
48
|
+
signal,
|
|
49
|
+
model: options.model,
|
|
50
|
+
promptText,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
console.error("memory LCM summarization failed", error);
|
|
56
|
+
syncContextState(state, messages);
|
|
57
|
+
this.persistContextState(sessionKey, state);
|
|
58
|
+
this.persistSessionState(sessionKey, state);
|
|
59
|
+
return assembleWithinBudget(state, settings, options.model);
|
|
60
|
+
}
|
|
61
|
+
this.persistContextState(sessionKey, state);
|
|
62
|
+
this.persistSessionState(sessionKey, state);
|
|
63
|
+
return assembleWithinBudget(state, settings, options.model);
|
|
64
|
+
}
|
|
65
|
+
async serviceCompactionDebt(sessionKey, signal, options = {}) {
|
|
66
|
+
const state = this.contextState(sessionKey);
|
|
67
|
+
await this.serviceCompactionDebtForState({
|
|
68
|
+
state,
|
|
69
|
+
sessionKey,
|
|
70
|
+
sessionId: options.sessionId,
|
|
71
|
+
signal,
|
|
72
|
+
model: options.model,
|
|
73
|
+
});
|
|
74
|
+
this.persistContextState(sessionKey, state);
|
|
75
|
+
this.persistSessionState(sessionKey, state);
|
|
76
|
+
}
|
|
77
|
+
async serviceCompactionDebtForState(input) {
|
|
78
|
+
for (let round = 0; input.state.compactionDebt > 0 && round < this.settings.maxRounds; round += 1) {
|
|
79
|
+
const pressure = this.evaluateCompactionPressure(input.state, input.model, input.promptText ?? "");
|
|
80
|
+
if (!pressure.candidate.shouldCompact) {
|
|
81
|
+
if (pressure.thresholdOverflowTokens > 0) {
|
|
82
|
+
const condensed = await this.condenseRuntimeSummaries({
|
|
83
|
+
state: input.state,
|
|
84
|
+
sessionKey: input.sessionKey,
|
|
85
|
+
signal: input.signal,
|
|
86
|
+
});
|
|
87
|
+
if (condensed.length > 0) {
|
|
88
|
+
input.state.compactionDebt = Math.max(0, input.state.compactionDebt - pressure.thresholdOverflowTokens);
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
input.state.compactionDebt = 0;
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
const progress = await this.compactLcmCandidate({
|
|
96
|
+
state: input.state,
|
|
97
|
+
candidate: pressure.candidate,
|
|
98
|
+
sessionKey: input.sessionKey,
|
|
99
|
+
sessionId: input.sessionId,
|
|
100
|
+
signal: input.signal,
|
|
101
|
+
});
|
|
102
|
+
if (!progress.compacted)
|
|
103
|
+
break;
|
|
104
|
+
input.state.compactionDebt = Math.max(0, input.state.compactionDebt - progress.tokensSaved);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
evaluateCompactionPressure(state, model, promptText = "") {
|
|
108
|
+
const rawItems = state.items.filter((item) => item.type === "raw");
|
|
109
|
+
const summaryTokens = state.items
|
|
110
|
+
.filter((item) => item.type === "summary")
|
|
111
|
+
.reduce((total, item) => total + item.tokens, 0);
|
|
112
|
+
const candidate = selectLcmCompactionCandidatePromptAware(rawItems, {
|
|
113
|
+
contextThreshold: this.settings.contextThreshold,
|
|
114
|
+
freshTailCount: this.settings.freshTailCount,
|
|
115
|
+
freshTailMaxTokens: this.settings.freshTailMaxTokens,
|
|
116
|
+
leafChunkTokens: this.settings.leafChunkTokens,
|
|
117
|
+
promptAwareEvictionEnabled: this.settings.promptAwareEvictionEnabled,
|
|
118
|
+
}, model?.contextWindow ?? 200_000, promptText, summaryTokens);
|
|
119
|
+
const evictableTokens = candidate.shouldCompact ? candidate.rawTokensOutsideTail : 0;
|
|
120
|
+
const thresholdOverflowTokens = Math.max(0, candidate.totalTokens - candidate.contextThresholdTokens);
|
|
121
|
+
return {
|
|
122
|
+
candidate,
|
|
123
|
+
pressureScore: Math.max(0, evictableTokens - this.settings.leafTargetTokens, thresholdOverflowTokens),
|
|
124
|
+
thresholdOverflowTokens,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
async compactLcmCandidate(input) {
|
|
128
|
+
let compacted = false;
|
|
129
|
+
let tokensSaved = 0;
|
|
130
|
+
const run = async () => {
|
|
131
|
+
const { state, candidate } = input;
|
|
132
|
+
const sourceIds = new Set(candidate.chunk.map((item) => item.id));
|
|
133
|
+
const startIndex = state.items.findIndex((item) => item.type === "raw" && sourceIds.has(item.id));
|
|
134
|
+
if (startIndex < 0)
|
|
135
|
+
return;
|
|
136
|
+
const removeCount = countContiguousRawSources(state.items, startIndex, sourceIds);
|
|
137
|
+
if (removeCount <= 0)
|
|
138
|
+
return;
|
|
139
|
+
const chunkItems = state.items
|
|
140
|
+
.slice(startIndex, startIndex + removeCount)
|
|
141
|
+
.filter((item) => item.type === "raw");
|
|
142
|
+
if (chunkItems.length === 0)
|
|
143
|
+
return;
|
|
144
|
+
const previousSummary = findPreviousSummaryText(state.items, startIndex);
|
|
145
|
+
const text = renderLcmSummaryInput(chunkItems);
|
|
146
|
+
const summaryText = await this.summarizer.summarizeLeaf({
|
|
147
|
+
text,
|
|
148
|
+
targetTokens: this.settings.leafTargetTokens,
|
|
149
|
+
mode: candidate.reasons.includes("context_threshold") ? "aggressive" : "normal",
|
|
150
|
+
previousSummary,
|
|
151
|
+
}, input.signal);
|
|
152
|
+
const summaryId = `${input.sessionKey}:summary-${++state.summaryCounter}`;
|
|
153
|
+
const message = createSyntheticLcmSummaryMessage(renderLcmSummaryMessage(summaryText), this.now());
|
|
154
|
+
const summaryItem = {
|
|
155
|
+
type: "summary",
|
|
156
|
+
id: summaryId,
|
|
157
|
+
sourceIds: chunkItems.map((item) => item.id),
|
|
158
|
+
depth: 1,
|
|
159
|
+
message,
|
|
160
|
+
tokens: estimateAgentMessageTokens(message),
|
|
161
|
+
};
|
|
162
|
+
state.items.splice(startIndex, removeCount, summaryItem);
|
|
163
|
+
compacted = true;
|
|
164
|
+
tokensSaved = Math.max(0, candidate.chunkTokens - summaryItem.tokens);
|
|
165
|
+
const persisted = await this.persistRuntimeSummary({
|
|
166
|
+
text: summaryText,
|
|
167
|
+
sourceItems: chunkItems,
|
|
168
|
+
sessionKey: input.sessionKey,
|
|
169
|
+
sessionId: input.sessionId,
|
|
170
|
+
signal: input.signal,
|
|
171
|
+
});
|
|
172
|
+
if (persisted?.summaryId !== undefined)
|
|
173
|
+
summaryItem.persistedSummaryId = persisted.summaryId;
|
|
174
|
+
await this.condenseRuntimeSummaries({ state, sessionKey: input.sessionKey, signal: input.signal });
|
|
175
|
+
};
|
|
176
|
+
input.state.compactionQueue = input.state.compactionQueue.then(run, run);
|
|
177
|
+
await input.state.compactionQueue;
|
|
178
|
+
return { compacted, tokensSaved };
|
|
179
|
+
}
|
|
180
|
+
async persistRuntimeSummary(input) {
|
|
181
|
+
const segmentId = this.segmentManager.activeSegmentId(input.sessionKey);
|
|
182
|
+
const recordIds = input.sourceItems.map((item) => item.recordId).filter((id) => id !== null);
|
|
183
|
+
if (recordIds.length === 0)
|
|
184
|
+
return null;
|
|
185
|
+
const summaryId = this.lcmStore.insertSummary({
|
|
186
|
+
segmentId,
|
|
187
|
+
depth: 1,
|
|
188
|
+
status: "ready",
|
|
189
|
+
text: input.text,
|
|
190
|
+
coversFromRecordId: recordIds[0],
|
|
191
|
+
coversToRecordId: recordIds[recordIds.length - 1],
|
|
192
|
+
source: { sourceType: "manual", sourceRef: `lcm_record:${recordIds[0]}-${recordIds[recordIds.length - 1]}` },
|
|
193
|
+
sourceItems: input.sourceItems.map((item) => ({
|
|
194
|
+
recordId: item.recordId,
|
|
195
|
+
sourceRef: item.id,
|
|
196
|
+
snapshot: {
|
|
197
|
+
role: item.message.role ?? null,
|
|
198
|
+
timestamp: item.message.timestamp ?? null,
|
|
199
|
+
},
|
|
200
|
+
})),
|
|
201
|
+
metadata: {
|
|
202
|
+
sessionKey: input.sessionKey,
|
|
203
|
+
sessionId: input.sessionId ?? null,
|
|
204
|
+
source: "transformContext",
|
|
205
|
+
...coverageMetadataFromRawItems(input.sourceItems),
|
|
206
|
+
},
|
|
207
|
+
});
|
|
208
|
+
const summary = this.lcmStore.getSummary(summaryId);
|
|
209
|
+
if (!summary)
|
|
210
|
+
return null;
|
|
211
|
+
await indexLcmSummaries({ indexer: this.indexer, summaries: [summary] }).catch((error) => console.error("memory LCM summary indexing failed", error));
|
|
212
|
+
return { summaryId };
|
|
213
|
+
}
|
|
214
|
+
async condenseRuntimeSummaries(input) {
|
|
215
|
+
const candidateIds = contiguousRuntimeSummaryCandidateIds(input.state.items, 1, this.settings.condenseGroupSize);
|
|
216
|
+
if (candidateIds.length === 0)
|
|
217
|
+
return [];
|
|
218
|
+
const created = await condense({
|
|
219
|
+
segmentId: this.segmentManager.activeSegmentId(input.sessionKey),
|
|
220
|
+
depth: 1,
|
|
221
|
+
store: this.lcmStore,
|
|
222
|
+
summarizer: this.summarizer,
|
|
223
|
+
config: this.settings,
|
|
224
|
+
candidateIds,
|
|
225
|
+
indexer: this.indexer,
|
|
226
|
+
signal: input.signal,
|
|
227
|
+
});
|
|
228
|
+
applyCondensedRuntimeSummaries(input.state, created, input.sessionKey);
|
|
229
|
+
return created;
|
|
230
|
+
}
|
|
231
|
+
contextState(sessionKey) {
|
|
232
|
+
let state = this.contextStates.get(sessionKey);
|
|
233
|
+
if (!state) {
|
|
234
|
+
state = {
|
|
235
|
+
items: [],
|
|
236
|
+
summaryCounter: 0,
|
|
237
|
+
compactionDebt: 0,
|
|
238
|
+
cacheTouchedAt: null,
|
|
239
|
+
compactionQueue: Promise.resolve(),
|
|
240
|
+
rehydrated: false,
|
|
241
|
+
};
|
|
242
|
+
this.contextStates.set(sessionKey, state);
|
|
243
|
+
}
|
|
244
|
+
if (!state.rehydrated) {
|
|
245
|
+
if (state.items.length === 0)
|
|
246
|
+
this.rehydrateContextState(sessionKey, state);
|
|
247
|
+
this.rehydrateSessionState(sessionKey, state);
|
|
248
|
+
}
|
|
249
|
+
state.rehydrated = true;
|
|
250
|
+
return state;
|
|
251
|
+
}
|
|
252
|
+
invalidateSession(sessionKey) {
|
|
253
|
+
this.contextStates.delete(sessionKey);
|
|
254
|
+
}
|
|
255
|
+
projectContextState(sessionKey, sessionId, state) {
|
|
256
|
+
const segmentId = this.segmentManager.activeSegmentId(sessionKey);
|
|
257
|
+
const inserts = state.items
|
|
258
|
+
.filter((item) => item.type === "raw" && item.recordId === null)
|
|
259
|
+
.map((item) => ({ item, input: rawItemToRecordInput(item, segmentId, sessionKey, sessionId) }));
|
|
260
|
+
if (inserts.length === 0)
|
|
261
|
+
return;
|
|
262
|
+
this.lcmStore.db
|
|
263
|
+
.transaction(() => {
|
|
264
|
+
for (const insert of inserts) {
|
|
265
|
+
insert.item.recordId = this.lcmStore.insertRecord(insert.input);
|
|
266
|
+
insert.item.record = this.lcmStore.getRecord(insert.item.recordId);
|
|
267
|
+
}
|
|
268
|
+
})
|
|
269
|
+
.immediate();
|
|
270
|
+
}
|
|
271
|
+
rehydrateContextState(sessionKey, state) {
|
|
272
|
+
const rows = this.lcmStore.listContextItems(sessionKey);
|
|
273
|
+
if (rows.length === 0)
|
|
274
|
+
return;
|
|
275
|
+
const items = [];
|
|
276
|
+
for (const row of rows) {
|
|
277
|
+
if (row.type === "raw") {
|
|
278
|
+
const record = this.lcmStore.getRecord(row.recordId);
|
|
279
|
+
if (!record) {
|
|
280
|
+
console.error(`memory LCM context item dropped because record ${row.recordId} is missing`);
|
|
281
|
+
continue;
|
|
282
|
+
}
|
|
283
|
+
const message = lcmRecordToAgentMessage(record);
|
|
284
|
+
items.push({
|
|
285
|
+
type: "raw",
|
|
286
|
+
id: row.fingerprint,
|
|
287
|
+
recordId: record.id,
|
|
288
|
+
record,
|
|
289
|
+
message,
|
|
290
|
+
tokens: estimateAgentMessageTokens(message),
|
|
291
|
+
});
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
const summary = this.lcmStore.getSummary(row.summaryId);
|
|
295
|
+
if (!summary) {
|
|
296
|
+
console.error(`memory LCM context item dropped because summary ${row.summaryId} is missing`);
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
items.push(summaryToContextItem(summary, row.fingerprint, sessionKey, this.summaryCoveredSourceIds(summary.id)));
|
|
300
|
+
}
|
|
301
|
+
state.items = items;
|
|
302
|
+
}
|
|
303
|
+
persistContextState(sessionKey, state) {
|
|
304
|
+
const items = contextItemsForStorage(state.items);
|
|
305
|
+
this.lcmStore.replaceContextItems(sessionKey, items);
|
|
306
|
+
}
|
|
307
|
+
rehydrateSessionState(sessionKey, state) {
|
|
308
|
+
const persisted = this.lcmStore.getSessionState(sessionKey);
|
|
309
|
+
if (!persisted)
|
|
310
|
+
return;
|
|
311
|
+
state.compactionDebt = persisted.compactionDebt;
|
|
312
|
+
state.cacheTouchedAt = persisted.cacheTouchedAt;
|
|
313
|
+
}
|
|
314
|
+
persistSessionState(sessionKey, state) {
|
|
315
|
+
this.lcmStore.upsertSessionState({
|
|
316
|
+
sessionKey,
|
|
317
|
+
compactionDebt: state.compactionDebt,
|
|
318
|
+
cacheTouchedAt: state.cacheTouchedAt,
|
|
319
|
+
updatedAt: this.now(),
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
summaryCoveredSourceIds(summaryId, seen = new Set()) {
|
|
323
|
+
if (seen.has(summaryId))
|
|
324
|
+
return [];
|
|
325
|
+
seen.add(summaryId);
|
|
326
|
+
const parents = this.lcmStore.getSummaryParents(summaryId);
|
|
327
|
+
if (parents.length > 0) {
|
|
328
|
+
// Condensed summaries use canonical parent edges; source rows are legacy advisory lineage.
|
|
329
|
+
return parents.flatMap((parentId) => this.summaryCoveredSourceIds(parentId, seen));
|
|
330
|
+
}
|
|
331
|
+
const ids = [];
|
|
332
|
+
for (const source of this.lcmStore.getSummarySources(summaryId)) {
|
|
333
|
+
if (source.sourceSummaryId !== null) {
|
|
334
|
+
ids.push(...this.summaryCoveredSourceIds(source.sourceSummaryId, seen));
|
|
335
|
+
}
|
|
336
|
+
else if (source.sourceRef) {
|
|
337
|
+
ids.push(source.sourceRef);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
return ids;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
function syncContextState(state, messages) {
|
|
344
|
+
const existingRecords = new Map(state.items.filter((item) => item.type === "raw").map((item) => [item.id, item]));
|
|
345
|
+
const rawItems = createRawContextItems(messages).map((item) => {
|
|
346
|
+
const existing = existingRecords.get(item.id);
|
|
347
|
+
return { ...item, type: "raw", recordId: existing?.recordId ?? null, record: existing?.record ?? null };
|
|
348
|
+
});
|
|
349
|
+
const rawById = new Map(rawItems.map((item) => [item.id, item]));
|
|
350
|
+
const next = [];
|
|
351
|
+
const covered = new Set();
|
|
352
|
+
for (const item of state.items) {
|
|
353
|
+
if (item.type === "summary") {
|
|
354
|
+
next.push(item);
|
|
355
|
+
for (const id of item.sourceIds)
|
|
356
|
+
covered.add(id);
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
const replacement = rawById.get(item.id);
|
|
360
|
+
if (replacement && !covered.has(item.id))
|
|
361
|
+
next.push(replacement);
|
|
362
|
+
else if (item.recordId !== null && !covered.has(item.id))
|
|
363
|
+
next.push(item);
|
|
364
|
+
}
|
|
365
|
+
for (const item of rawItems) {
|
|
366
|
+
if (!covered.has(item.id) && !next.some((existing) => existing.type === "raw" && existing.id === item.id)) {
|
|
367
|
+
next.push(item);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
state.items = next;
|
|
371
|
+
}
|
|
372
|
+
function lastUserText(messages) {
|
|
373
|
+
for (let index = messages.length - 1; index >= 0; index -= 1) {
|
|
374
|
+
const message = messages[index];
|
|
375
|
+
if (!message || message.role !== "user")
|
|
376
|
+
continue;
|
|
377
|
+
if (typeof message.content === "string")
|
|
378
|
+
return message.content.trim();
|
|
379
|
+
return message.content
|
|
380
|
+
.filter((item) => item.type === "text")
|
|
381
|
+
.map((item) => item.text)
|
|
382
|
+
.join("\n")
|
|
383
|
+
.trim();
|
|
384
|
+
}
|
|
385
|
+
return "";
|
|
386
|
+
}
|
|
387
|
+
function contextItemsForStorage(items) {
|
|
388
|
+
const stored = [];
|
|
389
|
+
for (const item of items) {
|
|
390
|
+
const timestamp = item.message.timestamp;
|
|
391
|
+
const happenedAt = typeof timestamp === "number" && Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
|
|
392
|
+
if (item.type === "raw") {
|
|
393
|
+
if (item.recordId !== null)
|
|
394
|
+
stored.push({ type: "raw", recordId: item.recordId, fingerprint: item.id, happenedAt });
|
|
395
|
+
continue;
|
|
396
|
+
}
|
|
397
|
+
if (item.persistedSummaryId !== undefined) {
|
|
398
|
+
stored.push({ type: "summary", summaryId: item.persistedSummaryId, fingerprint: item.id, happenedAt });
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
return stored;
|
|
402
|
+
}
|
|
403
|
+
function rawItemToRecordInput(item, segmentId, sessionKey, sessionId) {
|
|
404
|
+
const role = item.message.role;
|
|
405
|
+
const parts = lcmRecordPartsFromAgentMessage(item.message);
|
|
406
|
+
const text = renderPartsAsPlainText(parts).trim() || `[${role ?? "message"}]`;
|
|
407
|
+
const timestamp = item.message.timestamp;
|
|
408
|
+
return {
|
|
409
|
+
segmentId,
|
|
410
|
+
kind: role === "assistant" ? "assistant" : role === "user" ? "user" : role === "toolResult" ? "tool" : "note",
|
|
411
|
+
text,
|
|
412
|
+
parts: parts.length ? parts : undefined,
|
|
413
|
+
happenedAt: typeof timestamp === "number" && Number.isFinite(timestamp)
|
|
414
|
+
? new Date(timestamp).toISOString()
|
|
415
|
+
: new Date().toISOString(),
|
|
416
|
+
sessionId: sessionId ?? null,
|
|
417
|
+
channelKey: sessionKey,
|
|
418
|
+
source: { sourceType: "manual", sourceRef: `runtime:${item.id}` },
|
|
419
|
+
metadata: { source: "transformContext", fingerprint: item.id },
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
function coverageMetadataFromRawItems(items) {
|
|
423
|
+
const happenedAts = items
|
|
424
|
+
.map((item) => item.record?.happenedAt)
|
|
425
|
+
.filter((value) => typeof value === "string" && Number.isFinite(Date.parse(value)));
|
|
426
|
+
const from = happenedAts[0];
|
|
427
|
+
const to = happenedAts.at(-1);
|
|
428
|
+
return {
|
|
429
|
+
...(from ? { coverageFromHappenedAt: from } : {}),
|
|
430
|
+
...(to ? { coverageToHappenedAt: to, timestamp: to } : {}),
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
function countContiguousRawSources(items, startIndex, sourceIds) {
|
|
434
|
+
let count = 0;
|
|
435
|
+
for (let index = startIndex; index < items.length; index += 1) {
|
|
436
|
+
const item = items[index];
|
|
437
|
+
if (!item || item.type !== "raw" || !sourceIds.has(item.id))
|
|
438
|
+
break;
|
|
439
|
+
count += 1;
|
|
440
|
+
}
|
|
441
|
+
return count;
|
|
442
|
+
}
|
|
443
|
+
function findPreviousSummaryText(items, beforeIndex) {
|
|
444
|
+
for (let index = beforeIndex - 1; index >= 0; index -= 1) {
|
|
445
|
+
const item = items[index];
|
|
446
|
+
if (item?.type !== "summary")
|
|
447
|
+
continue;
|
|
448
|
+
return extractTextFromMessage(item.message);
|
|
449
|
+
}
|
|
450
|
+
return undefined;
|
|
451
|
+
}
|
|
452
|
+
function replaceCondensedRuntimeSummary(state, condensed, sessionKey) {
|
|
453
|
+
const parentIndexes = state.items
|
|
454
|
+
.map((item, index) => ({ item, index }))
|
|
455
|
+
.filter((entry) => entry.item.type === "summary" &&
|
|
456
|
+
entry.item.persistedSummaryId !== undefined &&
|
|
457
|
+
condensed.parents.includes(entry.item.persistedSummaryId));
|
|
458
|
+
if (parentIndexes.length !== condensed.parents.length)
|
|
459
|
+
return;
|
|
460
|
+
const indexes = parentIndexes.map((entry) => entry.index).sort((a, b) => a - b);
|
|
461
|
+
if (!indexes.every((index, offset) => offset === 0 || index === indexes[offset - 1] + 1))
|
|
462
|
+
return;
|
|
463
|
+
const first = indexes[0];
|
|
464
|
+
if (first === undefined)
|
|
465
|
+
return;
|
|
466
|
+
const sourceIds = parentIndexes.flatMap((entry) => entry.item.sourceIds);
|
|
467
|
+
const message = createSyntheticLcmSummaryMessage(renderLcmSummaryMessage(condensed.text), Date.now());
|
|
468
|
+
const item = {
|
|
469
|
+
type: "summary",
|
|
470
|
+
id: `${sessionKey}:summary-${condensed.id}`,
|
|
471
|
+
persistedSummaryId: condensed.id,
|
|
472
|
+
depth: condensed.depth,
|
|
473
|
+
sourceIds,
|
|
474
|
+
message,
|
|
475
|
+
tokens: estimateAgentMessageTokens(message),
|
|
476
|
+
};
|
|
477
|
+
state.items.splice(first, indexes.length, item);
|
|
478
|
+
}
|
|
479
|
+
function applyCondensedRuntimeSummaries(state, created, sessionKey) {
|
|
480
|
+
for (const summary of [...created].sort((a, b) => a.depth - b.depth || a.id - b.id)) {
|
|
481
|
+
replaceCondensedRuntimeSummary(state, summary, sessionKey);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
function contiguousRuntimeSummaryCandidateIds(items, depth, groupSize) {
|
|
485
|
+
const ids = [];
|
|
486
|
+
for (let index = 0; index < items.length; index += 1) {
|
|
487
|
+
const group = items.slice(index, index + groupSize);
|
|
488
|
+
if (group.length === groupSize &&
|
|
489
|
+
group.every((item) => item.type === "summary" && item.depth === depth && item.persistedSummaryId !== undefined)) {
|
|
490
|
+
ids.push(...group.map((item) => item.persistedSummaryId));
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
return ids;
|
|
494
|
+
}
|
|
495
|
+
function summaryToContextItem(summary, fingerprint, sessionKey, sourceIds) {
|
|
496
|
+
const message = createSyntheticLcmSummaryMessage(renderLcmSummaryMessage(summary.text), summary.createdAt * 1000);
|
|
497
|
+
return {
|
|
498
|
+
type: "summary",
|
|
499
|
+
id: fingerprint || `${sessionKey}:summary-${summary.id}`,
|
|
500
|
+
persistedSummaryId: summary.id,
|
|
501
|
+
depth: summary.depth,
|
|
502
|
+
sourceIds,
|
|
503
|
+
message,
|
|
504
|
+
tokens: estimateAgentMessageTokens(message),
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
function shouldServiceCompactionDebt(input) {
|
|
508
|
+
if (input.previousCacheTouchedAt === null)
|
|
509
|
+
return true;
|
|
510
|
+
if (input.pressureScore >= input.settings.criticalOverflowTokens)
|
|
511
|
+
return true;
|
|
512
|
+
const coldBoundaryMs = Math.max(0, input.settings.cacheTtlMs - input.settings.cacheTouchSlackMs);
|
|
513
|
+
const cacheAgeMs = input.now - input.previousCacheTouchedAt;
|
|
514
|
+
if (cacheAgeMs >= coldBoundaryMs)
|
|
515
|
+
return true;
|
|
516
|
+
return false;
|
|
517
|
+
}
|
|
518
|
+
function assembleWithinBudget(state, settings, model) {
|
|
519
|
+
const budget = Math.max(1, Math.floor((model?.contextWindow ?? 200_000) * settings.contextThreshold));
|
|
520
|
+
if (sumItemTokens(state.items) <= budget)
|
|
521
|
+
return state.items.map((item) => item.message);
|
|
522
|
+
const freshTail = state.items.slice(resolveFreshTailStartIndexForState(state.items, settings));
|
|
523
|
+
const selected = new Set(freshTail);
|
|
524
|
+
let tokens = sumItemTokens(freshTail);
|
|
525
|
+
const summaries = state.items
|
|
526
|
+
.filter((item) => item.type === "summary" && !selected.has(item))
|
|
527
|
+
.sort((a, b) => b.depth - a.depth || state.items.indexOf(b) - state.items.indexOf(a));
|
|
528
|
+
for (const item of summaries) {
|
|
529
|
+
if (tokens + item.tokens > budget && selected.size > 0)
|
|
530
|
+
continue;
|
|
531
|
+
selected.add(item);
|
|
532
|
+
tokens += item.tokens;
|
|
533
|
+
}
|
|
534
|
+
for (let index = state.items.length - 1; index >= 0; index -= 1) {
|
|
535
|
+
const item = state.items[index];
|
|
536
|
+
if (!item || selected.has(item) || item.type !== "raw")
|
|
537
|
+
continue;
|
|
538
|
+
if (tokens + item.tokens > budget && selected.size > 0)
|
|
539
|
+
continue;
|
|
540
|
+
selected.add(item);
|
|
541
|
+
tokens += item.tokens;
|
|
542
|
+
}
|
|
543
|
+
return state.items.filter((item) => selected.has(item)).map((item) => item.message);
|
|
544
|
+
}
|
|
545
|
+
function resolveFreshTailStartIndexForState(items, settings) {
|
|
546
|
+
return resolveFreshTailStartIndex(items, settings);
|
|
547
|
+
}
|
|
548
|
+
function sumItemTokens(items) {
|
|
549
|
+
return items.reduce((total, item) => total + item.tokens, 0);
|
|
550
|
+
}
|
|
551
|
+
function renderLcmSummaryInput(items) {
|
|
552
|
+
return items
|
|
553
|
+
.map((item) => renderMessageForSummary(item.message))
|
|
554
|
+
.filter(Boolean)
|
|
555
|
+
.join("\n\n");
|
|
556
|
+
}
|
|
557
|
+
function renderMessageForSummary(message) {
|
|
558
|
+
const role = message.role ?? "message";
|
|
559
|
+
const timestamp = message.timestamp;
|
|
560
|
+
const date = typeof timestamp === "number" && Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : "";
|
|
561
|
+
const text = extractTextFromMessage(message).trim();
|
|
562
|
+
if (!text)
|
|
563
|
+
return "";
|
|
564
|
+
return [`[${role}${date ? ` ${date}` : ""}]`, text].join("\n");
|
|
565
|
+
}
|
|
566
|
+
function renderLcmSummaryMessage(text) {
|
|
567
|
+
return `${LCM_SUMMARY_OPEN_TAG}\n${text.trim()}\n${LCM_SUMMARY_CLOSE_TAG}`;
|
|
568
|
+
}
|
|
569
|
+
function extractTextFromMessage(message) {
|
|
570
|
+
if (!("content" in message))
|
|
571
|
+
return "";
|
|
572
|
+
const content = message.content;
|
|
573
|
+
if (typeof content === "string")
|
|
574
|
+
return content;
|
|
575
|
+
if (!Array.isArray(content))
|
|
576
|
+
return "";
|
|
577
|
+
const parts = lcmRecordPartsFromAgentMessage(message);
|
|
578
|
+
return parts.length ? renderLcmRecordPartsForSummary(parts) : renderUnknownContent(content);
|
|
579
|
+
}
|
|
580
|
+
function lcmRecordPartsFromAgentMessage(message) {
|
|
581
|
+
if (!("content" in message))
|
|
582
|
+
return [];
|
|
583
|
+
if (message.role === "toolResult") {
|
|
584
|
+
const toolResult = message;
|
|
585
|
+
return [
|
|
586
|
+
{
|
|
587
|
+
kind: "tool_result",
|
|
588
|
+
toolCallId: toolResult.toolCallId ?? "",
|
|
589
|
+
toolName: toolResult.toolName ?? "tool",
|
|
590
|
+
output: toolResult.details ?? textFromContent(toolResult.content),
|
|
591
|
+
...(toolResult.isError ? { isError: true } : {}),
|
|
592
|
+
},
|
|
593
|
+
];
|
|
594
|
+
}
|
|
595
|
+
const content = message.content;
|
|
596
|
+
if (typeof content === "string")
|
|
597
|
+
return content ? [{ kind: "text", text: content }] : [];
|
|
598
|
+
if (!Array.isArray(content))
|
|
599
|
+
return [];
|
|
600
|
+
const parts = [];
|
|
601
|
+
for (const item of content) {
|
|
602
|
+
if (item.type === "text")
|
|
603
|
+
parts.push({ kind: "text", text: item.text });
|
|
604
|
+
else if (item.type === "thinking") {
|
|
605
|
+
parts.push({
|
|
606
|
+
kind: "thinking",
|
|
607
|
+
text: item.thinking,
|
|
608
|
+
...(item.thinkingSignature ? { signature: item.thinkingSignature } : {}),
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
else if (item.type === "toolCall") {
|
|
612
|
+
parts.push({ kind: "tool_call", toolCallId: item.id, toolName: item.name, arguments: item.arguments });
|
|
613
|
+
}
|
|
614
|
+
else if (item.type === "image") {
|
|
615
|
+
parts.push({ kind: "text", text: `[image: ${item.mimeType}]` });
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
return parts;
|
|
619
|
+
}
|
|
620
|
+
function renderPartsAsPlainText(parts) {
|
|
621
|
+
return parts
|
|
622
|
+
.map((part) => {
|
|
623
|
+
if (part.kind === "text")
|
|
624
|
+
return part.text;
|
|
625
|
+
if (part.kind === "thinking")
|
|
626
|
+
return part.text ? `[thinking] ${part.text}` : "";
|
|
627
|
+
if (part.kind === "tool_call")
|
|
628
|
+
return `[tool_call: ${part.toolName}(${JSON.stringify(part.arguments)})]`;
|
|
629
|
+
return `[tool_result: ${part.toolName} -> ${stringifyUnknown(part.output)}]`;
|
|
630
|
+
})
|
|
631
|
+
.filter(Boolean)
|
|
632
|
+
.join("\n");
|
|
633
|
+
}
|
|
634
|
+
function renderUnknownContent(content) {
|
|
635
|
+
return content
|
|
636
|
+
.map((item) => (item && typeof item === "object" && "type" in item ? `[${String(item.type)}]` : ""))
|
|
637
|
+
.filter(Boolean)
|
|
638
|
+
.join("\n");
|
|
639
|
+
}
|
|
640
|
+
function textFromContent(content) {
|
|
641
|
+
if (!Array.isArray(content))
|
|
642
|
+
return content;
|
|
643
|
+
return content
|
|
644
|
+
.map((item) => {
|
|
645
|
+
if (item && typeof item === "object" && item.type === "text") {
|
|
646
|
+
return item.text;
|
|
647
|
+
}
|
|
648
|
+
return "";
|
|
649
|
+
})
|
|
650
|
+
.filter((item) => typeof item === "string" && item.length > 0)
|
|
651
|
+
.join("\n");
|
|
652
|
+
}
|
|
653
|
+
function stringifyUnknown(value) {
|
|
654
|
+
if (typeof value === "string")
|
|
655
|
+
return value;
|
|
656
|
+
try {
|
|
657
|
+
return JSON.stringify(value);
|
|
658
|
+
}
|
|
659
|
+
catch {
|
|
660
|
+
return String(value);
|
|
661
|
+
}
|
|
662
|
+
}
|