@martian-engineering/lossless-claw 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/configuration.md +7 -0
- package/docs/tui.md +5 -0
- package/package.json +1 -1
- package/src/compaction.ts +36 -1
- package/src/db/migration.ts +58 -6
- package/src/expansion-auth.ts +53 -1
- package/src/retrieval.ts +44 -1
- package/src/store/summary-store.ts +122 -3
- package/src/tools/lcm-describe-tool.ts +104 -17
- package/src/tools/lcm-expand-query-tool.ts +128 -16
- package/src/tools/lcm-expand-tool.delegation.ts +96 -0
- package/src/tools/lcm-expansion-recursion-guard.ts +286 -0
package/docs/configuration.md
CHANGED
|
@@ -102,6 +102,13 @@ export LCM_SUMMARY_PROVIDER=anthropic
|
|
|
102
102
|
|
|
103
103
|
Using a cheaper/faster model for summarization can reduce costs, but quality matters — poor summaries compound as they're condensed into higher-level nodes.
|
|
104
104
|
|
|
105
|
+
## TUI conversation window size
|
|
106
|
+
|
|
107
|
+
`LCM_TUI_CONVERSATION_WINDOW_SIZE` (default `200`) controls how many messages `lcm-tui` loads per keyset-paged conversation window when a session has an LCM `conversation_id`.
|
|
108
|
+
|
|
109
|
+
- Smaller values reduce render/query cost for very large conversations.
|
|
110
|
+
- Larger values show more context per page but increase render time.
|
|
111
|
+
|
|
105
112
|
## Database management
|
|
106
113
|
|
|
107
114
|
The SQLite database lives at `LCM_DATABASE_PATH` (default `~/.openclaw/lcm.db`).
|
package/docs/tui.md
CHANGED
|
@@ -72,12 +72,16 @@ A scrollable, color-coded view of the raw session messages. Each message shows i
|
|
|
72
72
|
|
|
73
73
|
This is the raw session data, not the LCM-managed context. Use it to understand what actually happened in the conversation.
|
|
74
74
|
|
|
75
|
+
For sessions with an LCM `conv_id`, the conversation view uses keyset-paged windows by `message_id` (newest window first) instead of hydrating full history.
|
|
76
|
+
|
|
75
77
|
| Key | Action |
|
|
76
78
|
|-----|--------|
|
|
77
79
|
| `↑`/`↓` or `k`/`j` | Scroll one line |
|
|
78
80
|
| `PgUp`/`PgDn` | Scroll half page |
|
|
79
81
|
| `g` | Jump to top |
|
|
80
82
|
| `G` | Jump to bottom |
|
|
83
|
+
| `[` | Load older message window |
|
|
84
|
+
| `]` | Load newer message window |
|
|
81
85
|
| `l` | Open **Summary DAG** view |
|
|
82
86
|
| `c` | Open **Context** view |
|
|
83
87
|
| `f` | Open **Large Files** view |
|
|
@@ -475,6 +479,7 @@ If the provider auth profile mode is `oauth` (not `api_key`), set the provider A
|
|
|
475
479
|
Interactive rewrite (`w`/`W`) can be configured with:
|
|
476
480
|
- `LCM_TUI_SUMMARY_PROVIDER`
|
|
477
481
|
- `LCM_TUI_SUMMARY_MODEL`
|
|
482
|
+
- `LCM_TUI_CONVERSATION_WINDOW_SIZE` (default `200`)
|
|
478
483
|
|
|
479
484
|
It also honors `LCM_SUMMARY_PROVIDER` / `LCM_SUMMARY_MODEL` as fallback.
|
|
480
485
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@martian-engineering/lossless-claw",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"description": "Lossless Context Management plugin for OpenClaw — DAG-based conversation summarization with incremental compaction",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.ts",
|
package/src/compaction.ts
CHANGED
|
@@ -747,6 +747,18 @@ export class CompactionEngine {
|
|
|
747
747
|
return estimateTokens(summary.content);
|
|
748
748
|
}
|
|
749
749
|
|
|
750
|
+
/** Resolve message token count with content-length fallback. */
|
|
751
|
+
private resolveMessageTokenCount(message: { tokenCount: number; content: string }): number {
|
|
752
|
+
if (
|
|
753
|
+
typeof message.tokenCount === "number" &&
|
|
754
|
+
Number.isFinite(message.tokenCount) &&
|
|
755
|
+
message.tokenCount > 0
|
|
756
|
+
) {
|
|
757
|
+
return message.tokenCount;
|
|
758
|
+
}
|
|
759
|
+
return estimateTokens(message.content);
|
|
760
|
+
}
|
|
761
|
+
|
|
750
762
|
private resolveLeafMinFanout(): number {
|
|
751
763
|
if (
|
|
752
764
|
typeof this.config.leafMinFanout === "number" &&
|
|
@@ -995,7 +1007,8 @@ export class CompactionEngine {
|
|
|
995
1007
|
previousSummaryContent?: string,
|
|
996
1008
|
): Promise<{ summaryId: string; level: CompactionLevel; content: string }> {
|
|
997
1009
|
// Fetch full message content for each context item
|
|
998
|
-
const messageContents: { messageId: number; content: string; createdAt: Date }[] =
|
|
1010
|
+
const messageContents: { messageId: number; content: string; createdAt: Date; tokenCount: number }[] =
|
|
1011
|
+
[];
|
|
999
1012
|
for (const item of messageItems) {
|
|
1000
1013
|
if (item.messageId == null) {
|
|
1001
1014
|
continue;
|
|
@@ -1006,6 +1019,7 @@ export class CompactionEngine {
|
|
|
1006
1019
|
messageId: msg.messageId,
|
|
1007
1020
|
content: msg.content,
|
|
1008
1021
|
createdAt: msg.createdAt,
|
|
1022
|
+
tokenCount: this.resolveMessageTokenCount(msg),
|
|
1009
1023
|
});
|
|
1010
1024
|
}
|
|
1011
1025
|
}
|
|
@@ -1046,6 +1060,11 @@ export class CompactionEngine {
|
|
|
1046
1060
|
? new Date(Math.max(...messageContents.map((message) => message.createdAt.getTime())))
|
|
1047
1061
|
: undefined,
|
|
1048
1062
|
descendantCount: 0,
|
|
1063
|
+
descendantTokenCount: 0,
|
|
1064
|
+
sourceMessageTokenCount: messageContents.reduce(
|
|
1065
|
+
(sum, message) => sum + Math.max(0, Math.floor(message.tokenCount)),
|
|
1066
|
+
0,
|
|
1067
|
+
),
|
|
1049
1068
|
});
|
|
1050
1069
|
|
|
1051
1070
|
// Link to source messages
|
|
@@ -1156,6 +1175,22 @@ export class CompactionEngine {
|
|
|
1156
1175
|
: 0;
|
|
1157
1176
|
return count + childDescendants + 1;
|
|
1158
1177
|
}, 0),
|
|
1178
|
+
descendantTokenCount: summaryRecords.reduce((count, summary) => {
|
|
1179
|
+
const childDescendantTokens =
|
|
1180
|
+
typeof summary.descendantTokenCount === "number" &&
|
|
1181
|
+
Number.isFinite(summary.descendantTokenCount)
|
|
1182
|
+
? Math.max(0, Math.floor(summary.descendantTokenCount))
|
|
1183
|
+
: 0;
|
|
1184
|
+
return count + Math.max(0, Math.floor(summary.tokenCount)) + childDescendantTokens;
|
|
1185
|
+
}, 0),
|
|
1186
|
+
sourceMessageTokenCount: summaryRecords.reduce((count, summary) => {
|
|
1187
|
+
const sourceTokens =
|
|
1188
|
+
typeof summary.sourceMessageTokenCount === "number" &&
|
|
1189
|
+
Number.isFinite(summary.sourceMessageTokenCount)
|
|
1190
|
+
? Math.max(0, Math.floor(summary.sourceMessageTokenCount))
|
|
1191
|
+
: 0;
|
|
1192
|
+
return count + sourceTokens;
|
|
1193
|
+
}, 0),
|
|
1159
1194
|
});
|
|
1160
1195
|
|
|
1161
1196
|
// Link to parent summaries
|
package/src/db/migration.ts
CHANGED
|
@@ -9,6 +9,7 @@ type SummaryDepthRow = {
|
|
|
9
9
|
conversation_id: number;
|
|
10
10
|
kind: "leaf" | "condensed";
|
|
11
11
|
depth: number;
|
|
12
|
+
token_count: number;
|
|
12
13
|
created_at: string;
|
|
13
14
|
};
|
|
14
15
|
|
|
@@ -16,6 +17,7 @@ type SummaryMessageTimeRangeRow = {
|
|
|
16
17
|
summary_id: string;
|
|
17
18
|
earliest_at: string | null;
|
|
18
19
|
latest_at: string | null;
|
|
20
|
+
source_message_token_count: number | null;
|
|
19
21
|
};
|
|
20
22
|
|
|
21
23
|
type SummaryParentEdgeRow = {
|
|
@@ -36,6 +38,10 @@ function ensureSummaryMetadataColumns(db: DatabaseSync): void {
|
|
|
36
38
|
const hasEarliestAt = summaryColumns.some((col) => col.name === "earliest_at");
|
|
37
39
|
const hasLatestAt = summaryColumns.some((col) => col.name === "latest_at");
|
|
38
40
|
const hasDescendantCount = summaryColumns.some((col) => col.name === "descendant_count");
|
|
41
|
+
const hasDescendantTokenCount = summaryColumns.some((col) => col.name === "descendant_token_count");
|
|
42
|
+
const hasSourceMessageTokenCount = summaryColumns.some(
|
|
43
|
+
(col) => col.name === "source_message_token_count",
|
|
44
|
+
);
|
|
39
45
|
|
|
40
46
|
if (!hasEarliestAt) {
|
|
41
47
|
db.exec(`ALTER TABLE summaries ADD COLUMN earliest_at TEXT`);
|
|
@@ -46,6 +52,12 @@ function ensureSummaryMetadataColumns(db: DatabaseSync): void {
|
|
|
46
52
|
if (!hasDescendantCount) {
|
|
47
53
|
db.exec(`ALTER TABLE summaries ADD COLUMN descendant_count INTEGER NOT NULL DEFAULT 0`);
|
|
48
54
|
}
|
|
55
|
+
if (!hasDescendantTokenCount) {
|
|
56
|
+
db.exec(`ALTER TABLE summaries ADD COLUMN descendant_token_count INTEGER NOT NULL DEFAULT 0`);
|
|
57
|
+
}
|
|
58
|
+
if (!hasSourceMessageTokenCount) {
|
|
59
|
+
db.exec(`ALTER TABLE summaries ADD COLUMN source_message_token_count INTEGER NOT NULL DEFAULT 0`);
|
|
60
|
+
}
|
|
49
61
|
}
|
|
50
62
|
|
|
51
63
|
function parseTimestamp(value: string | null | undefined): Date | null {
|
|
@@ -84,7 +96,7 @@ function backfillSummaryDepths(db: DatabaseSync): void {
|
|
|
84
96
|
const conversationId = row.conversation_id;
|
|
85
97
|
const summaries = db
|
|
86
98
|
.prepare(
|
|
87
|
-
`SELECT summary_id, conversation_id, kind, depth, created_at
|
|
99
|
+
`SELECT summary_id, conversation_id, kind, depth, token_count, created_at
|
|
88
100
|
FROM summaries
|
|
89
101
|
WHERE conversation_id = ?`,
|
|
90
102
|
)
|
|
@@ -180,7 +192,8 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
180
192
|
|
|
181
193
|
const updateMetadataStmt = db.prepare(
|
|
182
194
|
`UPDATE summaries
|
|
183
|
-
SET earliest_at = ?, latest_at = ?, descendant_count =
|
|
195
|
+
SET earliest_at = ?, latest_at = ?, descendant_count = ?,
|
|
196
|
+
descendant_token_count = ?, source_message_token_count = ?
|
|
184
197
|
WHERE summary_id = ?`,
|
|
185
198
|
);
|
|
186
199
|
|
|
@@ -188,7 +201,7 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
188
201
|
const conversationId = conversationRow.conversation_id;
|
|
189
202
|
const summaries = db
|
|
190
203
|
.prepare(
|
|
191
|
-
`SELECT summary_id, conversation_id, kind, depth, created_at
|
|
204
|
+
`SELECT summary_id, conversation_id, kind, depth, token_count, created_at
|
|
192
205
|
FROM summaries
|
|
193
206
|
WHERE conversation_id = ?
|
|
194
207
|
ORDER BY depth ASC, created_at ASC`,
|
|
@@ -200,7 +213,11 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
200
213
|
|
|
201
214
|
const leafRanges = db
|
|
202
215
|
.prepare(
|
|
203
|
-
`SELECT
|
|
216
|
+
`SELECT
|
|
217
|
+
sm.summary_id,
|
|
218
|
+
MIN(m.created_at) AS earliest_at,
|
|
219
|
+
MAX(m.created_at) AS latest_at,
|
|
220
|
+
COALESCE(SUM(m.token_count), 0) AS source_message_token_count
|
|
204
221
|
FROM summary_messages sm
|
|
205
222
|
JOIN messages m ON m.message_id = sm.message_id
|
|
206
223
|
JOIN summaries s ON s.summary_id = sm.summary_id
|
|
@@ -209,7 +226,14 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
209
226
|
)
|
|
210
227
|
.all(conversationId) as SummaryMessageTimeRangeRow[];
|
|
211
228
|
const leafRangeBySummaryId = new Map(
|
|
212
|
-
leafRanges.map((row) => [
|
|
229
|
+
leafRanges.map((row) => [
|
|
230
|
+
row.summary_id,
|
|
231
|
+
{
|
|
232
|
+
earliestAt: row.earliest_at,
|
|
233
|
+
latestAt: row.latest_at,
|
|
234
|
+
sourceMessageTokenCount: row.source_message_token_count,
|
|
235
|
+
},
|
|
236
|
+
]),
|
|
213
237
|
);
|
|
214
238
|
|
|
215
239
|
const edges = db
|
|
@@ -230,8 +254,17 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
230
254
|
|
|
231
255
|
const metadataBySummaryId = new Map<
|
|
232
256
|
string,
|
|
233
|
-
{
|
|
257
|
+
{
|
|
258
|
+
earliestAt: Date | null;
|
|
259
|
+
latestAt: Date | null;
|
|
260
|
+
descendantCount: number;
|
|
261
|
+
descendantTokenCount: number;
|
|
262
|
+
sourceMessageTokenCount: number;
|
|
263
|
+
}
|
|
234
264
|
>();
|
|
265
|
+
const tokenCountBySummaryId = new Map(
|
|
266
|
+
summaries.map((summary) => [summary.summary_id, Math.max(0, Math.floor(summary.token_count ?? 0))]),
|
|
267
|
+
);
|
|
235
268
|
|
|
236
269
|
for (const summary of summaries) {
|
|
237
270
|
const fallbackDate = parseTimestamp(summary.created_at);
|
|
@@ -244,6 +277,11 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
244
277
|
earliestAt,
|
|
245
278
|
latestAt,
|
|
246
279
|
descendantCount: 0,
|
|
280
|
+
descendantTokenCount: 0,
|
|
281
|
+
sourceMessageTokenCount: Math.max(
|
|
282
|
+
0,
|
|
283
|
+
Math.floor(range?.sourceMessageTokenCount ?? 0),
|
|
284
|
+
),
|
|
247
285
|
});
|
|
248
286
|
continue;
|
|
249
287
|
}
|
|
@@ -254,6 +292,8 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
254
292
|
earliestAt: fallbackDate,
|
|
255
293
|
latestAt: fallbackDate,
|
|
256
294
|
descendantCount: 0,
|
|
295
|
+
descendantTokenCount: 0,
|
|
296
|
+
sourceMessageTokenCount: 0,
|
|
257
297
|
});
|
|
258
298
|
continue;
|
|
259
299
|
}
|
|
@@ -261,6 +301,8 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
261
301
|
let earliestAt: Date | null = null;
|
|
262
302
|
let latestAt: Date | null = null;
|
|
263
303
|
let descendantCount = 0;
|
|
304
|
+
let descendantTokenCount = 0;
|
|
305
|
+
let sourceMessageTokenCount = 0;
|
|
264
306
|
|
|
265
307
|
for (const parentId of parentIds) {
|
|
266
308
|
const parentMetadata = metadataBySummaryId.get(parentId);
|
|
@@ -279,12 +321,18 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
279
321
|
}
|
|
280
322
|
|
|
281
323
|
descendantCount += Math.max(0, parentMetadata.descendantCount) + 1;
|
|
324
|
+
const parentTokenCount = tokenCountBySummaryId.get(parentId) ?? 0;
|
|
325
|
+
descendantTokenCount +=
|
|
326
|
+
Math.max(0, parentTokenCount) + Math.max(0, parentMetadata.descendantTokenCount);
|
|
327
|
+
sourceMessageTokenCount += Math.max(0, parentMetadata.sourceMessageTokenCount);
|
|
282
328
|
}
|
|
283
329
|
|
|
284
330
|
metadataBySummaryId.set(summary.summary_id, {
|
|
285
331
|
earliestAt: earliestAt ?? fallbackDate,
|
|
286
332
|
latestAt: latestAt ?? fallbackDate,
|
|
287
333
|
descendantCount: Math.max(0, descendantCount),
|
|
334
|
+
descendantTokenCount: Math.max(0, descendantTokenCount),
|
|
335
|
+
sourceMessageTokenCount: Math.max(0, sourceMessageTokenCount),
|
|
288
336
|
});
|
|
289
337
|
}
|
|
290
338
|
|
|
@@ -298,6 +346,8 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
298
346
|
isoStringOrNull(metadata.earliestAt),
|
|
299
347
|
isoStringOrNull(metadata.latestAt),
|
|
300
348
|
Math.max(0, metadata.descendantCount),
|
|
349
|
+
Math.max(0, metadata.descendantTokenCount),
|
|
350
|
+
Math.max(0, metadata.sourceMessageTokenCount),
|
|
301
351
|
summary.summary_id,
|
|
302
352
|
);
|
|
303
353
|
}
|
|
@@ -336,6 +386,8 @@ export function runLcmMigrations(db: DatabaseSync): void {
|
|
|
336
386
|
earliest_at TEXT,
|
|
337
387
|
latest_at TEXT,
|
|
338
388
|
descendant_count INTEGER NOT NULL DEFAULT 0,
|
|
389
|
+
descendant_token_count INTEGER NOT NULL DEFAULT 0,
|
|
390
|
+
source_message_token_count INTEGER NOT NULL DEFAULT 0,
|
|
339
391
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
340
392
|
file_ids TEXT NOT NULL DEFAULT '[]'
|
|
341
393
|
);
|
package/src/expansion-auth.ts
CHANGED
|
@@ -56,6 +56,7 @@ const DEFAULT_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
|
|
56
56
|
|
|
57
57
|
export class ExpansionAuthManager {
|
|
58
58
|
private grants: Map<string, ExpansionGrant> = new Map();
|
|
59
|
+
private consumedTokensByGrantId: Map<string, number> = new Map();
|
|
59
60
|
|
|
60
61
|
/**
|
|
61
62
|
* Create a new expansion grant with the given parameters.
|
|
@@ -79,6 +80,7 @@ export class ExpansionAuthManager {
|
|
|
79
80
|
};
|
|
80
81
|
|
|
81
82
|
this.grants.set(grantId, grant);
|
|
83
|
+
this.consumedTokensByGrantId.set(grantId, 0);
|
|
82
84
|
return grant;
|
|
83
85
|
}
|
|
84
86
|
|
|
@@ -113,6 +115,36 @@ export class ExpansionAuthManager {
|
|
|
113
115
|
return true;
|
|
114
116
|
}
|
|
115
117
|
|
|
118
|
+
/**
|
|
119
|
+
* Resolve remaining token budget for an active grant.
|
|
120
|
+
*/
|
|
121
|
+
getRemainingTokenBudget(grantId: string): number | null {
|
|
122
|
+
const grant = this.getGrant(grantId);
|
|
123
|
+
if (!grant) {
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
const consumed = Math.max(0, this.consumedTokensByGrantId.get(grantId) ?? 0);
|
|
127
|
+
return Math.max(0, Math.floor(grant.tokenCap) - consumed);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Consume token budget for a grant, clamped to the grant token cap.
|
|
132
|
+
*/
|
|
133
|
+
consumeTokenBudget(grantId: string, consumedTokens: number): number | null {
|
|
134
|
+
const grant = this.getGrant(grantId);
|
|
135
|
+
if (!grant) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
const safeConsumed =
|
|
139
|
+
typeof consumedTokens === "number" && Number.isFinite(consumedTokens)
|
|
140
|
+
? Math.max(0, Math.floor(consumedTokens))
|
|
141
|
+
: 0;
|
|
142
|
+
const previous = Math.max(0, this.consumedTokensByGrantId.get(grantId) ?? 0);
|
|
143
|
+
const next = Math.min(Math.max(1, Math.floor(grant.tokenCap)), previous + safeConsumed);
|
|
144
|
+
this.consumedTokensByGrantId.set(grantId, next);
|
|
145
|
+
return Math.max(0, Math.floor(grant.tokenCap) - next);
|
|
146
|
+
}
|
|
147
|
+
|
|
116
148
|
/**
|
|
117
149
|
* Validate an expansion request against a grant.
|
|
118
150
|
* Checks existence, expiry, revocation, conversation scope, and summary scope.
|
|
@@ -177,6 +209,7 @@ export class ExpansionAuthManager {
|
|
|
177
209
|
for (const [grantId, grant] of this.grants) {
|
|
178
210
|
if (grant.revoked || grant.expiresAt.getTime() <= now) {
|
|
179
211
|
this.grants.delete(grantId);
|
|
212
|
+
this.consumedTokensByGrantId.delete(grantId);
|
|
180
213
|
removed++;
|
|
181
214
|
}
|
|
182
215
|
}
|
|
@@ -293,7 +326,26 @@ export function wrapWithAuth(
|
|
|
293
326
|
if (!validation.valid) {
|
|
294
327
|
throw new Error(`Expansion authorization failed: ${validation.reason}`);
|
|
295
328
|
}
|
|
296
|
-
|
|
329
|
+
|
|
330
|
+
const remainingBudget = authManager.getRemainingTokenBudget(grantId);
|
|
331
|
+
if (remainingBudget == null) {
|
|
332
|
+
throw new Error("Expansion authorization failed: Grant not found");
|
|
333
|
+
}
|
|
334
|
+
if (remainingBudget <= 0) {
|
|
335
|
+
throw new Error("Expansion authorization failed: Grant token budget exhausted");
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
const requestedTokenCap =
|
|
339
|
+
typeof request.tokenCap === "number" && Number.isFinite(request.tokenCap)
|
|
340
|
+
? Math.max(1, Math.trunc(request.tokenCap))
|
|
341
|
+
: remainingBudget;
|
|
342
|
+
const effectiveTokenCap = Math.max(1, Math.min(requestedTokenCap, remainingBudget));
|
|
343
|
+
const result = await orchestrator.expand({
|
|
344
|
+
...request,
|
|
345
|
+
tokenCap: effectiveTokenCap,
|
|
346
|
+
});
|
|
347
|
+
authManager.consumeTokenBudget(grantId, result.totalTokens);
|
|
348
|
+
return result;
|
|
297
349
|
},
|
|
298
350
|
};
|
|
299
351
|
}
|
package/src/retrieval.ts
CHANGED
|
@@ -20,11 +20,32 @@ export interface DescribeResult {
|
|
|
20
20
|
conversationId: number;
|
|
21
21
|
kind: "leaf" | "condensed";
|
|
22
22
|
content: string;
|
|
23
|
+
depth: number;
|
|
23
24
|
tokenCount: number;
|
|
25
|
+
descendantCount: number;
|
|
26
|
+
descendantTokenCount: number;
|
|
27
|
+
sourceMessageTokenCount: number;
|
|
24
28
|
fileIds: string[];
|
|
25
29
|
parentIds: string[];
|
|
26
30
|
childIds: string[];
|
|
27
31
|
messageIds: number[];
|
|
32
|
+
earliestAt: Date | null;
|
|
33
|
+
latestAt: Date | null;
|
|
34
|
+
subtree: Array<{
|
|
35
|
+
summaryId: string;
|
|
36
|
+
parentSummaryId: string | null;
|
|
37
|
+
depthFromRoot: number;
|
|
38
|
+
kind: "leaf" | "condensed";
|
|
39
|
+
depth: number;
|
|
40
|
+
tokenCount: number;
|
|
41
|
+
descendantCount: number;
|
|
42
|
+
descendantTokenCount: number;
|
|
43
|
+
sourceMessageTokenCount: number;
|
|
44
|
+
earliestAt: Date | null;
|
|
45
|
+
latestAt: Date | null;
|
|
46
|
+
childCount: number;
|
|
47
|
+
path: string;
|
|
48
|
+
}>;
|
|
28
49
|
createdAt: Date;
|
|
29
50
|
};
|
|
30
51
|
/** File-specific fields */
|
|
@@ -127,10 +148,11 @@ export class RetrievalEngine {
|
|
|
127
148
|
}
|
|
128
149
|
|
|
129
150
|
// Fetch lineage in parallel
|
|
130
|
-
const [parents, children, messageIds] = await Promise.all([
|
|
151
|
+
const [parents, children, messageIds, subtree] = await Promise.all([
|
|
131
152
|
this.summaryStore.getSummaryParents(id),
|
|
132
153
|
this.summaryStore.getSummaryChildren(id),
|
|
133
154
|
this.summaryStore.getSummaryMessages(id),
|
|
155
|
+
this.summaryStore.getSummarySubtree(id),
|
|
134
156
|
]);
|
|
135
157
|
|
|
136
158
|
return {
|
|
@@ -140,11 +162,32 @@ export class RetrievalEngine {
|
|
|
140
162
|
conversationId: summary.conversationId,
|
|
141
163
|
kind: summary.kind,
|
|
142
164
|
content: summary.content,
|
|
165
|
+
depth: summary.depth,
|
|
143
166
|
tokenCount: summary.tokenCount,
|
|
167
|
+
descendantCount: summary.descendantCount,
|
|
168
|
+
descendantTokenCount: summary.descendantTokenCount,
|
|
169
|
+
sourceMessageTokenCount: summary.sourceMessageTokenCount,
|
|
144
170
|
fileIds: summary.fileIds,
|
|
145
171
|
parentIds: parents.map((p) => p.summaryId),
|
|
146
172
|
childIds: children.map((c) => c.summaryId),
|
|
147
173
|
messageIds,
|
|
174
|
+
earliestAt: summary.earliestAt,
|
|
175
|
+
latestAt: summary.latestAt,
|
|
176
|
+
subtree: subtree.map((node) => ({
|
|
177
|
+
summaryId: node.summaryId,
|
|
178
|
+
parentSummaryId: node.parentSummaryId,
|
|
179
|
+
depthFromRoot: node.depthFromRoot,
|
|
180
|
+
kind: node.kind,
|
|
181
|
+
depth: node.depth,
|
|
182
|
+
tokenCount: node.tokenCount,
|
|
183
|
+
descendantCount: node.descendantCount,
|
|
184
|
+
descendantTokenCount: node.descendantTokenCount,
|
|
185
|
+
sourceMessageTokenCount: node.sourceMessageTokenCount,
|
|
186
|
+
earliestAt: node.earliestAt,
|
|
187
|
+
latestAt: node.latestAt,
|
|
188
|
+
childCount: node.childCount,
|
|
189
|
+
path: node.path,
|
|
190
|
+
})),
|
|
148
191
|
createdAt: summary.createdAt,
|
|
149
192
|
},
|
|
150
193
|
};
|
|
@@ -15,6 +15,8 @@ export type CreateSummaryInput = {
|
|
|
15
15
|
earliestAt?: Date;
|
|
16
16
|
latestAt?: Date;
|
|
17
17
|
descendantCount?: number;
|
|
18
|
+
descendantTokenCount?: number;
|
|
19
|
+
sourceMessageTokenCount?: number;
|
|
18
20
|
};
|
|
19
21
|
|
|
20
22
|
export type SummaryRecord = {
|
|
@@ -28,9 +30,18 @@ export type SummaryRecord = {
|
|
|
28
30
|
earliestAt: Date | null;
|
|
29
31
|
latestAt: Date | null;
|
|
30
32
|
descendantCount: number;
|
|
33
|
+
descendantTokenCount: number;
|
|
34
|
+
sourceMessageTokenCount: number;
|
|
31
35
|
createdAt: Date;
|
|
32
36
|
};
|
|
33
37
|
|
|
38
|
+
export type SummarySubtreeNodeRecord = SummaryRecord & {
|
|
39
|
+
depthFromRoot: number;
|
|
40
|
+
parentSummaryId: string | null;
|
|
41
|
+
path: string;
|
|
42
|
+
childCount: number;
|
|
43
|
+
};
|
|
44
|
+
|
|
34
45
|
export type ContextItemRecord = {
|
|
35
46
|
conversationId: number;
|
|
36
47
|
ordinal: number;
|
|
@@ -92,9 +103,18 @@ interface SummaryRow {
|
|
|
92
103
|
earliest_at: string | null;
|
|
93
104
|
latest_at: string | null;
|
|
94
105
|
descendant_count: number | null;
|
|
106
|
+
descendant_token_count: number | null;
|
|
107
|
+
source_message_token_count: number | null;
|
|
95
108
|
created_at: string;
|
|
96
109
|
}
|
|
97
110
|
|
|
111
|
+
interface SummarySubtreeRow extends SummaryRow {
|
|
112
|
+
depth_from_root: number;
|
|
113
|
+
parent_summary_id: string | null;
|
|
114
|
+
path: string;
|
|
115
|
+
child_count: number | null;
|
|
116
|
+
}
|
|
117
|
+
|
|
98
118
|
interface ContextItemRow {
|
|
99
119
|
conversation_id: number;
|
|
100
120
|
ordinal: number;
|
|
@@ -165,6 +185,18 @@ function toSummaryRecord(row: SummaryRow): SummaryRecord {
|
|
|
165
185
|
row.descendant_count >= 0
|
|
166
186
|
? Math.floor(row.descendant_count)
|
|
167
187
|
: 0,
|
|
188
|
+
descendantTokenCount:
|
|
189
|
+
typeof row.descendant_token_count === "number" &&
|
|
190
|
+
Number.isFinite(row.descendant_token_count) &&
|
|
191
|
+
row.descendant_token_count >= 0
|
|
192
|
+
? Math.floor(row.descendant_token_count)
|
|
193
|
+
: 0,
|
|
194
|
+
sourceMessageTokenCount:
|
|
195
|
+
typeof row.source_message_token_count === "number" &&
|
|
196
|
+
Number.isFinite(row.source_message_token_count) &&
|
|
197
|
+
row.source_message_token_count >= 0
|
|
198
|
+
? Math.floor(row.source_message_token_count)
|
|
199
|
+
: 0,
|
|
168
200
|
createdAt: new Date(row.created_at),
|
|
169
201
|
};
|
|
170
202
|
}
|
|
@@ -221,6 +253,18 @@ export class SummaryStore {
|
|
|
221
253
|
input.descendantCount >= 0
|
|
222
254
|
? Math.floor(input.descendantCount)
|
|
223
255
|
: 0;
|
|
256
|
+
const descendantTokenCount =
|
|
257
|
+
typeof input.descendantTokenCount === "number" &&
|
|
258
|
+
Number.isFinite(input.descendantTokenCount) &&
|
|
259
|
+
input.descendantTokenCount >= 0
|
|
260
|
+
? Math.floor(input.descendantTokenCount)
|
|
261
|
+
: 0;
|
|
262
|
+
const sourceMessageTokenCount =
|
|
263
|
+
typeof input.sourceMessageTokenCount === "number" &&
|
|
264
|
+
Number.isFinite(input.sourceMessageTokenCount) &&
|
|
265
|
+
input.sourceMessageTokenCount >= 0
|
|
266
|
+
? Math.floor(input.sourceMessageTokenCount)
|
|
267
|
+
: 0;
|
|
224
268
|
const depth =
|
|
225
269
|
typeof input.depth === "number" && Number.isFinite(input.depth) && input.depth >= 0
|
|
226
270
|
? Math.floor(input.depth)
|
|
@@ -240,9 +284,11 @@ export class SummaryStore {
|
|
|
240
284
|
file_ids,
|
|
241
285
|
earliest_at,
|
|
242
286
|
latest_at,
|
|
243
|
-
descendant_count
|
|
287
|
+
descendant_count,
|
|
288
|
+
descendant_token_count,
|
|
289
|
+
source_message_token_count
|
|
244
290
|
)
|
|
245
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
291
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
246
292
|
)
|
|
247
293
|
.run(
|
|
248
294
|
input.summaryId,
|
|
@@ -255,6 +301,8 @@ export class SummaryStore {
|
|
|
255
301
|
earliestAt,
|
|
256
302
|
latestAt,
|
|
257
303
|
descendantCount,
|
|
304
|
+
descendantTokenCount,
|
|
305
|
+
sourceMessageTokenCount,
|
|
258
306
|
);
|
|
259
307
|
|
|
260
308
|
// Index in FTS5 as best-effort; compaction flow must continue even if
|
|
@@ -272,6 +320,7 @@ export class SummaryStore {
|
|
|
272
320
|
.prepare(
|
|
273
321
|
`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
274
322
|
earliest_at, latest_at, descendant_count, created_at
|
|
323
|
+
, descendant_token_count, source_message_token_count
|
|
275
324
|
FROM summaries WHERE summary_id = ?`,
|
|
276
325
|
)
|
|
277
326
|
.get(input.summaryId) as unknown as SummaryRow;
|
|
@@ -284,6 +333,7 @@ export class SummaryStore {
|
|
|
284
333
|
.prepare(
|
|
285
334
|
`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
286
335
|
earliest_at, latest_at, descendant_count, created_at
|
|
336
|
+
, descendant_token_count, source_message_token_count
|
|
287
337
|
FROM summaries WHERE summary_id = ?`,
|
|
288
338
|
)
|
|
289
339
|
.get(summaryId) as unknown as SummaryRow | undefined;
|
|
@@ -295,6 +345,7 @@ export class SummaryStore {
|
|
|
295
345
|
.prepare(
|
|
296
346
|
`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
297
347
|
earliest_at, latest_at, descendant_count, created_at
|
|
348
|
+
, descendant_token_count, source_message_token_count
|
|
298
349
|
FROM summaries
|
|
299
350
|
WHERE conversation_id = ?
|
|
300
351
|
ORDER BY created_at`,
|
|
@@ -353,6 +404,7 @@ export class SummaryStore {
|
|
|
353
404
|
.prepare(
|
|
354
405
|
`SELECT s.summary_id, s.conversation_id, s.kind, s.depth, s.content, s.token_count,
|
|
355
406
|
s.file_ids, s.earliest_at, s.latest_at, s.descendant_count, s.created_at
|
|
407
|
+
, s.descendant_token_count, s.source_message_token_count
|
|
356
408
|
FROM summaries s
|
|
357
409
|
JOIN summary_parents sp ON sp.summary_id = s.summary_id
|
|
358
410
|
WHERE sp.parent_summary_id = ?
|
|
@@ -367,6 +419,7 @@ export class SummaryStore {
|
|
|
367
419
|
.prepare(
|
|
368
420
|
`SELECT s.summary_id, s.conversation_id, s.kind, s.depth, s.content, s.token_count,
|
|
369
421
|
s.file_ids, s.earliest_at, s.latest_at, s.descendant_count, s.created_at
|
|
422
|
+
, s.descendant_token_count, s.source_message_token_count
|
|
370
423
|
FROM summaries s
|
|
371
424
|
JOIN summary_parents sp ON sp.parent_summary_id = s.summary_id
|
|
372
425
|
WHERE sp.summary_id = ?
|
|
@@ -376,6 +429,71 @@ export class SummaryStore {
|
|
|
376
429
|
return rows.map(toSummaryRecord);
|
|
377
430
|
}
|
|
378
431
|
|
|
432
|
+
async getSummarySubtree(summaryId: string): Promise<SummarySubtreeNodeRecord[]> {
|
|
433
|
+
const rows = this.db
|
|
434
|
+
.prepare(
|
|
435
|
+
`WITH RECURSIVE subtree(summary_id, parent_summary_id, depth_from_root, path) AS (
|
|
436
|
+
SELECT ?, NULL, 0, ''
|
|
437
|
+
UNION ALL
|
|
438
|
+
SELECT
|
|
439
|
+
sp.summary_id,
|
|
440
|
+
sp.parent_summary_id,
|
|
441
|
+
subtree.depth_from_root + 1,
|
|
442
|
+
CASE
|
|
443
|
+
WHEN subtree.path = '' THEN printf('%04d', sp.ordinal)
|
|
444
|
+
ELSE subtree.path || '.' || printf('%04d', sp.ordinal)
|
|
445
|
+
END
|
|
446
|
+
FROM summary_parents sp
|
|
447
|
+
JOIN subtree ON sp.parent_summary_id = subtree.summary_id
|
|
448
|
+
)
|
|
449
|
+
SELECT
|
|
450
|
+
s.summary_id,
|
|
451
|
+
s.conversation_id,
|
|
452
|
+
s.kind,
|
|
453
|
+
s.depth,
|
|
454
|
+
s.content,
|
|
455
|
+
s.token_count,
|
|
456
|
+
s.file_ids,
|
|
457
|
+
s.earliest_at,
|
|
458
|
+
s.latest_at,
|
|
459
|
+
s.descendant_count,
|
|
460
|
+
s.descendant_token_count,
|
|
461
|
+
s.source_message_token_count,
|
|
462
|
+
s.created_at,
|
|
463
|
+
subtree.depth_from_root,
|
|
464
|
+
subtree.parent_summary_id,
|
|
465
|
+
subtree.path,
|
|
466
|
+
(
|
|
467
|
+
SELECT COUNT(*) FROM summary_parents sp2
|
|
468
|
+
WHERE sp2.parent_summary_id = s.summary_id
|
|
469
|
+
) AS child_count
|
|
470
|
+
FROM subtree
|
|
471
|
+
JOIN summaries s ON s.summary_id = subtree.summary_id
|
|
472
|
+
ORDER BY subtree.depth_from_root ASC, subtree.path ASC, s.created_at ASC`,
|
|
473
|
+
)
|
|
474
|
+
.all(summaryId) as unknown as SummarySubtreeRow[];
|
|
475
|
+
|
|
476
|
+
const seen = new Set<string>();
|
|
477
|
+
const output: SummarySubtreeNodeRecord[] = [];
|
|
478
|
+
for (const row of rows) {
|
|
479
|
+
if (seen.has(row.summary_id)) {
|
|
480
|
+
continue;
|
|
481
|
+
}
|
|
482
|
+
seen.add(row.summary_id);
|
|
483
|
+
output.push({
|
|
484
|
+
...toSummaryRecord(row),
|
|
485
|
+
depthFromRoot: Math.max(0, Math.floor(row.depth_from_root ?? 0)),
|
|
486
|
+
parentSummaryId: row.parent_summary_id ?? null,
|
|
487
|
+
path: typeof row.path === "string" ? row.path : "",
|
|
488
|
+
childCount:
|
|
489
|
+
typeof row.child_count === "number" && Number.isFinite(row.child_count)
|
|
490
|
+
? Math.max(0, Math.floor(row.child_count))
|
|
491
|
+
: 0,
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
return output;
|
|
495
|
+
}
|
|
496
|
+
|
|
379
497
|
// ── Context items ─────────────────────────────────────────────────────────
|
|
380
498
|
|
|
381
499
|
async getContextItems(conversationId: number): Promise<ContextItemRecord[]> {
|
|
@@ -644,7 +762,8 @@ export class SummaryStore {
|
|
|
644
762
|
const rows = this.db
|
|
645
763
|
.prepare(
|
|
646
764
|
`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
647
|
-
earliest_at, latest_at, descendant_count,
|
|
765
|
+
earliest_at, latest_at, descendant_count, descendant_token_count,
|
|
766
|
+
source_message_token_count, created_at
|
|
648
767
|
FROM summaries
|
|
649
768
|
${whereClause}
|
|
650
769
|
ORDER BY created_at DESC`,
|