@martian-engineering/lossless-claw 0.7.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +19 -3
  2. package/dist/index.js +19240 -0
  3. package/docs/agent-tools.md +9 -4
  4. package/docs/configuration.md +24 -5
  5. package/openclaw.plugin.json +27 -3
  6. package/package.json +7 -6
  7. package/skills/lossless-claw/SKILL.md +3 -2
  8. package/skills/lossless-claw/references/architecture.md +12 -0
  9. package/skills/lossless-claw/references/config.md +37 -0
  10. package/skills/lossless-claw/references/diagnostics.md +13 -0
  11. package/index.ts +0 -2
  12. package/src/assembler.ts +0 -1188
  13. package/src/compaction.ts +0 -1756
  14. package/src/db/config.ts +0 -345
  15. package/src/db/connection.ts +0 -141
  16. package/src/db/features.ts +0 -42
  17. package/src/db/migration.ts +0 -746
  18. package/src/engine.ts +0 -4306
  19. package/src/expansion-auth.ts +0 -365
  20. package/src/expansion-policy.ts +0 -303
  21. package/src/expansion.ts +0 -383
  22. package/src/integrity.ts +0 -600
  23. package/src/large-files.ts +0 -546
  24. package/src/lcm-log.ts +0 -37
  25. package/src/openclaw-bridge.ts +0 -22
  26. package/src/plugin/index.ts +0 -1960
  27. package/src/plugin/lcm-command.ts +0 -765
  28. package/src/plugin/lcm-doctor-apply.ts +0 -542
  29. package/src/plugin/lcm-doctor-shared.ts +0 -210
  30. package/src/plugin/shared-init.ts +0 -59
  31. package/src/prune.ts +0 -391
  32. package/src/retrieval.ts +0 -363
  33. package/src/session-patterns.ts +0 -23
  34. package/src/startup-banner-log.ts +0 -49
  35. package/src/store/compaction-telemetry-store.ts +0 -156
  36. package/src/store/conversation-store.ts +0 -929
  37. package/src/store/fts5-sanitize.ts +0 -50
  38. package/src/store/full-text-fallback.ts +0 -83
  39. package/src/store/full-text-sort.ts +0 -21
  40. package/src/store/index.ts +0 -39
  41. package/src/store/parse-utc-timestamp.ts +0 -25
  42. package/src/store/summary-store.ts +0 -1519
  43. package/src/summarize.ts +0 -1511
  44. package/src/tools/common.ts +0 -53
  45. package/src/tools/lcm-conversation-scope.ts +0 -127
  46. package/src/tools/lcm-describe-tool.ts +0 -245
  47. package/src/tools/lcm-expand-query-tool.ts +0 -831
  48. package/src/tools/lcm-expand-tool.delegation.ts +0 -580
  49. package/src/tools/lcm-expand-tool.ts +0 -453
  50. package/src/tools/lcm-expansion-recursion-guard.ts +0 -373
  51. package/src/tools/lcm-grep-tool.ts +0 -228
  52. package/src/transaction-mutex.ts +0 -136
  53. package/src/transcript-repair.ts +0 -301
  54. package/src/types.ts +0 -165
@@ -1,746 +0,0 @@
1
- import type { DatabaseSync } from "node:sqlite";
2
- import { getLcmDbFeatures } from "./features.js";
3
- import { parseUtcTimestampOrNull } from "../store/parse-utc-timestamp.js";
4
-
5
- type SummaryColumnInfo = {
6
- name?: string;
7
- };
8
-
9
- type SummaryDepthRow = {
10
- summary_id: string;
11
- conversation_id: number;
12
- kind: "leaf" | "condensed";
13
- depth: number;
14
- token_count: number;
15
- created_at: string;
16
- };
17
-
18
- type SummaryMessageTimeRangeRow = {
19
- summary_id: string;
20
- earliest_at: string | null;
21
- latest_at: string | null;
22
- source_message_token_count: number | null;
23
- };
24
-
25
- type SummaryParentEdgeRow = {
26
- summary_id: string;
27
- parent_summary_id: string;
28
- };
29
-
30
- function ensureSummaryDepthColumn(db: DatabaseSync): void {
31
- const summaryColumns = db.prepare(`PRAGMA table_info(summaries)`).all() as SummaryColumnInfo[];
32
- const hasDepth = summaryColumns.some((col) => col.name === "depth");
33
- if (!hasDepth) {
34
- db.exec(`ALTER TABLE summaries ADD COLUMN depth INTEGER NOT NULL DEFAULT 0`);
35
- }
36
- }
37
-
38
- function ensureSummaryMetadataColumns(db: DatabaseSync): void {
39
- const summaryColumns = db.prepare(`PRAGMA table_info(summaries)`).all() as SummaryColumnInfo[];
40
- const hasEarliestAt = summaryColumns.some((col) => col.name === "earliest_at");
41
- const hasLatestAt = summaryColumns.some((col) => col.name === "latest_at");
42
- const hasDescendantCount = summaryColumns.some((col) => col.name === "descendant_count");
43
- const hasDescendantTokenCount = summaryColumns.some((col) => col.name === "descendant_token_count");
44
- const hasSourceMessageTokenCount = summaryColumns.some(
45
- (col) => col.name === "source_message_token_count",
46
- );
47
-
48
- if (!hasEarliestAt) {
49
- db.exec(`ALTER TABLE summaries ADD COLUMN earliest_at TEXT`);
50
- }
51
- if (!hasLatestAt) {
52
- db.exec(`ALTER TABLE summaries ADD COLUMN latest_at TEXT`);
53
- }
54
- if (!hasDescendantCount) {
55
- db.exec(`ALTER TABLE summaries ADD COLUMN descendant_count INTEGER NOT NULL DEFAULT 0`);
56
- }
57
- if (!hasDescendantTokenCount) {
58
- db.exec(`ALTER TABLE summaries ADD COLUMN descendant_token_count INTEGER NOT NULL DEFAULT 0`);
59
- }
60
- if (!hasSourceMessageTokenCount) {
61
- db.exec(`ALTER TABLE summaries ADD COLUMN source_message_token_count INTEGER NOT NULL DEFAULT 0`);
62
- }
63
- }
64
-
65
- function parseTimestamp(value: string | null | undefined): Date | null {
66
- return parseUtcTimestampOrNull(value);
67
- }
68
-
69
- function isoStringOrNull(value: Date | null): string | null {
70
- return value ? value.toISOString() : null;
71
- }
72
-
73
- function ensureSummaryModelColumn(db: DatabaseSync): void {
74
- const summaryColumns = db.prepare(`PRAGMA table_info(summaries)`).all() as SummaryColumnInfo[];
75
- const hasModel = summaryColumns.some((col) => col.name === "model");
76
- if (!hasModel) {
77
- db.exec(`ALTER TABLE summaries ADD COLUMN model TEXT NOT NULL DEFAULT 'unknown'`);
78
- }
79
- }
80
-
81
- function ensureCompactionTelemetryColumns(db: DatabaseSync): void {
82
- const telemetryColumns = db.prepare(`PRAGMA table_info(conversation_compaction_telemetry)`).all() as SummaryColumnInfo[];
83
- const hasLastLeafCompactionAt = telemetryColumns.some((col) => col.name === "last_leaf_compaction_at");
84
- const hasTurnsSinceLeafCompaction = telemetryColumns.some((col) => col.name === "turns_since_leaf_compaction");
85
- const hasTokensAccumulatedSinceLeafCompaction = telemetryColumns.some(
86
- (col) => col.name === "tokens_accumulated_since_leaf_compaction",
87
- );
88
- const hasLastActivityBand = telemetryColumns.some((col) => col.name === "last_activity_band");
89
-
90
- if (!hasLastLeafCompactionAt) {
91
- db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN last_leaf_compaction_at TEXT`);
92
- }
93
- if (!hasTurnsSinceLeafCompaction) {
94
- db.exec(
95
- `ALTER TABLE conversation_compaction_telemetry ADD COLUMN turns_since_leaf_compaction INTEGER NOT NULL DEFAULT 0`,
96
- );
97
- }
98
- if (!hasTokensAccumulatedSinceLeafCompaction) {
99
- db.exec(
100
- `ALTER TABLE conversation_compaction_telemetry ADD COLUMN tokens_accumulated_since_leaf_compaction INTEGER NOT NULL DEFAULT 0`,
101
- );
102
- }
103
- if (!hasLastActivityBand) {
104
- db.exec(
105
- `ALTER TABLE conversation_compaction_telemetry ADD COLUMN last_activity_band TEXT NOT NULL DEFAULT 'low' CHECK (last_activity_band IN ('low', 'medium', 'high'))`,
106
- );
107
- }
108
- }
109
-
110
- function backfillSummaryDepths(db: DatabaseSync): void {
111
- // Leaves are always depth 0, even if legacy rows had malformed values.
112
- db.exec(`UPDATE summaries SET depth = 0 WHERE kind = 'leaf'`);
113
-
114
- const conversationRows = db
115
- .prepare(`SELECT DISTINCT conversation_id FROM summaries WHERE kind = 'condensed'`)
116
- .all() as Array<{ conversation_id: number }>;
117
- if (conversationRows.length === 0) {
118
- return;
119
- }
120
-
121
- const updateDepthStmt = db.prepare(`UPDATE summaries SET depth = ? WHERE summary_id = ?`);
122
-
123
- for (const row of conversationRows) {
124
- const conversationId = row.conversation_id;
125
- const summaries = db
126
- .prepare(
127
- `SELECT summary_id, conversation_id, kind, depth, token_count, created_at
128
- FROM summaries
129
- WHERE conversation_id = ?`,
130
- )
131
- .all(conversationId) as SummaryDepthRow[];
132
-
133
- const depthBySummaryId = new Map<string, number>();
134
- const unresolvedCondensedIds = new Set<string>();
135
- for (const summary of summaries) {
136
- if (summary.kind === "leaf") {
137
- depthBySummaryId.set(summary.summary_id, 0);
138
- continue;
139
- }
140
- unresolvedCondensedIds.add(summary.summary_id);
141
- }
142
-
143
- const edges = db
144
- .prepare(
145
- `SELECT summary_id, parent_summary_id
146
- FROM summary_parents
147
- WHERE summary_id IN (
148
- SELECT summary_id FROM summaries
149
- WHERE conversation_id = ? AND kind = 'condensed'
150
- )`,
151
- )
152
- .all(conversationId) as SummaryParentEdgeRow[];
153
- const parentsBySummaryId = new Map<string, string[]>();
154
- for (const edge of edges) {
155
- const existing = parentsBySummaryId.get(edge.summary_id) ?? [];
156
- existing.push(edge.parent_summary_id);
157
- parentsBySummaryId.set(edge.summary_id, existing);
158
- }
159
-
160
- while (unresolvedCondensedIds.size > 0) {
161
- let progressed = false;
162
-
163
- for (const summaryId of [...unresolvedCondensedIds]) {
164
- const parentIds = parentsBySummaryId.get(summaryId) ?? [];
165
- if (parentIds.length === 0) {
166
- depthBySummaryId.set(summaryId, 1);
167
- unresolvedCondensedIds.delete(summaryId);
168
- progressed = true;
169
- continue;
170
- }
171
-
172
- let maxParentDepth = -1;
173
- let allParentsResolved = true;
174
- for (const parentId of parentIds) {
175
- const parentDepth = depthBySummaryId.get(parentId);
176
- if (parentDepth == null) {
177
- allParentsResolved = false;
178
- break;
179
- }
180
- if (parentDepth > maxParentDepth) {
181
- maxParentDepth = parentDepth;
182
- }
183
- }
184
-
185
- if (!allParentsResolved) {
186
- continue;
187
- }
188
-
189
- depthBySummaryId.set(summaryId, maxParentDepth + 1);
190
- unresolvedCondensedIds.delete(summaryId);
191
- progressed = true;
192
- }
193
-
194
- // Guard against malformed cycles/cross-conversation references.
195
- if (!progressed) {
196
- for (const summaryId of unresolvedCondensedIds) {
197
- depthBySummaryId.set(summaryId, 1);
198
- }
199
- unresolvedCondensedIds.clear();
200
- }
201
- }
202
-
203
- for (const summary of summaries) {
204
- const depth = depthBySummaryId.get(summary.summary_id);
205
- if (depth == null) {
206
- continue;
207
- }
208
- updateDepthStmt.run(depth, summary.summary_id);
209
- }
210
- }
211
- }
212
-
213
- function backfillSummaryMetadata(db: DatabaseSync): void {
214
- const conversationRows = db
215
- .prepare(`SELECT DISTINCT conversation_id FROM summaries`)
216
- .all() as Array<{ conversation_id: number }>;
217
- if (conversationRows.length === 0) {
218
- return;
219
- }
220
-
221
- const updateMetadataStmt = db.prepare(
222
- `UPDATE summaries
223
- SET earliest_at = ?, latest_at = ?, descendant_count = ?,
224
- descendant_token_count = ?, source_message_token_count = ?
225
- WHERE summary_id = ?`,
226
- );
227
-
228
- for (const conversationRow of conversationRows) {
229
- const conversationId = conversationRow.conversation_id;
230
- const summaries = db
231
- .prepare(
232
- `SELECT summary_id, conversation_id, kind, depth, token_count, created_at
233
- FROM summaries
234
- WHERE conversation_id = ?
235
- ORDER BY depth ASC, created_at ASC`,
236
- )
237
- .all(conversationId) as SummaryDepthRow[];
238
- if (summaries.length === 0) {
239
- continue;
240
- }
241
-
242
- const leafRanges = db
243
- .prepare(
244
- `SELECT
245
- sm.summary_id,
246
- MIN(m.created_at) AS earliest_at,
247
- MAX(m.created_at) AS latest_at,
248
- COALESCE(SUM(m.token_count), 0) AS source_message_token_count
249
- FROM summary_messages sm
250
- JOIN messages m ON m.message_id = sm.message_id
251
- JOIN summaries s ON s.summary_id = sm.summary_id
252
- WHERE s.conversation_id = ? AND s.kind = 'leaf'
253
- GROUP BY sm.summary_id`,
254
- )
255
- .all(conversationId) as SummaryMessageTimeRangeRow[];
256
- const leafRangeBySummaryId = new Map(
257
- leafRanges.map((row) => [
258
- row.summary_id,
259
- {
260
- earliestAt: row.earliest_at,
261
- latestAt: row.latest_at,
262
- sourceMessageTokenCount: row.source_message_token_count,
263
- },
264
- ]),
265
- );
266
-
267
- const edges = db
268
- .prepare(
269
- `SELECT summary_id, parent_summary_id
270
- FROM summary_parents
271
- WHERE summary_id IN (
272
- SELECT summary_id FROM summaries WHERE conversation_id = ?
273
- )`,
274
- )
275
- .all(conversationId) as SummaryParentEdgeRow[];
276
- const parentsBySummaryId = new Map<string, string[]>();
277
- for (const edge of edges) {
278
- const existing = parentsBySummaryId.get(edge.summary_id) ?? [];
279
- existing.push(edge.parent_summary_id);
280
- parentsBySummaryId.set(edge.summary_id, existing);
281
- }
282
-
283
- const metadataBySummaryId = new Map<
284
- string,
285
- {
286
- earliestAt: Date | null;
287
- latestAt: Date | null;
288
- descendantCount: number;
289
- descendantTokenCount: number;
290
- sourceMessageTokenCount: number;
291
- }
292
- >();
293
- const tokenCountBySummaryId = new Map(
294
- summaries.map((summary) => [summary.summary_id, Math.max(0, Math.floor(summary.token_count ?? 0))]),
295
- );
296
-
297
- for (const summary of summaries) {
298
- const fallbackDate = parseTimestamp(summary.created_at);
299
- if (summary.kind === "leaf") {
300
- const range = leafRangeBySummaryId.get(summary.summary_id);
301
- const earliestAt = parseTimestamp(range?.earliestAt ?? summary.created_at) ?? fallbackDate;
302
- const latestAt = parseTimestamp(range?.latestAt ?? summary.created_at) ?? fallbackDate;
303
-
304
- metadataBySummaryId.set(summary.summary_id, {
305
- earliestAt,
306
- latestAt,
307
- descendantCount: 0,
308
- descendantTokenCount: 0,
309
- sourceMessageTokenCount: Math.max(
310
- 0,
311
- Math.floor(range?.sourceMessageTokenCount ?? 0),
312
- ),
313
- });
314
- continue;
315
- }
316
-
317
- const parentIds = parentsBySummaryId.get(summary.summary_id) ?? [];
318
- if (parentIds.length === 0) {
319
- metadataBySummaryId.set(summary.summary_id, {
320
- earliestAt: fallbackDate,
321
- latestAt: fallbackDate,
322
- descendantCount: 0,
323
- descendantTokenCount: 0,
324
- sourceMessageTokenCount: 0,
325
- });
326
- continue;
327
- }
328
-
329
- let earliestAt: Date | null = null;
330
- let latestAt: Date | null = null;
331
- let descendantCount = 0;
332
- let descendantTokenCount = 0;
333
- let sourceMessageTokenCount = 0;
334
-
335
- for (const parentId of parentIds) {
336
- const parentMetadata = metadataBySummaryId.get(parentId);
337
- if (!parentMetadata) {
338
- continue;
339
- }
340
-
341
- const parentEarliest = parentMetadata.earliestAt;
342
- if (parentEarliest && (!earliestAt || parentEarliest < earliestAt)) {
343
- earliestAt = parentEarliest;
344
- }
345
-
346
- const parentLatest = parentMetadata.latestAt;
347
- if (parentLatest && (!latestAt || parentLatest > latestAt)) {
348
- latestAt = parentLatest;
349
- }
350
-
351
- descendantCount += Math.max(0, parentMetadata.descendantCount) + 1;
352
- const parentTokenCount = tokenCountBySummaryId.get(parentId) ?? 0;
353
- descendantTokenCount +=
354
- Math.max(0, parentTokenCount) + Math.max(0, parentMetadata.descendantTokenCount);
355
- sourceMessageTokenCount += Math.max(0, parentMetadata.sourceMessageTokenCount);
356
- }
357
-
358
- metadataBySummaryId.set(summary.summary_id, {
359
- earliestAt: earliestAt ?? fallbackDate,
360
- latestAt: latestAt ?? fallbackDate,
361
- descendantCount: Math.max(0, descendantCount),
362
- descendantTokenCount: Math.max(0, descendantTokenCount),
363
- sourceMessageTokenCount: Math.max(0, sourceMessageTokenCount),
364
- });
365
- }
366
-
367
- for (const summary of summaries) {
368
- const metadata = metadataBySummaryId.get(summary.summary_id);
369
- if (!metadata) {
370
- continue;
371
- }
372
-
373
- updateMetadataStmt.run(
374
- isoStringOrNull(metadata.earliestAt),
375
- isoStringOrNull(metadata.latestAt),
376
- Math.max(0, metadata.descendantCount),
377
- Math.max(0, metadata.descendantTokenCount),
378
- Math.max(0, metadata.sourceMessageTokenCount),
379
- summary.summary_id,
380
- );
381
- }
382
- }
383
- }
384
-
385
- /**
386
- * Backfill tool_call_id, tool_name, and tool_input from metadata JSON for rows
387
- * where the DB columns are NULL but the values exist in metadata. This covers
388
- * legacy text-type parts where the string-content ingestion path stored tool
389
- * info only in the metadata JSON (see #158).
390
- */
391
- function backfillToolCallColumns(db: DatabaseSync): void {
392
- db.exec(
393
- `UPDATE message_parts
394
- SET tool_call_id = COALESCE(
395
- json_extract(metadata, '$.toolCallId'),
396
- json_extract(metadata, '$.raw.id'),
397
- json_extract(metadata, '$.raw.call_id'),
398
- json_extract(metadata, '$.raw.toolCallId'),
399
- json_extract(metadata, '$.raw.tool_call_id')
400
- )
401
- WHERE tool_call_id IS NULL
402
- AND metadata IS NOT NULL
403
- AND COALESCE(
404
- json_extract(metadata, '$.toolCallId'),
405
- json_extract(metadata, '$.raw.id'),
406
- json_extract(metadata, '$.raw.call_id'),
407
- json_extract(metadata, '$.raw.toolCallId'),
408
- json_extract(metadata, '$.raw.tool_call_id')
409
- ) IS NOT NULL`,
410
- );
411
-
412
- db.exec(
413
- `UPDATE message_parts
414
- SET tool_name = COALESCE(
415
- json_extract(metadata, '$.toolName'),
416
- json_extract(metadata, '$.raw.name'),
417
- json_extract(metadata, '$.raw.toolName'),
418
- json_extract(metadata, '$.raw.tool_name')
419
- )
420
- WHERE tool_name IS NULL
421
- AND metadata IS NOT NULL
422
- AND COALESCE(
423
- json_extract(metadata, '$.toolName'),
424
- json_extract(metadata, '$.raw.name'),
425
- json_extract(metadata, '$.raw.toolName'),
426
- json_extract(metadata, '$.raw.tool_name')
427
- ) IS NOT NULL`,
428
- );
429
-
430
- db.exec(
431
- `UPDATE message_parts
432
- SET tool_input = COALESCE(
433
- json_extract(metadata, '$.raw.input'),
434
- json_extract(metadata, '$.raw.arguments'),
435
- json_extract(metadata, '$.raw.toolInput')
436
- )
437
- WHERE tool_input IS NULL
438
- AND metadata IS NOT NULL
439
- AND COALESCE(
440
- json_extract(metadata, '$.raw.input'),
441
- json_extract(metadata, '$.raw.arguments'),
442
- json_extract(metadata, '$.raw.toolInput')
443
- ) IS NOT NULL`,
444
- );
445
- }
446
-
447
- export function runLcmMigrations(
448
- db: DatabaseSync,
449
- options?: { fts5Available?: boolean },
450
- ): void {
451
- db.exec(`
452
- CREATE TABLE IF NOT EXISTS conversations (
453
- conversation_id INTEGER PRIMARY KEY AUTOINCREMENT,
454
- session_id TEXT NOT NULL,
455
- session_key TEXT,
456
- active INTEGER NOT NULL DEFAULT 1,
457
- archived_at TEXT,
458
- title TEXT,
459
- bootstrapped_at TEXT,
460
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
461
- updated_at TEXT NOT NULL DEFAULT (datetime('now'))
462
- );
463
-
464
- CREATE TABLE IF NOT EXISTS messages (
465
- message_id INTEGER PRIMARY KEY AUTOINCREMENT,
466
- conversation_id INTEGER NOT NULL REFERENCES conversations(conversation_id) ON DELETE CASCADE,
467
- seq INTEGER NOT NULL,
468
- role TEXT NOT NULL CHECK (role IN ('system', 'user', 'assistant', 'tool')),
469
- content TEXT NOT NULL,
470
- token_count INTEGER NOT NULL,
471
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
472
- UNIQUE (conversation_id, seq)
473
- );
474
-
475
- CREATE TABLE IF NOT EXISTS summaries (
476
- summary_id TEXT PRIMARY KEY,
477
- conversation_id INTEGER NOT NULL REFERENCES conversations(conversation_id) ON DELETE CASCADE,
478
- kind TEXT NOT NULL CHECK (kind IN ('leaf', 'condensed')),
479
- depth INTEGER NOT NULL DEFAULT 0,
480
- content TEXT NOT NULL,
481
- token_count INTEGER NOT NULL,
482
- earliest_at TEXT,
483
- latest_at TEXT,
484
- descendant_count INTEGER NOT NULL DEFAULT 0,
485
- descendant_token_count INTEGER NOT NULL DEFAULT 0,
486
- source_message_token_count INTEGER NOT NULL DEFAULT 0,
487
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
488
- file_ids TEXT NOT NULL DEFAULT '[]'
489
- );
490
-
491
- CREATE TABLE IF NOT EXISTS message_parts (
492
- part_id TEXT PRIMARY KEY,
493
- message_id INTEGER NOT NULL REFERENCES messages(message_id) ON DELETE CASCADE,
494
- session_id TEXT NOT NULL,
495
- part_type TEXT NOT NULL CHECK (part_type IN (
496
- 'text', 'reasoning', 'tool', 'patch', 'file',
497
- 'subtask', 'compaction', 'step_start', 'step_finish',
498
- 'snapshot', 'agent', 'retry'
499
- )),
500
- ordinal INTEGER NOT NULL,
501
- text_content TEXT,
502
- is_ignored INTEGER,
503
- is_synthetic INTEGER,
504
- tool_call_id TEXT,
505
- tool_name TEXT,
506
- tool_status TEXT,
507
- tool_input TEXT,
508
- tool_output TEXT,
509
- tool_error TEXT,
510
- tool_title TEXT,
511
- patch_hash TEXT,
512
- patch_files TEXT,
513
- file_mime TEXT,
514
- file_name TEXT,
515
- file_url TEXT,
516
- subtask_prompt TEXT,
517
- subtask_desc TEXT,
518
- subtask_agent TEXT,
519
- step_reason TEXT,
520
- step_cost REAL,
521
- step_tokens_in INTEGER,
522
- step_tokens_out INTEGER,
523
- snapshot_hash TEXT,
524
- compaction_auto INTEGER,
525
- metadata TEXT,
526
- UNIQUE (message_id, ordinal)
527
- );
528
-
529
- CREATE TABLE IF NOT EXISTS summary_messages (
530
- summary_id TEXT NOT NULL REFERENCES summaries(summary_id) ON DELETE CASCADE,
531
- message_id INTEGER NOT NULL REFERENCES messages(message_id) ON DELETE RESTRICT,
532
- ordinal INTEGER NOT NULL,
533
- PRIMARY KEY (summary_id, message_id)
534
- );
535
-
536
- CREATE TABLE IF NOT EXISTS summary_parents (
537
- summary_id TEXT NOT NULL REFERENCES summaries(summary_id) ON DELETE CASCADE,
538
- parent_summary_id TEXT NOT NULL REFERENCES summaries(summary_id) ON DELETE RESTRICT,
539
- ordinal INTEGER NOT NULL,
540
- PRIMARY KEY (summary_id, parent_summary_id)
541
- );
542
-
543
- CREATE TABLE IF NOT EXISTS context_items (
544
- conversation_id INTEGER NOT NULL REFERENCES conversations(conversation_id) ON DELETE CASCADE,
545
- ordinal INTEGER NOT NULL,
546
- item_type TEXT NOT NULL CHECK (item_type IN ('message', 'summary')),
547
- message_id INTEGER REFERENCES messages(message_id) ON DELETE RESTRICT,
548
- summary_id TEXT REFERENCES summaries(summary_id) ON DELETE RESTRICT,
549
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
550
- PRIMARY KEY (conversation_id, ordinal),
551
- CHECK (
552
- (item_type = 'message' AND message_id IS NOT NULL AND summary_id IS NULL) OR
553
- (item_type = 'summary' AND summary_id IS NOT NULL AND message_id IS NULL)
554
- )
555
- );
556
-
557
- CREATE TABLE IF NOT EXISTS large_files (
558
- file_id TEXT PRIMARY KEY,
559
- conversation_id INTEGER NOT NULL REFERENCES conversations(conversation_id) ON DELETE CASCADE,
560
- file_name TEXT,
561
- mime_type TEXT,
562
- byte_size INTEGER,
563
- storage_uri TEXT NOT NULL,
564
- exploration_summary TEXT,
565
- created_at TEXT NOT NULL DEFAULT (datetime('now'))
566
- );
567
-
568
- CREATE TABLE IF NOT EXISTS conversation_bootstrap_state (
569
- conversation_id INTEGER PRIMARY KEY REFERENCES conversations(conversation_id) ON DELETE CASCADE,
570
- session_file_path TEXT NOT NULL,
571
- last_seen_size INTEGER NOT NULL,
572
- last_seen_mtime_ms INTEGER NOT NULL,
573
- last_processed_offset INTEGER NOT NULL,
574
- last_processed_entry_hash TEXT,
575
- updated_at TEXT NOT NULL DEFAULT (datetime('now'))
576
- );
577
-
578
- CREATE TABLE IF NOT EXISTS conversation_compaction_telemetry (
579
- conversation_id INTEGER PRIMARY KEY REFERENCES conversations(conversation_id) ON DELETE CASCADE,
580
- last_observed_cache_read INTEGER,
581
- last_observed_cache_write INTEGER,
582
- last_observed_cache_hit_at TEXT,
583
- last_observed_cache_break_at TEXT,
584
- cache_state TEXT NOT NULL DEFAULT 'unknown'
585
- CHECK (cache_state IN ('hot', 'cold', 'unknown')),
586
- retention TEXT,
587
- last_leaf_compaction_at TEXT,
588
- turns_since_leaf_compaction INTEGER NOT NULL DEFAULT 0,
589
- tokens_accumulated_since_leaf_compaction INTEGER NOT NULL DEFAULT 0,
590
- last_activity_band TEXT NOT NULL DEFAULT 'low'
591
- CHECK (last_activity_band IN ('low', 'medium', 'high')),
592
- updated_at TEXT NOT NULL DEFAULT (datetime('now'))
593
- );
594
-
595
- -- Indexes
596
- CREATE INDEX IF NOT EXISTS messages_conv_seq_idx ON messages (conversation_id, seq);
597
- CREATE INDEX IF NOT EXISTS summaries_conv_created_idx ON summaries (conversation_id, created_at);
598
- CREATE INDEX IF NOT EXISTS summary_messages_message_idx ON summary_messages (message_id);
599
- CREATE INDEX IF NOT EXISTS summary_parents_parent_summary_idx ON summary_parents (parent_summary_id);
600
- CREATE INDEX IF NOT EXISTS message_parts_message_idx ON message_parts (message_id);
601
- CREATE INDEX IF NOT EXISTS message_parts_type_idx ON message_parts (part_type);
602
- CREATE INDEX IF NOT EXISTS context_items_conv_idx ON context_items (conversation_id, ordinal);
603
- CREATE INDEX IF NOT EXISTS large_files_conv_idx ON large_files (conversation_id, created_at);
604
- CREATE INDEX IF NOT EXISTS bootstrap_state_path_idx
605
- ON conversation_bootstrap_state (session_file_path, updated_at);
606
- CREATE INDEX IF NOT EXISTS compaction_telemetry_state_idx
607
- ON conversation_compaction_telemetry (cache_state, updated_at);
608
-
609
- -- Speed up summary_messages lookups by message_id (PK is summary_id,message_id)
610
- CREATE INDEX IF NOT EXISTS summary_messages_message_idx ON summary_messages (message_id);
611
- `);
612
-
613
- // Forward-compatible conversations migration for existing DBs.
614
- const conversationColumns = db.prepare(`PRAGMA table_info(conversations)`).all() as Array<{
615
- name?: string;
616
- }>;
617
- const hasBootstrappedAt = conversationColumns.some((col) => col.name === "bootstrapped_at");
618
- if (!hasBootstrappedAt) {
619
- db.exec(`ALTER TABLE conversations ADD COLUMN bootstrapped_at TEXT`);
620
- }
621
-
622
- const hasSessionKey = conversationColumns.some((col) => col.name === "session_key");
623
- if (!hasSessionKey) {
624
- db.exec(`ALTER TABLE conversations ADD COLUMN session_key TEXT`);
625
- }
626
-
627
- const hasActive = conversationColumns.some((col) => col.name === "active");
628
- if (!hasActive) {
629
- db.exec(`ALTER TABLE conversations ADD COLUMN active INTEGER NOT NULL DEFAULT 1`);
630
- }
631
-
632
- const hasArchivedAt = conversationColumns.some((col) => col.name === "archived_at");
633
- if (!hasArchivedAt) {
634
- db.exec(`ALTER TABLE conversations ADD COLUMN archived_at TEXT`);
635
- }
636
-
637
- db.exec(`UPDATE conversations SET active = 1 WHERE active IS NULL`);
638
- db.exec(`
639
- CREATE UNIQUE INDEX IF NOT EXISTS conversations_active_session_key_idx
640
- ON conversations (session_key)
641
- WHERE session_key IS NOT NULL AND active = 1
642
- `);
643
- db.exec(`
644
- CREATE INDEX IF NOT EXISTS conversations_session_key_active_created_idx
645
- ON conversations (session_key, active, created_at)
646
- `);
647
- db.exec(`DROP INDEX IF EXISTS conversations_session_key_idx`);
648
- ensureSummaryDepthColumn(db);
649
- ensureSummaryMetadataColumns(db);
650
- ensureSummaryModelColumn(db);
651
- ensureCompactionTelemetryColumns(db);
652
- backfillSummaryDepths(db);
653
- // Index on depth — created AFTER backfillSummaryDepths to avoid index
654
- // maintenance overhead during bulk depth updates on large existing DBs.
655
- db.exec(`CREATE INDEX IF NOT EXISTS summaries_conv_depth_kind_idx ON summaries (conversation_id, depth, kind)`);
656
- backfillSummaryMetadata(db);
657
- backfillToolCallColumns(db);
658
-
659
- const fts5Available = options?.fts5Available ?? getLcmDbFeatures(db).fts5Available;
660
- if (!fts5Available) {
661
- return;
662
- }
663
-
664
- // FTS5 virtual tables for full-text search (cannot use IF NOT EXISTS, so check manually)
665
- const hasFts = db
666
- .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='messages_fts'")
667
- .get();
668
-
669
- if (hasFts) {
670
- // Check for stale schema: external-content FTS tables with content_rowid cause errors.
671
- // Drop and recreate as standalone FTS if the old schema is detected.
672
- const ftsSchema = (
673
- db
674
- .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='messages_fts'")
675
- .get() as { sql: string } | undefined
676
- )?.sql;
677
- if (ftsSchema && ftsSchema.includes("content_rowid")) {
678
- db.exec("DROP TABLE messages_fts");
679
- db.exec(`
680
- CREATE VIRTUAL TABLE messages_fts USING fts5(
681
- content,
682
- tokenize='porter unicode61'
683
- );
684
- INSERT INTO messages_fts(rowid, content) SELECT message_id, content FROM messages;
685
- `);
686
- }
687
- } else {
688
- db.exec(`
689
- CREATE VIRTUAL TABLE messages_fts USING fts5(
690
- content,
691
- tokenize='porter unicode61'
692
- );
693
- `);
694
- }
695
-
696
- const summariesFtsInfo = db
697
- .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='summaries_fts'")
698
- .get() as { sql?: string } | undefined;
699
- const summariesFtsSql = summariesFtsInfo?.sql ?? "";
700
- const summariesFtsColumns = db.prepare(`PRAGMA table_info(summaries_fts)`).all() as Array<{
701
- name?: string;
702
- }>;
703
- const hasSummaryIdColumn = summariesFtsColumns.some((col) => col.name === "summary_id");
704
- const shouldRecreateSummariesFts =
705
- !summariesFtsInfo ||
706
- !hasSummaryIdColumn ||
707
- summariesFtsSql.includes("content_rowid='summary_id'") ||
708
- summariesFtsSql.includes('content_rowid="summary_id"');
709
- if (shouldRecreateSummariesFts) {
710
- db.exec(`
711
- DROP TABLE IF EXISTS summaries_fts;
712
- CREATE VIRTUAL TABLE summaries_fts USING fts5(
713
- summary_id UNINDEXED,
714
- content,
715
- tokenize='porter unicode61'
716
- );
717
- INSERT INTO summaries_fts(summary_id, content)
718
- SELECT summary_id, content FROM summaries;
719
- `);
720
- }
721
-
722
- // ── CJK trigram FTS table ────────────────────────────────────────────────
723
- // FTS5 unicode61 (porter) tokenizer cannot segment CJK ideographs, so CJK
724
- // queries currently fall back to a LIKE path with AND logic. When the user's
725
- // phrasing doesn't match the summary verbatim (e.g. "端到端测试结果" vs
726
- // "端到端测试"), ALL terms must match and the query returns 0 candidates.
727
- //
728
- // A trigram-tokenized table indexes every 3-character substring, enabling
729
- // native CJK substring matching via FTS5 MATCH with OR semantics.
730
- const cjkTableExists = db
731
- .prepare(
732
- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='summaries_fts_cjk'",
733
- )
734
- .get();
735
- if (!cjkTableExists) {
736
- db.exec(`
737
- CREATE VIRTUAL TABLE summaries_fts_cjk USING fts5(
738
- summary_id UNINDEXED,
739
- content,
740
- tokenize='trigram'
741
- );
742
- INSERT INTO summaries_fts_cjk(summary_id, content)
743
- SELECT summary_id, content FROM summaries;
744
- `);
745
- }
746
- }