@martian-engineering/lossless-claw 0.8.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +8 -0
  2. package/dist/index.js +19240 -0
  3. package/docs/configuration.md +15 -5
  4. package/openclaw.plugin.json +27 -3
  5. package/package.json +7 -6
  6. package/skills/lossless-claw/references/config.md +37 -0
  7. package/index.ts +0 -2
  8. package/src/assembler.ts +0 -1196
  9. package/src/compaction.ts +0 -1753
  10. package/src/db/config.ts +0 -345
  11. package/src/db/connection.ts +0 -151
  12. package/src/db/features.ts +0 -61
  13. package/src/db/migration.ts +0 -868
  14. package/src/engine.ts +0 -4486
  15. package/src/estimate-tokens.ts +0 -80
  16. package/src/expansion-auth.ts +0 -365
  17. package/src/expansion-policy.ts +0 -303
  18. package/src/expansion.ts +0 -383
  19. package/src/integrity.ts +0 -600
  20. package/src/large-files.ts +0 -546
  21. package/src/lcm-log.ts +0 -37
  22. package/src/openclaw-bridge.ts +0 -22
  23. package/src/plugin/index.ts +0 -2037
  24. package/src/plugin/lcm-command.ts +0 -1040
  25. package/src/plugin/lcm-doctor-apply.ts +0 -540
  26. package/src/plugin/lcm-doctor-cleaners.ts +0 -655
  27. package/src/plugin/lcm-doctor-shared.ts +0 -210
  28. package/src/plugin/shared-init.ts +0 -59
  29. package/src/prune.ts +0 -391
  30. package/src/retrieval.ts +0 -360
  31. package/src/session-patterns.ts +0 -23
  32. package/src/startup-banner-log.ts +0 -49
  33. package/src/store/compaction-telemetry-store.ts +0 -156
  34. package/src/store/conversation-store.ts +0 -929
  35. package/src/store/fts5-sanitize.ts +0 -50
  36. package/src/store/full-text-fallback.ts +0 -83
  37. package/src/store/full-text-sort.ts +0 -21
  38. package/src/store/index.ts +0 -39
  39. package/src/store/parse-utc-timestamp.ts +0 -25
  40. package/src/store/summary-store.ts +0 -1519
  41. package/src/summarize.ts +0 -1508
  42. package/src/tools/common.ts +0 -53
  43. package/src/tools/lcm-conversation-scope.ts +0 -127
  44. package/src/tools/lcm-describe-tool.ts +0 -245
  45. package/src/tools/lcm-expand-query-tool.ts +0 -1235
  46. package/src/tools/lcm-expand-tool.delegation.ts +0 -580
  47. package/src/tools/lcm-expand-tool.ts +0 -453
  48. package/src/tools/lcm-expansion-recursion-guard.ts +0 -373
  49. package/src/tools/lcm-grep-tool.ts +0 -228
  50. package/src/transaction-mutex.ts +0 -136
  51. package/src/transcript-repair.ts +0 -301
  52. package/src/types.ts +0 -165
@@ -1,929 +0,0 @@
1
- import type { DatabaseSync } from "node:sqlite";
2
- import { randomUUID } from "node:crypto";
3
- import { withDatabaseTransaction } from "../transaction-mutex.js";
4
- import { sanitizeFts5Query } from "./fts5-sanitize.js";
5
- import { buildLikeSearchPlan, containsCjk, createFallbackSnippet } from "./full-text-fallback.js";
6
- import { parseUtcTimestamp, parseUtcTimestampOrNull } from "./parse-utc-timestamp.js";
7
- import { buildFtsOrderBy, type SearchSort } from "./full-text-sort.js";
8
-
9
- export type ConversationId = number;
10
- export type MessageId = number;
11
- export type SummaryId = string;
12
- export type MessageRole = "system" | "user" | "assistant" | "tool";
13
- export type MessagePartType =
14
- | "text"
15
- | "reasoning"
16
- | "tool"
17
- | "patch"
18
- | "file"
19
- | "subtask"
20
- | "compaction"
21
- | "step_start"
22
- | "step_finish"
23
- | "snapshot"
24
- | "agent"
25
- | "retry";
26
-
27
- export type CreateMessageInput = {
28
- conversationId: ConversationId;
29
- seq: number;
30
- role: MessageRole;
31
- content: string;
32
- tokenCount: number;
33
- };
34
-
35
- export type MessageRecord = {
36
- messageId: MessageId;
37
- conversationId: ConversationId;
38
- seq: number;
39
- role: MessageRole;
40
- content: string;
41
- tokenCount: number;
42
- createdAt: Date;
43
- };
44
-
45
- export type CreateMessagePartInput = {
46
- sessionId: string;
47
- partType: MessagePartType;
48
- ordinal: number;
49
- textContent?: string | null;
50
- toolCallId?: string | null;
51
- toolName?: string | null;
52
- toolInput?: string | null;
53
- toolOutput?: string | null;
54
- metadata?: string | null;
55
- };
56
-
57
- export type MessagePartRecord = {
58
- partId: string;
59
- messageId: MessageId;
60
- sessionId: string;
61
- partType: MessagePartType;
62
- ordinal: number;
63
- textContent: string | null;
64
- toolCallId: string | null;
65
- toolName: string | null;
66
- toolInput: string | null;
67
- toolOutput: string | null;
68
- metadata: string | null;
69
- };
70
-
71
- export type CreateConversationInput = {
72
- sessionId: string;
73
- sessionKey?: string;
74
- title?: string;
75
- active?: boolean;
76
- archivedAt?: Date | null;
77
- };
78
-
79
- export type ConversationRecord = {
80
- conversationId: ConversationId;
81
- sessionId: string;
82
- sessionKey: string | null;
83
- active: boolean;
84
- archivedAt: Date | null;
85
- title: string | null;
86
- bootstrappedAt: Date | null;
87
- createdAt: Date;
88
- updatedAt: Date;
89
- };
90
-
91
- export type MessageSearchInput = {
92
- conversationId?: ConversationId;
93
- query: string;
94
- mode: "regex" | "full_text";
95
- since?: Date;
96
- before?: Date;
97
- limit?: number;
98
- sort?: SearchSort;
99
- };
100
-
101
- export type MessageSearchResult = {
102
- messageId: MessageId;
103
- conversationId: ConversationId;
104
- role: MessageRole;
105
- snippet: string;
106
- createdAt: Date;
107
- rank?: number;
108
- };
109
-
110
- // ── DB row shapes (snake_case) ────────────────────────────────────────────────
111
-
112
- interface ConversationRow {
113
- conversation_id: number;
114
- session_id: string;
115
- session_key: string | null;
116
- active: number;
117
- archived_at: string | null;
118
- title: string | null;
119
- bootstrapped_at: string | null;
120
- created_at: string;
121
- updated_at: string;
122
- }
123
-
124
- interface MessageRow {
125
- message_id: number;
126
- conversation_id: number;
127
- seq: number;
128
- role: MessageRole;
129
- content: string;
130
- token_count: number;
131
- created_at: string;
132
- }
133
-
134
- interface MessageSearchRow {
135
- message_id: number;
136
- conversation_id: number;
137
- role: MessageRole;
138
- snippet: string;
139
- rank: number;
140
- created_at: string;
141
- }
142
-
143
- interface MessagePartRow {
144
- part_id: string;
145
- message_id: number;
146
- session_id: string;
147
- part_type: MessagePartType;
148
- ordinal: number;
149
- text_content: string | null;
150
- tool_call_id: string | null;
151
- tool_name: string | null;
152
- tool_input: string | null;
153
- tool_output: string | null;
154
- metadata: string | null;
155
- }
156
-
157
- interface CountRow {
158
- count: number;
159
- }
160
-
161
- interface MaxSeqRow {
162
- max_seq: number;
163
- }
164
-
165
- // ── Row mappers ───────────────────────────────────────────────────────────────
166
-
167
- function toConversationRecord(row: ConversationRow): ConversationRecord {
168
- return {
169
- conversationId: row.conversation_id,
170
- sessionId: row.session_id,
171
- sessionKey: row.session_key ?? null,
172
- active: row.active === 1,
173
- archivedAt: parseUtcTimestampOrNull(row.archived_at),
174
- title: row.title,
175
- bootstrappedAt: parseUtcTimestampOrNull(row.bootstrapped_at),
176
- createdAt: parseUtcTimestamp(row.created_at),
177
- updatedAt: parseUtcTimestamp(row.updated_at),
178
- };
179
- }
180
-
181
- function toMessageRecord(row: MessageRow): MessageRecord {
182
- return {
183
- messageId: row.message_id,
184
- conversationId: row.conversation_id,
185
- seq: row.seq,
186
- role: row.role,
187
- content: row.content,
188
- tokenCount: row.token_count,
189
- createdAt: parseUtcTimestamp(row.created_at),
190
- };
191
- }
192
-
193
- function toSearchResult(row: MessageSearchRow): MessageSearchResult {
194
- return {
195
- messageId: row.message_id,
196
- conversationId: row.conversation_id,
197
- role: row.role,
198
- snippet: row.snippet,
199
- createdAt: parseUtcTimestamp(row.created_at),
200
- rank: row.rank,
201
- };
202
- }
203
-
204
- function toMessagePartRecord(row: MessagePartRow): MessagePartRecord {
205
- return {
206
- partId: row.part_id,
207
- messageId: row.message_id,
208
- sessionId: row.session_id,
209
- partType: row.part_type,
210
- ordinal: row.ordinal,
211
- textContent: row.text_content,
212
- toolCallId: row.tool_call_id,
213
- toolName: row.tool_name,
214
- toolInput: row.tool_input,
215
- toolOutput: row.tool_output,
216
- metadata: row.metadata,
217
- };
218
- }
219
-
220
- function normalizeMessageContentForFullTextIndex(content: string): string | null {
221
- const trimmed = content.trim();
222
- if (!trimmed) {
223
- return null;
224
- }
225
-
226
- const isExternalizedReference =
227
- trimmed.startsWith("[LCM File:") || trimmed.startsWith("[LCM Tool Output:");
228
- if (!isExternalizedReference) {
229
- return content;
230
- }
231
-
232
- const lines = trimmed
233
- .split(/\r?\n/)
234
- .map((line) => line.trim())
235
- .filter((line) => line.length > 0);
236
- if (lines.length === 0) {
237
- return null;
238
- }
239
-
240
- const header = lines[0] ?? "";
241
- const summaryLines: string[] = [];
242
- let inSummary = false;
243
- for (let index = 1; index < lines.length; index += 1) {
244
- const line = lines[index]!;
245
- if (line === "Exploration Summary:") {
246
- inSummary = true;
247
- continue;
248
- }
249
- if (line.startsWith("Use lcm_describe")) {
250
- continue;
251
- }
252
- if (inSummary) {
253
- summaryLines.push(line);
254
- }
255
- }
256
-
257
- const normalized = [header, ...summaryLines].filter((line) => line.length > 0).join("\n");
258
- return normalized || null;
259
- }
260
-
261
- // ── ConversationStore ─────────────────────────────────────────────────────────
262
-
263
- export class ConversationStore {
264
- private readonly fts5Available: boolean;
265
-
266
- constructor(
267
- private db: DatabaseSync,
268
- options?: { fts5Available?: boolean },
269
- ) {
270
- this.fts5Available = options?.fts5Available ?? true;
271
- }
272
-
273
- // ── Transaction helpers ──────────────────────────────────────────────────
274
-
275
- async withTransaction<T>(operation: () => Promise<T> | T): Promise<T> {
276
- return withDatabaseTransaction(this.db, "BEGIN IMMEDIATE", operation);
277
- }
278
-
279
- // ── Conversation operations ───────────────────────────────────────────────
280
-
281
- async createConversation(input: CreateConversationInput): Promise<ConversationRecord> {
282
- const result = this.db
283
- .prepare(
284
- `INSERT INTO conversations (session_id, session_key, active, archived_at, title)
285
- VALUES (?, ?, ?, ?, ?)`,
286
- )
287
- .run(
288
- input.sessionId,
289
- input.sessionKey ?? null,
290
- input.active === false ? 0 : 1,
291
- input.archivedAt?.toISOString() ?? null,
292
- input.title ?? null,
293
- );
294
-
295
- const row = this.db
296
- .prepare(
297
- `SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
298
- FROM conversations WHERE conversation_id = ?`,
299
- )
300
- .get(Number(result.lastInsertRowid)) as unknown as ConversationRow;
301
-
302
- return toConversationRecord(row);
303
- }
304
-
305
- async getConversation(conversationId: ConversationId): Promise<ConversationRecord | null> {
306
- const row = this.db
307
- .prepare(
308
- `SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
309
- FROM conversations WHERE conversation_id = ?`,
310
- )
311
- .get(conversationId) as unknown as ConversationRow | undefined;
312
-
313
- return row ? toConversationRecord(row) : null;
314
- }
315
-
316
- async getConversationBySessionId(sessionId: string): Promise<ConversationRecord | null> {
317
- const row = this.db
318
- .prepare(
319
- `SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
320
- FROM conversations
321
- WHERE session_id = ?
322
- ORDER BY active DESC, created_at DESC
323
- LIMIT 1`,
324
- )
325
- .get(sessionId) as unknown as ConversationRow | undefined;
326
-
327
- return row ? toConversationRecord(row) : null;
328
- }
329
-
330
- async getConversationBySessionKey(sessionKey: string): Promise<ConversationRecord | null> {
331
- const row = this.db
332
- .prepare(
333
- `SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
334
- FROM conversations
335
- WHERE session_key = ?
336
- AND active = 1
337
- ORDER BY created_at DESC
338
- LIMIT 1`,
339
- )
340
- .get(sessionKey) as unknown as ConversationRow | undefined;
341
-
342
- return row ? toConversationRecord(row) : null;
343
- }
344
-
345
- /** Resolve a conversation by stable session identity. */
346
- async getConversationForSession(input: {
347
- sessionId?: string;
348
- sessionKey?: string;
349
- }): Promise<ConversationRecord | null> {
350
- const normalizedSessionKey = input.sessionKey?.trim();
351
- if (normalizedSessionKey) {
352
- const byKey = await this.getConversationBySessionKey(normalizedSessionKey);
353
- if (byKey) {
354
- return byKey;
355
- }
356
- }
357
-
358
- const normalizedSessionId = input.sessionId?.trim();
359
- if (!normalizedSessionId) {
360
- return null;
361
- }
362
-
363
- return this.getConversationBySessionId(normalizedSessionId);
364
- }
365
-
366
- async getOrCreateConversation(
367
- sessionId: string,
368
- titleOrOpts?: string | { title?: string; sessionKey?: string },
369
- ): Promise<ConversationRecord> {
370
- const opts = typeof titleOrOpts === "string" ? { title: titleOrOpts } : titleOrOpts ?? {};
371
- const normalizedSessionKey = opts.sessionKey?.trim();
372
- if (normalizedSessionKey) {
373
- const byKey = await this.getConversationBySessionKey(normalizedSessionKey);
374
- if (byKey) {
375
- if (byKey.sessionId !== sessionId) {
376
- this.db
377
- .prepare(
378
- `UPDATE conversations SET session_id = ?, updated_at = datetime('now') WHERE conversation_id = ?`,
379
- )
380
- .run(sessionId, byKey.conversationId);
381
- byKey.sessionId = sessionId;
382
- }
383
- return byKey;
384
- }
385
- }
386
-
387
- const existing = await this.getConversationBySessionId(sessionId);
388
- if (existing) {
389
- if (!normalizedSessionKey) {
390
- return existing;
391
- }
392
- if (existing.active && !existing.sessionKey) {
393
- this.db
394
- .prepare(
395
- `UPDATE conversations SET session_key = ?, updated_at = datetime('now') WHERE conversation_id = ?`,
396
- )
397
- .run(normalizedSessionKey, existing.conversationId);
398
- existing.sessionKey = normalizedSessionKey;
399
- return existing;
400
- }
401
- if (existing.active && existing.sessionKey === normalizedSessionKey) {
402
- return existing;
403
- }
404
- }
405
-
406
- return this.createConversation({ sessionId, title: opts.title, sessionKey: normalizedSessionKey });
407
- }
408
-
409
- async markConversationBootstrapped(conversationId: ConversationId): Promise<void> {
410
- this.db
411
- .prepare(
412
- `UPDATE conversations
413
- SET bootstrapped_at = COALESCE(bootstrapped_at, datetime('now')),
414
- updated_at = datetime('now')
415
- WHERE conversation_id = ?`,
416
- )
417
- .run(conversationId);
418
- }
419
-
420
- async archiveConversation(conversationId: ConversationId): Promise<void> {
421
- this.db
422
- .prepare(
423
- `UPDATE conversations
424
- SET active = 0,
425
- archived_at = COALESCE(archived_at, datetime('now')),
426
- updated_at = datetime('now')
427
- WHERE conversation_id = ?`,
428
- )
429
- .run(conversationId);
430
- }
431
-
432
- // ── Message operations ────────────────────────────────────────────────────
433
-
434
- async createMessage(input: CreateMessageInput): Promise<MessageRecord> {
435
- const result = this.db
436
- .prepare(
437
- `INSERT INTO messages (conversation_id, seq, role, content, token_count)
438
- VALUES (?, ?, ?, ?, ?)`,
439
- )
440
- .run(input.conversationId, input.seq, input.role, input.content, input.tokenCount);
441
-
442
- const messageId = Number(result.lastInsertRowid);
443
-
444
- this.indexMessageForFullText(messageId, input.content);
445
-
446
- const row = this.db
447
- .prepare(
448
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
449
- FROM messages WHERE message_id = ?`,
450
- )
451
- .get(messageId) as unknown as MessageRow;
452
-
453
- return toMessageRecord(row);
454
- }
455
-
456
- async createMessagesBulk(inputs: CreateMessageInput[]): Promise<MessageRecord[]> {
457
- if (inputs.length === 0) {
458
- return [];
459
- }
460
- const insertStmt = this.db.prepare(
461
- `INSERT INTO messages (conversation_id, seq, role, content, token_count)
462
- VALUES (?, ?, ?, ?, ?)`,
463
- );
464
- const selectStmt = this.db.prepare(
465
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
466
- FROM messages WHERE message_id = ?`,
467
- );
468
-
469
- const records: MessageRecord[] = [];
470
- for (const input of inputs) {
471
- const result = insertStmt.run(
472
- input.conversationId,
473
- input.seq,
474
- input.role,
475
- input.content,
476
- input.tokenCount,
477
- );
478
-
479
- const messageId = Number(result.lastInsertRowid);
480
- this.indexMessageForFullText(messageId, input.content);
481
- const row = selectStmt.get(messageId) as unknown as MessageRow;
482
- records.push(toMessageRecord(row));
483
- }
484
-
485
- return records;
486
- }
487
-
488
- async getMessages(
489
- conversationId: ConversationId,
490
- opts?: { afterSeq?: number; limit?: number },
491
- ): Promise<MessageRecord[]> {
492
- const afterSeq = opts?.afterSeq ?? -1;
493
- const limit = opts?.limit;
494
-
495
- if (limit != null) {
496
- const rows = this.db
497
- .prepare(
498
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
499
- FROM messages
500
- WHERE conversation_id = ? AND seq > ?
501
- ORDER BY seq
502
- LIMIT ?`,
503
- )
504
- .all(conversationId, afterSeq, limit) as unknown as MessageRow[];
505
- return rows.map(toMessageRecord);
506
- }
507
-
508
- const rows = this.db
509
- .prepare(
510
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
511
- FROM messages
512
- WHERE conversation_id = ? AND seq > ?
513
- ORDER BY seq`,
514
- )
515
- .all(conversationId, afterSeq) as unknown as MessageRow[];
516
- return rows.map(toMessageRecord);
517
- }
518
-
519
- async getLastMessage(conversationId: ConversationId): Promise<MessageRecord | null> {
520
- const row = this.db
521
- .prepare(
522
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
523
- FROM messages
524
- WHERE conversation_id = ?
525
- ORDER BY seq DESC
526
- LIMIT 1`,
527
- )
528
- .get(conversationId) as unknown as MessageRow | undefined;
529
-
530
- return row ? toMessageRecord(row) : null;
531
- }
532
-
533
- async hasMessage(
534
- conversationId: ConversationId,
535
- role: MessageRole,
536
- content: string,
537
- ): Promise<boolean> {
538
- const row = this.db
539
- .prepare(
540
- `SELECT 1 AS count
541
- FROM messages
542
- WHERE conversation_id = ? AND role = ? AND content = ?
543
- LIMIT 1`,
544
- )
545
- .get(conversationId, role, content) as unknown as CountRow | undefined;
546
-
547
- return row?.count === 1;
548
- }
549
-
550
- async countMessagesByIdentity(
551
- conversationId: ConversationId,
552
- role: MessageRole,
553
- content: string,
554
- ): Promise<number> {
555
- const row = this.db
556
- .prepare(
557
- `SELECT COUNT(*) AS count
558
- FROM messages
559
- WHERE conversation_id = ? AND role = ? AND content = ?`,
560
- )
561
- .get(conversationId, role, content) as unknown as CountRow | undefined;
562
-
563
- return row?.count ?? 0;
564
- }
565
-
566
- async getMessageById(messageId: MessageId): Promise<MessageRecord | null> {
567
- const row = this.db
568
- .prepare(
569
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
570
- FROM messages WHERE message_id = ?`,
571
- )
572
- .get(messageId) as unknown as MessageRow | undefined;
573
- return row ? toMessageRecord(row) : null;
574
- }
575
-
576
- async createMessageParts(messageId: MessageId, parts: CreateMessagePartInput[]): Promise<void> {
577
- if (parts.length === 0) {
578
- return;
579
- }
580
-
581
- const stmt = this.db.prepare(
582
- `INSERT INTO message_parts (
583
- part_id,
584
- message_id,
585
- session_id,
586
- part_type,
587
- ordinal,
588
- text_content,
589
- tool_call_id,
590
- tool_name,
591
- tool_input,
592
- tool_output,
593
- metadata
594
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
595
- );
596
-
597
- for (const part of parts) {
598
- stmt.run(
599
- randomUUID(),
600
- messageId,
601
- part.sessionId,
602
- part.partType,
603
- part.ordinal,
604
- part.textContent ?? null,
605
- part.toolCallId ?? null,
606
- part.toolName ?? null,
607
- part.toolInput ?? null,
608
- part.toolOutput ?? null,
609
- part.metadata ?? null,
610
- );
611
- }
612
- }
613
-
614
- async getMessageParts(messageId: MessageId): Promise<MessagePartRecord[]> {
615
- const rows = this.db
616
- .prepare(
617
- `SELECT
618
- part_id,
619
- message_id,
620
- session_id,
621
- part_type,
622
- ordinal,
623
- text_content,
624
- tool_call_id,
625
- tool_name,
626
- tool_input,
627
- tool_output,
628
- metadata
629
- FROM message_parts
630
- WHERE message_id = ?
631
- ORDER BY ordinal`,
632
- )
633
- .all(messageId) as unknown as MessagePartRow[];
634
-
635
- return rows.map(toMessagePartRecord);
636
- }
637
-
638
- async getMessageCount(conversationId: ConversationId): Promise<number> {
639
- const row = this.db
640
- .prepare(`SELECT COUNT(*) AS count FROM messages WHERE conversation_id = ?`)
641
- .get(conversationId) as unknown as CountRow;
642
- return row?.count ?? 0;
643
- }
644
-
645
- async getMaxSeq(conversationId: ConversationId): Promise<number> {
646
- const row = this.db
647
- .prepare(
648
- `SELECT COALESCE(MAX(seq), 0) AS max_seq
649
- FROM messages WHERE conversation_id = ?`,
650
- )
651
- .get(conversationId) as unknown as MaxSeqRow;
652
- return row?.max_seq ?? 0;
653
- }
654
-
655
- // ── Deletion ──────────────────────────────────────────────────────────────
656
-
657
- /**
658
- * Delete messages and their associated records (context_items, FTS, message_parts).
659
- *
660
- * Skips messages referenced in summary_messages (already compacted) to avoid
661
- * breaking the summary DAG. Returns the count of actually deleted messages.
662
- */
663
- async deleteMessages(messageIds: MessageId[]): Promise<number> {
664
- if (messageIds.length === 0) {
665
- return 0;
666
- }
667
-
668
- let deleted = 0;
669
- for (const messageId of messageIds) {
670
- // Skip if referenced by a summary (ON DELETE RESTRICT would fail anyway)
671
- const refRow = this.db
672
- .prepare(`SELECT 1 AS found FROM summary_messages WHERE message_id = ? LIMIT 1`)
673
- .get(messageId) as unknown as { found: number } | undefined;
674
- if (refRow) {
675
- continue;
676
- }
677
-
678
- // Remove from context_items first (RESTRICT constraint)
679
- this.db
680
- .prepare(`DELETE FROM context_items WHERE item_type = 'message' AND message_id = ?`)
681
- .run(messageId);
682
-
683
- this.deleteMessageFromFullText(messageId);
684
-
685
- // Delete the message (message_parts cascade via ON DELETE CASCADE)
686
- this.db.prepare(`DELETE FROM messages WHERE message_id = ?`).run(messageId);
687
-
688
- deleted += 1;
689
- }
690
-
691
- return deleted;
692
- }
693
-
694
- // ── Search ────────────────────────────────────────────────────────────────
695
-
696
- async searchMessages(input: MessageSearchInput): Promise<MessageSearchResult[]> {
697
- const limit = input.limit ?? 50;
698
-
699
- if (input.mode === "full_text") {
700
- // FTS5 unicode61 can return incomplete matches for CJK text, so route
701
- // those queries through the existing LIKE fallback path immediately.
702
- if (containsCjk(input.query)) {
703
- return this.searchLike(
704
- input.query,
705
- limit,
706
- input.conversationId,
707
- input.since,
708
- input.before,
709
- );
710
- }
711
- if (this.fts5Available) {
712
- try {
713
- return this.searchFullText(
714
- input.query,
715
- limit,
716
- input.conversationId,
717
- input.since,
718
- input.before,
719
- input.sort,
720
- );
721
- } catch {
722
- return this.searchLike(
723
- input.query,
724
- limit,
725
- input.conversationId,
726
- input.since,
727
- input.before,
728
- );
729
- }
730
- }
731
- return this.searchLike(input.query, limit, input.conversationId, input.since, input.before);
732
- }
733
- return this.searchRegex(input.query, limit, input.conversationId, input.since, input.before);
734
- }
735
-
736
- private indexMessageForFullText(messageId: MessageId, content: string): void {
737
- if (!this.fts5Available) {
738
- return;
739
- }
740
- const normalizedContent = normalizeMessageContentForFullTextIndex(content);
741
- if (!normalizedContent) {
742
- return;
743
- }
744
- try {
745
- this.db
746
- .prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`)
747
- .run(messageId, normalizedContent);
748
- } catch {
749
- // Full-text indexing is optional. Message persistence must still succeed.
750
- }
751
- }
752
-
753
- private deleteMessageFromFullText(messageId: MessageId): void {
754
- if (!this.fts5Available) {
755
- return;
756
- }
757
- try {
758
- this.db.prepare(`DELETE FROM messages_fts WHERE rowid = ?`).run(messageId);
759
- } catch {
760
- // Ignore FTS cleanup failures; the source row deletion is authoritative.
761
- }
762
- }
763
-
764
- private searchFullText(
765
- query: string,
766
- limit: number,
767
- conversationId?: ConversationId,
768
- since?: Date,
769
- before?: Date,
770
- sort?: SearchSort,
771
- ): MessageSearchResult[] {
772
- const where: string[] = ["messages_fts MATCH ?"];
773
- const args: Array<string | number> = [sanitizeFts5Query(query)];
774
- if (conversationId != null) {
775
- where.push("m.conversation_id = ?");
776
- args.push(conversationId);
777
- }
778
- if (since) {
779
- where.push("julianday(m.created_at) >= julianday(?)");
780
- args.push(since.toISOString());
781
- }
782
- if (before) {
783
- where.push("julianday(m.created_at) < julianday(?)");
784
- args.push(before.toISOString());
785
- }
786
- args.push(limit);
787
- const orderBy = buildFtsOrderBy(sort, "m.created_at");
788
-
789
- const sql = `SELECT
790
- m.message_id,
791
- m.conversation_id,
792
- m.role,
793
- snippet(messages_fts, 0, '', '', '...', 32) AS snippet,
794
- rank,
795
- m.created_at
796
- FROM messages_fts
797
- JOIN messages m ON m.message_id = messages_fts.rowid
798
- WHERE ${where.join(" AND ")}
799
- ORDER BY ${orderBy}
800
- LIMIT ?`;
801
- const rows = this.db.prepare(sql).all(...args) as unknown as MessageSearchRow[];
802
- return rows.map(toSearchResult);
803
- }
804
-
805
- private searchLike(
806
- query: string,
807
- limit: number,
808
- conversationId?: ConversationId,
809
- since?: Date,
810
- before?: Date,
811
- ): MessageSearchResult[] {
812
- const plan = buildLikeSearchPlan("content", query);
813
- if (plan.terms.length === 0) {
814
- return [];
815
- }
816
-
817
- const where: string[] = [...plan.where];
818
- const args: Array<string | number> = [...plan.args];
819
- if (conversationId != null) {
820
- where.push("conversation_id = ?");
821
- args.push(conversationId);
822
- }
823
- if (since) {
824
- where.push("julianday(created_at) >= julianday(?)");
825
- args.push(since.toISOString());
826
- }
827
- if (before) {
828
- where.push("julianday(created_at) < julianday(?)");
829
- args.push(before.toISOString());
830
- }
831
- args.push(limit);
832
-
833
- const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
834
- const rows = this.db
835
- .prepare(
836
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
837
- FROM messages
838
- ${whereClause}
839
- ORDER BY created_at DESC
840
- LIMIT ?`,
841
- )
842
- .all(...args) as unknown as MessageRow[];
843
-
844
- return rows
845
- .map((row) => {
846
- const normalizedContent = normalizeMessageContentForFullTextIndex(row.content) ?? row.content;
847
- const haystack = normalizedContent.toLowerCase();
848
- const matchesAllTerms = plan.terms.every((term) => haystack.includes(term));
849
- if (!matchesAllTerms) {
850
- return null;
851
- }
852
- return {
853
- messageId: row.message_id,
854
- conversationId: row.conversation_id,
855
- role: row.role,
856
- snippet: createFallbackSnippet(normalizedContent, plan.terms),
857
- createdAt: parseUtcTimestamp(row.created_at),
858
- rank: 0,
859
- };
860
- })
861
- .filter((row): row is MessageSearchResult => row !== null);
862
- }
863
-
864
- private searchRegex(
865
- pattern: string,
866
- limit: number,
867
- conversationId?: ConversationId,
868
- since?: Date,
869
- before?: Date,
870
- ): MessageSearchResult[] {
871
- // SQLite has no native POSIX regex; fetch candidates and filter in JS
872
- // Guard against ReDoS: reject patterns with nested quantifiers or excessive length
873
- if (pattern.length > 500 || /(\+|\*|\?)\)(\+|\*|\?|\{\d)/.test(pattern)) {
874
- return [];
875
- }
876
- let re: RegExp;
877
- try {
878
- re = new RegExp(pattern);
879
- } catch {
880
- return [];
881
- }
882
-
883
- const where: string[] = [];
884
- const args: Array<string | number> = [];
885
- if (conversationId != null) {
886
- where.push("conversation_id = ?");
887
- args.push(conversationId);
888
- }
889
- if (since) {
890
- where.push("julianday(created_at) >= julianday(?)");
891
- args.push(since.toISOString());
892
- }
893
- if (before) {
894
- where.push("julianday(created_at) < julianday(?)");
895
- args.push(before.toISOString());
896
- }
897
- const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
898
- const rows = this.db
899
- .prepare(
900
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
901
- FROM messages
902
- ${whereClause}
903
- ORDER BY created_at DESC`,
904
- )
905
- .all(...args) as unknown as MessageRow[];
906
-
907
- const MAX_ROW_SCAN = 10_000;
908
- const results: MessageSearchResult[] = [];
909
- let scanned = 0;
910
- for (const row of rows) {
911
- if (results.length >= limit || scanned >= MAX_ROW_SCAN) {
912
- break;
913
- }
914
- scanned++;
915
- const match = re.exec(row.content);
916
- if (match) {
917
- results.push({
918
- messageId: row.message_id,
919
- conversationId: row.conversation_id,
920
- role: row.role,
921
- snippet: match[0],
922
- createdAt: parseUtcTimestamp(row.created_at),
923
- rank: 0,
924
- });
925
- }
926
- }
927
- return results;
928
- }
929
- }