@martian-engineering/lossless-claw 0.7.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +19 -3
  2. package/dist/index.js +19240 -0
  3. package/docs/agent-tools.md +9 -4
  4. package/docs/configuration.md +24 -5
  5. package/openclaw.plugin.json +27 -3
  6. package/package.json +7 -6
  7. package/skills/lossless-claw/SKILL.md +3 -2
  8. package/skills/lossless-claw/references/architecture.md +12 -0
  9. package/skills/lossless-claw/references/config.md +37 -0
  10. package/skills/lossless-claw/references/diagnostics.md +13 -0
  11. package/index.ts +0 -2
  12. package/src/assembler.ts +0 -1188
  13. package/src/compaction.ts +0 -1756
  14. package/src/db/config.ts +0 -345
  15. package/src/db/connection.ts +0 -141
  16. package/src/db/features.ts +0 -42
  17. package/src/db/migration.ts +0 -746
  18. package/src/engine.ts +0 -4306
  19. package/src/expansion-auth.ts +0 -365
  20. package/src/expansion-policy.ts +0 -303
  21. package/src/expansion.ts +0 -383
  22. package/src/integrity.ts +0 -600
  23. package/src/large-files.ts +0 -546
  24. package/src/lcm-log.ts +0 -37
  25. package/src/openclaw-bridge.ts +0 -22
  26. package/src/plugin/index.ts +0 -1960
  27. package/src/plugin/lcm-command.ts +0 -765
  28. package/src/plugin/lcm-doctor-apply.ts +0 -542
  29. package/src/plugin/lcm-doctor-shared.ts +0 -210
  30. package/src/plugin/shared-init.ts +0 -59
  31. package/src/prune.ts +0 -391
  32. package/src/retrieval.ts +0 -363
  33. package/src/session-patterns.ts +0 -23
  34. package/src/startup-banner-log.ts +0 -49
  35. package/src/store/compaction-telemetry-store.ts +0 -156
  36. package/src/store/conversation-store.ts +0 -929
  37. package/src/store/fts5-sanitize.ts +0 -50
  38. package/src/store/full-text-fallback.ts +0 -83
  39. package/src/store/full-text-sort.ts +0 -21
  40. package/src/store/index.ts +0 -39
  41. package/src/store/parse-utc-timestamp.ts +0 -25
  42. package/src/store/summary-store.ts +0 -1519
  43. package/src/summarize.ts +0 -1511
  44. package/src/tools/common.ts +0 -53
  45. package/src/tools/lcm-conversation-scope.ts +0 -127
  46. package/src/tools/lcm-describe-tool.ts +0 -245
  47. package/src/tools/lcm-expand-query-tool.ts +0 -831
  48. package/src/tools/lcm-expand-tool.delegation.ts +0 -580
  49. package/src/tools/lcm-expand-tool.ts +0 -453
  50. package/src/tools/lcm-expansion-recursion-guard.ts +0 -373
  51. package/src/tools/lcm-grep-tool.ts +0 -228
  52. package/src/transaction-mutex.ts +0 -136
  53. package/src/transcript-repair.ts +0 -301
  54. package/src/types.ts +0 -165
@@ -1,929 +0,0 @@
1
- import type { DatabaseSync } from "node:sqlite";
2
- import { randomUUID } from "node:crypto";
3
- import { withDatabaseTransaction } from "../transaction-mutex.js";
4
- import { sanitizeFts5Query } from "./fts5-sanitize.js";
5
- import { buildLikeSearchPlan, containsCjk, createFallbackSnippet } from "./full-text-fallback.js";
6
- import { parseUtcTimestamp, parseUtcTimestampOrNull } from "./parse-utc-timestamp.js";
7
- import { buildFtsOrderBy, type SearchSort } from "./full-text-sort.js";
8
-
9
- export type ConversationId = number;
10
- export type MessageId = number;
11
- export type SummaryId = string;
12
- export type MessageRole = "system" | "user" | "assistant" | "tool";
13
- export type MessagePartType =
14
- | "text"
15
- | "reasoning"
16
- | "tool"
17
- | "patch"
18
- | "file"
19
- | "subtask"
20
- | "compaction"
21
- | "step_start"
22
- | "step_finish"
23
- | "snapshot"
24
- | "agent"
25
- | "retry";
26
-
27
- export type CreateMessageInput = {
28
- conversationId: ConversationId;
29
- seq: number;
30
- role: MessageRole;
31
- content: string;
32
- tokenCount: number;
33
- };
34
-
35
- export type MessageRecord = {
36
- messageId: MessageId;
37
- conversationId: ConversationId;
38
- seq: number;
39
- role: MessageRole;
40
- content: string;
41
- tokenCount: number;
42
- createdAt: Date;
43
- };
44
-
45
- export type CreateMessagePartInput = {
46
- sessionId: string;
47
- partType: MessagePartType;
48
- ordinal: number;
49
- textContent?: string | null;
50
- toolCallId?: string | null;
51
- toolName?: string | null;
52
- toolInput?: string | null;
53
- toolOutput?: string | null;
54
- metadata?: string | null;
55
- };
56
-
57
- export type MessagePartRecord = {
58
- partId: string;
59
- messageId: MessageId;
60
- sessionId: string;
61
- partType: MessagePartType;
62
- ordinal: number;
63
- textContent: string | null;
64
- toolCallId: string | null;
65
- toolName: string | null;
66
- toolInput: string | null;
67
- toolOutput: string | null;
68
- metadata: string | null;
69
- };
70
-
71
- export type CreateConversationInput = {
72
- sessionId: string;
73
- sessionKey?: string;
74
- title?: string;
75
- active?: boolean;
76
- archivedAt?: Date | null;
77
- };
78
-
79
- export type ConversationRecord = {
80
- conversationId: ConversationId;
81
- sessionId: string;
82
- sessionKey: string | null;
83
- active: boolean;
84
- archivedAt: Date | null;
85
- title: string | null;
86
- bootstrappedAt: Date | null;
87
- createdAt: Date;
88
- updatedAt: Date;
89
- };
90
-
91
- export type MessageSearchInput = {
92
- conversationId?: ConversationId;
93
- query: string;
94
- mode: "regex" | "full_text";
95
- since?: Date;
96
- before?: Date;
97
- limit?: number;
98
- sort?: SearchSort;
99
- };
100
-
101
- export type MessageSearchResult = {
102
- messageId: MessageId;
103
- conversationId: ConversationId;
104
- role: MessageRole;
105
- snippet: string;
106
- createdAt: Date;
107
- rank?: number;
108
- };
109
-
110
- // ── DB row shapes (snake_case) ────────────────────────────────────────────────
111
-
112
- interface ConversationRow {
113
- conversation_id: number;
114
- session_id: string;
115
- session_key: string | null;
116
- active: number;
117
- archived_at: string | null;
118
- title: string | null;
119
- bootstrapped_at: string | null;
120
- created_at: string;
121
- updated_at: string;
122
- }
123
-
124
- interface MessageRow {
125
- message_id: number;
126
- conversation_id: number;
127
- seq: number;
128
- role: MessageRole;
129
- content: string;
130
- token_count: number;
131
- created_at: string;
132
- }
133
-
134
- interface MessageSearchRow {
135
- message_id: number;
136
- conversation_id: number;
137
- role: MessageRole;
138
- snippet: string;
139
- rank: number;
140
- created_at: string;
141
- }
142
-
143
- interface MessagePartRow {
144
- part_id: string;
145
- message_id: number;
146
- session_id: string;
147
- part_type: MessagePartType;
148
- ordinal: number;
149
- text_content: string | null;
150
- tool_call_id: string | null;
151
- tool_name: string | null;
152
- tool_input: string | null;
153
- tool_output: string | null;
154
- metadata: string | null;
155
- }
156
-
157
- interface CountRow {
158
- count: number;
159
- }
160
-
161
- interface MaxSeqRow {
162
- max_seq: number;
163
- }
164
-
165
- // ── Row mappers ───────────────────────────────────────────────────────────────
166
-
167
- function toConversationRecord(row: ConversationRow): ConversationRecord {
168
- return {
169
- conversationId: row.conversation_id,
170
- sessionId: row.session_id,
171
- sessionKey: row.session_key ?? null,
172
- active: row.active === 1,
173
- archivedAt: parseUtcTimestampOrNull(row.archived_at),
174
- title: row.title,
175
- bootstrappedAt: parseUtcTimestampOrNull(row.bootstrapped_at),
176
- createdAt: parseUtcTimestamp(row.created_at),
177
- updatedAt: parseUtcTimestamp(row.updated_at),
178
- };
179
- }
180
-
181
- function toMessageRecord(row: MessageRow): MessageRecord {
182
- return {
183
- messageId: row.message_id,
184
- conversationId: row.conversation_id,
185
- seq: row.seq,
186
- role: row.role,
187
- content: row.content,
188
- tokenCount: row.token_count,
189
- createdAt: parseUtcTimestamp(row.created_at),
190
- };
191
- }
192
-
193
- function toSearchResult(row: MessageSearchRow): MessageSearchResult {
194
- return {
195
- messageId: row.message_id,
196
- conversationId: row.conversation_id,
197
- role: row.role,
198
- snippet: row.snippet,
199
- createdAt: parseUtcTimestamp(row.created_at),
200
- rank: row.rank,
201
- };
202
- }
203
-
204
- function toMessagePartRecord(row: MessagePartRow): MessagePartRecord {
205
- return {
206
- partId: row.part_id,
207
- messageId: row.message_id,
208
- sessionId: row.session_id,
209
- partType: row.part_type,
210
- ordinal: row.ordinal,
211
- textContent: row.text_content,
212
- toolCallId: row.tool_call_id,
213
- toolName: row.tool_name,
214
- toolInput: row.tool_input,
215
- toolOutput: row.tool_output,
216
- metadata: row.metadata,
217
- };
218
- }
219
-
220
- function normalizeMessageContentForFullTextIndex(content: string): string | null {
221
- const trimmed = content.trim();
222
- if (!trimmed) {
223
- return null;
224
- }
225
-
226
- const isExternalizedReference =
227
- trimmed.startsWith("[LCM File:") || trimmed.startsWith("[LCM Tool Output:");
228
- if (!isExternalizedReference) {
229
- return content;
230
- }
231
-
232
- const lines = trimmed
233
- .split(/\r?\n/)
234
- .map((line) => line.trim())
235
- .filter((line) => line.length > 0);
236
- if (lines.length === 0) {
237
- return null;
238
- }
239
-
240
- const header = lines[0] ?? "";
241
- const summaryLines: string[] = [];
242
- let inSummary = false;
243
- for (let index = 1; index < lines.length; index += 1) {
244
- const line = lines[index]!;
245
- if (line === "Exploration Summary:") {
246
- inSummary = true;
247
- continue;
248
- }
249
- if (line.startsWith("Use lcm_describe")) {
250
- continue;
251
- }
252
- if (inSummary) {
253
- summaryLines.push(line);
254
- }
255
- }
256
-
257
- const normalized = [header, ...summaryLines].filter((line) => line.length > 0).join("\n");
258
- return normalized || null;
259
- }
260
-
261
- // ── ConversationStore ─────────────────────────────────────────────────────────
262
-
263
- export class ConversationStore {
264
- private readonly fts5Available: boolean;
265
-
266
- constructor(
267
- private db: DatabaseSync,
268
- options?: { fts5Available?: boolean },
269
- ) {
270
- this.fts5Available = options?.fts5Available ?? true;
271
- }
272
-
273
- // ── Transaction helpers ──────────────────────────────────────────────────
274
-
275
- async withTransaction<T>(operation: () => Promise<T> | T): Promise<T> {
276
- return withDatabaseTransaction(this.db, "BEGIN IMMEDIATE", operation);
277
- }
278
-
279
- // ── Conversation operations ───────────────────────────────────────────────
280
-
281
- async createConversation(input: CreateConversationInput): Promise<ConversationRecord> {
282
- const result = this.db
283
- .prepare(
284
- `INSERT INTO conversations (session_id, session_key, active, archived_at, title)
285
- VALUES (?, ?, ?, ?, ?)`,
286
- )
287
- .run(
288
- input.sessionId,
289
- input.sessionKey ?? null,
290
- input.active === false ? 0 : 1,
291
- input.archivedAt?.toISOString() ?? null,
292
- input.title ?? null,
293
- );
294
-
295
- const row = this.db
296
- .prepare(
297
- `SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
298
- FROM conversations WHERE conversation_id = ?`,
299
- )
300
- .get(Number(result.lastInsertRowid)) as unknown as ConversationRow;
301
-
302
- return toConversationRecord(row);
303
- }
304
-
305
- async getConversation(conversationId: ConversationId): Promise<ConversationRecord | null> {
306
- const row = this.db
307
- .prepare(
308
- `SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
309
- FROM conversations WHERE conversation_id = ?`,
310
- )
311
- .get(conversationId) as unknown as ConversationRow | undefined;
312
-
313
- return row ? toConversationRecord(row) : null;
314
- }
315
-
316
- async getConversationBySessionId(sessionId: string): Promise<ConversationRecord | null> {
317
- const row = this.db
318
- .prepare(
319
- `SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
320
- FROM conversations
321
- WHERE session_id = ?
322
- ORDER BY active DESC, created_at DESC
323
- LIMIT 1`,
324
- )
325
- .get(sessionId) as unknown as ConversationRow | undefined;
326
-
327
- return row ? toConversationRecord(row) : null;
328
- }
329
-
330
- async getConversationBySessionKey(sessionKey: string): Promise<ConversationRecord | null> {
331
- const row = this.db
332
- .prepare(
333
- `SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
334
- FROM conversations
335
- WHERE session_key = ?
336
- AND active = 1
337
- ORDER BY created_at DESC
338
- LIMIT 1`,
339
- )
340
- .get(sessionKey) as unknown as ConversationRow | undefined;
341
-
342
- return row ? toConversationRecord(row) : null;
343
- }
344
-
345
- /** Resolve a conversation by stable session identity. */
346
- async getConversationForSession(input: {
347
- sessionId?: string;
348
- sessionKey?: string;
349
- }): Promise<ConversationRecord | null> {
350
- const normalizedSessionKey = input.sessionKey?.trim();
351
- if (normalizedSessionKey) {
352
- const byKey = await this.getConversationBySessionKey(normalizedSessionKey);
353
- if (byKey) {
354
- return byKey;
355
- }
356
- }
357
-
358
- const normalizedSessionId = input.sessionId?.trim();
359
- if (!normalizedSessionId) {
360
- return null;
361
- }
362
-
363
- return this.getConversationBySessionId(normalizedSessionId);
364
- }
365
-
366
- async getOrCreateConversation(
367
- sessionId: string,
368
- titleOrOpts?: string | { title?: string; sessionKey?: string },
369
- ): Promise<ConversationRecord> {
370
- const opts = typeof titleOrOpts === "string" ? { title: titleOrOpts } : titleOrOpts ?? {};
371
- const normalizedSessionKey = opts.sessionKey?.trim();
372
- if (normalizedSessionKey) {
373
- const byKey = await this.getConversationBySessionKey(normalizedSessionKey);
374
- if (byKey) {
375
- if (byKey.sessionId !== sessionId) {
376
- this.db
377
- .prepare(
378
- `UPDATE conversations SET session_id = ?, updated_at = datetime('now') WHERE conversation_id = ?`,
379
- )
380
- .run(sessionId, byKey.conversationId);
381
- byKey.sessionId = sessionId;
382
- }
383
- return byKey;
384
- }
385
- }
386
-
387
- const existing = await this.getConversationBySessionId(sessionId);
388
- if (existing) {
389
- if (!normalizedSessionKey) {
390
- return existing;
391
- }
392
- if (existing.active && !existing.sessionKey) {
393
- this.db
394
- .prepare(
395
- `UPDATE conversations SET session_key = ?, updated_at = datetime('now') WHERE conversation_id = ?`,
396
- )
397
- .run(normalizedSessionKey, existing.conversationId);
398
- existing.sessionKey = normalizedSessionKey;
399
- return existing;
400
- }
401
- if (existing.active && existing.sessionKey === normalizedSessionKey) {
402
- return existing;
403
- }
404
- }
405
-
406
- return this.createConversation({ sessionId, title: opts.title, sessionKey: normalizedSessionKey });
407
- }
408
-
409
- async markConversationBootstrapped(conversationId: ConversationId): Promise<void> {
410
- this.db
411
- .prepare(
412
- `UPDATE conversations
413
- SET bootstrapped_at = COALESCE(bootstrapped_at, datetime('now')),
414
- updated_at = datetime('now')
415
- WHERE conversation_id = ?`,
416
- )
417
- .run(conversationId);
418
- }
419
-
420
- async archiveConversation(conversationId: ConversationId): Promise<void> {
421
- this.db
422
- .prepare(
423
- `UPDATE conversations
424
- SET active = 0,
425
- archived_at = COALESCE(archived_at, datetime('now')),
426
- updated_at = datetime('now')
427
- WHERE conversation_id = ?`,
428
- )
429
- .run(conversationId);
430
- }
431
-
432
- // ── Message operations ────────────────────────────────────────────────────
433
-
434
- async createMessage(input: CreateMessageInput): Promise<MessageRecord> {
435
- const result = this.db
436
- .prepare(
437
- `INSERT INTO messages (conversation_id, seq, role, content, token_count)
438
- VALUES (?, ?, ?, ?, ?)`,
439
- )
440
- .run(input.conversationId, input.seq, input.role, input.content, input.tokenCount);
441
-
442
- const messageId = Number(result.lastInsertRowid);
443
-
444
- this.indexMessageForFullText(messageId, input.content);
445
-
446
- const row = this.db
447
- .prepare(
448
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
449
- FROM messages WHERE message_id = ?`,
450
- )
451
- .get(messageId) as unknown as MessageRow;
452
-
453
- return toMessageRecord(row);
454
- }
455
-
456
- async createMessagesBulk(inputs: CreateMessageInput[]): Promise<MessageRecord[]> {
457
- if (inputs.length === 0) {
458
- return [];
459
- }
460
- const insertStmt = this.db.prepare(
461
- `INSERT INTO messages (conversation_id, seq, role, content, token_count)
462
- VALUES (?, ?, ?, ?, ?)`,
463
- );
464
- const selectStmt = this.db.prepare(
465
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
466
- FROM messages WHERE message_id = ?`,
467
- );
468
-
469
- const records: MessageRecord[] = [];
470
- for (const input of inputs) {
471
- const result = insertStmt.run(
472
- input.conversationId,
473
- input.seq,
474
- input.role,
475
- input.content,
476
- input.tokenCount,
477
- );
478
-
479
- const messageId = Number(result.lastInsertRowid);
480
- this.indexMessageForFullText(messageId, input.content);
481
- const row = selectStmt.get(messageId) as unknown as MessageRow;
482
- records.push(toMessageRecord(row));
483
- }
484
-
485
- return records;
486
- }
487
-
488
- async getMessages(
489
- conversationId: ConversationId,
490
- opts?: { afterSeq?: number; limit?: number },
491
- ): Promise<MessageRecord[]> {
492
- const afterSeq = opts?.afterSeq ?? -1;
493
- const limit = opts?.limit;
494
-
495
- if (limit != null) {
496
- const rows = this.db
497
- .prepare(
498
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
499
- FROM messages
500
- WHERE conversation_id = ? AND seq > ?
501
- ORDER BY seq
502
- LIMIT ?`,
503
- )
504
- .all(conversationId, afterSeq, limit) as unknown as MessageRow[];
505
- return rows.map(toMessageRecord);
506
- }
507
-
508
- const rows = this.db
509
- .prepare(
510
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
511
- FROM messages
512
- WHERE conversation_id = ? AND seq > ?
513
- ORDER BY seq`,
514
- )
515
- .all(conversationId, afterSeq) as unknown as MessageRow[];
516
- return rows.map(toMessageRecord);
517
- }
518
-
519
- async getLastMessage(conversationId: ConversationId): Promise<MessageRecord | null> {
520
- const row = this.db
521
- .prepare(
522
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
523
- FROM messages
524
- WHERE conversation_id = ?
525
- ORDER BY seq DESC
526
- LIMIT 1`,
527
- )
528
- .get(conversationId) as unknown as MessageRow | undefined;
529
-
530
- return row ? toMessageRecord(row) : null;
531
- }
532
-
533
- async hasMessage(
534
- conversationId: ConversationId,
535
- role: MessageRole,
536
- content: string,
537
- ): Promise<boolean> {
538
- const row = this.db
539
- .prepare(
540
- `SELECT 1 AS count
541
- FROM messages
542
- WHERE conversation_id = ? AND role = ? AND content = ?
543
- LIMIT 1`,
544
- )
545
- .get(conversationId, role, content) as unknown as CountRow | undefined;
546
-
547
- return row?.count === 1;
548
- }
549
-
550
- async countMessagesByIdentity(
551
- conversationId: ConversationId,
552
- role: MessageRole,
553
- content: string,
554
- ): Promise<number> {
555
- const row = this.db
556
- .prepare(
557
- `SELECT COUNT(*) AS count
558
- FROM messages
559
- WHERE conversation_id = ? AND role = ? AND content = ?`,
560
- )
561
- .get(conversationId, role, content) as unknown as CountRow | undefined;
562
-
563
- return row?.count ?? 0;
564
- }
565
-
566
- async getMessageById(messageId: MessageId): Promise<MessageRecord | null> {
567
- const row = this.db
568
- .prepare(
569
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
570
- FROM messages WHERE message_id = ?`,
571
- )
572
- .get(messageId) as unknown as MessageRow | undefined;
573
- return row ? toMessageRecord(row) : null;
574
- }
575
-
576
- async createMessageParts(messageId: MessageId, parts: CreateMessagePartInput[]): Promise<void> {
577
- if (parts.length === 0) {
578
- return;
579
- }
580
-
581
- const stmt = this.db.prepare(
582
- `INSERT INTO message_parts (
583
- part_id,
584
- message_id,
585
- session_id,
586
- part_type,
587
- ordinal,
588
- text_content,
589
- tool_call_id,
590
- tool_name,
591
- tool_input,
592
- tool_output,
593
- metadata
594
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
595
- );
596
-
597
- for (const part of parts) {
598
- stmt.run(
599
- randomUUID(),
600
- messageId,
601
- part.sessionId,
602
- part.partType,
603
- part.ordinal,
604
- part.textContent ?? null,
605
- part.toolCallId ?? null,
606
- part.toolName ?? null,
607
- part.toolInput ?? null,
608
- part.toolOutput ?? null,
609
- part.metadata ?? null,
610
- );
611
- }
612
- }
613
-
614
- async getMessageParts(messageId: MessageId): Promise<MessagePartRecord[]> {
615
- const rows = this.db
616
- .prepare(
617
- `SELECT
618
- part_id,
619
- message_id,
620
- session_id,
621
- part_type,
622
- ordinal,
623
- text_content,
624
- tool_call_id,
625
- tool_name,
626
- tool_input,
627
- tool_output,
628
- metadata
629
- FROM message_parts
630
- WHERE message_id = ?
631
- ORDER BY ordinal`,
632
- )
633
- .all(messageId) as unknown as MessagePartRow[];
634
-
635
- return rows.map(toMessagePartRecord);
636
- }
637
-
638
- async getMessageCount(conversationId: ConversationId): Promise<number> {
639
- const row = this.db
640
- .prepare(`SELECT COUNT(*) AS count FROM messages WHERE conversation_id = ?`)
641
- .get(conversationId) as unknown as CountRow;
642
- return row?.count ?? 0;
643
- }
644
-
645
- async getMaxSeq(conversationId: ConversationId): Promise<number> {
646
- const row = this.db
647
- .prepare(
648
- `SELECT COALESCE(MAX(seq), 0) AS max_seq
649
- FROM messages WHERE conversation_id = ?`,
650
- )
651
- .get(conversationId) as unknown as MaxSeqRow;
652
- return row?.max_seq ?? 0;
653
- }
654
-
655
- // ── Deletion ──────────────────────────────────────────────────────────────
656
-
657
- /**
658
- * Delete messages and their associated records (context_items, FTS, message_parts).
659
- *
660
- * Skips messages referenced in summary_messages (already compacted) to avoid
661
- * breaking the summary DAG. Returns the count of actually deleted messages.
662
- */
663
- async deleteMessages(messageIds: MessageId[]): Promise<number> {
664
- if (messageIds.length === 0) {
665
- return 0;
666
- }
667
-
668
- let deleted = 0;
669
- for (const messageId of messageIds) {
670
- // Skip if referenced by a summary (ON DELETE RESTRICT would fail anyway)
671
- const refRow = this.db
672
- .prepare(`SELECT 1 AS found FROM summary_messages WHERE message_id = ? LIMIT 1`)
673
- .get(messageId) as unknown as { found: number } | undefined;
674
- if (refRow) {
675
- continue;
676
- }
677
-
678
- // Remove from context_items first (RESTRICT constraint)
679
- this.db
680
- .prepare(`DELETE FROM context_items WHERE item_type = 'message' AND message_id = ?`)
681
- .run(messageId);
682
-
683
- this.deleteMessageFromFullText(messageId);
684
-
685
- // Delete the message (message_parts cascade via ON DELETE CASCADE)
686
- this.db.prepare(`DELETE FROM messages WHERE message_id = ?`).run(messageId);
687
-
688
- deleted += 1;
689
- }
690
-
691
- return deleted;
692
- }
693
-
694
- // ── Search ────────────────────────────────────────────────────────────────
695
-
696
- async searchMessages(input: MessageSearchInput): Promise<MessageSearchResult[]> {
697
- const limit = input.limit ?? 50;
698
-
699
- if (input.mode === "full_text") {
700
- // FTS5 unicode61 can return incomplete matches for CJK text, so route
701
- // those queries through the existing LIKE fallback path immediately.
702
- if (containsCjk(input.query)) {
703
- return this.searchLike(
704
- input.query,
705
- limit,
706
- input.conversationId,
707
- input.since,
708
- input.before,
709
- );
710
- }
711
- if (this.fts5Available) {
712
- try {
713
- return this.searchFullText(
714
- input.query,
715
- limit,
716
- input.conversationId,
717
- input.since,
718
- input.before,
719
- input.sort,
720
- );
721
- } catch {
722
- return this.searchLike(
723
- input.query,
724
- limit,
725
- input.conversationId,
726
- input.since,
727
- input.before,
728
- );
729
- }
730
- }
731
- return this.searchLike(input.query, limit, input.conversationId, input.since, input.before);
732
- }
733
- return this.searchRegex(input.query, limit, input.conversationId, input.since, input.before);
734
- }
735
-
736
- private indexMessageForFullText(messageId: MessageId, content: string): void {
737
- if (!this.fts5Available) {
738
- return;
739
- }
740
- const normalizedContent = normalizeMessageContentForFullTextIndex(content);
741
- if (!normalizedContent) {
742
- return;
743
- }
744
- try {
745
- this.db
746
- .prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`)
747
- .run(messageId, normalizedContent);
748
- } catch {
749
- // Full-text indexing is optional. Message persistence must still succeed.
750
- }
751
- }
752
-
753
- private deleteMessageFromFullText(messageId: MessageId): void {
754
- if (!this.fts5Available) {
755
- return;
756
- }
757
- try {
758
- this.db.prepare(`DELETE FROM messages_fts WHERE rowid = ?`).run(messageId);
759
- } catch {
760
- // Ignore FTS cleanup failures; the source row deletion is authoritative.
761
- }
762
- }
763
-
764
- private searchFullText(
765
- query: string,
766
- limit: number,
767
- conversationId?: ConversationId,
768
- since?: Date,
769
- before?: Date,
770
- sort?: SearchSort,
771
- ): MessageSearchResult[] {
772
- const where: string[] = ["messages_fts MATCH ?"];
773
- const args: Array<string | number> = [sanitizeFts5Query(query)];
774
- if (conversationId != null) {
775
- where.push("m.conversation_id = ?");
776
- args.push(conversationId);
777
- }
778
- if (since) {
779
- where.push("julianday(m.created_at) >= julianday(?)");
780
- args.push(since.toISOString());
781
- }
782
- if (before) {
783
- where.push("julianday(m.created_at) < julianday(?)");
784
- args.push(before.toISOString());
785
- }
786
- args.push(limit);
787
- const orderBy = buildFtsOrderBy(sort, "m.created_at");
788
-
789
- const sql = `SELECT
790
- m.message_id,
791
- m.conversation_id,
792
- m.role,
793
- snippet(messages_fts, 0, '', '', '...', 32) AS snippet,
794
- rank,
795
- m.created_at
796
- FROM messages_fts
797
- JOIN messages m ON m.message_id = messages_fts.rowid
798
- WHERE ${where.join(" AND ")}
799
- ORDER BY ${orderBy}
800
- LIMIT ?`;
801
- const rows = this.db.prepare(sql).all(...args) as unknown as MessageSearchRow[];
802
- return rows.map(toSearchResult);
803
- }
804
-
805
- private searchLike(
806
- query: string,
807
- limit: number,
808
- conversationId?: ConversationId,
809
- since?: Date,
810
- before?: Date,
811
- ): MessageSearchResult[] {
812
- const plan = buildLikeSearchPlan("content", query);
813
- if (plan.terms.length === 0) {
814
- return [];
815
- }
816
-
817
- const where: string[] = [...plan.where];
818
- const args: Array<string | number> = [...plan.args];
819
- if (conversationId != null) {
820
- where.push("conversation_id = ?");
821
- args.push(conversationId);
822
- }
823
- if (since) {
824
- where.push("julianday(created_at) >= julianday(?)");
825
- args.push(since.toISOString());
826
- }
827
- if (before) {
828
- where.push("julianday(created_at) < julianday(?)");
829
- args.push(before.toISOString());
830
- }
831
- args.push(limit);
832
-
833
- const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
834
- const rows = this.db
835
- .prepare(
836
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
837
- FROM messages
838
- ${whereClause}
839
- ORDER BY created_at DESC
840
- LIMIT ?`,
841
- )
842
- .all(...args) as unknown as MessageRow[];
843
-
844
- return rows
845
- .map((row) => {
846
- const normalizedContent = normalizeMessageContentForFullTextIndex(row.content) ?? row.content;
847
- const haystack = normalizedContent.toLowerCase();
848
- const matchesAllTerms = plan.terms.every((term) => haystack.includes(term));
849
- if (!matchesAllTerms) {
850
- return null;
851
- }
852
- return {
853
- messageId: row.message_id,
854
- conversationId: row.conversation_id,
855
- role: row.role,
856
- snippet: createFallbackSnippet(normalizedContent, plan.terms),
857
- createdAt: parseUtcTimestamp(row.created_at),
858
- rank: 0,
859
- };
860
- })
861
- .filter((row): row is MessageSearchResult => row !== null);
862
- }
863
-
864
- private searchRegex(
865
- pattern: string,
866
- limit: number,
867
- conversationId?: ConversationId,
868
- since?: Date,
869
- before?: Date,
870
- ): MessageSearchResult[] {
871
- // SQLite has no native POSIX regex; fetch candidates and filter in JS
872
- // Guard against ReDoS: reject patterns with nested quantifiers or excessive length
873
- if (pattern.length > 500 || /(\+|\*|\?)\)(\+|\*|\?|\{\d)/.test(pattern)) {
874
- return [];
875
- }
876
- let re: RegExp;
877
- try {
878
- re = new RegExp(pattern);
879
- } catch {
880
- return [];
881
- }
882
-
883
- const where: string[] = [];
884
- const args: Array<string | number> = [];
885
- if (conversationId != null) {
886
- where.push("conversation_id = ?");
887
- args.push(conversationId);
888
- }
889
- if (since) {
890
- where.push("julianday(created_at) >= julianday(?)");
891
- args.push(since.toISOString());
892
- }
893
- if (before) {
894
- where.push("julianday(created_at) < julianday(?)");
895
- args.push(before.toISOString());
896
- }
897
- const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
898
- const rows = this.db
899
- .prepare(
900
- `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
901
- FROM messages
902
- ${whereClause}
903
- ORDER BY created_at DESC`,
904
- )
905
- .all(...args) as unknown as MessageRow[];
906
-
907
- const MAX_ROW_SCAN = 10_000;
908
- const results: MessageSearchResult[] = [];
909
- let scanned = 0;
910
- for (const row of rows) {
911
- if (results.length >= limit || scanned >= MAX_ROW_SCAN) {
912
- break;
913
- }
914
- scanned++;
915
- const match = re.exec(row.content);
916
- if (match) {
917
- results.push({
918
- messageId: row.message_id,
919
- conversationId: row.conversation_id,
920
- role: row.role,
921
- snippet: match[0],
922
- createdAt: parseUtcTimestamp(row.created_at),
923
- rank: 0,
924
- });
925
- }
926
- }
927
- return results;
928
- }
929
- }