@martian-engineering/lossless-claw 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +163 -6
- package/index.ts +5 -5
- package/package.json +4 -4
- package/src/db/config.ts +7 -2
- package/src/db/features.ts +42 -0
- package/src/db/migration.ts +10 -1
- package/src/engine.ts +12 -3
- package/src/store/conversation-store.ts +105 -16
- package/src/store/full-text-fallback.ts +74 -0
- package/src/store/summary-store.ts +93 -17
package/README.md
CHANGED
|
@@ -68,6 +68,168 @@ If you need to set it manually, ensure the context engine slot points at lossles
|
|
|
68
68
|
|
|
69
69
|
Restart OpenClaw after configuration changes.
|
|
70
70
|
|
|
71
|
+
### Optional: enable FTS5 for fast full-text search
|
|
72
|
+
|
|
73
|
+
`lossless-claw` works without FTS5 as of the current release. When FTS5 is unavailable in the
|
|
74
|
+
Node runtime that runs the OpenClaw gateway, the plugin:
|
|
75
|
+
|
|
76
|
+
- keeps persisting messages and summaries
|
|
77
|
+
- falls back from `"full_text"` search to a slower `LIKE`-based search
|
|
78
|
+
- loses FTS ranking/snippet quality
|
|
79
|
+
|
|
80
|
+
If you want native FTS5 search performance and ranking, the **exact Node runtime that runs the
|
|
81
|
+
gateway** must have SQLite FTS5 compiled in.
|
|
82
|
+
|
|
83
|
+
#### Probe the gateway runtime
|
|
84
|
+
|
|
85
|
+
Run this with the same `node` binary your gateway uses:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
node --input-type=module - <<'NODE'
|
|
89
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
90
|
+
const db = new DatabaseSync(':memory:');
|
|
91
|
+
const options = db.prepare('pragma compile_options').all().map((row) => row.compile_options);
|
|
92
|
+
|
|
93
|
+
console.log(options.filter((value) => value.includes('FTS')).join('\n') || 'no fts compile options');
|
|
94
|
+
|
|
95
|
+
try {
|
|
96
|
+
db.exec("CREATE VIRTUAL TABLE t USING fts5(content)");
|
|
97
|
+
console.log("fts5: ok");
|
|
98
|
+
} catch (err) {
|
|
99
|
+
console.log("fts5: fail");
|
|
100
|
+
console.log(err instanceof Error ? err.message : String(err));
|
|
101
|
+
}
|
|
102
|
+
NODE
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Expected output:
|
|
106
|
+
|
|
107
|
+
```text
|
|
108
|
+
ENABLE_FTS5
|
|
109
|
+
fts5: ok
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
If you get `fts5: fail`, build or install an FTS5-capable Node and point the gateway at that runtime.
|
|
113
|
+
|
|
114
|
+
#### Build an FTS5-capable Node on macOS
|
|
115
|
+
|
|
116
|
+
This workflow was verified with Node `v22.15.0`.
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
cd ~/Projects
|
|
120
|
+
git clone --depth 1 --branch v22.15.0 https://github.com/nodejs/node.git node-fts5
|
|
121
|
+
cd node-fts5
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Edit `deps/sqlite/sqlite.gyp` and add `SQLITE_ENABLE_FTS5` to the `defines` list for the `sqlite`
|
|
125
|
+
target:
|
|
126
|
+
|
|
127
|
+
```diff
|
|
128
|
+
'defines': [
|
|
129
|
+
'SQLITE_DEFAULT_MEMSTATUS=0',
|
|
130
|
+
+ 'SQLITE_ENABLE_FTS5',
|
|
131
|
+
'SQLITE_ENABLE_MATH_FUNCTIONS',
|
|
132
|
+
'SQLITE_ENABLE_SESSION',
|
|
133
|
+
'SQLITE_ENABLE_PREUPDATE_HOOK'
|
|
134
|
+
],
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Important:
|
|
138
|
+
|
|
139
|
+
- patch `deps/sqlite/sqlite.gyp`, not only `node.gyp`
|
|
140
|
+
- `node:sqlite` uses the embedded SQLite built from `deps/sqlite/sqlite.gyp`
|
|
141
|
+
|
|
142
|
+
Build the runtime:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
./configure --prefix="$PWD/out-install"
|
|
146
|
+
make -j8 node
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Expose the binary under a Node-compatible basename that OpenClaw recognizes:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
mkdir -p ~/Projects/node-fts5/bin
|
|
153
|
+
ln -sfn ~/Projects/node-fts5/out/Release/node ~/Projects/node-fts5/bin/node-22.15.0
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Use a basename like `node-22.15.0`, `node`, or `nodejs`. Names like
|
|
157
|
+
`node-v22.15.0-fts5` may not be recognized correctly by OpenClaw's CLI/runtime parsing.
|
|
158
|
+
|
|
159
|
+
Verify the new runtime:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
~/Projects/node-fts5/bin/node-22.15.0 --version
|
|
163
|
+
~/Projects/node-fts5/bin/node-22.15.0 --input-type=module - <<'NODE'
|
|
164
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
165
|
+
const db = new DatabaseSync(':memory:');
|
|
166
|
+
db.exec("CREATE VIRTUAL TABLE t USING fts5(content)");
|
|
167
|
+
console.log("fts5: ok");
|
|
168
|
+
NODE
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
#### Point the OpenClaw gateway at that runtime on macOS
|
|
172
|
+
|
|
173
|
+
Back up the existing LaunchAgent plist first:
|
|
174
|
+
|
|
175
|
+
```bash
|
|
176
|
+
cp ~/Library/LaunchAgents/ai.openclaw.gateway.plist \
|
|
177
|
+
~/Library/LaunchAgents/ai.openclaw.gateway.plist.bak-$(date +%Y%m%d-%H%M%S)
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Replace the runtime path, then reload the agent:
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
/usr/libexec/PlistBuddy -c 'Set :ProgramArguments:0 /Users/youruser/Projects/node-fts5/bin/node-22.15.0' \
|
|
184
|
+
~/Library/LaunchAgents/ai.openclaw.gateway.plist
|
|
185
|
+
|
|
186
|
+
launchctl bootout gui/$UID ~/Library/LaunchAgents/ai.openclaw.gateway.plist 2>/dev/null || true
|
|
187
|
+
launchctl bootstrap gui/$UID ~/Library/LaunchAgents/ai.openclaw.gateway.plist
|
|
188
|
+
launchctl kickstart -k gui/$UID/ai.openclaw.gateway
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Verify the live runtime:
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
launchctl print gui/$UID/ai.openclaw.gateway | sed -n '1,80p'
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
You should see:
|
|
198
|
+
|
|
199
|
+
```text
|
|
200
|
+
program = /Users/youruser/Projects/node-fts5/bin/node-22.15.0
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
#### Verify `lossless-claw`
|
|
204
|
+
|
|
205
|
+
Check the logs:
|
|
206
|
+
|
|
207
|
+
```bash
|
|
208
|
+
tail -n 60 ~/.openclaw/logs/gateway.log
|
|
209
|
+
tail -n 60 ~/.openclaw/logs/gateway.err.log
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
You want:
|
|
213
|
+
|
|
214
|
+
- `[gateway] [lcm] Plugin loaded ...`
|
|
215
|
+
- no new `no such module: fts5`
|
|
216
|
+
|
|
217
|
+
Then force one turn through the gateway and verify the DB fills:
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
/Users/youruser/Projects/node-fts5/bin/node-22.15.0 \
|
|
221
|
+
/path/to/openclaw/dist/index.js \
|
|
222
|
+
agent --session-id fts5-smoke --message 'Reply with exactly: ok' --timeout 60
|
|
223
|
+
|
|
224
|
+
sqlite3 ~/.openclaw/lcm.db '
|
|
225
|
+
select count(*) as conversations from conversations;
|
|
226
|
+
select count(*) as messages from messages;
|
|
227
|
+
select count(*) as summaries from summaries;
|
|
228
|
+
'
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Those counts should increase after a real turn.
|
|
232
|
+
|
|
71
233
|
## Configuration
|
|
72
234
|
|
|
73
235
|
LCM is configured through a combination of plugin config and environment variables. Environment variables take precedence for backward compatibility.
|
|
@@ -81,12 +243,7 @@ Add a `lossless-claw` entry under `plugins.entries` in your OpenClaw config:
|
|
|
81
243
|
"plugins": {
|
|
82
244
|
"entries": {
|
|
83
245
|
"lossless-claw": {
|
|
84
|
-
"enabled": true
|
|
85
|
-
"config": {
|
|
86
|
-
"freshTailCount": 32,
|
|
87
|
-
"contextThreshold": 0.75,
|
|
88
|
-
"incrementalMaxDepth": -1
|
|
89
|
-
}
|
|
246
|
+
"enabled": true
|
|
90
247
|
}
|
|
91
248
|
}
|
|
92
249
|
}
|
package/index.ts
CHANGED
|
@@ -597,7 +597,11 @@ function readLatestAssistantReply(messages: unknown[]): string | undefined {
|
|
|
597
597
|
function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
|
|
598
598
|
const envSnapshot = snapshotPluginEnv();
|
|
599
599
|
const readEnv: ReadEnvFn = (key) => process.env[key];
|
|
600
|
-
const
|
|
600
|
+
const pluginConfig =
|
|
601
|
+
api.pluginConfig && typeof api.pluginConfig === "object" && !Array.isArray(api.pluginConfig)
|
|
602
|
+
? api.pluginConfig
|
|
603
|
+
: undefined;
|
|
604
|
+
const config = resolveLcmConfig(process.env, pluginConfig);
|
|
601
605
|
|
|
602
606
|
return {
|
|
603
607
|
config,
|
|
@@ -854,9 +858,6 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
|
|
|
854
858
|
};
|
|
855
859
|
}
|
|
856
860
|
|
|
857
|
-
/** Cached config from configSchema.parse — available by the time register() runs. */
|
|
858
|
-
let _parsedPluginConfig: Record<string, unknown> | undefined;
|
|
859
|
-
|
|
860
861
|
const lcmPlugin = {
|
|
861
862
|
id: "lossless-claw",
|
|
862
863
|
name: "Lossless Context Management",
|
|
@@ -869,7 +870,6 @@ const lcmPlugin = {
|
|
|
869
870
|
value && typeof value === "object" && !Array.isArray(value)
|
|
870
871
|
? (value as Record<string, unknown>)
|
|
871
872
|
: {};
|
|
872
|
-
_parsedPluginConfig = raw;
|
|
873
873
|
return resolveLcmConfig(process.env, raw);
|
|
874
874
|
},
|
|
875
875
|
},
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@martian-engineering/lossless-claw",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"description": "Lossless Context Management plugin for OpenClaw — DAG-based conversation summarization with incremental compaction",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.ts",
|
|
@@ -24,6 +24,8 @@
|
|
|
24
24
|
"LICENSE"
|
|
25
25
|
],
|
|
26
26
|
"dependencies": {
|
|
27
|
+
"@mariozechner/pi-agent-core": "*",
|
|
28
|
+
"@mariozechner/pi-ai": "*",
|
|
27
29
|
"@sinclair/typebox": "0.34.48"
|
|
28
30
|
},
|
|
29
31
|
"devDependencies": {
|
|
@@ -31,9 +33,7 @@
|
|
|
31
33
|
"vitest": "^3.0.0"
|
|
32
34
|
},
|
|
33
35
|
"peerDependencies": {
|
|
34
|
-
"openclaw": "*"
|
|
35
|
-
"@mariozechner/pi-agent-core": "*",
|
|
36
|
-
"@mariozechner/pi-ai": "*"
|
|
36
|
+
"openclaw": "*"
|
|
37
37
|
},
|
|
38
38
|
"openclaw": {
|
|
39
39
|
"extensions": [
|
package/src/db/config.ts
CHANGED
|
@@ -71,7 +71,10 @@ export function resolveLcmConfig(
|
|
|
71
71
|
? env.LCM_ENABLED !== "false"
|
|
72
72
|
: toBool(pc.enabled) ?? true,
|
|
73
73
|
databasePath:
|
|
74
|
-
env.LCM_DATABASE_PATH
|
|
74
|
+
env.LCM_DATABASE_PATH
|
|
75
|
+
?? toStr(pc.dbPath)
|
|
76
|
+
?? toStr(pc.databasePath)
|
|
77
|
+
?? join(homedir(), ".openclaw", "lcm.db"),
|
|
75
78
|
contextThreshold:
|
|
76
79
|
(env.LCM_CONTEXT_THRESHOLD !== undefined ? parseFloat(env.LCM_CONTEXT_THRESHOLD) : undefined)
|
|
77
80
|
?? toNumber(pc.contextThreshold) ?? 0.75,
|
|
@@ -104,7 +107,9 @@ export function resolveLcmConfig(
|
|
|
104
107
|
?? toNumber(pc.maxExpandTokens) ?? 4000,
|
|
105
108
|
largeFileTokenThreshold:
|
|
106
109
|
(env.LCM_LARGE_FILE_TOKEN_THRESHOLD !== undefined ? parseInt(env.LCM_LARGE_FILE_TOKEN_THRESHOLD, 10) : undefined)
|
|
107
|
-
?? toNumber(pc.
|
|
110
|
+
?? toNumber(pc.largeFileThresholdTokens)
|
|
111
|
+
?? toNumber(pc.largeFileTokenThreshold)
|
|
112
|
+
?? 25000,
|
|
108
113
|
largeFileSummaryProvider:
|
|
109
114
|
env.LCM_LARGE_FILE_SUMMARY_PROVIDER?.trim() ?? toStr(pc.largeFileSummaryProvider) ?? "",
|
|
110
115
|
largeFileSummaryModel:
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { DatabaseSync } from "node:sqlite";
|
|
2
|
+
|
|
3
|
+
export type LcmDbFeatures = {
|
|
4
|
+
fts5Available: boolean;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
const featureCache = new WeakMap<DatabaseSync, LcmDbFeatures>();
|
|
8
|
+
|
|
9
|
+
function probeFts5(db: DatabaseSync): boolean {
|
|
10
|
+
try {
|
|
11
|
+
db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
|
|
12
|
+
db.exec("CREATE VIRTUAL TABLE temp.__lcm_fts5_probe USING fts5(content)");
|
|
13
|
+
db.exec("DROP TABLE temp.__lcm_fts5_probe");
|
|
14
|
+
return true;
|
|
15
|
+
} catch {
|
|
16
|
+
try {
|
|
17
|
+
db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
|
|
18
|
+
} catch {
|
|
19
|
+
// Ignore cleanup failures after a failed probe.
|
|
20
|
+
}
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Detect SQLite features exposed by the current Node runtime.
|
|
27
|
+
*
|
|
28
|
+
* The result is cached per DatabaseSync handle because the probe is runtime-
|
|
29
|
+
* specific, not database-file-specific.
|
|
30
|
+
*/
|
|
31
|
+
export function getLcmDbFeatures(db: DatabaseSync): LcmDbFeatures {
|
|
32
|
+
const cached = featureCache.get(db);
|
|
33
|
+
if (cached) {
|
|
34
|
+
return cached;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const detected: LcmDbFeatures = {
|
|
38
|
+
fts5Available: probeFts5(db),
|
|
39
|
+
};
|
|
40
|
+
featureCache.set(db, detected);
|
|
41
|
+
return detected;
|
|
42
|
+
}
|
package/src/db/migration.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import { getLcmDbFeatures } from "./features.js";
|
|
2
3
|
|
|
3
4
|
type SummaryColumnInfo = {
|
|
4
5
|
name?: string;
|
|
@@ -354,7 +355,10 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
354
355
|
}
|
|
355
356
|
}
|
|
356
357
|
|
|
357
|
-
export function runLcmMigrations(
|
|
358
|
+
export function runLcmMigrations(
|
|
359
|
+
db: DatabaseSync,
|
|
360
|
+
options?: { fts5Available?: boolean },
|
|
361
|
+
): void {
|
|
358
362
|
db.exec(`
|
|
359
363
|
CREATE TABLE IF NOT EXISTS conversations (
|
|
360
364
|
conversation_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -492,6 +496,11 @@ export function runLcmMigrations(db: DatabaseSync): void {
|
|
|
492
496
|
backfillSummaryDepths(db);
|
|
493
497
|
backfillSummaryMetadata(db);
|
|
494
498
|
|
|
499
|
+
const fts5Available = options?.fts5Available ?? getLcmDbFeatures(db).fts5Available;
|
|
500
|
+
if (!fts5Available) {
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
|
|
495
504
|
// FTS5 virtual tables for full-text search (cannot use IF NOT EXISTS, so check manually)
|
|
496
505
|
const hasFts = db
|
|
497
506
|
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='messages_fts'")
|
package/src/engine.ts
CHANGED
|
@@ -18,6 +18,7 @@ import { ContextAssembler } from "./assembler.js";
|
|
|
18
18
|
import { CompactionEngine, type CompactionConfig } from "./compaction.js";
|
|
19
19
|
import type { LcmConfig } from "./db/config.js";
|
|
20
20
|
import { getLcmConnection, closeLcmConnection } from "./db/connection.js";
|
|
21
|
+
import { getLcmDbFeatures } from "./db/features.js";
|
|
21
22
|
import { runLcmMigrations } from "./db/migration.js";
|
|
22
23
|
import {
|
|
23
24
|
createDelegatedExpansionGrant,
|
|
@@ -513,6 +514,7 @@ export class LcmContextEngine implements ContextEngine {
|
|
|
513
514
|
private compaction: CompactionEngine;
|
|
514
515
|
private retrieval: RetrievalEngine;
|
|
515
516
|
private migrated = false;
|
|
517
|
+
private readonly fts5Available: boolean;
|
|
516
518
|
private sessionOperationQueues = new Map<string, Promise<void>>();
|
|
517
519
|
private largeFileTextSummarizerResolved = false;
|
|
518
520
|
private largeFileTextSummarizer?: (prompt: string) => Promise<string | null>;
|
|
@@ -523,9 +525,16 @@ export class LcmContextEngine implements ContextEngine {
|
|
|
523
525
|
this.config = deps.config;
|
|
524
526
|
|
|
525
527
|
const db = getLcmConnection(this.config.databasePath);
|
|
528
|
+
this.fts5Available = getLcmDbFeatures(db).fts5Available;
|
|
526
529
|
|
|
527
|
-
this.conversationStore = new ConversationStore(db);
|
|
528
|
-
this.summaryStore = new SummaryStore(db);
|
|
530
|
+
this.conversationStore = new ConversationStore(db, { fts5Available: this.fts5Available });
|
|
531
|
+
this.summaryStore = new SummaryStore(db, { fts5Available: this.fts5Available });
|
|
532
|
+
|
|
533
|
+
if (!this.fts5Available) {
|
|
534
|
+
this.deps.log.warn(
|
|
535
|
+
"[lcm] FTS5 unavailable in the current Node runtime; full_text search will fall back to LIKE and indexing is disabled",
|
|
536
|
+
);
|
|
537
|
+
}
|
|
529
538
|
|
|
530
539
|
this.assembler = new ContextAssembler(
|
|
531
540
|
this.conversationStore,
|
|
@@ -561,7 +570,7 @@ export class LcmContextEngine implements ContextEngine {
|
|
|
561
570
|
return;
|
|
562
571
|
}
|
|
563
572
|
const db = getLcmConnection(this.config.databasePath);
|
|
564
|
-
runLcmMigrations(db);
|
|
573
|
+
runLcmMigrations(db, { fts5Available: this.fts5Available });
|
|
565
574
|
this.migrated = true;
|
|
566
575
|
}
|
|
567
576
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { DatabaseSync } from "node:sqlite";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { sanitizeFts5Query } from "./fts5-sanitize.js";
|
|
4
|
+
import { buildLikeSearchPlan, createFallbackSnippet } from "./full-text-fallback.js";
|
|
4
5
|
|
|
5
6
|
export type ConversationId = number;
|
|
6
7
|
export type MessageId = number;
|
|
@@ -203,7 +204,14 @@ function toMessagePartRecord(row: MessagePartRow): MessagePartRecord {
|
|
|
203
204
|
// ── ConversationStore ─────────────────────────────────────────────────────────
|
|
204
205
|
|
|
205
206
|
export class ConversationStore {
|
|
206
|
-
|
|
207
|
+
private readonly fts5Available: boolean;
|
|
208
|
+
|
|
209
|
+
constructor(
|
|
210
|
+
private db: DatabaseSync,
|
|
211
|
+
options?: { fts5Available?: boolean },
|
|
212
|
+
) {
|
|
213
|
+
this.fts5Available = options?.fts5Available ?? true;
|
|
214
|
+
}
|
|
207
215
|
|
|
208
216
|
// ── Transaction helpers ──────────────────────────────────────────────────
|
|
209
217
|
|
|
@@ -292,10 +300,7 @@ export class ConversationStore {
|
|
|
292
300
|
|
|
293
301
|
const messageId = Number(result.lastInsertRowid);
|
|
294
302
|
|
|
295
|
-
|
|
296
|
-
this.db
|
|
297
|
-
.prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`)
|
|
298
|
-
.run(messageId, input.content);
|
|
303
|
+
this.indexMessageForFullText(messageId, input.content);
|
|
299
304
|
|
|
300
305
|
const row = this.db
|
|
301
306
|
.prepare(
|
|
@@ -315,7 +320,6 @@ export class ConversationStore {
|
|
|
315
320
|
`INSERT INTO messages (conversation_id, seq, role, content, token_count)
|
|
316
321
|
VALUES (?, ?, ?, ?, ?)`,
|
|
317
322
|
);
|
|
318
|
-
const insertFtsStmt = this.db.prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`);
|
|
319
323
|
const selectStmt = this.db.prepare(
|
|
320
324
|
`SELECT message_id, conversation_id, seq, role, content, token_count, created_at
|
|
321
325
|
FROM messages WHERE message_id = ?`,
|
|
@@ -332,7 +336,7 @@ export class ConversationStore {
|
|
|
332
336
|
);
|
|
333
337
|
|
|
334
338
|
const messageId = Number(result.lastInsertRowid);
|
|
335
|
-
|
|
339
|
+
this.indexMessageForFullText(messageId, input.content);
|
|
336
340
|
const row = selectStmt.get(messageId) as unknown as MessageRow;
|
|
337
341
|
records.push(toMessageRecord(row));
|
|
338
342
|
}
|
|
@@ -535,8 +539,7 @@ export class ConversationStore {
|
|
|
535
539
|
.prepare(`DELETE FROM context_items WHERE item_type = 'message' AND message_id = ?`)
|
|
536
540
|
.run(messageId);
|
|
537
541
|
|
|
538
|
-
|
|
539
|
-
this.db.prepare(`DELETE FROM messages_fts WHERE rowid = ?`).run(messageId);
|
|
542
|
+
this.deleteMessageFromFullText(messageId);
|
|
540
543
|
|
|
541
544
|
// Delete the message (message_parts cascade via ON DELETE CASCADE)
|
|
542
545
|
this.db.prepare(`DELETE FROM messages WHERE message_id = ?`).run(messageId);
|
|
@@ -553,17 +556,54 @@ export class ConversationStore {
|
|
|
553
556
|
const limit = input.limit ?? 50;
|
|
554
557
|
|
|
555
558
|
if (input.mode === "full_text") {
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
559
|
+
if (this.fts5Available) {
|
|
560
|
+
try {
|
|
561
|
+
return this.searchFullText(
|
|
562
|
+
input.query,
|
|
563
|
+
limit,
|
|
564
|
+
input.conversationId,
|
|
565
|
+
input.since,
|
|
566
|
+
input.before,
|
|
567
|
+
);
|
|
568
|
+
} catch {
|
|
569
|
+
return this.searchLike(
|
|
570
|
+
input.query,
|
|
571
|
+
limit,
|
|
572
|
+
input.conversationId,
|
|
573
|
+
input.since,
|
|
574
|
+
input.before,
|
|
575
|
+
);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
return this.searchLike(input.query, limit, input.conversationId, input.since, input.before);
|
|
563
579
|
}
|
|
564
580
|
return this.searchRegex(input.query, limit, input.conversationId, input.since, input.before);
|
|
565
581
|
}
|
|
566
582
|
|
|
583
|
+
private indexMessageForFullText(messageId: MessageId, content: string): void {
|
|
584
|
+
if (!this.fts5Available) {
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
try {
|
|
588
|
+
this.db
|
|
589
|
+
.prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`)
|
|
590
|
+
.run(messageId, content);
|
|
591
|
+
} catch {
|
|
592
|
+
// Full-text indexing is optional. Message persistence must still succeed.
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
private deleteMessageFromFullText(messageId: MessageId): void {
|
|
597
|
+
if (!this.fts5Available) {
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
try {
|
|
601
|
+
this.db.prepare(`DELETE FROM messages_fts WHERE rowid = ?`).run(messageId);
|
|
602
|
+
} catch {
|
|
603
|
+
// Ignore FTS cleanup failures; the source row deletion is authoritative.
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
|
|
567
607
|
private searchFullText(
|
|
568
608
|
query: string,
|
|
569
609
|
limit: number,
|
|
@@ -603,6 +643,55 @@ export class ConversationStore {
|
|
|
603
643
|
return rows.map(toSearchResult);
|
|
604
644
|
}
|
|
605
645
|
|
|
646
|
+
private searchLike(
|
|
647
|
+
query: string,
|
|
648
|
+
limit: number,
|
|
649
|
+
conversationId?: ConversationId,
|
|
650
|
+
since?: Date,
|
|
651
|
+
before?: Date,
|
|
652
|
+
): MessageSearchResult[] {
|
|
653
|
+
const plan = buildLikeSearchPlan("content", query);
|
|
654
|
+
if (plan.terms.length === 0) {
|
|
655
|
+
return [];
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
const where: string[] = [...plan.where];
|
|
659
|
+
const args: Array<string | number> = [...plan.args];
|
|
660
|
+
if (conversationId != null) {
|
|
661
|
+
where.push("conversation_id = ?");
|
|
662
|
+
args.push(conversationId);
|
|
663
|
+
}
|
|
664
|
+
if (since) {
|
|
665
|
+
where.push("julianday(created_at) >= julianday(?)");
|
|
666
|
+
args.push(since.toISOString());
|
|
667
|
+
}
|
|
668
|
+
if (before) {
|
|
669
|
+
where.push("julianday(created_at) < julianday(?)");
|
|
670
|
+
args.push(before.toISOString());
|
|
671
|
+
}
|
|
672
|
+
args.push(limit);
|
|
673
|
+
|
|
674
|
+
const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
|
|
675
|
+
const rows = this.db
|
|
676
|
+
.prepare(
|
|
677
|
+
`SELECT message_id, conversation_id, seq, role, content, token_count, created_at
|
|
678
|
+
FROM messages
|
|
679
|
+
${whereClause}
|
|
680
|
+
ORDER BY created_at DESC
|
|
681
|
+
LIMIT ?`,
|
|
682
|
+
)
|
|
683
|
+
.all(...args) as unknown as MessageRow[];
|
|
684
|
+
|
|
685
|
+
return rows.map((row) => ({
|
|
686
|
+
messageId: row.message_id,
|
|
687
|
+
conversationId: row.conversation_id,
|
|
688
|
+
role: row.role,
|
|
689
|
+
snippet: createFallbackSnippet(row.content, plan.terms),
|
|
690
|
+
createdAt: new Date(row.created_at),
|
|
691
|
+
rank: 0,
|
|
692
|
+
}));
|
|
693
|
+
}
|
|
694
|
+
|
|
606
695
|
private searchRegex(
|
|
607
696
|
pattern: string,
|
|
608
697
|
limit: number,
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
const RAW_TERM_RE = /"([^"]+)"|(\S+)/g;
|
|
2
|
+
const EDGE_PUNCTUATION_RE = /^[`'"()[\]{}<>.,:;!?*_+=|\\/-]+|[`'"()[\]{}<>.,:;!?*_+=|\\/-]+$/g;
|
|
3
|
+
|
|
4
|
+
export type LikeSearchPlan = {
|
|
5
|
+
terms: string[];
|
|
6
|
+
where: string[];
|
|
7
|
+
args: string[];
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
function normalizeFallbackTerm(raw: string): string {
|
|
11
|
+
return raw.trim().replace(EDGE_PUNCTUATION_RE, "").toLowerCase();
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function escapeLike(term: string): string {
|
|
15
|
+
return term.replace(/([\\%_])/g, "\\$1");
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Convert a free-text query into a conservative LIKE search plan.
|
|
20
|
+
*
|
|
21
|
+
* The fallback keeps phrase tokens when the query uses double quotes, and
|
|
22
|
+
* otherwise searches for all normalized tokens as case-insensitive substrings.
|
|
23
|
+
*/
|
|
24
|
+
export function buildLikeSearchPlan(column: string, query: string): LikeSearchPlan {
|
|
25
|
+
const terms: string[] = [];
|
|
26
|
+
for (const match of query.matchAll(RAW_TERM_RE)) {
|
|
27
|
+
const raw = match[1] ?? match[2] ?? "";
|
|
28
|
+
const normalized = normalizeFallbackTerm(raw);
|
|
29
|
+
if (normalized.length > 0 && !terms.includes(normalized)) {
|
|
30
|
+
terms.push(normalized);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (terms.length === 0) {
|
|
35
|
+
const fallback = normalizeFallbackTerm(query);
|
|
36
|
+
if (fallback.length > 0) {
|
|
37
|
+
terms.push(fallback);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return {
|
|
42
|
+
terms,
|
|
43
|
+
where: terms.map(() => `LOWER(${column}) LIKE ? ESCAPE '\\'`),
|
|
44
|
+
args: terms.map((term) => `%${escapeLike(term)}%`),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Build a compact snippet centered around the earliest matching term.
|
|
50
|
+
*/
|
|
51
|
+
export function createFallbackSnippet(content: string, terms: string[]): string {
|
|
52
|
+
const haystack = content.toLowerCase();
|
|
53
|
+
let matchIndex = -1;
|
|
54
|
+
let matchLength = 0;
|
|
55
|
+
|
|
56
|
+
for (const term of terms) {
|
|
57
|
+
const idx = haystack.indexOf(term);
|
|
58
|
+
if (idx !== -1 && (matchIndex === -1 || idx < matchIndex)) {
|
|
59
|
+
matchIndex = idx;
|
|
60
|
+
matchLength = term.length;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (matchIndex === -1) {
|
|
65
|
+
const head = content.trim();
|
|
66
|
+
return head.length <= 80 ? head : `${head.slice(0, 77).trimEnd()}...`;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const start = Math.max(0, matchIndex - 24);
|
|
70
|
+
const end = Math.min(content.length, matchIndex + Math.max(matchLength, 1) + 40);
|
|
71
|
+
const prefix = start > 0 ? "..." : "";
|
|
72
|
+
const suffix = end < content.length ? "..." : "";
|
|
73
|
+
return `${prefix}${content.slice(start, end).trim()}${suffix}`;
|
|
74
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { DatabaseSync } from "node:sqlite";
|
|
2
2
|
import { sanitizeFts5Query } from "./fts5-sanitize.js";
|
|
3
|
+
import { buildLikeSearchPlan, createFallbackSnippet } from "./full-text-fallback.js";
|
|
3
4
|
|
|
4
5
|
export type SummaryKind = "leaf" | "condensed";
|
|
5
6
|
export type ContextItemType = "message" | "summary";
|
|
@@ -239,7 +240,14 @@ function toLargeFileRecord(row: LargeFileRow): LargeFileRecord {
|
|
|
239
240
|
// ── SummaryStore ──────────────────────────────────────────────────────────────
|
|
240
241
|
|
|
241
242
|
export class SummaryStore {
|
|
242
|
-
|
|
243
|
+
private readonly fts5Available: boolean;
|
|
244
|
+
|
|
245
|
+
constructor(
|
|
246
|
+
private db: DatabaseSync,
|
|
247
|
+
options?: { fts5Available?: boolean },
|
|
248
|
+
) {
|
|
249
|
+
this.fts5Available = options?.fts5Available ?? true;
|
|
250
|
+
}
|
|
243
251
|
|
|
244
252
|
// ── Summary CRUD ──────────────────────────────────────────────────────────
|
|
245
253
|
|
|
@@ -305,8 +313,21 @@ export class SummaryStore {
|
|
|
305
313
|
sourceMessageTokenCount,
|
|
306
314
|
);
|
|
307
315
|
|
|
316
|
+
const row = this.db
|
|
317
|
+
.prepare(
|
|
318
|
+
`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
319
|
+
earliest_at, latest_at, descendant_count, created_at
|
|
320
|
+
, descendant_token_count, source_message_token_count
|
|
321
|
+
FROM summaries WHERE summary_id = ?`,
|
|
322
|
+
)
|
|
323
|
+
.get(input.summaryId) as unknown as SummaryRow;
|
|
324
|
+
|
|
308
325
|
// Index in FTS5 as best-effort; compaction flow must continue even if
|
|
309
326
|
// FTS indexing fails for any reason.
|
|
327
|
+
if (!this.fts5Available) {
|
|
328
|
+
return toSummaryRecord(row);
|
|
329
|
+
}
|
|
330
|
+
|
|
310
331
|
try {
|
|
311
332
|
this.db
|
|
312
333
|
.prepare(`INSERT INTO summaries_fts(summary_id, content) VALUES (?, ?)`)
|
|
@@ -316,15 +337,6 @@ export class SummaryStore {
|
|
|
316
337
|
// compaction and assembly will still work correctly.
|
|
317
338
|
}
|
|
318
339
|
|
|
319
|
-
const row = this.db
|
|
320
|
-
.prepare(
|
|
321
|
-
`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
322
|
-
earliest_at, latest_at, descendant_count, created_at
|
|
323
|
-
, descendant_token_count, source_message_token_count
|
|
324
|
-
FROM summaries WHERE summary_id = ?`,
|
|
325
|
-
)
|
|
326
|
-
.get(input.summaryId) as unknown as SummaryRow;
|
|
327
|
-
|
|
328
340
|
return toSummaryRecord(row);
|
|
329
341
|
}
|
|
330
342
|
|
|
@@ -685,13 +697,26 @@ export class SummaryStore {
|
|
|
685
697
|
const limit = input.limit ?? 50;
|
|
686
698
|
|
|
687
699
|
if (input.mode === "full_text") {
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
700
|
+
if (this.fts5Available) {
|
|
701
|
+
try {
|
|
702
|
+
return this.searchFullText(
|
|
703
|
+
input.query,
|
|
704
|
+
limit,
|
|
705
|
+
input.conversationId,
|
|
706
|
+
input.since,
|
|
707
|
+
input.before,
|
|
708
|
+
);
|
|
709
|
+
} catch {
|
|
710
|
+
return this.searchLike(
|
|
711
|
+
input.query,
|
|
712
|
+
limit,
|
|
713
|
+
input.conversationId,
|
|
714
|
+
input.since,
|
|
715
|
+
input.before,
|
|
716
|
+
);
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
return this.searchLike(input.query, limit, input.conversationId, input.since, input.before);
|
|
695
720
|
}
|
|
696
721
|
return this.searchRegex(input.query, limit, input.conversationId, input.since, input.before);
|
|
697
722
|
}
|
|
@@ -735,6 +760,57 @@ export class SummaryStore {
|
|
|
735
760
|
return rows.map(toSearchResult);
|
|
736
761
|
}
|
|
737
762
|
|
|
763
|
+
private searchLike(
|
|
764
|
+
query: string,
|
|
765
|
+
limit: number,
|
|
766
|
+
conversationId?: number,
|
|
767
|
+
since?: Date,
|
|
768
|
+
before?: Date,
|
|
769
|
+
): SummarySearchResult[] {
|
|
770
|
+
const plan = buildLikeSearchPlan("content", query);
|
|
771
|
+
if (plan.terms.length === 0) {
|
|
772
|
+
return [];
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
const where: string[] = [...plan.where];
|
|
776
|
+
const args: Array<string | number> = [...plan.args];
|
|
777
|
+
if (conversationId != null) {
|
|
778
|
+
where.push("conversation_id = ?");
|
|
779
|
+
args.push(conversationId);
|
|
780
|
+
}
|
|
781
|
+
if (since) {
|
|
782
|
+
where.push("julianday(created_at) >= julianday(?)");
|
|
783
|
+
args.push(since.toISOString());
|
|
784
|
+
}
|
|
785
|
+
if (before) {
|
|
786
|
+
where.push("julianday(created_at) < julianday(?)");
|
|
787
|
+
args.push(before.toISOString());
|
|
788
|
+
}
|
|
789
|
+
args.push(limit);
|
|
790
|
+
|
|
791
|
+
const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
|
|
792
|
+
const rows = this.db
|
|
793
|
+
.prepare(
|
|
794
|
+
`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
795
|
+
earliest_at, latest_at, descendant_count, descendant_token_count,
|
|
796
|
+
source_message_token_count, created_at
|
|
797
|
+
FROM summaries
|
|
798
|
+
${whereClause}
|
|
799
|
+
ORDER BY created_at DESC
|
|
800
|
+
LIMIT ?`,
|
|
801
|
+
)
|
|
802
|
+
.all(...args) as unknown as SummaryRow[];
|
|
803
|
+
|
|
804
|
+
return rows.map((row) => ({
|
|
805
|
+
summaryId: row.summary_id,
|
|
806
|
+
conversationId: row.conversation_id,
|
|
807
|
+
kind: row.kind,
|
|
808
|
+
snippet: createFallbackSnippet(row.content, plan.terms),
|
|
809
|
+
createdAt: new Date(row.created_at),
|
|
810
|
+
rank: 0,
|
|
811
|
+
}));
|
|
812
|
+
}
|
|
813
|
+
|
|
738
814
|
private searchRegex(
|
|
739
815
|
pattern: string,
|
|
740
816
|
limit: number,
|