@martian-engineering/lossless-claw 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +172 -13
- package/docs/architecture.md +1 -1
- package/docs/configuration.md +4 -3
- package/index.ts +6 -8
- package/openclaw.plugin.json +2 -2
- package/package.json +4 -4
- package/src/db/config.ts +96 -19
- package/src/db/features.ts +42 -0
- package/src/db/migration.ts +10 -1
- package/src/engine.ts +12 -3
- package/src/store/conversation-store.ts +105 -16
- package/src/store/full-text-fallback.ts +74 -0
- package/src/store/summary-store.ts +93 -17
- package/src/summarize.ts +3 -5
package/README.md
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# lossless-claw
|
|
2
2
|
|
|
3
|
-
> ⚠️ **Current requirement:** This plugin currently requires a custom OpenClaw build with [PR #22201](https://github.com/openclaw/openclaw/pull/22201) applied until that PR is merged upstream.
|
|
4
|
-
|
|
5
3
|
Lossless Context Management plugin for [OpenClaw](https://github.com/openclaw/openclaw), based on the [LCM paper](https://papers.voltropy.com/LCM). Replaces OpenClaw's built-in sliding-window compaction with a DAG-based summarization system that preserves every message while keeping active context within model token limits.
|
|
6
4
|
|
|
7
5
|
## What it does
|
|
8
6
|
|
|
7
|
+
Two ways to learn: read the below, or [check out this super cool animated visualization](https://losslesscontext.ai).
|
|
8
|
+
|
|
9
9
|
When a conversation grows beyond the model's context window, OpenClaw (just like all of the other agents) normally truncates older messages. LCM instead:
|
|
10
10
|
|
|
11
11
|
1. **Persists every message** in a SQLite database, organized by conversation
|
|
@@ -68,23 +68,182 @@ If you need to set it manually, ensure the context engine slot points at lossles
|
|
|
68
68
|
|
|
69
69
|
Restart OpenClaw after configuration changes.
|
|
70
70
|
|
|
71
|
+
### Optional: enable FTS5 for fast full-text search
|
|
72
|
+
|
|
73
|
+
`lossless-claw` works without FTS5 as of the current release. When FTS5 is unavailable in the
|
|
74
|
+
Node runtime that runs the OpenClaw gateway, the plugin:
|
|
75
|
+
|
|
76
|
+
- keeps persisting messages and summaries
|
|
77
|
+
- falls back from `"full_text"` search to a slower `LIKE`-based search
|
|
78
|
+
- loses FTS ranking/snippet quality
|
|
79
|
+
|
|
80
|
+
If you want native FTS5 search performance and ranking, the **exact Node runtime that runs the
|
|
81
|
+
gateway** must have SQLite FTS5 compiled in.
|
|
82
|
+
|
|
83
|
+
#### Probe the gateway runtime
|
|
84
|
+
|
|
85
|
+
Run this with the same `node` binary your gateway uses:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
node --input-type=module - <<'NODE'
|
|
89
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
90
|
+
const db = new DatabaseSync(':memory:');
|
|
91
|
+
const options = db.prepare('pragma compile_options').all().map((row) => row.compile_options);
|
|
92
|
+
|
|
93
|
+
console.log(options.filter((value) => value.includes('FTS')).join('\n') || 'no fts compile options');
|
|
94
|
+
|
|
95
|
+
try {
|
|
96
|
+
db.exec("CREATE VIRTUAL TABLE t USING fts5(content)");
|
|
97
|
+
console.log("fts5: ok");
|
|
98
|
+
} catch (err) {
|
|
99
|
+
console.log("fts5: fail");
|
|
100
|
+
console.log(err instanceof Error ? err.message : String(err));
|
|
101
|
+
}
|
|
102
|
+
NODE
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Expected output:
|
|
106
|
+
|
|
107
|
+
```text
|
|
108
|
+
ENABLE_FTS5
|
|
109
|
+
fts5: ok
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
If you get `fts5: fail`, build or install an FTS5-capable Node and point the gateway at that runtime.
|
|
113
|
+
|
|
114
|
+
#### Build an FTS5-capable Node on macOS
|
|
115
|
+
|
|
116
|
+
This workflow was verified with Node `v22.15.0`.
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
cd ~/Projects
|
|
120
|
+
git clone --depth 1 --branch v22.15.0 https://github.com/nodejs/node.git node-fts5
|
|
121
|
+
cd node-fts5
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Edit `deps/sqlite/sqlite.gyp` and add `SQLITE_ENABLE_FTS5` to the `defines` list for the `sqlite`
|
|
125
|
+
target:
|
|
126
|
+
|
|
127
|
+
```diff
|
|
128
|
+
'defines': [
|
|
129
|
+
'SQLITE_DEFAULT_MEMSTATUS=0',
|
|
130
|
+
+ 'SQLITE_ENABLE_FTS5',
|
|
131
|
+
'SQLITE_ENABLE_MATH_FUNCTIONS',
|
|
132
|
+
'SQLITE_ENABLE_SESSION',
|
|
133
|
+
'SQLITE_ENABLE_PREUPDATE_HOOK'
|
|
134
|
+
],
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Important:
|
|
138
|
+
|
|
139
|
+
- patch `deps/sqlite/sqlite.gyp`, not only `node.gyp`
|
|
140
|
+
- `node:sqlite` uses the embedded SQLite built from `deps/sqlite/sqlite.gyp`
|
|
141
|
+
|
|
142
|
+
Build the runtime:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
./configure --prefix="$PWD/out-install"
|
|
146
|
+
make -j8 node
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Expose the binary under a Node-compatible basename that OpenClaw recognizes:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
mkdir -p ~/Projects/node-fts5/bin
|
|
153
|
+
ln -sfn ~/Projects/node-fts5/out/Release/node ~/Projects/node-fts5/bin/node-22.15.0
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Use a basename like `node-22.15.0`, `node`, or `nodejs`. Names like
|
|
157
|
+
`node-v22.15.0-fts5` may not be recognized correctly by OpenClaw's CLI/runtime parsing.
|
|
158
|
+
|
|
159
|
+
Verify the new runtime:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
~/Projects/node-fts5/bin/node-22.15.0 --version
|
|
163
|
+
~/Projects/node-fts5/bin/node-22.15.0 --input-type=module - <<'NODE'
|
|
164
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
165
|
+
const db = new DatabaseSync(':memory:');
|
|
166
|
+
db.exec("CREATE VIRTUAL TABLE t USING fts5(content)");
|
|
167
|
+
console.log("fts5: ok");
|
|
168
|
+
NODE
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
#### Point the OpenClaw gateway at that runtime on macOS
|
|
172
|
+
|
|
173
|
+
Back up the existing LaunchAgent plist first:
|
|
174
|
+
|
|
175
|
+
```bash
|
|
176
|
+
cp ~/Library/LaunchAgents/ai.openclaw.gateway.plist \
|
|
177
|
+
~/Library/LaunchAgents/ai.openclaw.gateway.plist.bak-$(date +%Y%m%d-%H%M%S)
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Replace the runtime path, then reload the agent:
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
/usr/libexec/PlistBuddy -c 'Set :ProgramArguments:0 /Users/youruser/Projects/node-fts5/bin/node-22.15.0' \
|
|
184
|
+
~/Library/LaunchAgents/ai.openclaw.gateway.plist
|
|
185
|
+
|
|
186
|
+
launchctl bootout gui/$UID ~/Library/LaunchAgents/ai.openclaw.gateway.plist 2>/dev/null || true
|
|
187
|
+
launchctl bootstrap gui/$UID ~/Library/LaunchAgents/ai.openclaw.gateway.plist
|
|
188
|
+
launchctl kickstart -k gui/$UID/ai.openclaw.gateway
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Verify the live runtime:
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
launchctl print gui/$UID/ai.openclaw.gateway | sed -n '1,80p'
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
You should see:
|
|
198
|
+
|
|
199
|
+
```text
|
|
200
|
+
program = /Users/youruser/Projects/node-fts5/bin/node-22.15.0
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
#### Verify `lossless-claw`
|
|
204
|
+
|
|
205
|
+
Check the logs:
|
|
206
|
+
|
|
207
|
+
```bash
|
|
208
|
+
tail -n 60 ~/.openclaw/logs/gateway.log
|
|
209
|
+
tail -n 60 ~/.openclaw/logs/gateway.err.log
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
You want:
|
|
213
|
+
|
|
214
|
+
- `[gateway] [lcm] Plugin loaded ...`
|
|
215
|
+
- no new `no such module: fts5`
|
|
216
|
+
|
|
217
|
+
Then force one turn through the gateway and verify the DB fills:
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
/Users/youruser/Projects/node-fts5/bin/node-22.15.0 \
|
|
221
|
+
/path/to/openclaw/dist/index.js \
|
|
222
|
+
agent --session-id fts5-smoke --message 'Reply with exactly: ok' --timeout 60
|
|
223
|
+
|
|
224
|
+
sqlite3 ~/.openclaw/lcm.db '
|
|
225
|
+
select count(*) as conversations from conversations;
|
|
226
|
+
select count(*) as messages from messages;
|
|
227
|
+
select count(*) as summaries from summaries;
|
|
228
|
+
'
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Those counts should increase after a real turn.
|
|
232
|
+
|
|
71
233
|
## Configuration
|
|
72
234
|
|
|
73
235
|
LCM is configured through a combination of plugin config and environment variables. Environment variables take precedence for backward compatibility.
|
|
74
236
|
|
|
75
237
|
### Plugin config
|
|
76
238
|
|
|
77
|
-
Add
|
|
239
|
+
Add a `lossless-claw` entry under `plugins.entries` in your OpenClaw config:
|
|
78
240
|
|
|
79
241
|
```json
|
|
80
242
|
{
|
|
81
243
|
"plugins": {
|
|
82
|
-
"
|
|
244
|
+
"entries": {
|
|
83
245
|
"lossless-claw": {
|
|
84
|
-
"enabled": true
|
|
85
|
-
"freshTailCount": 32,
|
|
86
|
-
"contextThreshold": 0.75,
|
|
87
|
-
"incrementalMaxDepth": 1
|
|
246
|
+
"enabled": true
|
|
88
247
|
}
|
|
89
248
|
}
|
|
90
249
|
}
|
|
@@ -102,7 +261,7 @@ Add an `lossless-claw` block under `plugins.config` in your OpenClaw config:
|
|
|
102
261
|
| `LCM_LEAF_MIN_FANOUT` | `8` | Minimum raw messages per leaf summary |
|
|
103
262
|
| `LCM_CONDENSED_MIN_FANOUT` | `4` | Minimum summaries per condensed node |
|
|
104
263
|
| `LCM_CONDENSED_MIN_FANOUT_HARD` | `2` | Relaxed fanout for forced compaction sweeps |
|
|
105
|
-
| `LCM_INCREMENTAL_MAX_DEPTH` | `0` | How deep incremental compaction goes (0 = leaf only) |
|
|
264
|
+
| `LCM_INCREMENTAL_MAX_DEPTH` | `0` | How deep incremental compaction goes (0 = leaf only, -1 = unlimited) |
|
|
106
265
|
| `LCM_LEAF_CHUNK_TOKENS` | `20000` | Max source tokens per leaf compaction chunk |
|
|
107
266
|
| `LCM_LEAF_TARGET_TOKENS` | `1200` | Target token count for leaf summaries |
|
|
108
267
|
| `LCM_CONDENSED_TARGET_TOKENS` | `2000` | Target token count for condensed summaries |
|
|
@@ -110,18 +269,18 @@ Add an `lossless-claw` block under `plugins.config` in your OpenClaw config:
|
|
|
110
269
|
| `LCM_LARGE_FILE_TOKEN_THRESHOLD` | `25000` | File blocks above this size are intercepted and stored separately |
|
|
111
270
|
| `LCM_SUMMARY_MODEL` | *(from OpenClaw)* | Model for summarization (e.g. `anthropic/claude-sonnet-4-20250514`) |
|
|
112
271
|
| `LCM_SUMMARY_PROVIDER` | *(from OpenClaw)* | Provider override for summarization |
|
|
113
|
-
| `LCM_INCREMENTAL_MAX_DEPTH` | `0` | Depth limit for incremental condensation after leaf passes |
|
|
272
|
+
| `LCM_INCREMENTAL_MAX_DEPTH` | `0` | Depth limit for incremental condensation after leaf passes (-1 = unlimited) |
|
|
114
273
|
|
|
115
274
|
### Recommended starting configuration
|
|
116
275
|
|
|
117
276
|
```
|
|
118
277
|
LCM_FRESH_TAIL_COUNT=32
|
|
119
|
-
LCM_INCREMENTAL_MAX_DEPTH
|
|
278
|
+
LCM_INCREMENTAL_MAX_DEPTH=-1
|
|
120
279
|
LCM_CONTEXT_THRESHOLD=0.75
|
|
121
280
|
```
|
|
122
281
|
|
|
123
282
|
- **freshTailCount=32** protects the last 32 messages from compaction, giving the model enough recent context for continuity.
|
|
124
|
-
- **incrementalMaxDepth
|
|
283
|
+
- **incrementalMaxDepth=-1** enables unlimited automatic condensation after each compaction pass — the DAG cascades as deep as needed. Set to `0` (default) for leaf-only, or a positive integer for a specific depth cap.
|
|
125
284
|
- **contextThreshold=0.75** triggers compaction when context reaches 75% of the model's window, leaving headroom for the model's response.
|
|
126
285
|
|
|
127
286
|
## How it works
|
|
@@ -171,7 +330,7 @@ This gives the model enough information to know what was discussed, when, and ho
|
|
|
171
330
|
|
|
172
331
|
Compaction runs in two modes:
|
|
173
332
|
|
|
174
|
-
- **Proactive (after each turn):** If raw messages outside the fresh tail exceed `leafChunkTokens`, a leaf pass runs. If `incrementalMaxDepth
|
|
333
|
+
- **Proactive (after each turn):** If raw messages outside the fresh tail exceed `leafChunkTokens`, a leaf pass runs. If `incrementalMaxDepth != 0`, condensation follows (cascading to the configured depth, or unlimited with `-1`).
|
|
175
334
|
- **Reactive (overflow/manual):** When total context exceeds `contextThreshold × tokenBudget`, a full sweep runs: all eligible leaf chunks are compacted, then condensation proceeds depth-by-depth until stable.
|
|
176
335
|
|
|
177
336
|
### Depth-aware prompts
|
package/docs/architecture.md
CHANGED
|
@@ -87,7 +87,7 @@ The **condensed pass** merges summaries at the same depth into a higher-level su
|
|
|
87
87
|
**Incremental (after each turn):**
|
|
88
88
|
- Checks if raw tokens outside the fresh tail exceed `leafChunkTokens`
|
|
89
89
|
- If so, runs one leaf pass
|
|
90
|
-
- If `incrementalMaxDepth
|
|
90
|
+
- If `incrementalMaxDepth != 0`, follows with condensation passes up to that depth (`-1` for unlimited)
|
|
91
91
|
- Best-effort: failures don't break the conversation
|
|
92
92
|
|
|
93
93
|
**Full sweep (manual `/compact` or overflow):**
|
package/docs/configuration.md
CHANGED
|
@@ -26,7 +26,7 @@ Set recommended environment variables:
|
|
|
26
26
|
|
|
27
27
|
```bash
|
|
28
28
|
export LCM_FRESH_TAIL_COUNT=32
|
|
29
|
-
export LCM_INCREMENTAL_MAX_DEPTH
|
|
29
|
+
export LCM_INCREMENTAL_MAX_DEPTH=-1
|
|
30
30
|
```
|
|
31
31
|
|
|
32
32
|
Restart OpenClaw.
|
|
@@ -70,8 +70,9 @@ For coding conversations with tool calls (which generate many messages per logic
|
|
|
70
70
|
`LCM_INCREMENTAL_MAX_DEPTH` (default `0`) controls whether condensation happens automatically after leaf passes.
|
|
71
71
|
|
|
72
72
|
- **0** — Only leaf summaries are created incrementally. Condensation only happens during manual `/compact` or overflow.
|
|
73
|
-
- **1** — After each leaf pass, attempt to condense d0 summaries into d1.
|
|
74
|
-
- **2+** — Deeper automatic condensation
|
|
73
|
+
- **1** — After each leaf pass, attempt to condense d0 summaries into d1.
|
|
74
|
+
- **2+** — Deeper automatic condensation up to the specified depth.
|
|
75
|
+
- **-1** — Unlimited depth. Condensation cascades as deep as needed after each leaf pass. Recommended for long-running sessions.
|
|
75
76
|
|
|
76
77
|
### Summary target tokens
|
|
77
78
|
|
package/index.ts
CHANGED
|
@@ -597,7 +597,11 @@ function readLatestAssistantReply(messages: unknown[]): string | undefined {
|
|
|
597
597
|
function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
|
|
598
598
|
const envSnapshot = snapshotPluginEnv();
|
|
599
599
|
const readEnv: ReadEnvFn = (key) => process.env[key];
|
|
600
|
-
const
|
|
600
|
+
const pluginConfig =
|
|
601
|
+
api.pluginConfig && typeof api.pluginConfig === "object" && !Array.isArray(api.pluginConfig)
|
|
602
|
+
? api.pluginConfig
|
|
603
|
+
: undefined;
|
|
604
|
+
const config = resolveLcmConfig(process.env, pluginConfig);
|
|
601
605
|
|
|
602
606
|
return {
|
|
603
607
|
config,
|
|
@@ -862,17 +866,11 @@ const lcmPlugin = {
|
|
|
862
866
|
|
|
863
867
|
configSchema: {
|
|
864
868
|
parse(value: unknown) {
|
|
865
|
-
// Merge plugin config with env vars — env vars take precedence for backward compat
|
|
866
869
|
const raw =
|
|
867
870
|
value && typeof value === "object" && !Array.isArray(value)
|
|
868
871
|
? (value as Record<string, unknown>)
|
|
869
872
|
: {};
|
|
870
|
-
|
|
871
|
-
const config = resolveLcmConfig();
|
|
872
|
-
if (enabled !== undefined) {
|
|
873
|
-
config.enabled = enabled;
|
|
874
|
-
}
|
|
875
|
-
return config;
|
|
873
|
+
return resolveLcmConfig(process.env, raw);
|
|
876
874
|
},
|
|
877
875
|
},
|
|
878
876
|
|
package/openclaw.plugin.json
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
},
|
|
8
8
|
"incrementalMaxDepth": {
|
|
9
9
|
"label": "Incremental Max Depth",
|
|
10
|
-
"help": "How deep incremental compaction goes (0 = leaf only)"
|
|
10
|
+
"help": "How deep incremental compaction goes (0 = leaf only, -1 = unlimited)"
|
|
11
11
|
},
|
|
12
12
|
"freshTailCount": {
|
|
13
13
|
"label": "Fresh Tail Count",
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
},
|
|
33
33
|
"incrementalMaxDepth": {
|
|
34
34
|
"type": "integer",
|
|
35
|
-
"minimum":
|
|
35
|
+
"minimum": -1
|
|
36
36
|
},
|
|
37
37
|
"freshTailCount": {
|
|
38
38
|
"type": "integer",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@martian-engineering/lossless-claw",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"description": "Lossless Context Management plugin for OpenClaw — DAG-based conversation summarization with incremental compaction",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.ts",
|
|
@@ -24,6 +24,8 @@
|
|
|
24
24
|
"LICENSE"
|
|
25
25
|
],
|
|
26
26
|
"dependencies": {
|
|
27
|
+
"@mariozechner/pi-agent-core": "*",
|
|
28
|
+
"@mariozechner/pi-ai": "*",
|
|
27
29
|
"@sinclair/typebox": "0.34.48"
|
|
28
30
|
},
|
|
29
31
|
"devDependencies": {
|
|
@@ -31,9 +33,7 @@
|
|
|
31
33
|
"vitest": "^3.0.0"
|
|
32
34
|
},
|
|
33
35
|
"peerDependencies": {
|
|
34
|
-
"openclaw": "*"
|
|
35
|
-
"@mariozechner/pi-agent-core": "*",
|
|
36
|
-
"@mariozechner/pi-ai": "*"
|
|
36
|
+
"openclaw": "*"
|
|
37
37
|
},
|
|
38
38
|
"openclaw": {
|
|
39
39
|
"extensions": [
|
package/src/db/config.ts
CHANGED
|
@@ -26,25 +26,102 @@ export type LcmConfig = {
|
|
|
26
26
|
pruneHeartbeatOk: boolean;
|
|
27
27
|
};
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
/** Safely coerce an unknown value to a finite number, or return undefined. */
|
|
30
|
+
function toNumber(value: unknown): number | undefined {
|
|
31
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
32
|
+
if (typeof value === "string") {
|
|
33
|
+
const n = Number(value);
|
|
34
|
+
if (Number.isFinite(n)) return n;
|
|
35
|
+
}
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Safely coerce an unknown value to a boolean, or return undefined. */
|
|
40
|
+
function toBool(value: unknown): boolean | undefined {
|
|
41
|
+
if (typeof value === "boolean") return value;
|
|
42
|
+
if (value === "true") return true;
|
|
43
|
+
if (value === "false") return false;
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Safely coerce an unknown value to a trimmed non-empty string, or return undefined. */
|
|
48
|
+
function toStr(value: unknown): string | undefined {
|
|
49
|
+
if (typeof value === "string") {
|
|
50
|
+
const trimmed = value.trim();
|
|
51
|
+
return trimmed.length > 0 ? trimmed : undefined;
|
|
52
|
+
}
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Resolve LCM configuration with three-tier precedence:
|
|
58
|
+
* 1. Environment variables (highest — backward compat)
|
|
59
|
+
* 2. Plugin config object (from plugins.entries.lossless-claw.config)
|
|
60
|
+
* 3. Hardcoded defaults (lowest)
|
|
61
|
+
*/
|
|
62
|
+
export function resolveLcmConfig(
|
|
63
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
64
|
+
pluginConfig?: Record<string, unknown>,
|
|
65
|
+
): LcmConfig {
|
|
66
|
+
const pc = pluginConfig ?? {};
|
|
67
|
+
|
|
30
68
|
return {
|
|
31
|
-
enabled:
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
69
|
+
enabled:
|
|
70
|
+
env.LCM_ENABLED !== undefined
|
|
71
|
+
? env.LCM_ENABLED !== "false"
|
|
72
|
+
: toBool(pc.enabled) ?? true,
|
|
73
|
+
databasePath:
|
|
74
|
+
env.LCM_DATABASE_PATH
|
|
75
|
+
?? toStr(pc.dbPath)
|
|
76
|
+
?? toStr(pc.databasePath)
|
|
77
|
+
?? join(homedir(), ".openclaw", "lcm.db"),
|
|
78
|
+
contextThreshold:
|
|
79
|
+
(env.LCM_CONTEXT_THRESHOLD !== undefined ? parseFloat(env.LCM_CONTEXT_THRESHOLD) : undefined)
|
|
80
|
+
?? toNumber(pc.contextThreshold) ?? 0.75,
|
|
81
|
+
freshTailCount:
|
|
82
|
+
(env.LCM_FRESH_TAIL_COUNT !== undefined ? parseInt(env.LCM_FRESH_TAIL_COUNT, 10) : undefined)
|
|
83
|
+
?? toNumber(pc.freshTailCount) ?? 32,
|
|
84
|
+
leafMinFanout:
|
|
85
|
+
(env.LCM_LEAF_MIN_FANOUT !== undefined ? parseInt(env.LCM_LEAF_MIN_FANOUT, 10) : undefined)
|
|
86
|
+
?? toNumber(pc.leafMinFanout) ?? 8,
|
|
87
|
+
condensedMinFanout:
|
|
88
|
+
(env.LCM_CONDENSED_MIN_FANOUT !== undefined ? parseInt(env.LCM_CONDENSED_MIN_FANOUT, 10) : undefined)
|
|
89
|
+
?? toNumber(pc.condensedMinFanout) ?? 4,
|
|
90
|
+
condensedMinFanoutHard:
|
|
91
|
+
(env.LCM_CONDENSED_MIN_FANOUT_HARD !== undefined ? parseInt(env.LCM_CONDENSED_MIN_FANOUT_HARD, 10) : undefined)
|
|
92
|
+
?? toNumber(pc.condensedMinFanoutHard) ?? 2,
|
|
93
|
+
incrementalMaxDepth:
|
|
94
|
+
(env.LCM_INCREMENTAL_MAX_DEPTH !== undefined ? parseInt(env.LCM_INCREMENTAL_MAX_DEPTH, 10) : undefined)
|
|
95
|
+
?? toNumber(pc.incrementalMaxDepth) ?? 0,
|
|
96
|
+
leafChunkTokens:
|
|
97
|
+
(env.LCM_LEAF_CHUNK_TOKENS !== undefined ? parseInt(env.LCM_LEAF_CHUNK_TOKENS, 10) : undefined)
|
|
98
|
+
?? toNumber(pc.leafChunkTokens) ?? 20000,
|
|
99
|
+
leafTargetTokens:
|
|
100
|
+
(env.LCM_LEAF_TARGET_TOKENS !== undefined ? parseInt(env.LCM_LEAF_TARGET_TOKENS, 10) : undefined)
|
|
101
|
+
?? toNumber(pc.leafTargetTokens) ?? 1200,
|
|
102
|
+
condensedTargetTokens:
|
|
103
|
+
(env.LCM_CONDENSED_TARGET_TOKENS !== undefined ? parseInt(env.LCM_CONDENSED_TARGET_TOKENS, 10) : undefined)
|
|
104
|
+
?? toNumber(pc.condensedTargetTokens) ?? 2000,
|
|
105
|
+
maxExpandTokens:
|
|
106
|
+
(env.LCM_MAX_EXPAND_TOKENS !== undefined ? parseInt(env.LCM_MAX_EXPAND_TOKENS, 10) : undefined)
|
|
107
|
+
?? toNumber(pc.maxExpandTokens) ?? 4000,
|
|
108
|
+
largeFileTokenThreshold:
|
|
109
|
+
(env.LCM_LARGE_FILE_TOKEN_THRESHOLD !== undefined ? parseInt(env.LCM_LARGE_FILE_TOKEN_THRESHOLD, 10) : undefined)
|
|
110
|
+
?? toNumber(pc.largeFileThresholdTokens)
|
|
111
|
+
?? toNumber(pc.largeFileTokenThreshold)
|
|
112
|
+
?? 25000,
|
|
113
|
+
largeFileSummaryProvider:
|
|
114
|
+
env.LCM_LARGE_FILE_SUMMARY_PROVIDER?.trim() ?? toStr(pc.largeFileSummaryProvider) ?? "",
|
|
115
|
+
largeFileSummaryModel:
|
|
116
|
+
env.LCM_LARGE_FILE_SUMMARY_MODEL?.trim() ?? toStr(pc.largeFileSummaryModel) ?? "",
|
|
117
|
+
autocompactDisabled:
|
|
118
|
+
env.LCM_AUTOCOMPACT_DISABLED !== undefined
|
|
119
|
+
? env.LCM_AUTOCOMPACT_DISABLED === "true"
|
|
120
|
+
: toBool(pc.autocompactDisabled) ?? false,
|
|
121
|
+
timezone: env.TZ ?? toStr(pc.timezone) ?? Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
122
|
+
pruneHeartbeatOk:
|
|
123
|
+
env.LCM_PRUNE_HEARTBEAT_OK !== undefined
|
|
124
|
+
? env.LCM_PRUNE_HEARTBEAT_OK === "true"
|
|
125
|
+
: toBool(pc.pruneHeartbeatOk) ?? false,
|
|
49
126
|
};
|
|
50
127
|
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { DatabaseSync } from "node:sqlite";
|
|
2
|
+
|
|
3
|
+
export type LcmDbFeatures = {
|
|
4
|
+
fts5Available: boolean;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
const featureCache = new WeakMap<DatabaseSync, LcmDbFeatures>();
|
|
8
|
+
|
|
9
|
+
function probeFts5(db: DatabaseSync): boolean {
|
|
10
|
+
try {
|
|
11
|
+
db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
|
|
12
|
+
db.exec("CREATE VIRTUAL TABLE temp.__lcm_fts5_probe USING fts5(content)");
|
|
13
|
+
db.exec("DROP TABLE temp.__lcm_fts5_probe");
|
|
14
|
+
return true;
|
|
15
|
+
} catch {
|
|
16
|
+
try {
|
|
17
|
+
db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
|
|
18
|
+
} catch {
|
|
19
|
+
// Ignore cleanup failures after a failed probe.
|
|
20
|
+
}
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Detect SQLite features exposed by the current Node runtime.
|
|
27
|
+
*
|
|
28
|
+
* The result is cached per DatabaseSync handle because the probe is runtime-
|
|
29
|
+
* specific, not database-file-specific.
|
|
30
|
+
*/
|
|
31
|
+
export function getLcmDbFeatures(db: DatabaseSync): LcmDbFeatures {
|
|
32
|
+
const cached = featureCache.get(db);
|
|
33
|
+
if (cached) {
|
|
34
|
+
return cached;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const detected: LcmDbFeatures = {
|
|
38
|
+
fts5Available: probeFts5(db),
|
|
39
|
+
};
|
|
40
|
+
featureCache.set(db, detected);
|
|
41
|
+
return detected;
|
|
42
|
+
}
|
package/src/db/migration.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import { getLcmDbFeatures } from "./features.js";
|
|
2
3
|
|
|
3
4
|
type SummaryColumnInfo = {
|
|
4
5
|
name?: string;
|
|
@@ -354,7 +355,10 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
|
|
|
354
355
|
}
|
|
355
356
|
}
|
|
356
357
|
|
|
357
|
-
export function runLcmMigrations(
|
|
358
|
+
export function runLcmMigrations(
|
|
359
|
+
db: DatabaseSync,
|
|
360
|
+
options?: { fts5Available?: boolean },
|
|
361
|
+
): void {
|
|
358
362
|
db.exec(`
|
|
359
363
|
CREATE TABLE IF NOT EXISTS conversations (
|
|
360
364
|
conversation_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -492,6 +496,11 @@ export function runLcmMigrations(db: DatabaseSync): void {
|
|
|
492
496
|
backfillSummaryDepths(db);
|
|
493
497
|
backfillSummaryMetadata(db);
|
|
494
498
|
|
|
499
|
+
const fts5Available = options?.fts5Available ?? getLcmDbFeatures(db).fts5Available;
|
|
500
|
+
if (!fts5Available) {
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
|
|
495
504
|
// FTS5 virtual tables for full-text search (cannot use IF NOT EXISTS, so check manually)
|
|
496
505
|
const hasFts = db
|
|
497
506
|
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='messages_fts'")
|
package/src/engine.ts
CHANGED
|
@@ -18,6 +18,7 @@ import { ContextAssembler } from "./assembler.js";
|
|
|
18
18
|
import { CompactionEngine, type CompactionConfig } from "./compaction.js";
|
|
19
19
|
import type { LcmConfig } from "./db/config.js";
|
|
20
20
|
import { getLcmConnection, closeLcmConnection } from "./db/connection.js";
|
|
21
|
+
import { getLcmDbFeatures } from "./db/features.js";
|
|
21
22
|
import { runLcmMigrations } from "./db/migration.js";
|
|
22
23
|
import {
|
|
23
24
|
createDelegatedExpansionGrant,
|
|
@@ -513,6 +514,7 @@ export class LcmContextEngine implements ContextEngine {
|
|
|
513
514
|
private compaction: CompactionEngine;
|
|
514
515
|
private retrieval: RetrievalEngine;
|
|
515
516
|
private migrated = false;
|
|
517
|
+
private readonly fts5Available: boolean;
|
|
516
518
|
private sessionOperationQueues = new Map<string, Promise<void>>();
|
|
517
519
|
private largeFileTextSummarizerResolved = false;
|
|
518
520
|
private largeFileTextSummarizer?: (prompt: string) => Promise<string | null>;
|
|
@@ -523,9 +525,16 @@ export class LcmContextEngine implements ContextEngine {
|
|
|
523
525
|
this.config = deps.config;
|
|
524
526
|
|
|
525
527
|
const db = getLcmConnection(this.config.databasePath);
|
|
528
|
+
this.fts5Available = getLcmDbFeatures(db).fts5Available;
|
|
526
529
|
|
|
527
|
-
this.conversationStore = new ConversationStore(db);
|
|
528
|
-
this.summaryStore = new SummaryStore(db);
|
|
530
|
+
this.conversationStore = new ConversationStore(db, { fts5Available: this.fts5Available });
|
|
531
|
+
this.summaryStore = new SummaryStore(db, { fts5Available: this.fts5Available });
|
|
532
|
+
|
|
533
|
+
if (!this.fts5Available) {
|
|
534
|
+
this.deps.log.warn(
|
|
535
|
+
"[lcm] FTS5 unavailable in the current Node runtime; full_text search will fall back to LIKE and indexing is disabled",
|
|
536
|
+
);
|
|
537
|
+
}
|
|
529
538
|
|
|
530
539
|
this.assembler = new ContextAssembler(
|
|
531
540
|
this.conversationStore,
|
|
@@ -561,7 +570,7 @@ export class LcmContextEngine implements ContextEngine {
|
|
|
561
570
|
return;
|
|
562
571
|
}
|
|
563
572
|
const db = getLcmConnection(this.config.databasePath);
|
|
564
|
-
runLcmMigrations(db);
|
|
573
|
+
runLcmMigrations(db, { fts5Available: this.fts5Available });
|
|
565
574
|
this.migrated = true;
|
|
566
575
|
}
|
|
567
576
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { DatabaseSync } from "node:sqlite";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { sanitizeFts5Query } from "./fts5-sanitize.js";
|
|
4
|
+
import { buildLikeSearchPlan, createFallbackSnippet } from "./full-text-fallback.js";
|
|
4
5
|
|
|
5
6
|
export type ConversationId = number;
|
|
6
7
|
export type MessageId = number;
|
|
@@ -203,7 +204,14 @@ function toMessagePartRecord(row: MessagePartRow): MessagePartRecord {
|
|
|
203
204
|
// ── ConversationStore ─────────────────────────────────────────────────────────
|
|
204
205
|
|
|
205
206
|
export class ConversationStore {
|
|
206
|
-
|
|
207
|
+
private readonly fts5Available: boolean;
|
|
208
|
+
|
|
209
|
+
constructor(
|
|
210
|
+
private db: DatabaseSync,
|
|
211
|
+
options?: { fts5Available?: boolean },
|
|
212
|
+
) {
|
|
213
|
+
this.fts5Available = options?.fts5Available ?? true;
|
|
214
|
+
}
|
|
207
215
|
|
|
208
216
|
// ── Transaction helpers ──────────────────────────────────────────────────
|
|
209
217
|
|
|
@@ -292,10 +300,7 @@ export class ConversationStore {
|
|
|
292
300
|
|
|
293
301
|
const messageId = Number(result.lastInsertRowid);
|
|
294
302
|
|
|
295
|
-
|
|
296
|
-
this.db
|
|
297
|
-
.prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`)
|
|
298
|
-
.run(messageId, input.content);
|
|
303
|
+
this.indexMessageForFullText(messageId, input.content);
|
|
299
304
|
|
|
300
305
|
const row = this.db
|
|
301
306
|
.prepare(
|
|
@@ -315,7 +320,6 @@ export class ConversationStore {
|
|
|
315
320
|
`INSERT INTO messages (conversation_id, seq, role, content, token_count)
|
|
316
321
|
VALUES (?, ?, ?, ?, ?)`,
|
|
317
322
|
);
|
|
318
|
-
const insertFtsStmt = this.db.prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`);
|
|
319
323
|
const selectStmt = this.db.prepare(
|
|
320
324
|
`SELECT message_id, conversation_id, seq, role, content, token_count, created_at
|
|
321
325
|
FROM messages WHERE message_id = ?`,
|
|
@@ -332,7 +336,7 @@ export class ConversationStore {
|
|
|
332
336
|
);
|
|
333
337
|
|
|
334
338
|
const messageId = Number(result.lastInsertRowid);
|
|
335
|
-
|
|
339
|
+
this.indexMessageForFullText(messageId, input.content);
|
|
336
340
|
const row = selectStmt.get(messageId) as unknown as MessageRow;
|
|
337
341
|
records.push(toMessageRecord(row));
|
|
338
342
|
}
|
|
@@ -535,8 +539,7 @@ export class ConversationStore {
|
|
|
535
539
|
.prepare(`DELETE FROM context_items WHERE item_type = 'message' AND message_id = ?`)
|
|
536
540
|
.run(messageId);
|
|
537
541
|
|
|
538
|
-
|
|
539
|
-
this.db.prepare(`DELETE FROM messages_fts WHERE rowid = ?`).run(messageId);
|
|
542
|
+
this.deleteMessageFromFullText(messageId);
|
|
540
543
|
|
|
541
544
|
// Delete the message (message_parts cascade via ON DELETE CASCADE)
|
|
542
545
|
this.db.prepare(`DELETE FROM messages WHERE message_id = ?`).run(messageId);
|
|
@@ -553,17 +556,54 @@ export class ConversationStore {
|
|
|
553
556
|
const limit = input.limit ?? 50;
|
|
554
557
|
|
|
555
558
|
if (input.mode === "full_text") {
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
559
|
+
if (this.fts5Available) {
|
|
560
|
+
try {
|
|
561
|
+
return this.searchFullText(
|
|
562
|
+
input.query,
|
|
563
|
+
limit,
|
|
564
|
+
input.conversationId,
|
|
565
|
+
input.since,
|
|
566
|
+
input.before,
|
|
567
|
+
);
|
|
568
|
+
} catch {
|
|
569
|
+
return this.searchLike(
|
|
570
|
+
input.query,
|
|
571
|
+
limit,
|
|
572
|
+
input.conversationId,
|
|
573
|
+
input.since,
|
|
574
|
+
input.before,
|
|
575
|
+
);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
return this.searchLike(input.query, limit, input.conversationId, input.since, input.before);
|
|
563
579
|
}
|
|
564
580
|
return this.searchRegex(input.query, limit, input.conversationId, input.since, input.before);
|
|
565
581
|
}
|
|
566
582
|
|
|
583
|
+
private indexMessageForFullText(messageId: MessageId, content: string): void {
|
|
584
|
+
if (!this.fts5Available) {
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
try {
|
|
588
|
+
this.db
|
|
589
|
+
.prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`)
|
|
590
|
+
.run(messageId, content);
|
|
591
|
+
} catch {
|
|
592
|
+
// Full-text indexing is optional. Message persistence must still succeed.
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
private deleteMessageFromFullText(messageId: MessageId): void {
|
|
597
|
+
if (!this.fts5Available) {
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
try {
|
|
601
|
+
this.db.prepare(`DELETE FROM messages_fts WHERE rowid = ?`).run(messageId);
|
|
602
|
+
} catch {
|
|
603
|
+
// Ignore FTS cleanup failures; the source row deletion is authoritative.
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
|
|
567
607
|
private searchFullText(
|
|
568
608
|
query: string,
|
|
569
609
|
limit: number,
|
|
@@ -603,6 +643,55 @@ export class ConversationStore {
|
|
|
603
643
|
return rows.map(toSearchResult);
|
|
604
644
|
}
|
|
605
645
|
|
|
646
|
+
private searchLike(
|
|
647
|
+
query: string,
|
|
648
|
+
limit: number,
|
|
649
|
+
conversationId?: ConversationId,
|
|
650
|
+
since?: Date,
|
|
651
|
+
before?: Date,
|
|
652
|
+
): MessageSearchResult[] {
|
|
653
|
+
const plan = buildLikeSearchPlan("content", query);
|
|
654
|
+
if (plan.terms.length === 0) {
|
|
655
|
+
return [];
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
const where: string[] = [...plan.where];
|
|
659
|
+
const args: Array<string | number> = [...plan.args];
|
|
660
|
+
if (conversationId != null) {
|
|
661
|
+
where.push("conversation_id = ?");
|
|
662
|
+
args.push(conversationId);
|
|
663
|
+
}
|
|
664
|
+
if (since) {
|
|
665
|
+
where.push("julianday(created_at) >= julianday(?)");
|
|
666
|
+
args.push(since.toISOString());
|
|
667
|
+
}
|
|
668
|
+
if (before) {
|
|
669
|
+
where.push("julianday(created_at) < julianday(?)");
|
|
670
|
+
args.push(before.toISOString());
|
|
671
|
+
}
|
|
672
|
+
args.push(limit);
|
|
673
|
+
|
|
674
|
+
const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
|
|
675
|
+
const rows = this.db
|
|
676
|
+
.prepare(
|
|
677
|
+
`SELECT message_id, conversation_id, seq, role, content, token_count, created_at
|
|
678
|
+
FROM messages
|
|
679
|
+
${whereClause}
|
|
680
|
+
ORDER BY created_at DESC
|
|
681
|
+
LIMIT ?`,
|
|
682
|
+
)
|
|
683
|
+
.all(...args) as unknown as MessageRow[];
|
|
684
|
+
|
|
685
|
+
return rows.map((row) => ({
|
|
686
|
+
messageId: row.message_id,
|
|
687
|
+
conversationId: row.conversation_id,
|
|
688
|
+
role: row.role,
|
|
689
|
+
snippet: createFallbackSnippet(row.content, plan.terms),
|
|
690
|
+
createdAt: new Date(row.created_at),
|
|
691
|
+
rank: 0,
|
|
692
|
+
}));
|
|
693
|
+
}
|
|
694
|
+
|
|
606
695
|
private searchRegex(
|
|
607
696
|
pattern: string,
|
|
608
697
|
limit: number,
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
const RAW_TERM_RE = /"([^"]+)"|(\S+)/g;
|
|
2
|
+
const EDGE_PUNCTUATION_RE = /^[`'"()[\]{}<>.,:;!?*_+=|\\/-]+|[`'"()[\]{}<>.,:;!?*_+=|\\/-]+$/g;
|
|
3
|
+
|
|
4
|
+
export type LikeSearchPlan = {
|
|
5
|
+
terms: string[];
|
|
6
|
+
where: string[];
|
|
7
|
+
args: string[];
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
function normalizeFallbackTerm(raw: string): string {
|
|
11
|
+
return raw.trim().replace(EDGE_PUNCTUATION_RE, "").toLowerCase();
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function escapeLike(term: string): string {
|
|
15
|
+
return term.replace(/([\\%_])/g, "\\$1");
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Convert a free-text query into a conservative LIKE search plan.
|
|
20
|
+
*
|
|
21
|
+
* The fallback keeps phrase tokens when the query uses double quotes, and
|
|
22
|
+
* otherwise searches for all normalized tokens as case-insensitive substrings.
|
|
23
|
+
*/
|
|
24
|
+
export function buildLikeSearchPlan(column: string, query: string): LikeSearchPlan {
|
|
25
|
+
const terms: string[] = [];
|
|
26
|
+
for (const match of query.matchAll(RAW_TERM_RE)) {
|
|
27
|
+
const raw = match[1] ?? match[2] ?? "";
|
|
28
|
+
const normalized = normalizeFallbackTerm(raw);
|
|
29
|
+
if (normalized.length > 0 && !terms.includes(normalized)) {
|
|
30
|
+
terms.push(normalized);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (terms.length === 0) {
|
|
35
|
+
const fallback = normalizeFallbackTerm(query);
|
|
36
|
+
if (fallback.length > 0) {
|
|
37
|
+
terms.push(fallback);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return {
|
|
42
|
+
terms,
|
|
43
|
+
where: terms.map(() => `LOWER(${column}) LIKE ? ESCAPE '\\'`),
|
|
44
|
+
args: terms.map((term) => `%${escapeLike(term)}%`),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Build a compact snippet centered around the earliest matching term.
|
|
50
|
+
*/
|
|
51
|
+
export function createFallbackSnippet(content: string, terms: string[]): string {
|
|
52
|
+
const haystack = content.toLowerCase();
|
|
53
|
+
let matchIndex = -1;
|
|
54
|
+
let matchLength = 0;
|
|
55
|
+
|
|
56
|
+
for (const term of terms) {
|
|
57
|
+
const idx = haystack.indexOf(term);
|
|
58
|
+
if (idx !== -1 && (matchIndex === -1 || idx < matchIndex)) {
|
|
59
|
+
matchIndex = idx;
|
|
60
|
+
matchLength = term.length;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (matchIndex === -1) {
|
|
65
|
+
const head = content.trim();
|
|
66
|
+
return head.length <= 80 ? head : `${head.slice(0, 77).trimEnd()}...`;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const start = Math.max(0, matchIndex - 24);
|
|
70
|
+
const end = Math.min(content.length, matchIndex + Math.max(matchLength, 1) + 40);
|
|
71
|
+
const prefix = start > 0 ? "..." : "";
|
|
72
|
+
const suffix = end < content.length ? "..." : "";
|
|
73
|
+
return `${prefix}${content.slice(start, end).trim()}${suffix}`;
|
|
74
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { DatabaseSync } from "node:sqlite";
|
|
2
2
|
import { sanitizeFts5Query } from "./fts5-sanitize.js";
|
|
3
|
+
import { buildLikeSearchPlan, createFallbackSnippet } from "./full-text-fallback.js";
|
|
3
4
|
|
|
4
5
|
export type SummaryKind = "leaf" | "condensed";
|
|
5
6
|
export type ContextItemType = "message" | "summary";
|
|
@@ -239,7 +240,14 @@ function toLargeFileRecord(row: LargeFileRow): LargeFileRecord {
|
|
|
239
240
|
// ── SummaryStore ──────────────────────────────────────────────────────────────
|
|
240
241
|
|
|
241
242
|
export class SummaryStore {
|
|
242
|
-
|
|
243
|
+
private readonly fts5Available: boolean;
|
|
244
|
+
|
|
245
|
+
constructor(
|
|
246
|
+
private db: DatabaseSync,
|
|
247
|
+
options?: { fts5Available?: boolean },
|
|
248
|
+
) {
|
|
249
|
+
this.fts5Available = options?.fts5Available ?? true;
|
|
250
|
+
}
|
|
243
251
|
|
|
244
252
|
// ── Summary CRUD ──────────────────────────────────────────────────────────
|
|
245
253
|
|
|
@@ -305,8 +313,21 @@ export class SummaryStore {
|
|
|
305
313
|
sourceMessageTokenCount,
|
|
306
314
|
);
|
|
307
315
|
|
|
316
|
+
const row = this.db
|
|
317
|
+
.prepare(
|
|
318
|
+
`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
319
|
+
earliest_at, latest_at, descendant_count, created_at
|
|
320
|
+
, descendant_token_count, source_message_token_count
|
|
321
|
+
FROM summaries WHERE summary_id = ?`,
|
|
322
|
+
)
|
|
323
|
+
.get(input.summaryId) as unknown as SummaryRow;
|
|
324
|
+
|
|
308
325
|
// Index in FTS5 as best-effort; compaction flow must continue even if
|
|
309
326
|
// FTS indexing fails for any reason.
|
|
327
|
+
if (!this.fts5Available) {
|
|
328
|
+
return toSummaryRecord(row);
|
|
329
|
+
}
|
|
330
|
+
|
|
310
331
|
try {
|
|
311
332
|
this.db
|
|
312
333
|
.prepare(`INSERT INTO summaries_fts(summary_id, content) VALUES (?, ?)`)
|
|
@@ -316,15 +337,6 @@ export class SummaryStore {
|
|
|
316
337
|
// compaction and assembly will still work correctly.
|
|
317
338
|
}
|
|
318
339
|
|
|
319
|
-
const row = this.db
|
|
320
|
-
.prepare(
|
|
321
|
-
`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
322
|
-
earliest_at, latest_at, descendant_count, created_at
|
|
323
|
-
, descendant_token_count, source_message_token_count
|
|
324
|
-
FROM summaries WHERE summary_id = ?`,
|
|
325
|
-
)
|
|
326
|
-
.get(input.summaryId) as unknown as SummaryRow;
|
|
327
|
-
|
|
328
340
|
return toSummaryRecord(row);
|
|
329
341
|
}
|
|
330
342
|
|
|
@@ -685,13 +697,26 @@ export class SummaryStore {
|
|
|
685
697
|
const limit = input.limit ?? 50;
|
|
686
698
|
|
|
687
699
|
if (input.mode === "full_text") {
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
700
|
+
if (this.fts5Available) {
|
|
701
|
+
try {
|
|
702
|
+
return this.searchFullText(
|
|
703
|
+
input.query,
|
|
704
|
+
limit,
|
|
705
|
+
input.conversationId,
|
|
706
|
+
input.since,
|
|
707
|
+
input.before,
|
|
708
|
+
);
|
|
709
|
+
} catch {
|
|
710
|
+
return this.searchLike(
|
|
711
|
+
input.query,
|
|
712
|
+
limit,
|
|
713
|
+
input.conversationId,
|
|
714
|
+
input.since,
|
|
715
|
+
input.before,
|
|
716
|
+
);
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
return this.searchLike(input.query, limit, input.conversationId, input.since, input.before);
|
|
695
720
|
}
|
|
696
721
|
return this.searchRegex(input.query, limit, input.conversationId, input.since, input.before);
|
|
697
722
|
}
|
|
@@ -735,6 +760,57 @@ export class SummaryStore {
|
|
|
735
760
|
return rows.map(toSearchResult);
|
|
736
761
|
}
|
|
737
762
|
|
|
763
|
+
private searchLike(
|
|
764
|
+
query: string,
|
|
765
|
+
limit: number,
|
|
766
|
+
conversationId?: number,
|
|
767
|
+
since?: Date,
|
|
768
|
+
before?: Date,
|
|
769
|
+
): SummarySearchResult[] {
|
|
770
|
+
const plan = buildLikeSearchPlan("content", query);
|
|
771
|
+
if (plan.terms.length === 0) {
|
|
772
|
+
return [];
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
const where: string[] = [...plan.where];
|
|
776
|
+
const args: Array<string | number> = [...plan.args];
|
|
777
|
+
if (conversationId != null) {
|
|
778
|
+
where.push("conversation_id = ?");
|
|
779
|
+
args.push(conversationId);
|
|
780
|
+
}
|
|
781
|
+
if (since) {
|
|
782
|
+
where.push("julianday(created_at) >= julianday(?)");
|
|
783
|
+
args.push(since.toISOString());
|
|
784
|
+
}
|
|
785
|
+
if (before) {
|
|
786
|
+
where.push("julianday(created_at) < julianday(?)");
|
|
787
|
+
args.push(before.toISOString());
|
|
788
|
+
}
|
|
789
|
+
args.push(limit);
|
|
790
|
+
|
|
791
|
+
const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
|
|
792
|
+
const rows = this.db
|
|
793
|
+
.prepare(
|
|
794
|
+
`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
795
|
+
earliest_at, latest_at, descendant_count, descendant_token_count,
|
|
796
|
+
source_message_token_count, created_at
|
|
797
|
+
FROM summaries
|
|
798
|
+
${whereClause}
|
|
799
|
+
ORDER BY created_at DESC
|
|
800
|
+
LIMIT ?`,
|
|
801
|
+
)
|
|
802
|
+
.all(...args) as unknown as SummaryRow[];
|
|
803
|
+
|
|
804
|
+
return rows.map((row) => ({
|
|
805
|
+
summaryId: row.summary_id,
|
|
806
|
+
conversationId: row.conversation_id,
|
|
807
|
+
kind: row.kind,
|
|
808
|
+
snippet: createFallbackSnippet(row.content, plan.terms),
|
|
809
|
+
createdAt: new Date(row.created_at),
|
|
810
|
+
rank: 0,
|
|
811
|
+
}));
|
|
812
|
+
}
|
|
813
|
+
|
|
738
814
|
private searchRegex(
|
|
739
815
|
pattern: string,
|
|
740
816
|
limit: number,
|
package/src/summarize.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { resolveLcmConfig } from "./db/config.js";
|
|
2
1
|
import type { LcmDependencies } from "./types.js";
|
|
3
2
|
|
|
4
3
|
export type LcmSummarizeOptions = {
|
|
@@ -675,11 +674,10 @@ export async function createLcmSummarizeFromLegacyParams(params: {
|
|
|
675
674
|
|
|
676
675
|
const apiKey = params.deps.getApiKey(provider, model);
|
|
677
676
|
|
|
678
|
-
const runtimeLcmConfig = resolveLcmConfig();
|
|
679
677
|
const condensedTargetTokens =
|
|
680
|
-
Number.isFinite(
|
|
681
|
-
|
|
682
|
-
?
|
|
678
|
+
Number.isFinite(params.deps.config.condensedTargetTokens) &&
|
|
679
|
+
params.deps.config.condensedTargetTokens > 0
|
|
680
|
+
? params.deps.config.condensedTargetTokens
|
|
683
681
|
: DEFAULT_CONDENSED_TARGET_TOKENS;
|
|
684
682
|
|
|
685
683
|
return async (
|