@martian-engineering/lossless-claw 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,11 +1,11 @@
1
1
  # lossless-claw
2
2
 
3
- > ⚠️ **Current requirement:** This plugin currently requires a custom OpenClaw build with [PR #22201](https://github.com/openclaw/openclaw/pull/22201) applied until that PR is merged upstream.
4
-
5
3
  Lossless Context Management plugin for [OpenClaw](https://github.com/openclaw/openclaw), based on the [LCM paper](https://papers.voltropy.com/LCM). Replaces OpenClaw's built-in sliding-window compaction with a DAG-based summarization system that preserves every message while keeping active context within model token limits.
6
4
 
7
5
  ## What it does
8
6
 
7
+ Two ways to learn: read the below, or [check out this super cool animated visualization](https://losslesscontext.ai).
8
+
9
9
  When a conversation grows beyond the model's context window, OpenClaw (just like all of the other agents) normally truncates older messages. LCM instead:
10
10
 
11
11
  1. **Persists every message** in a SQLite database, organized by conversation
@@ -68,23 +68,182 @@ If you need to set it manually, ensure the context engine slot points at lossles
68
68
 
69
69
  Restart OpenClaw after configuration changes.
70
70
 
71
+ ### Optional: enable FTS5 for fast full-text search
72
+
73
+ `lossless-claw` works without FTS5 as of the current release. When FTS5 is unavailable in the
74
+ Node runtime that runs the OpenClaw gateway, the plugin:
75
+
76
+ - keeps persisting messages and summaries
77
+ - falls back from `"full_text"` search to a slower `LIKE`-based search
78
+ - loses FTS ranking/snippet quality
79
+
80
+ If you want native FTS5 search performance and ranking, the **exact Node runtime that runs the
81
+ gateway** must have SQLite FTS5 compiled in.
82
+
83
+ #### Probe the gateway runtime
84
+
85
+ Run this with the same `node` binary your gateway uses:
86
+
87
+ ```bash
88
+ node --input-type=module - <<'NODE'
89
+ import { DatabaseSync } from 'node:sqlite';
90
+ const db = new DatabaseSync(':memory:');
91
+ const options = db.prepare('pragma compile_options').all().map((row) => row.compile_options);
92
+
93
+ console.log(options.filter((value) => value.includes('FTS')).join('\n') || 'no fts compile options');
94
+
95
+ try {
96
+ db.exec("CREATE VIRTUAL TABLE t USING fts5(content)");
97
+ console.log("fts5: ok");
98
+ } catch (err) {
99
+ console.log("fts5: fail");
100
+ console.log(err instanceof Error ? err.message : String(err));
101
+ }
102
+ NODE
103
+ ```
104
+
105
+ Expected output:
106
+
107
+ ```text
108
+ ENABLE_FTS5
109
+ fts5: ok
110
+ ```
111
+
112
+ If you get `fts5: fail`, build or install an FTS5-capable Node and point the gateway at that runtime.
113
+
114
+ #### Build an FTS5-capable Node on macOS
115
+
116
+ This workflow was verified with Node `v22.15.0`.
117
+
118
+ ```bash
119
+ cd ~/Projects
120
+ git clone --depth 1 --branch v22.15.0 https://github.com/nodejs/node.git node-fts5
121
+ cd node-fts5
122
+ ```
123
+
124
+ Edit `deps/sqlite/sqlite.gyp` and add `SQLITE_ENABLE_FTS5` to the `defines` list for the `sqlite`
125
+ target:
126
+
127
+ ```diff
128
+ 'defines': [
129
+ 'SQLITE_DEFAULT_MEMSTATUS=0',
130
+ + 'SQLITE_ENABLE_FTS5',
131
+ 'SQLITE_ENABLE_MATH_FUNCTIONS',
132
+ 'SQLITE_ENABLE_SESSION',
133
+ 'SQLITE_ENABLE_PREUPDATE_HOOK'
134
+ ],
135
+ ```
136
+
137
+ Important:
138
+
139
+ - patch `deps/sqlite/sqlite.gyp`, not only `node.gyp`
140
+ - `node:sqlite` uses the embedded SQLite built from `deps/sqlite/sqlite.gyp`
141
+
142
+ Build the runtime:
143
+
144
+ ```bash
145
+ ./configure --prefix="$PWD/out-install"
146
+ make -j8 node
147
+ ```
148
+
149
+ Expose the binary under a Node-compatible basename that OpenClaw recognizes:
150
+
151
+ ```bash
152
+ mkdir -p ~/Projects/node-fts5/bin
153
+ ln -sfn ~/Projects/node-fts5/out/Release/node ~/Projects/node-fts5/bin/node-22.15.0
154
+ ```
155
+
156
+ Use a basename like `node-22.15.0`, `node`, or `nodejs`. Names like
157
+ `node-v22.15.0-fts5` may not be recognized correctly by OpenClaw's CLI/runtime parsing.
158
+
159
+ Verify the new runtime:
160
+
161
+ ```bash
162
+ ~/Projects/node-fts5/bin/node-22.15.0 --version
163
+ ~/Projects/node-fts5/bin/node-22.15.0 --input-type=module - <<'NODE'
164
+ import { DatabaseSync } from 'node:sqlite';
165
+ const db = new DatabaseSync(':memory:');
166
+ db.exec("CREATE VIRTUAL TABLE t USING fts5(content)");
167
+ console.log("fts5: ok");
168
+ NODE
169
+ ```
170
+
171
+ #### Point the OpenClaw gateway at that runtime on macOS
172
+
173
+ Back up the existing LaunchAgent plist first:
174
+
175
+ ```bash
176
+ cp ~/Library/LaunchAgents/ai.openclaw.gateway.plist \
177
+ ~/Library/LaunchAgents/ai.openclaw.gateway.plist.bak-$(date +%Y%m%d-%H%M%S)
178
+ ```
179
+
180
+ Replace the runtime path, then reload the agent:
181
+
182
+ ```bash
183
+ /usr/libexec/PlistBuddy -c 'Set :ProgramArguments:0 /Users/youruser/Projects/node-fts5/bin/node-22.15.0' \
184
+ ~/Library/LaunchAgents/ai.openclaw.gateway.plist
185
+
186
+ launchctl bootout gui/$UID ~/Library/LaunchAgents/ai.openclaw.gateway.plist 2>/dev/null || true
187
+ launchctl bootstrap gui/$UID ~/Library/LaunchAgents/ai.openclaw.gateway.plist
188
+ launchctl kickstart -k gui/$UID/ai.openclaw.gateway
189
+ ```
190
+
191
+ Verify the live runtime:
192
+
193
+ ```bash
194
+ launchctl print gui/$UID/ai.openclaw.gateway | sed -n '1,80p'
195
+ ```
196
+
197
+ You should see:
198
+
199
+ ```text
200
+ program = /Users/youruser/Projects/node-fts5/bin/node-22.15.0
201
+ ```
202
+
203
+ #### Verify `lossless-claw`
204
+
205
+ Check the logs:
206
+
207
+ ```bash
208
+ tail -n 60 ~/.openclaw/logs/gateway.log
209
+ tail -n 60 ~/.openclaw/logs/gateway.err.log
210
+ ```
211
+
212
+ You want:
213
+
214
+ - `[gateway] [lcm] Plugin loaded ...`
215
+ - no new `no such module: fts5`
216
+
217
+ Then force one turn through the gateway and verify the DB fills:
218
+
219
+ ```bash
220
+ /Users/youruser/Projects/node-fts5/bin/node-22.15.0 \
221
+ /path/to/openclaw/dist/index.js \
222
+ agent --session-id fts5-smoke --message 'Reply with exactly: ok' --timeout 60
223
+
224
+ sqlite3 ~/.openclaw/lcm.db '
225
+ select count(*) as conversations from conversations;
226
+ select count(*) as messages from messages;
227
+ select count(*) as summaries from summaries;
228
+ '
229
+ ```
230
+
231
+ Those counts should increase after a real turn.
232
+
71
233
  ## Configuration
72
234
 
73
235
  LCM is configured through a combination of plugin config and environment variables. Environment variables take precedence for backward compatibility.
74
236
 
75
237
  ### Plugin config
76
238
 
77
- Add an `lossless-claw` block under `plugins.config` in your OpenClaw config:
239
+ Add a `lossless-claw` entry under `plugins.entries` in your OpenClaw config:
78
240
 
79
241
  ```json
80
242
  {
81
243
  "plugins": {
82
- "config": {
244
+ "entries": {
83
245
  "lossless-claw": {
84
- "enabled": true,
85
- "freshTailCount": 32,
86
- "contextThreshold": 0.75,
87
- "incrementalMaxDepth": 1
246
+ "enabled": true
88
247
  }
89
248
  }
90
249
  }
@@ -102,7 +261,7 @@ Add an `lossless-claw` block under `plugins.config` in your OpenClaw config:
102
261
  | `LCM_LEAF_MIN_FANOUT` | `8` | Minimum raw messages per leaf summary |
103
262
  | `LCM_CONDENSED_MIN_FANOUT` | `4` | Minimum summaries per condensed node |
104
263
  | `LCM_CONDENSED_MIN_FANOUT_HARD` | `2` | Relaxed fanout for forced compaction sweeps |
105
- | `LCM_INCREMENTAL_MAX_DEPTH` | `0` | How deep incremental compaction goes (0 = leaf only) |
264
+ | `LCM_INCREMENTAL_MAX_DEPTH` | `0` | How deep incremental compaction goes (0 = leaf only, -1 = unlimited) |
106
265
  | `LCM_LEAF_CHUNK_TOKENS` | `20000` | Max source tokens per leaf compaction chunk |
107
266
  | `LCM_LEAF_TARGET_TOKENS` | `1200` | Target token count for leaf summaries |
108
267
  | `LCM_CONDENSED_TARGET_TOKENS` | `2000` | Target token count for condensed summaries |
@@ -110,18 +269,18 @@ Add an `lossless-claw` block under `plugins.config` in your OpenClaw config:
110
269
  | `LCM_LARGE_FILE_TOKEN_THRESHOLD` | `25000` | File blocks above this size are intercepted and stored separately |
111
270
  | `LCM_SUMMARY_MODEL` | *(from OpenClaw)* | Model for summarization (e.g. `anthropic/claude-sonnet-4-20250514`) |
112
271
  | `LCM_SUMMARY_PROVIDER` | *(from OpenClaw)* | Provider override for summarization |
113
- | `LCM_INCREMENTAL_MAX_DEPTH` | `0` | Depth limit for incremental condensation after leaf passes |
272
+ | `LCM_INCREMENTAL_MAX_DEPTH` | `0` | Depth limit for incremental condensation after leaf passes (-1 = unlimited) |
114
273
 
115
274
  ### Recommended starting configuration
116
275
 
117
276
  ```
118
277
  LCM_FRESH_TAIL_COUNT=32
119
- LCM_INCREMENTAL_MAX_DEPTH=1
278
+ LCM_INCREMENTAL_MAX_DEPTH=-1
120
279
  LCM_CONTEXT_THRESHOLD=0.75
121
280
  ```
122
281
 
123
282
  - **freshTailCount=32** protects the last 32 messages from compaction, giving the model enough recent context for continuity.
124
- - **incrementalMaxDepth=1** enables automatic condensation of leaf summaries after each compaction pass (without this, only leaf summaries are created and condensation only happens during manual `/compact` or overflow).
283
+ - **incrementalMaxDepth=-1** enables unlimited automatic condensation after each compaction pass the DAG cascades as deep as needed. Set to `0` (default) for leaf-only, or a positive integer for a specific depth cap.
125
284
  - **contextThreshold=0.75** triggers compaction when context reaches 75% of the model's window, leaving headroom for the model's response.
126
285
 
127
286
  ## How it works
@@ -171,7 +330,7 @@ This gives the model enough information to know what was discussed, when, and ho
171
330
 
172
331
  Compaction runs in two modes:
173
332
 
174
- - **Proactive (after each turn):** If raw messages outside the fresh tail exceed `leafChunkTokens`, a leaf pass runs. If `incrementalMaxDepth > 0`, condensation follows.
333
+ - **Proactive (after each turn):** If raw messages outside the fresh tail exceed `leafChunkTokens`, a leaf pass runs. If `incrementalMaxDepth != 0`, condensation follows (cascading to the configured depth, or unlimited with `-1`).
175
334
  - **Reactive (overflow/manual):** When total context exceeds `contextThreshold × tokenBudget`, a full sweep runs: all eligible leaf chunks are compacted, then condensation proceeds depth-by-depth until stable.
176
335
 
177
336
  ### Depth-aware prompts
@@ -87,7 +87,7 @@ The **condensed pass** merges summaries at the same depth into a higher-level su
87
87
  **Incremental (after each turn):**
88
88
  - Checks if raw tokens outside the fresh tail exceed `leafChunkTokens`
89
89
  - If so, runs one leaf pass
90
- - If `incrementalMaxDepth > 0`, follows with condensation passes up to that depth
90
+ - If `incrementalMaxDepth != 0`, follows with condensation passes up to that depth (`-1` for unlimited)
91
91
  - Best-effort: failures don't break the conversation
92
92
 
93
93
  **Full sweep (manual `/compact` or overflow):**
@@ -26,7 +26,7 @@ Set recommended environment variables:
26
26
 
27
27
  ```bash
28
28
  export LCM_FRESH_TAIL_COUNT=32
29
- export LCM_INCREMENTAL_MAX_DEPTH=1
29
+ export LCM_INCREMENTAL_MAX_DEPTH=-1
30
30
  ```
31
31
 
32
32
  Restart OpenClaw.
@@ -70,8 +70,9 @@ For coding conversations with tool calls (which generate many messages per logic
70
70
  `LCM_INCREMENTAL_MAX_DEPTH` (default `0`) controls whether condensation happens automatically after leaf passes.
71
71
 
72
72
  - **0** — Only leaf summaries are created incrementally. Condensation only happens during manual `/compact` or overflow.
73
- - **1** — After each leaf pass, attempt to condense d0 summaries into d1. Good default for active conversations.
74
- - **2+** — Deeper automatic condensation. Rarely needed; the full sweep handles this during overflow.
73
+ - **1** — After each leaf pass, attempt to condense d0 summaries into d1.
74
+ - **2+** — Deeper automatic condensation up to the specified depth.
75
+ - **-1** — Unlimited depth. Condensation cascades as deep as needed after each leaf pass. Recommended for long-running sessions.
75
76
 
76
77
  ### Summary target tokens
77
78
 
package/index.ts CHANGED
@@ -597,7 +597,11 @@ function readLatestAssistantReply(messages: unknown[]): string | undefined {
597
597
  function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
598
598
  const envSnapshot = snapshotPluginEnv();
599
599
  const readEnv: ReadEnvFn = (key) => process.env[key];
600
- const config = resolveLcmConfig();
600
+ const pluginConfig =
601
+ api.pluginConfig && typeof api.pluginConfig === "object" && !Array.isArray(api.pluginConfig)
602
+ ? api.pluginConfig
603
+ : undefined;
604
+ const config = resolveLcmConfig(process.env, pluginConfig);
601
605
 
602
606
  return {
603
607
  config,
@@ -862,17 +866,11 @@ const lcmPlugin = {
862
866
 
863
867
  configSchema: {
864
868
  parse(value: unknown) {
865
- // Merge plugin config with env vars — env vars take precedence for backward compat
866
869
  const raw =
867
870
  value && typeof value === "object" && !Array.isArray(value)
868
871
  ? (value as Record<string, unknown>)
869
872
  : {};
870
- const enabled = typeof raw.enabled === "boolean" ? raw.enabled : undefined;
871
- const config = resolveLcmConfig();
872
- if (enabled !== undefined) {
873
- config.enabled = enabled;
874
- }
875
- return config;
873
+ return resolveLcmConfig(process.env, raw);
876
874
  },
877
875
  },
878
876
 
@@ -7,7 +7,7 @@
7
7
  },
8
8
  "incrementalMaxDepth": {
9
9
  "label": "Incremental Max Depth",
10
- "help": "How deep incremental compaction goes (0 = leaf only)"
10
+ "help": "How deep incremental compaction goes (0 = leaf only, -1 = unlimited)"
11
11
  },
12
12
  "freshTailCount": {
13
13
  "label": "Fresh Tail Count",
@@ -32,7 +32,7 @@
32
32
  },
33
33
  "incrementalMaxDepth": {
34
34
  "type": "integer",
35
- "minimum": 0
35
+ "minimum": -1
36
36
  },
37
37
  "freshTailCount": {
38
38
  "type": "integer",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@martian-engineering/lossless-claw",
3
- "version": "0.2.1",
3
+ "version": "0.2.3",
4
4
  "description": "Lossless Context Management plugin for OpenClaw — DAG-based conversation summarization with incremental compaction",
5
5
  "type": "module",
6
6
  "main": "index.ts",
@@ -24,6 +24,8 @@
24
24
  "LICENSE"
25
25
  ],
26
26
  "dependencies": {
27
+ "@mariozechner/pi-agent-core": "*",
28
+ "@mariozechner/pi-ai": "*",
27
29
  "@sinclair/typebox": "0.34.48"
28
30
  },
29
31
  "devDependencies": {
@@ -31,9 +33,7 @@
31
33
  "vitest": "^3.0.0"
32
34
  },
33
35
  "peerDependencies": {
34
- "openclaw": "*",
35
- "@mariozechner/pi-agent-core": "*",
36
- "@mariozechner/pi-ai": "*"
36
+ "openclaw": "*"
37
37
  },
38
38
  "openclaw": {
39
39
  "extensions": [
package/src/db/config.ts CHANGED
@@ -26,25 +26,102 @@ export type LcmConfig = {
26
26
  pruneHeartbeatOk: boolean;
27
27
  };
28
28
 
29
- export function resolveLcmConfig(env: NodeJS.ProcessEnv = process.env): LcmConfig {
29
+ /** Safely coerce an unknown value to a finite number, or return undefined. */
30
+ function toNumber(value: unknown): number | undefined {
31
+ if (typeof value === "number" && Number.isFinite(value)) return value;
32
+ if (typeof value === "string") {
33
+ const n = Number(value);
34
+ if (Number.isFinite(n)) return n;
35
+ }
36
+ return undefined;
37
+ }
38
+
39
+ /** Safely coerce an unknown value to a boolean, or return undefined. */
40
+ function toBool(value: unknown): boolean | undefined {
41
+ if (typeof value === "boolean") return value;
42
+ if (value === "true") return true;
43
+ if (value === "false") return false;
44
+ return undefined;
45
+ }
46
+
47
+ /** Safely coerce an unknown value to a trimmed non-empty string, or return undefined. */
48
+ function toStr(value: unknown): string | undefined {
49
+ if (typeof value === "string") {
50
+ const trimmed = value.trim();
51
+ return trimmed.length > 0 ? trimmed : undefined;
52
+ }
53
+ return undefined;
54
+ }
55
+
56
+ /**
57
+ * Resolve LCM configuration with three-tier precedence:
58
+ * 1. Environment variables (highest — backward compat)
59
+ * 2. Plugin config object (from plugins.entries.lossless-claw.config)
60
+ * 3. Hardcoded defaults (lowest)
61
+ */
62
+ export function resolveLcmConfig(
63
+ env: NodeJS.ProcessEnv = process.env,
64
+ pluginConfig?: Record<string, unknown>,
65
+ ): LcmConfig {
66
+ const pc = pluginConfig ?? {};
67
+
30
68
  return {
31
- enabled: env.LCM_ENABLED !== "false",
32
- databasePath: env.LCM_DATABASE_PATH ?? join(homedir(), ".openclaw", "lcm.db"),
33
- contextThreshold: parseFloat(env.LCM_CONTEXT_THRESHOLD ?? "0.75"),
34
- freshTailCount: parseInt(env.LCM_FRESH_TAIL_COUNT ?? "32", 10),
35
- leafMinFanout: parseInt(env.LCM_LEAF_MIN_FANOUT ?? "8", 10),
36
- condensedMinFanout: parseInt(env.LCM_CONDENSED_MIN_FANOUT ?? "4", 10),
37
- condensedMinFanoutHard: parseInt(env.LCM_CONDENSED_MIN_FANOUT_HARD ?? "2", 10),
38
- incrementalMaxDepth: parseInt(env.LCM_INCREMENTAL_MAX_DEPTH ?? "0", 10),
39
- leafChunkTokens: parseInt(env.LCM_LEAF_CHUNK_TOKENS ?? "20000", 10),
40
- leafTargetTokens: parseInt(env.LCM_LEAF_TARGET_TOKENS ?? "1200", 10),
41
- condensedTargetTokens: parseInt(env.LCM_CONDENSED_TARGET_TOKENS ?? "2000", 10),
42
- maxExpandTokens: parseInt(env.LCM_MAX_EXPAND_TOKENS ?? "4000", 10),
43
- largeFileTokenThreshold: parseInt(env.LCM_LARGE_FILE_TOKEN_THRESHOLD ?? "25000", 10),
44
- largeFileSummaryProvider: env.LCM_LARGE_FILE_SUMMARY_PROVIDER?.trim() ?? "",
45
- largeFileSummaryModel: env.LCM_LARGE_FILE_SUMMARY_MODEL?.trim() ?? "",
46
- autocompactDisabled: env.LCM_AUTOCOMPACT_DISABLED === "true",
47
- timezone: env.TZ ?? Intl.DateTimeFormat().resolvedOptions().timeZone,
48
- pruneHeartbeatOk: env.LCM_PRUNE_HEARTBEAT_OK === "true",
69
+ enabled:
70
+ env.LCM_ENABLED !== undefined
71
+ ? env.LCM_ENABLED !== "false"
72
+ : toBool(pc.enabled) ?? true,
73
+ databasePath:
74
+ env.LCM_DATABASE_PATH
75
+ ?? toStr(pc.dbPath)
76
+ ?? toStr(pc.databasePath)
77
+ ?? join(homedir(), ".openclaw", "lcm.db"),
78
+ contextThreshold:
79
+ (env.LCM_CONTEXT_THRESHOLD !== undefined ? parseFloat(env.LCM_CONTEXT_THRESHOLD) : undefined)
80
+ ?? toNumber(pc.contextThreshold) ?? 0.75,
81
+ freshTailCount:
82
+ (env.LCM_FRESH_TAIL_COUNT !== undefined ? parseInt(env.LCM_FRESH_TAIL_COUNT, 10) : undefined)
83
+ ?? toNumber(pc.freshTailCount) ?? 32,
84
+ leafMinFanout:
85
+ (env.LCM_LEAF_MIN_FANOUT !== undefined ? parseInt(env.LCM_LEAF_MIN_FANOUT, 10) : undefined)
86
+ ?? toNumber(pc.leafMinFanout) ?? 8,
87
+ condensedMinFanout:
88
+ (env.LCM_CONDENSED_MIN_FANOUT !== undefined ? parseInt(env.LCM_CONDENSED_MIN_FANOUT, 10) : undefined)
89
+ ?? toNumber(pc.condensedMinFanout) ?? 4,
90
+ condensedMinFanoutHard:
91
+ (env.LCM_CONDENSED_MIN_FANOUT_HARD !== undefined ? parseInt(env.LCM_CONDENSED_MIN_FANOUT_HARD, 10) : undefined)
92
+ ?? toNumber(pc.condensedMinFanoutHard) ?? 2,
93
+ incrementalMaxDepth:
94
+ (env.LCM_INCREMENTAL_MAX_DEPTH !== undefined ? parseInt(env.LCM_INCREMENTAL_MAX_DEPTH, 10) : undefined)
95
+ ?? toNumber(pc.incrementalMaxDepth) ?? 0,
96
+ leafChunkTokens:
97
+ (env.LCM_LEAF_CHUNK_TOKENS !== undefined ? parseInt(env.LCM_LEAF_CHUNK_TOKENS, 10) : undefined)
98
+ ?? toNumber(pc.leafChunkTokens) ?? 20000,
99
+ leafTargetTokens:
100
+ (env.LCM_LEAF_TARGET_TOKENS !== undefined ? parseInt(env.LCM_LEAF_TARGET_TOKENS, 10) : undefined)
101
+ ?? toNumber(pc.leafTargetTokens) ?? 1200,
102
+ condensedTargetTokens:
103
+ (env.LCM_CONDENSED_TARGET_TOKENS !== undefined ? parseInt(env.LCM_CONDENSED_TARGET_TOKENS, 10) : undefined)
104
+ ?? toNumber(pc.condensedTargetTokens) ?? 2000,
105
+ maxExpandTokens:
106
+ (env.LCM_MAX_EXPAND_TOKENS !== undefined ? parseInt(env.LCM_MAX_EXPAND_TOKENS, 10) : undefined)
107
+ ?? toNumber(pc.maxExpandTokens) ?? 4000,
108
+ largeFileTokenThreshold:
109
+ (env.LCM_LARGE_FILE_TOKEN_THRESHOLD !== undefined ? parseInt(env.LCM_LARGE_FILE_TOKEN_THRESHOLD, 10) : undefined)
110
+ ?? toNumber(pc.largeFileThresholdTokens)
111
+ ?? toNumber(pc.largeFileTokenThreshold)
112
+ ?? 25000,
113
+ largeFileSummaryProvider:
114
+ env.LCM_LARGE_FILE_SUMMARY_PROVIDER?.trim() ?? toStr(pc.largeFileSummaryProvider) ?? "",
115
+ largeFileSummaryModel:
116
+ env.LCM_LARGE_FILE_SUMMARY_MODEL?.trim() ?? toStr(pc.largeFileSummaryModel) ?? "",
117
+ autocompactDisabled:
118
+ env.LCM_AUTOCOMPACT_DISABLED !== undefined
119
+ ? env.LCM_AUTOCOMPACT_DISABLED === "true"
120
+ : toBool(pc.autocompactDisabled) ?? false,
121
+ timezone: env.TZ ?? toStr(pc.timezone) ?? Intl.DateTimeFormat().resolvedOptions().timeZone,
122
+ pruneHeartbeatOk:
123
+ env.LCM_PRUNE_HEARTBEAT_OK !== undefined
124
+ ? env.LCM_PRUNE_HEARTBEAT_OK === "true"
125
+ : toBool(pc.pruneHeartbeatOk) ?? false,
49
126
  };
50
127
  }
@@ -0,0 +1,42 @@
1
+ import type { DatabaseSync } from "node:sqlite";
2
+
3
+ export type LcmDbFeatures = {
4
+ fts5Available: boolean;
5
+ };
6
+
7
+ const featureCache = new WeakMap<DatabaseSync, LcmDbFeatures>();
8
+
9
+ function probeFts5(db: DatabaseSync): boolean {
10
+ try {
11
+ db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
12
+ db.exec("CREATE VIRTUAL TABLE temp.__lcm_fts5_probe USING fts5(content)");
13
+ db.exec("DROP TABLE temp.__lcm_fts5_probe");
14
+ return true;
15
+ } catch {
16
+ try {
17
+ db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
18
+ } catch {
19
+ // Ignore cleanup failures after a failed probe.
20
+ }
21
+ return false;
22
+ }
23
+ }
24
+
25
+ /**
26
+ * Detect SQLite features exposed by the current Node runtime.
27
+ *
28
+ * The result is cached per DatabaseSync handle because the probe is runtime-
29
+ * specific, not database-file-specific.
30
+ */
31
+ export function getLcmDbFeatures(db: DatabaseSync): LcmDbFeatures {
32
+ const cached = featureCache.get(db);
33
+ if (cached) {
34
+ return cached;
35
+ }
36
+
37
+ const detected: LcmDbFeatures = {
38
+ fts5Available: probeFts5(db),
39
+ };
40
+ featureCache.set(db, detected);
41
+ return detected;
42
+ }
@@ -1,4 +1,5 @@
1
1
  import type { DatabaseSync } from "node:sqlite";
2
+ import { getLcmDbFeatures } from "./features.js";
2
3
 
3
4
  type SummaryColumnInfo = {
4
5
  name?: string;
@@ -354,7 +355,10 @@ function backfillSummaryMetadata(db: DatabaseSync): void {
354
355
  }
355
356
  }
356
357
 
357
- export function runLcmMigrations(db: DatabaseSync): void {
358
+ export function runLcmMigrations(
359
+ db: DatabaseSync,
360
+ options?: { fts5Available?: boolean },
361
+ ): void {
358
362
  db.exec(`
359
363
  CREATE TABLE IF NOT EXISTS conversations (
360
364
  conversation_id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -492,6 +496,11 @@ export function runLcmMigrations(db: DatabaseSync): void {
492
496
  backfillSummaryDepths(db);
493
497
  backfillSummaryMetadata(db);
494
498
 
499
+ const fts5Available = options?.fts5Available ?? getLcmDbFeatures(db).fts5Available;
500
+ if (!fts5Available) {
501
+ return;
502
+ }
503
+
495
504
  // FTS5 virtual tables for full-text search (cannot use IF NOT EXISTS, so check manually)
496
505
  const hasFts = db
497
506
  .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='messages_fts'")
package/src/engine.ts CHANGED
@@ -18,6 +18,7 @@ import { ContextAssembler } from "./assembler.js";
18
18
  import { CompactionEngine, type CompactionConfig } from "./compaction.js";
19
19
  import type { LcmConfig } from "./db/config.js";
20
20
  import { getLcmConnection, closeLcmConnection } from "./db/connection.js";
21
+ import { getLcmDbFeatures } from "./db/features.js";
21
22
  import { runLcmMigrations } from "./db/migration.js";
22
23
  import {
23
24
  createDelegatedExpansionGrant,
@@ -513,6 +514,7 @@ export class LcmContextEngine implements ContextEngine {
513
514
  private compaction: CompactionEngine;
514
515
  private retrieval: RetrievalEngine;
515
516
  private migrated = false;
517
+ private readonly fts5Available: boolean;
516
518
  private sessionOperationQueues = new Map<string, Promise<void>>();
517
519
  private largeFileTextSummarizerResolved = false;
518
520
  private largeFileTextSummarizer?: (prompt: string) => Promise<string | null>;
@@ -523,9 +525,16 @@ export class LcmContextEngine implements ContextEngine {
523
525
  this.config = deps.config;
524
526
 
525
527
  const db = getLcmConnection(this.config.databasePath);
528
+ this.fts5Available = getLcmDbFeatures(db).fts5Available;
526
529
 
527
- this.conversationStore = new ConversationStore(db);
528
- this.summaryStore = new SummaryStore(db);
530
+ this.conversationStore = new ConversationStore(db, { fts5Available: this.fts5Available });
531
+ this.summaryStore = new SummaryStore(db, { fts5Available: this.fts5Available });
532
+
533
+ if (!this.fts5Available) {
534
+ this.deps.log.warn(
535
+ "[lcm] FTS5 unavailable in the current Node runtime; full_text search will fall back to LIKE and indexing is disabled",
536
+ );
537
+ }
529
538
 
530
539
  this.assembler = new ContextAssembler(
531
540
  this.conversationStore,
@@ -561,7 +570,7 @@ export class LcmContextEngine implements ContextEngine {
561
570
  return;
562
571
  }
563
572
  const db = getLcmConnection(this.config.databasePath);
564
- runLcmMigrations(db);
573
+ runLcmMigrations(db, { fts5Available: this.fts5Available });
565
574
  this.migrated = true;
566
575
  }
567
576
 
@@ -1,6 +1,7 @@
1
1
  import type { DatabaseSync } from "node:sqlite";
2
2
  import { randomUUID } from "node:crypto";
3
3
  import { sanitizeFts5Query } from "./fts5-sanitize.js";
4
+ import { buildLikeSearchPlan, createFallbackSnippet } from "./full-text-fallback.js";
4
5
 
5
6
  export type ConversationId = number;
6
7
  export type MessageId = number;
@@ -203,7 +204,14 @@ function toMessagePartRecord(row: MessagePartRow): MessagePartRecord {
203
204
  // ── ConversationStore ─────────────────────────────────────────────────────────
204
205
 
205
206
  export class ConversationStore {
206
- constructor(private db: DatabaseSync) {}
207
+ private readonly fts5Available: boolean;
208
+
209
+ constructor(
210
+ private db: DatabaseSync,
211
+ options?: { fts5Available?: boolean },
212
+ ) {
213
+ this.fts5Available = options?.fts5Available ?? true;
214
+ }
207
215
 
208
216
  // ── Transaction helpers ──────────────────────────────────────────────────
209
217
 
@@ -292,10 +300,7 @@ export class ConversationStore {
292
300
 
293
301
  const messageId = Number(result.lastInsertRowid);
294
302
 
295
- // Index in FTS5
296
- this.db
297
- .prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`)
298
- .run(messageId, input.content);
303
+ this.indexMessageForFullText(messageId, input.content);
299
304
 
300
305
  const row = this.db
301
306
  .prepare(
@@ -315,7 +320,6 @@ export class ConversationStore {
315
320
  `INSERT INTO messages (conversation_id, seq, role, content, token_count)
316
321
  VALUES (?, ?, ?, ?, ?)`,
317
322
  );
318
- const insertFtsStmt = this.db.prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`);
319
323
  const selectStmt = this.db.prepare(
320
324
  `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
321
325
  FROM messages WHERE message_id = ?`,
@@ -332,7 +336,7 @@ export class ConversationStore {
332
336
  );
333
337
 
334
338
  const messageId = Number(result.lastInsertRowid);
335
- insertFtsStmt.run(messageId, input.content);
339
+ this.indexMessageForFullText(messageId, input.content);
336
340
  const row = selectStmt.get(messageId) as unknown as MessageRow;
337
341
  records.push(toMessageRecord(row));
338
342
  }
@@ -535,8 +539,7 @@ export class ConversationStore {
535
539
  .prepare(`DELETE FROM context_items WHERE item_type = 'message' AND message_id = ?`)
536
540
  .run(messageId);
537
541
 
538
- // Remove from FTS index
539
- this.db.prepare(`DELETE FROM messages_fts WHERE rowid = ?`).run(messageId);
542
+ this.deleteMessageFromFullText(messageId);
540
543
 
541
544
  // Delete the message (message_parts cascade via ON DELETE CASCADE)
542
545
  this.db.prepare(`DELETE FROM messages WHERE message_id = ?`).run(messageId);
@@ -553,17 +556,54 @@ export class ConversationStore {
553
556
  const limit = input.limit ?? 50;
554
557
 
555
558
  if (input.mode === "full_text") {
556
- return this.searchFullText(
557
- input.query,
558
- limit,
559
- input.conversationId,
560
- input.since,
561
- input.before,
562
- );
559
+ if (this.fts5Available) {
560
+ try {
561
+ return this.searchFullText(
562
+ input.query,
563
+ limit,
564
+ input.conversationId,
565
+ input.since,
566
+ input.before,
567
+ );
568
+ } catch {
569
+ return this.searchLike(
570
+ input.query,
571
+ limit,
572
+ input.conversationId,
573
+ input.since,
574
+ input.before,
575
+ );
576
+ }
577
+ }
578
+ return this.searchLike(input.query, limit, input.conversationId, input.since, input.before);
563
579
  }
564
580
  return this.searchRegex(input.query, limit, input.conversationId, input.since, input.before);
565
581
  }
566
582
 
583
+ private indexMessageForFullText(messageId: MessageId, content: string): void {
584
+ if (!this.fts5Available) {
585
+ return;
586
+ }
587
+ try {
588
+ this.db
589
+ .prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`)
590
+ .run(messageId, content);
591
+ } catch {
592
+ // Full-text indexing is optional. Message persistence must still succeed.
593
+ }
594
+ }
595
+
596
+ private deleteMessageFromFullText(messageId: MessageId): void {
597
+ if (!this.fts5Available) {
598
+ return;
599
+ }
600
+ try {
601
+ this.db.prepare(`DELETE FROM messages_fts WHERE rowid = ?`).run(messageId);
602
+ } catch {
603
+ // Ignore FTS cleanup failures; the source row deletion is authoritative.
604
+ }
605
+ }
606
+
567
607
  private searchFullText(
568
608
  query: string,
569
609
  limit: number,
@@ -603,6 +643,55 @@ export class ConversationStore {
603
643
  return rows.map(toSearchResult);
604
644
  }
605
645
 
646
+ private searchLike(
647
+ query: string,
648
+ limit: number,
649
+ conversationId?: ConversationId,
650
+ since?: Date,
651
+ before?: Date,
652
+ ): MessageSearchResult[] {
653
+ const plan = buildLikeSearchPlan("content", query);
654
+ if (plan.terms.length === 0) {
655
+ return [];
656
+ }
657
+
658
+ const where: string[] = [...plan.where];
659
+ const args: Array<string | number> = [...plan.args];
660
+ if (conversationId != null) {
661
+ where.push("conversation_id = ?");
662
+ args.push(conversationId);
663
+ }
664
+ if (since) {
665
+ where.push("julianday(created_at) >= julianday(?)");
666
+ args.push(since.toISOString());
667
+ }
668
+ if (before) {
669
+ where.push("julianday(created_at) < julianday(?)");
670
+ args.push(before.toISOString());
671
+ }
672
+ args.push(limit);
673
+
674
+ const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
675
+ const rows = this.db
676
+ .prepare(
677
+ `SELECT message_id, conversation_id, seq, role, content, token_count, created_at
678
+ FROM messages
679
+ ${whereClause}
680
+ ORDER BY created_at DESC
681
+ LIMIT ?`,
682
+ )
683
+ .all(...args) as unknown as MessageRow[];
684
+
685
+ return rows.map((row) => ({
686
+ messageId: row.message_id,
687
+ conversationId: row.conversation_id,
688
+ role: row.role,
689
+ snippet: createFallbackSnippet(row.content, plan.terms),
690
+ createdAt: new Date(row.created_at),
691
+ rank: 0,
692
+ }));
693
+ }
694
+
606
695
  private searchRegex(
607
696
  pattern: string,
608
697
  limit: number,
@@ -0,0 +1,74 @@
1
+ const RAW_TERM_RE = /"([^"]+)"|(\S+)/g;
2
+ const EDGE_PUNCTUATION_RE = /^[`'"()[\]{}<>.,:;!?*_+=|\\/-]+|[`'"()[\]{}<>.,:;!?*_+=|\\/-]+$/g;
3
+
4
+ export type LikeSearchPlan = {
5
+ terms: string[];
6
+ where: string[];
7
+ args: string[];
8
+ };
9
+
10
+ function normalizeFallbackTerm(raw: string): string {
11
+ return raw.trim().replace(EDGE_PUNCTUATION_RE, "").toLowerCase();
12
+ }
13
+
14
+ function escapeLike(term: string): string {
15
+ return term.replace(/([\\%_])/g, "\\$1");
16
+ }
17
+
18
+ /**
19
+ * Convert a free-text query into a conservative LIKE search plan.
20
+ *
21
+ * The fallback keeps phrase tokens when the query uses double quotes, and
22
+ * otherwise searches for all normalized tokens as case-insensitive substrings.
23
+ */
24
+ export function buildLikeSearchPlan(column: string, query: string): LikeSearchPlan {
25
+ const terms: string[] = [];
26
+ for (const match of query.matchAll(RAW_TERM_RE)) {
27
+ const raw = match[1] ?? match[2] ?? "";
28
+ const normalized = normalizeFallbackTerm(raw);
29
+ if (normalized.length > 0 && !terms.includes(normalized)) {
30
+ terms.push(normalized);
31
+ }
32
+ }
33
+
34
+ if (terms.length === 0) {
35
+ const fallback = normalizeFallbackTerm(query);
36
+ if (fallback.length > 0) {
37
+ terms.push(fallback);
38
+ }
39
+ }
40
+
41
+ return {
42
+ terms,
43
+ where: terms.map(() => `LOWER(${column}) LIKE ? ESCAPE '\\'`),
44
+ args: terms.map((term) => `%${escapeLike(term)}%`),
45
+ };
46
+ }
47
+
48
+ /**
49
+ * Build a compact snippet centered around the earliest matching term.
50
+ */
51
+ export function createFallbackSnippet(content: string, terms: string[]): string {
52
+ const haystack = content.toLowerCase();
53
+ let matchIndex = -1;
54
+ let matchLength = 0;
55
+
56
+ for (const term of terms) {
57
+ const idx = haystack.indexOf(term);
58
+ if (idx !== -1 && (matchIndex === -1 || idx < matchIndex)) {
59
+ matchIndex = idx;
60
+ matchLength = term.length;
61
+ }
62
+ }
63
+
64
+ if (matchIndex === -1) {
65
+ const head = content.trim();
66
+ return head.length <= 80 ? head : `${head.slice(0, 77).trimEnd()}...`;
67
+ }
68
+
69
+ const start = Math.max(0, matchIndex - 24);
70
+ const end = Math.min(content.length, matchIndex + Math.max(matchLength, 1) + 40);
71
+ const prefix = start > 0 ? "..." : "";
72
+ const suffix = end < content.length ? "..." : "";
73
+ return `${prefix}${content.slice(start, end).trim()}${suffix}`;
74
+ }
@@ -1,5 +1,6 @@
1
1
  import type { DatabaseSync } from "node:sqlite";
2
2
  import { sanitizeFts5Query } from "./fts5-sanitize.js";
3
+ import { buildLikeSearchPlan, createFallbackSnippet } from "./full-text-fallback.js";
3
4
 
4
5
  export type SummaryKind = "leaf" | "condensed";
5
6
  export type ContextItemType = "message" | "summary";
@@ -239,7 +240,14 @@ function toLargeFileRecord(row: LargeFileRow): LargeFileRecord {
239
240
  // ── SummaryStore ──────────────────────────────────────────────────────────────
240
241
 
241
242
  export class SummaryStore {
242
- constructor(private db: DatabaseSync) {}
243
+ private readonly fts5Available: boolean;
244
+
245
+ constructor(
246
+ private db: DatabaseSync,
247
+ options?: { fts5Available?: boolean },
248
+ ) {
249
+ this.fts5Available = options?.fts5Available ?? true;
250
+ }
243
251
 
244
252
  // ── Summary CRUD ──────────────────────────────────────────────────────────
245
253
 
@@ -305,8 +313,21 @@ export class SummaryStore {
305
313
  sourceMessageTokenCount,
306
314
  );
307
315
 
316
+ const row = this.db
317
+ .prepare(
318
+ `SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
319
+ earliest_at, latest_at, descendant_count, created_at
320
+ , descendant_token_count, source_message_token_count
321
+ FROM summaries WHERE summary_id = ?`,
322
+ )
323
+ .get(input.summaryId) as unknown as SummaryRow;
324
+
308
325
  // Index in FTS5 as best-effort; compaction flow must continue even if
309
326
  // FTS indexing fails for any reason.
327
+ if (!this.fts5Available) {
328
+ return toSummaryRecord(row);
329
+ }
330
+
310
331
  try {
311
332
  this.db
312
333
  .prepare(`INSERT INTO summaries_fts(summary_id, content) VALUES (?, ?)`)
@@ -316,15 +337,6 @@ export class SummaryStore {
316
337
  // compaction and assembly will still work correctly.
317
338
  }
318
339
 
319
- const row = this.db
320
- .prepare(
321
- `SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
322
- earliest_at, latest_at, descendant_count, created_at
323
- , descendant_token_count, source_message_token_count
324
- FROM summaries WHERE summary_id = ?`,
325
- )
326
- .get(input.summaryId) as unknown as SummaryRow;
327
-
328
340
  return toSummaryRecord(row);
329
341
  }
330
342
 
@@ -685,13 +697,26 @@ export class SummaryStore {
685
697
  const limit = input.limit ?? 50;
686
698
 
687
699
  if (input.mode === "full_text") {
688
- return this.searchFullText(
689
- input.query,
690
- limit,
691
- input.conversationId,
692
- input.since,
693
- input.before,
694
- );
700
+ if (this.fts5Available) {
701
+ try {
702
+ return this.searchFullText(
703
+ input.query,
704
+ limit,
705
+ input.conversationId,
706
+ input.since,
707
+ input.before,
708
+ );
709
+ } catch {
710
+ return this.searchLike(
711
+ input.query,
712
+ limit,
713
+ input.conversationId,
714
+ input.since,
715
+ input.before,
716
+ );
717
+ }
718
+ }
719
+ return this.searchLike(input.query, limit, input.conversationId, input.since, input.before);
695
720
  }
696
721
  return this.searchRegex(input.query, limit, input.conversationId, input.since, input.before);
697
722
  }
@@ -735,6 +760,57 @@ export class SummaryStore {
735
760
  return rows.map(toSearchResult);
736
761
  }
737
762
 
763
+ private searchLike(
764
+ query: string,
765
+ limit: number,
766
+ conversationId?: number,
767
+ since?: Date,
768
+ before?: Date,
769
+ ): SummarySearchResult[] {
770
+ const plan = buildLikeSearchPlan("content", query);
771
+ if (plan.terms.length === 0) {
772
+ return [];
773
+ }
774
+
775
+ const where: string[] = [...plan.where];
776
+ const args: Array<string | number> = [...plan.args];
777
+ if (conversationId != null) {
778
+ where.push("conversation_id = ?");
779
+ args.push(conversationId);
780
+ }
781
+ if (since) {
782
+ where.push("julianday(created_at) >= julianday(?)");
783
+ args.push(since.toISOString());
784
+ }
785
+ if (before) {
786
+ where.push("julianday(created_at) < julianday(?)");
787
+ args.push(before.toISOString());
788
+ }
789
+ args.push(limit);
790
+
791
+ const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
792
+ const rows = this.db
793
+ .prepare(
794
+ `SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
795
+ earliest_at, latest_at, descendant_count, descendant_token_count,
796
+ source_message_token_count, created_at
797
+ FROM summaries
798
+ ${whereClause}
799
+ ORDER BY created_at DESC
800
+ LIMIT ?`,
801
+ )
802
+ .all(...args) as unknown as SummaryRow[];
803
+
804
+ return rows.map((row) => ({
805
+ summaryId: row.summary_id,
806
+ conversationId: row.conversation_id,
807
+ kind: row.kind,
808
+ snippet: createFallbackSnippet(row.content, plan.terms),
809
+ createdAt: new Date(row.created_at),
810
+ rank: 0,
811
+ }));
812
+ }
813
+
738
814
  private searchRegex(
739
815
  pattern: string,
740
816
  limit: number,
package/src/summarize.ts CHANGED
@@ -1,4 +1,3 @@
1
- import { resolveLcmConfig } from "./db/config.js";
2
1
  import type { LcmDependencies } from "./types.js";
3
2
 
4
3
  export type LcmSummarizeOptions = {
@@ -675,11 +674,10 @@ export async function createLcmSummarizeFromLegacyParams(params: {
675
674
 
676
675
  const apiKey = params.deps.getApiKey(provider, model);
677
676
 
678
- const runtimeLcmConfig = resolveLcmConfig();
679
677
  const condensedTargetTokens =
680
- Number.isFinite(runtimeLcmConfig.condensedTargetTokens) &&
681
- runtimeLcmConfig.condensedTargetTokens > 0
682
- ? runtimeLcmConfig.condensedTargetTokens
678
+ Number.isFinite(params.deps.config.condensedTargetTokens) &&
679
+ params.deps.config.condensedTargetTokens > 0
680
+ ? params.deps.config.condensedTargetTokens
683
681
  : DEFAULT_CONDENSED_TARGET_TOKENS;
684
682
 
685
683
  return async (