clawmem 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/AGENTS.md CHANGED
@@ -19,6 +19,7 @@ Three services — **embedding**, **LLM** (query expansion / intent / A-MEM), **
19
19
  - The zerank-2 **GGUF is inert** — llama.cpp drops the score head → ranking silently collapses to RRF. Use the **seq-cls sidecar**; verify with `clawmem rerank-health` (liveness ≠ correctness).
20
20
  - `-ub` must equal `-b` for embedding/reranking (non-causal attention) or `llama-server` asserts.
21
21
  - Changing embedding dimensions → `clawmem embed --force` (full re-embed).
22
+ - Changing the embedding **model** (even at the same dimension) → `clawmem embed --force`; querying otherwise now throws `VecReadModelMismatchError` instead of serving cosine-meaningless results (v0.18.0).
22
23
  - `CLAWMEM_NO_LOCAL_MODELS=true` to fail fast instead of silent CPU fallback.
23
24
 
24
25
  → Stack decision matrix + server setup: [docs/guides/inference-services.md](docs/guides/inference-services.md) · cloud: [docs/guides/cloud-embedding.md](docs/guides/cloud-embedding.md) · all env vars: [docs/reference/configuration.md](docs/reference/configuration.md).
package/README.md CHANGED
@@ -782,10 +782,12 @@ Notes referenced by the agent during a session get boosted (`access_count++`). U
782
782
  | `CLAWMEM_EMBED_TPM_LIMIT` | `100000` | Tokens-per-minute limit for cloud embedding pacing. Match to your provider tier. |
783
783
  | `CLAWMEM_EMBED_DIMENSIONS` | (none) | Output dimensions for OpenAI `text-embedding-3-*` Matryoshka models (e.g. `512`, `1024`). |
784
784
  | `CLAWMEM_LLM_URL` | `http://localhost:8089` | LLM server URL for intent/query/A-MEM. Without it, falls to `node-llama-cpp` (if allowed). |
785
+ | `CLAWMEM_LLM_API_KEY` | (none) | Bearer token for an authenticated remote LLM endpoint. Independent of the embed and rerank keys. |
785
786
  | `CLAWMEM_LLM_MODEL` | `qwen3` | Model name sent to the configured LLM endpoint. Override this for OpenAI-compatible proxies such as `gpt-5.4-mini`. |
786
787
  | `CLAWMEM_LLM_REASONING_EFFORT` | (none) | Optional top-level `reasoning_effort` field for Chat Completions endpoints that support it (for example OpenAI reasoning models). Leave unset for llama-server/vLLM unless your serving stack explicitly accepts that field. |
787
788
  | `CLAWMEM_LLM_NO_THINK` | `true` | Append `/no_think` to remote LLM prompts. Set to `false` for standard OpenAI models and other endpoints that reject or treat the Qwen-style suffix as literal prompt text. |
788
789
  | `CLAWMEM_RERANK_URL` | `http://localhost:8090` | Reranker server URL. Without it, falls to `node-llama-cpp` (if allowed). |
790
+ | `CLAWMEM_RERANK_API_KEY` | (none) | Bearer token for an authenticated remote reranker endpoint. Independent of the embed and LLM keys. |
789
791
  | `CLAWMEM_NO_LOCAL_MODELS` | `false` | Block `node-llama-cpp` from auto-downloading GGUF models. Set `true` for remote-only setups where you want fail-fast on unreachable endpoints. |
790
792
  | `CLAWMEM_MERGE_SCORE_NORMAL` | `0.93` | **v0.7.1.** Phase 2 consolidation merge-safety threshold when candidate and existing anchors align. Merges above this normalized 3-gram cosine score are allowed. |
791
793
  | `CLAWMEM_MERGE_SCORE_STRICT` | `0.98` | **v0.7.1.** Strictest merge-safety threshold — fallback when anchor sets are ambiguous. |
@@ -169,10 +169,12 @@ The session ID is resolved from `--session-id <id>`, then `CLAUDE_SESSION_ID`, t
169
169
  | `CLAWMEM_EMBED_TPM_LIMIT` | `100000` | Tokens-per-minute limit for cloud embedding pacing |
170
170
  | `CLAWMEM_EMBED_DIMENSIONS` | — | Output dimensions for OpenAI `text-embedding-3-*` models |
171
171
  | `CLAWMEM_LLM_URL` | `http://localhost:8089` | LLM server |
172
+ | `CLAWMEM_LLM_API_KEY` | — | Bearer token for an authenticated remote LLM endpoint |
172
173
  | `CLAWMEM_LLM_MODEL` | `qwen3` | Model name sent to the configured LLM endpoint |
173
174
  | `CLAWMEM_LLM_REASONING_EFFORT` | — | Optional top-level `reasoning_effort` field for Chat Completions endpoints that support it (for example OpenAI reasoning models). Leave unset for llama-server/vLLM unless explicitly supported. |
174
175
  | `CLAWMEM_LLM_NO_THINK` | `true` | Append `/no_think` to remote prompts; set `false` for standard OpenAI models and other endpoints that reject or treat it as literal prompt text |
175
176
  | `CLAWMEM_RERANK_URL` | `http://localhost:8090` | Reranker server |
177
+ | `CLAWMEM_RERANK_API_KEY` | — | Bearer token for an authenticated remote reranker endpoint |
176
178
  | `CLAWMEM_NO_LOCAL_MODELS` | `false` | Block node-llama-cpp auto-downloads |
177
179
  | `CLAWMEM_PROFILE` | `balanced` | Performance profile: `speed` (BM25 only), `balanced` (BM25+vector), `deep` (BM25+vector+expansion+reranking) |
178
180
  | `CLAWMEM_VAULTS` | — | JSON map of vault name to SQLite path |
@@ -12,7 +12,9 @@ See also: [../guides/inference-services.md](../guides/inference-services.md) (st
12
12
  |---|---|---|
13
13
  | `CLAWMEM_EMBED_URL` | `http://localhost:8088` | Embedding server URL. Local `llama-server`, cloud API, or in-process `node-llama-cpp` fallback if unset. |
14
14
  | `CLAWMEM_LLM_URL` | `http://localhost:8089` | LLM server for intent, expansion, A-MEM, entity extraction. Falls to `node-llama-cpp` if unset + `NO_LOCAL_MODELS=false`. Point at a 7B+ model or cloud API during `reindex --enrich` for better entity extraction. |
15
+ | `CLAWMEM_LLM_API_KEY` | (none) | Bearer token for an authenticated remote LLM endpoint. Independent of the embed/rerank keys — set it when the LLM points at a different authenticated host. |
15
16
  | `CLAWMEM_RERANK_URL` | `http://localhost:8090` | Reranker server. Falls to `node-llama-cpp` if unset + `NO_LOCAL_MODELS=false`. |
17
+ | `CLAWMEM_RERANK_API_KEY` | (none) | Bearer token for an authenticated remote reranker endpoint. Independent of the embed/LLM keys. |
16
18
  | `CLAWMEM_LLM_MODEL` | `qwen3` | Model name sent on LLM requests. |
17
19
  | `CLAWMEM_LLM_REASONING_EFFORT` | (none) | Top-level `reasoning_effort` for Chat Completions endpoints that support it (e.g. a remote reasoning model). Optional. |
18
20
  | `CLAWMEM_LLM_NO_THINK` | enabled | Appends `/no_think` to remote LLM prompts (Qwen3 emits thinking tokens by default). Set `false` for standard OpenAI-compatible models that would treat `/no_think` as literal text. |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "clawmem",
3
- "version": "0.17.0",
3
+ "version": "0.18.0",
4
4
  "description": "On-device memory layer for AI agents. Claude Code, OpenClaw, and Hermes. Hooks + MCP server + hybrid RAG search.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -15,6 +15,7 @@
15
15
  import type { Store } from "./store.ts";
16
16
  import type { LlamaCpp } from "./llm.ts";
17
17
  import { extractJsonFromLLM } from "./amem.ts";
18
+ import { isSchemaPlaceholder } from "./schema-placeholder.ts";
18
19
  import { hashContent } from "./indexer.ts";
19
20
  import { passesMergeSafety } from "./text-similarity.ts";
20
21
  import { withWorkerLease } from "./worker-lease.ts";
@@ -68,6 +69,8 @@ export interface DeductiveSynthesisStats {
68
69
  unsupportedRejects: number;
69
70
  /** Drafts rejected because the conclusion was empty/trivial */
70
71
  emptyRejects: number;
72
+ /** Drafts rejected because the conclusion echoed the schema skeleton residue (anti-parrot) */
73
+ placeholderRejects: number;
71
74
  /** Accepted drafts that were then skipped as deductive dedupe duplicates */
72
75
  dedupSkipped: number;
73
76
  /**
@@ -93,6 +96,7 @@ function emptyDeductiveStats(considered: number = 0): DeductiveSynthesisStats {
93
96
  invalidIndexRejects: 0,
94
97
  unsupportedRejects: 0,
95
98
  emptyRejects: 0,
99
+ placeholderRejects: 0,
96
100
  dedupSkipped: 0,
97
101
  validatorFallbackAccepts: 0,
98
102
  };
@@ -1030,11 +1034,12 @@ For each valid deduction:
1030
1034
  2. List the premises (which observations support it)
1031
1035
  3. List the source indices (1-indexed)
1032
1036
 
1033
- Return ONLY valid JSON array:
1037
+ Return ONLY a JSON array in this shape (structure only — replace every {{...}} with real content
1038
+ drawn from the observations above; never emit the {{...}} tokens or any schema text literally):
1034
1039
  [
1035
1040
  {
1036
- "conclusion": "Clear deductive statement",
1037
- "premises": ["Premise from obs 1", "Premise from obs 3"],
1041
+ "conclusion": "{{new conclusion combining 2+ observations, 1-2 sentences}}",
1042
+ "premises": ["{{fact from one source observation}}", "{{fact from another}}"],
1038
1043
  "source_indices": [1, 3]
1039
1044
  }
1040
1045
  ]
@@ -1073,6 +1078,18 @@ Return ONLY the JSON array. /no_think`;
1073
1078
  continue;
1074
1079
  }
1075
1080
 
1081
+ // Anti-parrot: reject a draft whose conclusion echoes the JSON skeleton's placeholder residue
1082
+ // ({{...}} tokens / "clear deductive statement") before the expensive validator runs. Premises
1083
+ // that are placeholder residue are filtered (non-fatal — the conclusion carries the deduction).
1084
+ if (isSchemaPlaceholder(deduction.conclusion)) {
1085
+ stats.rejected++;
1086
+ stats.placeholderRejects++;
1087
+ continue;
1088
+ }
1089
+ if (Array.isArray(deduction.premises)) {
1090
+ deduction.premises = deduction.premises.filter(p => typeof p === "string" && !isSchemaPlaceholder(p));
1091
+ }
1092
+
1076
1093
  const sourceDocIds = [...new Set(
1077
1094
  deduction.source_indices
1078
1095
  .filter(i => i >= 1 && i <= recentObs.length)
@@ -28,6 +28,7 @@
28
28
  import type { Store } from "./store.ts";
29
29
  import type { LlamaCpp } from "./llm.ts";
30
30
  import { extractJsonFromLLM } from "./amem.ts";
31
+ import { isSchemaPlaceholder } from "./schema-placeholder.ts";
31
32
  import type { ContentType } from "./memory.ts";
32
33
 
33
34
  // =============================================================================
@@ -179,7 +180,7 @@ For each fact provide:
179
180
  - contentType: one of [decision, preference, milestone, problem]
180
181
  - narrative: 1-3 sentence description of the fact in context
181
182
  - facts: optional array of supporting fact strings (evidence)
182
- - aliases: optional alternative titles for linking (e.g., ["OAuth choice"] for "Use OAuth 2.0")
183
+ - aliases: optional alternative titles for linking (help match this fact to related ones)
183
184
  - links: optional array of cross-fact references. Each link is
184
185
  {targetTitle, relationType, weight}
185
186
  - targetTitle may refer to another fact extracted from this conversation OR from
@@ -189,18 +190,20 @@ For each fact provide:
189
190
  - weight is 0.0-1.0 (default 0.6)
190
191
 
191
192
  Only extract facts the conversation clearly supports. Do NOT fabricate.
193
+ Never copy the {{...}} tokens or any schema text below into your output — replace every {{...}}
194
+ with real content drawn from the conversation. Emit only real extracted facts.
192
195
  Return ONLY valid JSON array. Return empty array [] if no structured facts found.
193
196
 
194
- Example output:
197
+ Shape (structure only — not example content):
195
198
  [
196
199
  {
197
- "title": "Use OAuth 2.0 with PKCE",
198
- "contentType": "decision",
199
- "narrative": "Team decided to use OAuth 2.0 with PKCE for user authentication, replacing session cookies.",
200
- "facts": ["PKCE chosen for mobile support", "Legacy session auth to be deprecated Q2"],
201
- "aliases": ["OAuth decision", "switch to OAuth"],
200
+ "title": "{{concise 3-8 word title}}",
201
+ "contentType": "{{one of: decision, preference, milestone, problem}}",
202
+ "narrative": "{{1-3 sentence description of the fact in context}}",
203
+ "facts": ["{{supporting evidence string}}"],
204
+ "aliases": ["{{optional alternative title}}"],
202
205
  "links": [
203
- { "targetTitle": "Deprecate session auth", "relationType": "causal", "weight": 0.8 }
206
+ { "targetTitle": "{{another fact's title}}", "relationType": "{{one of: semantic, supporting, contradicts, causal, temporal, entity}}", "weight": 0.6 }
204
207
  ]
205
208
  }
206
209
  ]`;
@@ -219,6 +222,8 @@ function normalizeExtractedFact(
219
222
 
220
223
  const title = typeof obj.title === "string" ? obj.title.trim() : "";
221
224
  if (!title) return null;
225
+ // Anti-parrot: reject a fact whose title is echoed schema/skeleton residue.
226
+ if (isSchemaPlaceholder(title)) return null;
222
227
 
223
228
  const contentType = obj.contentType;
224
229
  if (typeof contentType !== "string") return null;
@@ -226,13 +231,14 @@ function normalizeExtractedFact(
226
231
 
227
232
  const narrative = typeof obj.narrative === "string" ? obj.narrative.trim() : "";
228
233
  if (!narrative) return null;
234
+ if (isSchemaPlaceholder(narrative)) return null;
229
235
 
230
236
  const facts: string[] = Array.isArray(obj.facts)
231
- ? obj.facts.filter((f): f is string => typeof f === "string" && f.trim().length > 0)
237
+ ? obj.facts.filter((f): f is string => typeof f === "string" && f.trim().length > 0 && !isSchemaPlaceholder(f))
232
238
  : [];
233
239
 
234
240
  const aliases: string[] = Array.isArray(obj.aliases)
235
- ? obj.aliases.filter((a): a is string => typeof a === "string" && a.trim().length > 0)
241
+ ? obj.aliases.filter((a): a is string => typeof a === "string" && a.trim().length > 0 && !isSchemaPlaceholder(a))
236
242
  : [];
237
243
 
238
244
  const links: ExtractedFactLink[] = Array.isArray(obj.links)
@@ -7,7 +7,7 @@
7
7
  */
8
8
 
9
9
  import type { Store, SearchResult } from "../store.ts";
10
- import { DEFAULT_EMBED_MODEL, DEFAULT_QUERY_MODEL, DEFAULT_RERANK_MODEL, extractSnippet, resolveStore } from "../store.ts";
10
+ import { DEFAULT_EMBED_MODEL, DEFAULT_QUERY_MODEL, DEFAULT_RERANK_MODEL, warnOnceOnVectorModelMismatch, extractSnippet, resolveStore } from "../store.ts";
11
11
  import { getVaultPath, getActiveProfile } from "../config.ts";
12
12
  import type { HookInput, HookOutput } from "../hooks.ts";
13
13
  import {
@@ -210,8 +210,11 @@ export async function contextSurfacing(
210
210
  vectorTimer = setTimeout(() => reject(new Error("vector timeout")), profile.vectorTimeout);
211
211
  });
212
212
  results = await Promise.race([vectorPromise, timeoutPromise]);
213
- } catch {
214
- // Vector search unavailable, timed out, or errored — fall back to BM25
213
+ } catch (e) {
214
+ // Vector search unavailable, timed out, or errored — fall back to BM25. A vault-wide
215
+ // embedding-model mismatch is a persistent config error, not a transient miss: surface it
216
+ // loudly once, then degrade (the hook stays fail-open).
217
+ warnOnceOnVectorModelMismatch(e);
215
218
  } finally {
216
219
  // Clear the timer when the vector promise won the race: a pending (ref'd) setTimeout keeps the
217
220
  // Bun hook process alive for the full vectorTimeout after results are already in hand.
@@ -307,7 +310,7 @@ export async function contextSurfacing(
307
310
  deepTimer = setTimeout(() => reject(new Error("vector timeout")), remainingMs);
308
311
  });
309
312
  hits = await Promise.race([deepVec, deepTimeout]);
310
- } catch { /* vector leg non-fatal (timed out or errored) */ }
313
+ } catch (e) { warnOnceOnVectorModelMismatch(e); /* vector leg non-fatal (timed out or errored) */ }
311
314
  finally { if (deepTimer) clearTimeout(deepTimer); } // don't let a pending timer keep the hook process alive
312
315
  }
313
316
  for (const r of hits) {
@@ -23,7 +23,7 @@ import { loadConfig } from "../collections.ts";
23
23
  import { getDefaultLlamaCpp } from "../llm.ts";
24
24
  import type { ObservationWithDoc } from "../amem.ts";
25
25
  import { extractJsonFromLLM } from "../amem.ts";
26
- import { DEFAULT_EMBED_MODEL, extractSnippet, type SearchResult } from "../store.ts";
26
+ import { DEFAULT_EMBED_MODEL, warnOnceOnVectorModelMismatch, extractSnippet, type SearchResult } from "../store.ts";
27
27
  import { ensureEntityCanonical, resolveEntityTypeExact } from "../entity.ts";
28
28
 
29
29
  // Observation types that are allowed to contribute SPO triples. Widened from the
@@ -92,7 +92,8 @@ export async function checkMergePolicy(
92
92
  if (sameType.length > 0) {
93
93
  return { action: 'skip' };
94
94
  }
95
- } catch {
95
+ } catch (e) {
96
+ warnOnceOnVectorModelMismatch(e);
96
97
  // Vector search unavailable — fall through to insert
97
98
  }
98
99
  return { action: 'insert' };
@@ -211,7 +212,8 @@ async function detectContradictions(
211
212
  let existingDocs: SearchResult[];
212
213
  try {
213
214
  existingDocs = await store.searchVec(queryText, DEFAULT_EMBED_MODEL, 5);
214
- } catch {
215
+ } catch (e) {
216
+ warnOnceOnVectorModelMismatch(e);
215
217
  existingDocs = store.searchFTS(queryText, 5);
216
218
  }
217
219
 
package/src/llm.ts CHANGED
@@ -327,6 +327,13 @@ export type LlamaCppConfig = {
327
327
  * When set, generate() calls /v1/chat/completions instead of local node-llama-cpp.
328
328
  */
329
329
  remoteLlmUrl?: string;
330
+ /**
331
+ * API key for the remote LLM service (independent of the embed/rerank keys —
332
+ * the LLM endpoint may point at a different authenticated host than embedding).
333
+ * When set, sent as Authorization: Bearer header with chat completion requests.
334
+ * Env: CLAWMEM_LLM_API_KEY
335
+ */
336
+ remoteLlmApiKey?: string;
330
337
  /**
331
338
  * Remote LLM model name to send with chat completion requests.
332
339
  * Env: CLAWMEM_LLM_MODEL
@@ -401,6 +408,7 @@ export class LlamaCpp implements LLM {
401
408
  private remoteEmbedApiKey: string | null;
402
409
  private remoteEmbedModel: string;
403
410
  private remoteLlmUrl: string | null;
411
+ private remoteLlmApiKey: string | null;
404
412
  private remoteLlmModel: string;
405
413
  private remoteLlmReasoningEffort: string | null;
406
414
  private remoteLlmNoThink: boolean;
@@ -436,6 +444,7 @@ export class LlamaCpp implements LLM {
436
444
  this.remoteEmbedApiKey = config.remoteEmbedApiKey || null;
437
445
  this.remoteEmbedModel = config.remoteEmbedModel || "embedding";
438
446
  this.remoteLlmUrl = config.remoteLlmUrl || null;
447
+ this.remoteLlmApiKey = config.remoteLlmApiKey || null;
439
448
  const normalizedRemoteLlmModel = config.remoteLlmModel?.trim();
440
449
  this.remoteLlmModel = normalizedRemoteLlmModel || "qwen3";
441
450
  this.remoteLlmReasoningEffort = normalizeRemoteLlmReasoningEffort(config.remoteLlmReasoningEffort);
@@ -944,6 +953,14 @@ export class LlamaCpp implements LLM {
944
953
  return headers;
945
954
  }
946
955
 
956
+ private getLlmHeaders(): Record<string, string> {
957
+ const headers: Record<string, string> = { "Content-Type": "application/json" };
958
+ if (this.remoteLlmApiKey) {
959
+ headers["Authorization"] = `Bearer ${this.remoteLlmApiKey}`;
960
+ }
961
+ return headers;
962
+ }
963
+
947
964
  private truncateForEmbed(text: string): string {
948
965
  // Cloud providers handle their own context window limits
949
966
  if (this.isCloudEmbedding()) return text;
@@ -1167,7 +1184,7 @@ export class LlamaCpp implements LLM {
1167
1184
  }
1168
1185
  const resp = await fetch(buildRemoteChatCompletionsUrl(this.remoteLlmUrl!), {
1169
1186
  method: "POST",
1170
- headers: { "Content-Type": "application/json" },
1187
+ headers: this.getLlmHeaders(),
1171
1188
  body: JSON.stringify(body),
1172
1189
  signal,
1173
1190
  });
@@ -1485,6 +1502,7 @@ export function getDefaultLlamaCpp(): LlamaCpp {
1485
1502
  remoteEmbedApiKey: embedApiKey,
1486
1503
  remoteEmbedModel: process.env.CLAWMEM_EMBED_MODEL || undefined,
1487
1504
  remoteLlmUrl: process.env.CLAWMEM_LLM_URL || undefined,
1505
+ remoteLlmApiKey: process.env.CLAWMEM_LLM_API_KEY || undefined,
1488
1506
  remoteLlmModel: process.env.CLAWMEM_LLM_MODEL?.trim() || undefined,
1489
1507
  remoteLlmReasoningEffort: process.env.CLAWMEM_LLM_REASONING_EFFORT || undefined,
1490
1508
  remoteLlmNoThink: (() => {
@@ -479,6 +479,7 @@ export async function runHeavyMaintenanceTick(
479
479
  invalidIndexRejects: stats.invalidIndexRejects,
480
480
  unsupportedRejects: stats.unsupportedRejects,
481
481
  emptyRejects: stats.emptyRejects,
482
+ placeholderRejects: stats.placeholderRejects,
482
483
  dedupSkipped: stats.dedupSkipped,
483
484
  validatorFallbackAccepts: stats.validatorFallbackAccepts,
484
485
  },
package/src/mcp.ts CHANGED
@@ -20,6 +20,7 @@ import {
20
20
  DEFAULT_QUERY_MODEL,
21
21
  DEFAULT_RERANK_MODEL,
22
22
  DEFAULT_MULTI_GET_MAX_BYTES,
23
+ rethrowIfFatalVectorError,
23
24
  type Store,
24
25
  type SearchResult,
25
26
  type CausalLink,
@@ -283,7 +284,7 @@ This is the recommended entry point for ALL memory queries.`,
283
284
  const intent = await classifyIntent(query, llm, store.db);
284
285
  const bm25Results = store.searchFTS(query, 30);
285
286
  let vecResults: SearchResult[] = [];
286
- try { vecResults = await store.searchVec(query, DEFAULT_EMBED_MODEL, 30); } catch { /* no vectors */ }
287
+ try { vecResults = await store.searchVec(query, DEFAULT_EMBED_MODEL, 30); } catch (e) { rethrowIfFatalVectorError(e); /* else: no vectors */ }
287
288
  const rrfWeights = intent.intent === 'WHY' ? [1.0, 1.5] : intent.intent === 'WHEN' ? [1.5, 1.0] : [1.0, 1.0];
288
289
  const fusedRanked = reciprocalRankFusion([bm25Results.map(toRanked), vecResults.map(toRanked)], rrfWeights);
289
290
  const allSearch = [...bm25Results, ...vecResults];
@@ -354,7 +355,7 @@ This is the recommended entry point for ALL memory queries.`,
354
355
  for (const clause of clauses.sort((a, b) => a.priority - b.priority)) {
355
356
  let results: SearchResult[] = [];
356
357
  if (clause.type === 'bm25') results = store.searchFTS(clause.query, 20, undefined, clause.collections);
357
- else if (clause.type === 'vector') { try { results = await store.searchVec(clause.query, DEFAULT_EMBED_MODEL, 20, undefined, clause.collections); } catch { /* */ } }
358
+ else if (clause.type === 'vector') { try { results = await store.searchVec(clause.query, DEFAULT_EMBED_MODEL, 20, undefined, clause.collections); } catch (e) { rethrowIfFatalVectorError(e); /* */ } }
358
359
  else if (clause.type === 'graph') { results = store.searchFTS(clause.query, 15, undefined, clause.collections); }
359
360
  allResults.push(...results);
360
361
  }
@@ -378,12 +379,12 @@ This is the recommended entry point for ALL memory queries.`,
378
379
  if (effectiveMode === "keyword") {
379
380
  results = store.searchFTS(query, lim);
380
381
  } else if (effectiveMode === "semantic" || effectiveMode === "discovery") {
381
- try { results = await store.searchVec(query, DEFAULT_EMBED_MODEL, lim); } catch { results = store.searchFTS(query, lim); }
382
+ try { results = await store.searchVec(query, DEFAULT_EMBED_MODEL, lim); } catch (e) { rethrowIfFatalVectorError(e); results = store.searchFTS(query, lim); }
382
383
  } else {
383
384
  // Hybrid: BM25 + vector + RRF
384
385
  const bm25 = store.searchFTS(query, 30);
385
386
  let vec: SearchResult[] = [];
386
- try { vec = await store.searchVec(query, DEFAULT_EMBED_MODEL, 30); } catch { /* */ }
387
+ try { vec = await store.searchVec(query, DEFAULT_EMBED_MODEL, 30); } catch (e) { rethrowIfFatalVectorError(e); /* */ }
387
388
  if (vec.length > 0) {
388
389
  const fusedRanked = reciprocalRankFusion([bm25.map(toRanked), vec.map(toRanked)], [1.0, 1.0]);
389
390
  const allSearch = [...bm25, ...vec];
@@ -961,7 +962,8 @@ This is the recommended entry point for ALL memory queries.`,
961
962
  }));
962
963
  }
963
964
  }
964
- } catch {
965
+ } catch (e) {
966
+ rethrowIfFatalVectorError(e);
965
967
  // Vector search unavailable — degrade gracefully
966
968
  }
967
969
 
package/src/observer.ts CHANGED
@@ -9,6 +9,7 @@
9
9
  import type { TranscriptMessage } from "./hooks.ts";
10
10
  import { getDefaultLlamaCpp } from "./llm.ts";
11
11
  import { MAX_LLM_GENERATE_TIMEOUT_MS } from "./limits.ts";
12
+ import { isSchemaPlaceholder } from "./schema-placeholder.ts";
12
13
 
13
14
  // =============================================================================
14
15
  // Types
@@ -179,31 +180,9 @@ export const VALID_PREDICATES = new Set([
179
180
  // Predicates whose <object> should be stored as a literal (not resolved to an entity).
180
181
  export const LITERAL_PREDICATES = new Set(["prefers", "avoids"]);
181
182
 
182
- // Exact placeholder strings that must never be persisted as facts or triple components.
183
- // Defense-in-depth: even though the prompt no longer places example text inside
184
- // <fact>/<subject>/<object> tags, a weak model could still echo these phrases.
185
- const SCHEMA_PLACEHOLDER_STRINGS = new Set([
186
- "individual atomic fact",
187
- "atomic fact",
188
- "one atomic claim per fact element",
189
- "brief descriptive title",
190
- "canonical entity name",
191
- ]);
192
-
193
- // Regex for template placeholder markers: {{...}}, <!--...-->, ${...}.
194
- // Intentionally narrow — earlier drafts rejected any line starting with
195
- // "example:" / "placeholder:", which false-positived legitimate facts like
196
- // "Example: QMD switched to Bun in v0.2". Shape-only matching avoids that
197
- // drift; the exact-string blocklist above handles known echoed placeholders.
198
- const PLACEHOLDER_REGEX = /^(\{\{.*\}\}|<!--.*-->|\$\{.*\})/;
199
-
200
- function isSchemaPlaceholder(text: string): boolean {
201
- if (!text) return true;
202
- const normalized = text.trim().toLowerCase();
203
- if (SCHEMA_PLACEHOLDER_STRINGS.has(normalized)) return true;
204
- if (PLACEHOLDER_REGEX.test(normalized)) return true;
205
- return false;
206
- }
183
+ // Anti-parrot residue guard (SCHEMA_PLACEHOLDER_STRINGS / PLACEHOLDER_REGEX / isSchemaPlaceholder)
184
+ // now lives in ./schema-placeholder.ts, shared with the consolidation + conversation-synthesis
185
+ // extraction paths. Imported at the top of this file.
207
186
 
208
187
  export function parseObservationXml(xml: string): Observation | null {
209
188
  const typeMatch = xml.match(/<type>\s*(.*?)\s*<\/type>/s);
@@ -0,0 +1,56 @@
1
+ /**
2
+ * Shared anti-parrot guard.
3
+ *
4
+ * A weak local extraction model, run out-of-distribution, may echo the schema example text or
5
+ * template placeholders from its prompt verbatim instead of extracting real content — the confirmed
6
+ * root cause of a failed mining run with the 1.7B model. This guard rejects that residue on output.
7
+ *
8
+ * Extracted from observer.ts so all three extraction paths — observer (Stop-hook observations),
9
+ * consolidation (Phase-3 deductive synthesis), and conversation-synthesis (mining/import) — share
10
+ * one residue check instead of each re-implementing (or omitting) it.
11
+ *
12
+ * Design note on what does NOT belong here: this blocklist holds only strings that no real fact
13
+ * could legitimately be (schema-instruction words, obviously-synthetic skeleton phrases). It must
14
+ * NOT hold plausible real content — an earlier draft rejected any line starting with
15
+ * "example:" / "placeholder:", which false-positived legitimate facts like
16
+ * "Example: QMD switched to Bun in v0.2". The prompts kill the parrot at the source by using
17
+ * {{...}} skeleton tokens (caught generically by PLACEHOLDER_REGEX) rather than copyable
18
+ * real-looking examples, so this blocklist stays narrow.
19
+ */
20
+
21
+ // Exact placeholder strings that must never be persisted as facts, titles, or triple components.
22
+ // Defense-in-depth: even though prompts now use structure-only skeletons (no copyable example
23
+ // content), a weak model could still echo these phrases from the schema description itself.
24
+ export const SCHEMA_PLACEHOLDER_STRINGS = new Set([
25
+ // observer.ts schema-instruction residue
26
+ "individual atomic fact",
27
+ "atomic fact",
28
+ "one atomic claim per fact element",
29
+ "brief descriptive title",
30
+ "canonical entity name",
31
+ // consolidation.ts deductive-skeleton residue (former copyable example — safe to blocklist
32
+ // because no genuine deduction is literally "clear deductive statement" / "premise from obs N")
33
+ "clear deductive statement",
34
+ "premise from obs 1",
35
+ "premise from obs 3",
36
+ ]);
37
+
38
+ // Regex for template placeholder markers: {{...}}, <!--...-->, ${...}.
39
+ // Intentionally narrow — earlier drafts rejected any line starting with
40
+ // "example:" / "placeholder:", which false-positived legitimate facts like
41
+ // "Example: QMD switched to Bun in v0.2". Shape-only matching avoids that
42
+ // drift; the exact-string blocklist above handles known echoed placeholders.
43
+ export const PLACEHOLDER_REGEX = /^(\{\{.*\}\}|<!--.*-->|\$\{.*\})/;
44
+
45
+ /**
46
+ * True when `text` is empty, a known schema-placeholder string, or a template marker — i.e.
47
+ * residue a model echoed from its prompt rather than real extracted content.
48
+ */
49
+ export function isSchemaPlaceholder(text: string): boolean {
50
+ if (!text) return true;
51
+ const normalized = text.trim().toLowerCase();
52
+ if (!normalized) return true; // whitespace-only is effectively empty content
53
+ if (SCHEMA_PLACEHOLDER_STRINGS.has(normalized)) return true;
54
+ if (PLACEHOLDER_REGEX.test(normalized)) return true;
55
+ return false;
56
+ }
package/src/server.ts CHANGED
@@ -23,6 +23,7 @@ import {
23
23
  DEFAULT_QUERY_MODEL,
24
24
  DEFAULT_RERANK_MODEL,
25
25
  extractSnippet,
26
+ rethrowIfFatalVectorError,
26
27
  } from "./store.ts";
27
28
 
28
29
  // =============================================================================
@@ -149,7 +150,8 @@ async function handleSearch(req: Request, _url: URL, store: Store): Promise<Resp
149
150
  } else if (mode === "semantic") {
150
151
  try {
151
152
  results = await store.searchVec(query, DEFAULT_EMBED_MODEL, limit * 2, undefined, collections);
152
- } catch {
153
+ } catch (e) {
154
+ rethrowIfFatalVectorError(e);
153
155
  results = store.searchFTS(query, limit * 2, undefined, collections);
154
156
  }
155
157
  } else {
@@ -158,7 +160,7 @@ async function handleSearch(req: Request, _url: URL, store: Store): Promise<Resp
158
160
  let vecResults: SearchResult[] = [];
159
161
  try {
160
162
  vecResults = await store.searchVec(query, DEFAULT_EMBED_MODEL, limit * 2, undefined, collections);
161
- } catch { /* vector unavailable */ }
163
+ } catch (e) { rethrowIfFatalVectorError(e); /* vector unavailable */ }
162
164
  // Simple merge — dedupe by filepath, take max score
163
165
  const merged = new Map<string, SearchResult>();
164
166
  for (const r of [...ftsResults, ...vecResults]) {
@@ -612,7 +614,7 @@ async function handleRetrieve(req: Request, _url: URL, store: Store): Promise<Re
612
614
  let vec: SearchResult[] = [];
613
615
  try {
614
616
  vec = await store.searchVec(query, DEFAULT_EMBED_MODEL, limit * 2, undefined, collections);
615
- } catch { /* vector unavailable */ }
617
+ } catch (e) { rethrowIfFatalVectorError(e); /* vector unavailable */ }
616
618
  const weights = intent.intent === "WHEN" ? [1.5, 1.0] : [1.0, 1.5];
617
619
  const fused = reciprocalRankFusion([bm25.map(toRanked), vec.map(toRanked)], weights);
618
620
  const allResults = [...bm25, ...vec];
@@ -625,7 +627,8 @@ async function handleRetrieve(req: Request, _url: URL, store: Store): Promise<Re
625
627
  } else if (mode === "semantic") {
626
628
  try {
627
629
  results = await store.searchVec(query, DEFAULT_EMBED_MODEL, limit * 2, undefined, collections);
628
- } catch {
630
+ } catch (e) {
631
+ rethrowIfFatalVectorError(e);
629
632
  results = store.searchFTS(query, limit * 2, undefined, collections);
630
633
  }
631
634
  } else {
@@ -634,7 +637,7 @@ async function handleRetrieve(req: Request, _url: URL, store: Store): Promise<Re
634
637
  let vec: SearchResult[] = [];
635
638
  try {
636
639
  vec = await store.searchVec(query, DEFAULT_EMBED_MODEL, limit * 2, undefined, collections);
637
- } catch { /* vector unavailable */ }
640
+ } catch (e) { rethrowIfFatalVectorError(e); /* vector unavailable */ }
638
641
  const merged = new Map<string, SearchResult>();
639
642
  for (const r of [...fts, ...vec]) {
640
643
  const existing = merged.get(r.filepath);
package/src/store.ts CHANGED
@@ -1035,6 +1035,16 @@ const contextUsageHasQueryTextCache = new WeakMap<Database, boolean>();
1035
1035
  */
1036
1036
  export class FatalVectorError extends Error {}
1037
1037
 
1038
+ /**
1039
+ * Rethrow a fatal vector error (dimension / model / schema mismatch) so a searchVec() fallback
1040
+ * catch surfaces it instead of silently degrading to BM25. Transient conditions (timeout, absent
1041
+ * vectors) are NOT FatalVectorError and remain swallowed by the caller, as before. Apply this at the
1042
+ * FTS-fallback catch sites that wrap searchVec.
1043
+ */
1044
+ export function rethrowIfFatalVectorError(e: unknown): void {
1045
+ if (e instanceof FatalVectorError) throw e;
1046
+ }
1047
+
1038
1048
  export class VecDimensionMismatchError extends FatalVectorError {
1039
1049
  constructor(public readonly existingDim: number, public readonly requestedDim: number) {
1040
1050
  super(`Embedding dimension changed: vectors_vec is float[${existingDim}] but the model now returns float[${requestedDim}]. Run 'clawmem embed --force' to clear and rebuild the full vault.`);
@@ -1056,6 +1066,33 @@ export class VecModelMismatchError extends FatalVectorError {
1056
1066
  }
1057
1067
  }
1058
1068
 
1069
+ /**
1070
+ * Read-path sibling of VecModelMismatchError. VecModelMismatchError fires only mid-embed-run; this
1071
+ * one fires on the QUERY path when the vault's stored vectors were embedded with one model but the
1072
+ * active embedding endpoint now returns a different model at the SAME dimension (so the dimension
1073
+ * guard cannot catch it). Matching the new model's query vector against the old model's stored
1074
+ * vectors is cosine-meaningless, so searchVec throws this rather than serving corrupted results.
1075
+ */
1076
+ export class VecReadModelMismatchError extends FatalVectorError {
1077
+ constructor(public readonly storedModels: string[], public readonly activeModel: string) {
1078
+ super(`Embedding model mismatch on the query path: the vault's vectors were embedded with ${storedModels.map(m => `"${m}"`).join(", ")} but the active embedding endpoint now returns "${activeModel}". At the same dimension the dimension guard cannot catch this, and matching the new model's query vector against the old model's stored vectors makes cosine similarity meaningless. Run 'clawmem embed --force' to rebuild the vault with the current model.`);
1079
+ this.name = "VecReadModelMismatchError";
1080
+ }
1081
+ }
1082
+
1083
+ // Fail-open surfacing for a read-path model mismatch: warn LOUDLY once per process, then let the
1084
+ // caller degrade to BM25. For hooks that MUST NOT throw — context-surfacing (UserPromptSubmit) and
1085
+ // the Stop hooks (decision-extractor) — a throwing hook breaks that turn. Explicit query paths use
1086
+ // rethrowIfFatalVectorError instead. The once-flag is process-global so the warning fires exactly
1087
+ // once no matter which hook trips it first.
1088
+ let _warnedVectorModelMismatch = false;
1089
+ export function warnOnceOnVectorModelMismatch(e: unknown): void {
1090
+ if (e instanceof VecReadModelMismatchError && !_warnedVectorModelMismatch) {
1091
+ _warnedVectorModelMismatch = true;
1092
+ console.warn(`[clawmem] ${e.message}`);
1093
+ }
1094
+ }
1095
+
1059
1096
  // A FatalVectorError so it propagates out of the per-fragment catch and aborts the
1060
1097
  // embed run. Thrown both by cmdEmbed (between fragments) and INSIDE insertEmbedding's
1061
1098
  // write transaction (atomic lease-token fence) when the lease was reclaimed.
@@ -3448,12 +3485,54 @@ export function searchFTS(db: Database, query: string, limit: number = 20, colle
3448
3485
  // Vector Search
3449
3486
  // =============================================================================
3450
3487
 
3488
+ // W1 read-path model-consistency cache, keyed on SQLite's `data_version` so a CROSS-PROCESS vault
3489
+ // rebuild invalidates a stale OK verdict. `data_version` changes whenever ANOTHER connection commits
3490
+ // (e.g. a separate `clawmem embed --force` process re-embeds with a different model) — exactly the
3491
+ // multi-process staleness case — and is a cheap header read (no scan), unlike the getVecModels()
3492
+ // DISTINCT+JOIN it guards. Same-connection writes don't bump it, but in-process model swaps are
3493
+ // already blocked by the embed-time VecModelMismatchError / clearAllEmbeddings drop.
3494
+ const verifiedQueryEmbedModels = new WeakMap<Database, { dataVersion: number; model: string }>();
3495
+
3496
+ /**
3497
+ * Guard the query path against a same-dimension embedding-model swap. Compares the ENDPOINT-returned
3498
+ * model (NOT the caller's DEFAULT_EMBED_MODEL arg, which is a local alias unrelated to what the
3499
+ * endpoint actually serves) against the models the vault's active vectors were embedded with. The
3500
+ * vault is consistent ONLY when it holds EXACTLY ONE model equal to the endpoint's — a heterogeneous
3501
+ * vault (length > 1) is cosine-corrupt even if the endpoint matches one of the models, because the
3502
+ * other model's vectors still pollute the space. Throws VecReadModelMismatchError otherwise; no-ops
3503
+ * when the vault has no vectors yet.
3504
+ */
3505
+ function assertQueryEmbedModelConsistent(db: Database, endpointModel: string): void {
3506
+ const dataVersion = (db.prepare("PRAGMA data_version").get() as { data_version: number }).data_version;
3507
+ const cached = verifiedQueryEmbedModels.get(db);
3508
+ if (cached && cached.dataVersion === dataVersion && cached.model === endpointModel) return;
3509
+
3510
+ const storedModels = getVecModels(db);
3511
+ if (storedModels.length === 0) return; // nothing embedded yet — nothing to be inconsistent with
3512
+
3513
+ if (!(storedModels.length === 1 && storedModels[0] === endpointModel)) {
3514
+ throw new VecReadModelMismatchError(storedModels, endpointModel);
3515
+ }
3516
+
3517
+ // Consistent — memoize under the current data_version. A cross-process rebuild bumps data_version,
3518
+ // invalidating this entry so the next query re-reads content_vectors.
3519
+ verifiedQueryEmbedModels.set(db, { dataVersion, model: endpointModel });
3520
+ }
3521
+
3451
3522
  export async function searchVec(db: Database, query: string, model: string, limit: number = 20, collectionId?: number, collections?: string[], dateRange?: { start: string; end: string }, deadlineMs?: number): Promise<SearchResult[]> {
3452
3523
  const tableExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
3453
3524
  if (!tableExists) return [];
3454
3525
 
3455
- const embedding = await getEmbedding(query, model, true, deadlineMs);
3456
- if (!embedding) return [];
3526
+ const embedResult = await getEmbedding(query, model, true, deadlineMs);
3527
+ if (!embedResult) return [];
3528
+
3529
+ // W1: read-path embedding-model consistency gate. On a same-dimension model swap (which
3530
+ // VecDimensionMismatchError cannot catch) this throws VecReadModelMismatchError rather than
3531
+ // serving cosine-meaningless results. Cached per (db, model) — the DISTINCT runs at most once
3532
+ // per model per process.
3533
+ assertQueryEmbedModelConsistent(db, embedResult.model);
3534
+
3535
+ const embedding = embedResult.embedding;
3457
3536
 
3458
3537
  // Guard-defect fix: the caller's Promise.race(vectorTimeout) cannot interrupt the SYNCHRONOUS
3459
3538
  // sqlite-vec MATCH below (bun:sqlite blocks the event loop) and does NOT cancel this promise.
@@ -3560,7 +3639,7 @@ export async function searchVec(db: Database, query: string, model: string, limi
3560
3639
  // Embeddings
3561
3640
  // =============================================================================
3562
3641
 
3563
- async function getEmbedding(text: string, model: string, isQuery: boolean, deadlineMs?: number): Promise<number[] | null> {
3642
+ async function getEmbedding(text: string, model: string, isQuery: boolean, deadlineMs?: number): Promise<{ embedding: number[]; model: string } | null> {
3564
3643
  const llm = getDefaultLlamaCpp();
3565
3644
  // Format text using the appropriate prompt template
3566
3645
  const formattedText = isQuery ? formatQueryForEmbedding(text) : formatDocForEmbedding(text);
@@ -3576,7 +3655,8 @@ async function getEmbedding(text: string, model: string, isQuery: boolean, deadl
3576
3655
  signal = AbortSignal.timeout(remaining);
3577
3656
  }
3578
3657
  const result = await llm.embed(formattedText, { model, isQuery, signal });
3579
- return result?.embedding || null;
3658
+ if (!result?.embedding) return null;
3659
+ return { embedding: result.embedding, model: result.model };
3580
3660
  }
3581
3661
 
3582
3662
  /**
@@ -3879,19 +3959,24 @@ export async function rerank(query: string, documents: { file: string; text: str
3879
3959
  // Cap parallelism at 4 to prevent VRAM exhaustion
3880
3960
  if (uncachedDocs.length > 0) {
3881
3961
  const rerankUrl = Bun.env.CLAWMEM_RERANK_URL;
3962
+ const rerankApiKey = Bun.env.CLAWMEM_RERANK_API_KEY;
3882
3963
  let scored = false;
3883
3964
 
3884
3965
  // Try remote GPU reranker first
3885
3966
  // Truncate to ~400 chars per doc to fit within server's 512-token context
3886
3967
  // (query + document must fit in one pair; ~2 chars/token for mixed content)
3887
3968
  if (rerankUrl) {
3969
+ // Independent of the embed/LLM keys — the rerank endpoint may be a different
3970
+ // authenticated host. Sent as Authorization: Bearer when CLAWMEM_RERANK_API_KEY is set.
3971
+ const rerankHeaders: Record<string, string> = { "Content-Type": "application/json" };
3972
+ if (rerankApiKey) rerankHeaders["Authorization"] = `Bearer ${rerankApiKey}`;
3888
3973
  try {
3889
3974
  // Process in batches of 4 to prevent VRAM exhaustion
3890
3975
  for (let i = 0; i < uncachedDocs.length; i += 4) {
3891
3976
  const batch = uncachedDocs.slice(i, i + 4);
3892
3977
  const resp = await fetch(`${rerankUrl}/v1/rerank`, {
3893
3978
  method: "POST",
3894
- headers: { "Content-Type": "application/json" },
3979
+ headers: rerankHeaders,
3895
3980
  body: JSON.stringify({
3896
3981
  query: rerankQuery,
3897
3982
  documents: batch.map(d => d.text.slice(0, 400)),