@cerefox/memory 1.0.0-beta.3 → 1.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. package/README.md +10 -9
  2. package/dist/bin/cerefox.js +688 -385
  3. package/dist/frontend/assets/{index-BZdKiyrT.js → index-6DS5ujj2.js} +30 -30
  4. package/dist/frontend/assets/{index-BZdKiyrT.js.map → index-6DS5ujj2.js.map} +1 -1
  5. package/dist/frontend/index.html +1 -1
  6. package/dist/server-assets/_shared/ef-meta/index.ts +1 -1
  7. package/dist/server-assets/_shared/embeddings/index.ts +51 -1
  8. package/dist/server-assets/_shared/embeddings/onnx-embedder.ts +229 -0
  9. package/dist/server-assets/_shared/mcp-tools/ingest.ts +16 -8
  10. package/dist/server-assets/_shared/mcp-tools/search.ts +3 -3
  11. package/dist/server-assets/db/rpcs.sql +5 -3
  12. package/dist/server-assets/db/schema.sql +1 -1
  13. package/dist/server-assets/supabase/functions/cerefox-ingest/index.ts +10 -0
  14. package/docs/guides/configuration.md +8 -3
  15. package/docs/guides/setup-local.md +34 -2
  16. package/package.json +6 -2
package/dist/frontend/index.html CHANGED
@@ -15,7 +15,7 @@
15
15
  href="https://fonts.googleapis.com/css2?family=Geist:wght@300;400;500;600;700&display=swap"
16
16
  />
17
17
  <title>Cerefox</title>
18
- <script type="module" crossorigin src="/app/assets/index-BZdKiyrT.js"></script>
18
+ <script type="module" crossorigin src="/app/assets/index-6DS5ujj2.js"></script>
19
19
  <link rel="stylesheet" crossorigin href="/app/assets/index-Asx5wD7g.css">
20
20
  </head>
21
21
  <body>
package/dist/server-assets/_shared/ef-meta/index.ts CHANGED
@@ -18,7 +18,7 @@
18
18
  * doesn't touch `supabase/functions/` leaves it alone).
19
19
  */
20
20
 
21
- export const EF_VERSION = "1.0.0-beta.3";
21
+ export const EF_VERSION = "1.0.0-beta.4";
22
22
 
23
23
  /**
24
24
  * The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp
package/dist/server-assets/_shared/embeddings/index.ts CHANGED
@@ -84,8 +84,52 @@ export function capEmbeddingInput(text: string): string {
84
84
  const EMBEDDING_MAX_RETRIES = 3;
85
85
  const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms → 1s → 2s
86
86
 
87
- /** Embed a single string. Used for the query vector in `cerefox_search`. */
87
+ // ── Embedder selection (iter-31: local ONNX embedder, World B only) ─────────
88
+ //
89
+ // `CEREFOX_EMBEDDER` selects the backend: "openai" (default) or "local" (the
90
+ // in-container nomic ONNX model — Cerefox Local only; cloud embedding runs in
91
+ // the Edge Functions, which a local model can't serve). The ONNX module is
92
+ // reached ONLY via dynamic import so the cloud `cerefox-mcp` Edge Function
93
+ // (which loads this file via _shared/mcp-tools) never touches the Node-only
94
+ // onnxruntime dependency — a top-level import here would break that EF.
95
+ //
96
+ // The public signatures are unchanged (`apiKey` is ignored by the local path).
97
+ // The query/document role is implicit in the entry point — an invariant every
98
+ // call site was audited for (design doc §"role is reliably implicit"):
99
+ // getEmbedding = QUERY-only · embedBatch = DOCUMENT-only.
100
+ // If a future caller needs the other role from either function, add an explicit
101
+ // role argument instead of misusing the entry point — the nomic model embeds
102
+ // the two roles into different spaces (`search_query:` vs `search_document:`).
103
+
104
+ export type EmbedderKind = "openai" | "local";
105
+
106
+ export function resolveEmbedderKind(): EmbedderKind {
107
+ const env =
108
+ (globalThis as { process?: { env?: Record<string, string | undefined> } }).process?.env ?? {};
109
+ return env.CEREFOX_EMBEDDER === "local" ? "local" : "openai";
110
+ }
111
+
112
+ /**
113
+ * The embedder identifier recorded on chunks (`cerefox_chunks.embedder_primary`)
114
+ * and compared by the mismatch guard. Literal (not imported from onnx-embedder)
115
+ * so this file keeps ZERO top-level onnx references (EF safety test enforces).
116
+ * Keep in sync with ONNX_MODEL_NAME in onnx-embedder.ts.
117
+ */
118
+ export function activeEmbedderName(): string {
119
+ return resolveEmbedderKind() === "local" ? "nomic-embed-text-v1.5" : openaiEmbeddingConfig().model;
120
+ }
121
+
122
+ async function onnxModule(): Promise<typeof import("./onnx-embedder.ts")> {
123
+ return await import("./onnx-embedder.ts");
124
+ }
125
+
126
+ /** Embed a single string. QUERY role — used for the query vector in search. */
88
127
  export async function getEmbedding(text: string, apiKey: string): Promise<number[]> {
128
+ if (resolveEmbedderKind() === "local") {
129
+ const onnx = await onnxModule();
130
+ const [vec] = await onnx.onnxEmbed([capEmbeddingInput(text)], "query");
131
+ return vec;
132
+ }
89
133
  let lastError: Error | null = null;
90
134
  const cfg = openaiEmbeddingConfig();
91
135
  const input = capEmbeddingInput(text); // cap once, before the retry loop
@@ -229,6 +273,12 @@ export async function embedBatch(
229
273
  batchSize: number = EMBEDDING_BATCH_SIZE,
230
274
  ): Promise<number[][]> {
231
275
  if (texts.length === 0) return [];
276
+ if (resolveEmbedderKind() === "local") {
277
+ // DOCUMENT role (see the invariant note above). The ONNX runtime batches
278
+ // internally; the OpenAI per-request batch limit doesn't apply.
279
+ const onnx = await onnxModule();
280
+ return onnx.onnxEmbed(texts.map(capEmbeddingInput), "document");
281
+ }
232
282
  if (texts.length <= batchSize) {
233
283
  return embedBatchSingleCall(texts, apiKey);
234
284
  }
package/dist/server-assets/_shared/embeddings/onnx-embedder.ts ADDED
@@ -0,0 +1,229 @@
1
+ /**
2
+ * Local ONNX embedder (iter-31, World B / Cerefox Local only).
3
+ *
4
+ * Ported from cfcf's proven `OnnxEmbedder` (packages/core/src/clio/embedders/),
5
+ * adapted per docs/research/local-embedder-design.md:
6
+ * - single model, no catalogue: `nomic-ai/nomic-embed-text-v1.5` q8 — 768-dim
7
+ * (matches Cerefox's `vector(768)` schema; no schema change), ~130 MB.
8
+ * - nomic task prefixes per role: a query embeds as `search_query: <text>`,
9
+ * a stored chunk as `search_document: <text>` (asymmetric model).
10
+ * - returns `number[][]` (matches the OpenAI path's shape).
11
+ *
12
+ * LOADING RULES (must hold — the cerefox-mcp Edge Function imports
13
+ * `_shared/embeddings`):
14
+ * - This module's top level imports ONLY node builtins — cheap + Deno-safe.
15
+ * - `@huggingface/transformers` (and its `onnxruntime-node` backend) load via a
16
+ * VARIABLE-SPECIFIER dynamic import so no bundler (bun build, supabase eszip)
17
+ * tries to resolve/bundle them. At runtime (Node/Bun in the World-B image)
18
+ * they resolve from node_modules; the cloud EF never reaches this path.
19
+ *
20
+ * Model cache: `CEREFOX_MODELS_DIR` (the World-B image points it inside the data
21
+ * volume so models survive container recreate/upgrade); default `~/.cerefox/models`.
22
+ */
23
+
24
+ import { existsSync, mkdirSync } from "node:fs";
25
+ import { homedir } from "node:os";
26
+ import { join } from "node:path";
27
+
28
+ export const ONNX_MODEL_ID = "nomic-ai/nomic-embed-text-v1.5";
29
+ export const ONNX_MODEL_NAME = "nomic-embed-text-v1.5";
30
+ export const ONNX_MODEL_DTYPE = "q8";
31
+ export const ONNX_MODEL_DIM = 768;
32
+ export const ONNX_MODEL_APPROX_MB = 130;
33
+
34
+ export type EmbedRole = "query" | "document";
35
+
36
+ /**
37
+ * Nomic's asymmetric task prefixes. Applied ONLY by this embedder — OpenAI
38
+ * `text-embedding-3-small` is symmetric and must never see them.
39
+ */
40
+ export function nomicPrefix(role: EmbedRole): string {
41
+ return role === "query" ? "search_query: " : "search_document: ";
42
+ }
43
+
44
+ export function buildPrefixedInputs(texts: string[], role: EmbedRole): string[] {
45
+ const p = nomicPrefix(role);
46
+ return texts.map((t) => p + t);
47
+ }
48
+
49
+ function getCacheDir(): string {
50
+ const env = (globalThis as { process?: { env?: Record<string, string | undefined> } })
51
+ .process?.env ?? {};
52
+ if (env.CEREFOX_MODELS_DIR) return env.CEREFOX_MODELS_DIR;
53
+ return join(homedir(), ".cerefox", "models");
54
+ }
55
+
56
+ // transformers.js module handle — loaded lazily, cached for the process.
57
+ // Deliberately untyped (`any`): typing it would need the package's types at
58
+ // typecheck time, but it's an optionalDependency that cloud installs may skip.
59
+ // deno-lint-ignore no-explicit-any
60
+ let transformersModule: any = null;
61
+
62
+ async function loadTransformers(): Promise<typeof transformersModule> {
63
+ if (transformersModule) return transformersModule;
64
+ // Variable specifier: keeps every bundler (bun build for the CLI, supabase
65
+ // eszip for the EF) from statically resolving the ~30 MB package + native
66
+ // onnxruntime binary. Resolved at runtime from node_modules.
67
+ const spec = "@huggingface/transformers";
68
+ transformersModule = await import(spec);
69
+ const dir = getCacheDir();
70
+ if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
71
+ transformersModule.env.cacheDir = dir;
72
+ transformersModule.env.allowLocalModels = true;
73
+ transformersModule.env.allowRemoteModels = true;
74
+ return transformersModule;
75
+ }
76
+
77
+ function makeBar(pct: number, width = 20): string {
78
+ const clamped = Math.max(0, Math.min(100, pct));
79
+ const filled = Math.round((clamped / 100) * width);
80
+ return `[${"█".repeat(filled)}${"░".repeat(width - filled)}]`;
81
+ }
82
+
83
+ function l2Normalise(v: Float32Array): Float32Array {
84
+ let sum = 0;
85
+ for (let i = 0; i < v.length; i++) sum += v[i] * v[i];
86
+ const norm = Math.sqrt(sum);
87
+ if (norm === 0) return v;
88
+ const out = new Float32Array(v.length);
89
+ for (let i = 0; i < v.length; i++) out[i] = v[i] / norm;
90
+ return out;
91
+ }
92
+
93
+ // transformers.js `pipeline()` returns a callable; typed loosely (see above).
94
+ // deno-lint-ignore no-explicit-any
95
+ type FeaturePipeline = (texts: string[], opts?: unknown) => Promise<any>;
96
+
97
+ let pipelinePromise: Promise<FeaturePipeline> | null = null;
98
+
99
+ /**
100
+ * Download (first use) + materialise the inference pipeline, with progress to
101
+ * stderr. Ported from cfcf including its hard-won progress-rendering fixes
102
+ * (indeterminate totals under Bun, in-place TTY line finalisation).
103
+ */
104
+ async function ensurePipeline(): Promise<FeaturePipeline> {
105
+ if (pipelinePromise) return pipelinePromise;
106
+ pipelinePromise = (async () => {
107
+ const transformers = await loadTransformers();
108
+ const mb = ONNX_MODEL_APPROX_MB;
109
+ const fmt = (s: number) => (s < 60 ? `${s}s` : `${Math.round(s / 60)}m`);
110
+ process.stderr.write(
111
+ `[cerefox-embed] loading "${ONNX_MODEL_NAME}" from HuggingFace (~${mb} MB; ` +
112
+ `est. ${fmt(Math.round((mb * 8) / 50))}-${fmt(Math.round((mb * 8) / 10))} at 50-10 Mbps; first-run only)…\n`,
113
+ );
114
+
115
+ type FileState = {
116
+ loaded: number; total: number; done: boolean;
117
+ indeterminate: boolean; lastRenderAt: number; lastRenderedPct: number;
118
+ };
119
+ const progressState = new Map<string, FileState>();
120
+ let activeFile: string | null = null;
121
+ const isTty = !!process.stderr.isTTY;
122
+ const RENDER_INTERVAL_MS = 250;
123
+ const fmtMb = (n: number) => (n / 1024 / 1024).toFixed(1);
124
+ const renderInPlace = (line: string): void => {
125
+ if (isTty) process.stderr.write(`\r\x1b[K${line}`);
126
+ else process.stderr.write(`${line}\n`);
127
+ };
128
+ const finalizeLine = (): void => {
129
+ if (isTty && activeFile !== null) process.stderr.write("\n");
130
+ activeFile = null;
131
+ };
132
+
133
+ const progressCallback = (info: {
134
+ status?: string; file?: string; name?: string;
135
+ loaded?: number; total?: number;
136
+ }) => {
137
+ const file = info.file ?? info.name ?? "(unknown)";
138
+ const now = Date.now();
139
+ if (info.status === "progress") {
140
+ const total = info.total ?? 0;
141
+ const loaded = info.loaded ?? 0;
142
+ if (total === 0 && loaded === 0) return;
143
+ const prior = progressState.get(file) ?? {
144
+ loaded: 0, total: 0, done: false,
145
+ indeterminate: false, lastRenderAt: 0, lastRenderedPct: -1,
146
+ };
147
+ // Indeterminate: total grew across events (streaming, unknown size).
148
+ const indeterminate = prior.indeterminate || (prior.total > 0 && total > prior.total);
149
+ const next: FileState = {
150
+ loaded, total, done: false,
151
+ indeterminate, lastRenderAt: prior.lastRenderAt, lastRenderedPct: prior.lastRenderedPct,
152
+ };
153
+ if (activeFile !== file) { finalizeLine(); activeFile = file; }
154
+ if (indeterminate) {
155
+ if (now - prior.lastRenderAt >= RENDER_INTERVAL_MS) {
156
+ renderInPlace(`[cerefox-embed] [streaming...] ${fmtMb(loaded)} MB ${file}`);
157
+ next.lastRenderAt = now;
158
+ }
159
+ } else if (total > 0) {
160
+ const pct = Math.floor((loaded / total) * 100);
161
+ const stepBumped = pct >= prior.lastRenderedPct + 5;
162
+ const timeBumped = isTty && now - prior.lastRenderAt >= RENDER_INTERVAL_MS
163
+ && pct !== prior.lastRenderedPct;
164
+ if (stepBumped || timeBumped) {
165
+ renderInPlace(
166
+ `[cerefox-embed] ${makeBar(pct)} ${pct.toString().padStart(3)}% ${fmtMb(loaded)}/${fmtMb(total)} MB ${file}`,
167
+ );
168
+ next.lastRenderedPct = pct;
169
+ next.lastRenderAt = now;
170
+ }
171
+ }
172
+ progressState.set(file, next);
173
+ } else if (info.status === "done") {
174
+ const prior = progressState.get(file);
175
+ finalizeLine(); // land the ✓ on its own row regardless of who owns the bar
176
+ const finalSize = prior && prior.loaded > 0
177
+ ? `${fmtMb(prior.loaded)} MB`
178
+ : info.total && info.total > 0
179
+ ? `${fmtMb(info.total)} MB`
180
+ : "cached";
181
+ process.stderr.write(`[cerefox-embed] ✓ ${file} (${finalSize})\n`);
182
+ if (prior) progressState.set(file, { ...prior, done: true });
183
+ }
184
+ };
185
+
186
+ const pipe = await transformers.pipeline("feature-extraction", ONNX_MODEL_ID, {
187
+ dtype: ONNX_MODEL_DTYPE,
188
+ progress_callback: progressCallback,
189
+ });
190
+ process.stderr.write(`[cerefox-embed] embedder ready.\n`);
191
+ return pipe as unknown as FeaturePipeline;
192
+ })();
193
+ // On failure, clear the memo so a later call can retry (e.g. transient network).
194
+ pipelinePromise.catch(() => { pipelinePromise = null; });
195
+ return pipelinePromise;
196
+ }
197
+
198
+ /**
199
+ * Force the model download + pipeline materialisation now. Called at install
200
+ * time when the local embedder is selected so the first search isn't slow.
201
+ */
202
+ export async function warmup(): Promise<void> {
203
+ await ensurePipeline();
204
+ }
205
+
206
+ /**
207
+ * Embed texts with the local nomic model, applying the role prefix.
208
+ * Mean pooling + L2 normalisation (sentence-transformers convention; nomic
209
+ * expects both). Returns plain `number[][]` to match the OpenAI path.
210
+ */
211
+ export async function onnxEmbed(texts: string[], role: EmbedRole): Promise<number[][]> {
212
+ if (texts.length === 0) return [];
213
+ const pipeline = await ensurePipeline();
214
+ const inputs = buildPrefixedInputs(texts, role);
215
+ const out = await pipeline(inputs, { pooling: "mean", normalize: true });
216
+ // out.data: flat Float32Array (batch * dim); out.dims: [batch, dim].
217
+ const dim: number = out.dims[out.dims.length - 1];
218
+ if (dim !== ONNX_MODEL_DIM) {
219
+ throw new Error(
220
+ `OnnxEmbedder: expected dim=${ONNX_MODEL_DIM} (schema vector(768)), got ${dim} from model`,
221
+ );
222
+ }
223
+ const vectors: number[][] = [];
224
+ for (let i = 0; i < inputs.length; i++) {
225
+ const slice: Float32Array = out.data.slice(i * dim, (i + 1) * dim);
226
+ vectors.push(Array.from(l2Normalise(slice)));
227
+ }
228
+ return vectors;
229
+ }
package/dist/server-assets/_shared/mcp-tools/ingest.ts CHANGED
@@ -24,7 +24,7 @@ import {
24
24
  normalizeContent,
25
25
  sha256hex,
26
26
  } from "./_chunker.ts";
27
- import { embedBatch, OPENAI_MODEL } from "../embeddings/index.ts";
27
+ import { activeEmbedderName, embedBatch, resolveEmbedderKind } from "../embeddings/index.ts";
28
28
  import { ensureDocumentInProject, setDocumentProjectsByName } from "./_projects.ts";
29
29
  import { logUsage } from "./_utils.ts";
30
30
  import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
@@ -81,6 +81,14 @@ async function handler(
81
81
  // {} on create (v0.11.1 — defaulting to {} here used to wipe a document's
82
82
  // tags on every content update that didn't re-pass them).
83
83
  const metadata = (args.metadata as Record<string, unknown> | undefined) ?? null;
84
+ // Must be a plain JSON object (or absent). A scalar/array stored in the JSONB
85
+ // column poisons cerefox_list_metadata_keys for the whole dataset (issue #89),
86
+ // so reject it at the boundary too, not just in the RPC.
87
+ if (metadata !== null && (typeof metadata !== "object" || Array.isArray(metadata))) {
88
+ throw new McpInvalidParams(
89
+ 'metadata must be a JSON object of key/value pairs, e.g. {"type":"note"} — not a string, number, or array',
90
+ );
91
+ }
84
92
  const update_if_exists = (args.update_if_exists as boolean | undefined) ?? false;
85
93
  const author = (args.author as string | undefined) ?? "mcp-agent";
86
94
  const author_type = "agent"; // MCP path is always agent
@@ -103,7 +111,7 @@ async function handler(
103
111
  ? project_names_raw.filter((s): s is string => typeof s === "string" && s.length > 0)
104
112
  : null;
105
113
 
106
- if (!ctx.openaiApiKey) {
114
+ if (!ctx.openaiApiKey && resolveEmbedderKind() !== "local") {
107
115
  throw new Error(
108
116
  "OpenAI API key not configured. Set OPENAI_API_KEY (Edge Function) or CEREFOX_OPENAI_API_KEY (.env, local).",
109
117
  );
@@ -144,7 +152,7 @@ async function handler(
144
152
  if (chunks.length === 0) throw new Error("Content produced no chunks");
145
153
 
146
154
  const texts = chunks.map((c) => embeddingInputFor(title, c));
147
- const embeddings = await embedBatch(texts, ctx.openaiApiKey);
155
+ const embeddings = await embedBatch(texts, ctx.openaiApiKey ?? "");
148
156
  const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
149
157
 
150
158
  const chunkData = chunks.map((chunk, i) => ({
@@ -155,7 +163,7 @@ async function handler(
155
163
  content: chunk.content,
156
164
  char_count: chunk.char_count,
157
165
  embedding: embeddings[i],
158
- embedder: OPENAI_MODEL,
166
+ embedder: activeEmbedderName(),
159
167
  }));
160
168
 
161
169
  const { error: ingestErr } = await supabase.rpc("cerefox_ingest_document", {
@@ -222,7 +230,7 @@ async function handler(
222
230
  if (chunks.length === 0) throw new Error("Content produced no chunks");
223
231
 
224
232
  const texts = chunks.map((c) => embeddingInputFor(title, c));
225
- const embeddings = await embedBatch(texts, ctx.openaiApiKey);
233
+ const embeddings = await embedBatch(texts, ctx.openaiApiKey ?? "");
226
234
  const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
227
235
 
228
236
  const chunkData = chunks.map((chunk, i) => ({
@@ -233,7 +241,7 @@ async function handler(
233
241
  content: chunk.content,
234
242
  char_count: chunk.char_count,
235
243
  embedding: embeddings[i],
236
- embedder: OPENAI_MODEL,
244
+ embedder: activeEmbedderName(),
237
245
  }));
238
246
 
239
247
  const { error: ingestErr } = await supabase.rpc("cerefox_ingest_document", {
@@ -288,7 +296,7 @@ async function handler(
288
296
  if (chunks.length === 0) throw new Error("Content produced no chunks");
289
297
 
290
298
  const texts = chunks.map((c) => embeddingInputFor(title, c));
291
- const embeddings = await embedBatch(texts, ctx.openaiApiKey);
299
+ const embeddings = await embedBatch(texts, ctx.openaiApiKey ?? "");
292
300
  const totalChars = chunks.reduce((s, c) => s + c.char_count, 0);
293
301
 
294
302
  const chunkData = chunks.map((chunk, i) => ({
@@ -299,7 +307,7 @@ async function handler(
299
307
  content: chunk.content,
300
308
  char_count: chunk.char_count,
301
309
  embedding: embeddings[i],
302
- embedder: OPENAI_MODEL,
310
+ embedder: activeEmbedderName(),
303
311
  }));
304
312
 
305
313
  const { data: ingestResult, error: ingestErr } = await supabase.rpc("cerefox_ingest_document", {
package/dist/server-assets/_shared/mcp-tools/search.ts CHANGED
@@ -17,7 +17,7 @@
17
17
 
18
18
  import type { MCPSupabaseClient } from "./types.ts";
19
19
 
20
- import { getEmbedding } from "../embeddings/index.ts";
20
+ import { getEmbedding, resolveEmbedderKind } from "../embeddings/index.ts";
21
21
  import { applyByteBudget, getMaxResponseBytes, getMinSearchScore, logUsage } from "./_utils.ts";
22
22
  import { lookupProjectId } from "./_projects.ts";
23
23
  import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
@@ -50,7 +50,7 @@ async function handler(
50
50
 
51
51
  if (!query?.trim()) throw new McpInvalidParams("query is required");
52
52
 
53
- if (mode !== "fts" && !ctx.openaiApiKey) {
53
+ if (mode !== "fts" && !ctx.openaiApiKey && resolveEmbedderKind() !== "local") {
54
54
  throw new Error(
55
55
  "OpenAI API key not configured. Set OPENAI_API_KEY (Edge Function) or CEREFOX_OPENAI_API_KEY (.env, local).",
56
56
  );
@@ -66,7 +66,7 @@ async function handler(
66
66
  // FTS mode doesn't need an embedding
67
67
  let embedding: number[] | null = null;
68
68
  if (mode !== "fts") {
69
- embedding = await getEmbedding(query, ctx.openaiApiKey!);
69
+ embedding = await getEmbedding(query, ctx.openaiApiKey ?? "");
70
70
  }
71
71
 
72
72
  const metaFilterParam =
package/dist/server-assets/db/rpcs.sql CHANGED
@@ -1420,8 +1420,10 @@ AS $$
1420
1420
  (WHERE d.metadata ->> k.key IS NOT NULL))[1:5] AS example_values
1421
1421
  FROM cerefox_documents d,
1422
1422
  LATERAL jsonb_object_keys(d.metadata) AS k(key)
1423
- WHERE d.metadata IS NOT NULL
1424
- AND d.metadata != '{}'::jsonb
1423
+ -- jsonb_object_keys() throws on non-object jsonb (scalar/array), and one such
1424
+ -- row would poison the whole listing (issue #89). jsonb_typeof covers NULL
1425
+ -- (returns NULL → filtered), scalars, and arrays; '{}' yields no keys anyway.
1426
+ WHERE jsonb_typeof(d.metadata) = 'object'
1425
1427
  GROUP BY k.key
1426
1428
  ORDER BY doc_count DESC, k.key;
1427
1429
  $$;
@@ -1774,7 +1776,7 @@ SET search_path = public, pg_catalog
1774
1776
  AS $$
1775
1777
  -- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
1776
1778
  -- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
1777
- SELECT '0.8.0'::TEXT;
1779
+ SELECT '0.8.1'::TEXT;
1778
1780
  $$;
1779
1781
 
1780
1782
  -- ── cerefox_content_format_stats ─────────────────────────────────────────────
package/dist/server-assets/db/schema.sql CHANGED
@@ -5,7 +5,7 @@
5
5
  -- Requires extensions: vector (pgvector), uuid-ossp
6
6
  -- These are enabled at the top of db_deploy.py before this file is applied.
7
7
  --
8
- -- @version: 0.8.0
8
+ -- @version: 0.8.1
9
9
  -- The `@version` marker above is read by the schema-version-mismatch banner
10
10
  -- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
11
11
  -- changes in a way that requires `cerefox server deploy` to be re-run —
package/dist/server-assets/supabase/functions/cerefox-ingest/index.ts CHANGED
@@ -334,6 +334,16 @@ Deno.serve(async (req: Request) => {
334
334
  // a document's tags on every content update that didn't re-pass them).
335
335
  const { title, content, document_id = null, project_name, source = "agent", metadata = null, update_if_exists = false, author = "agent", author_type = "agent", expected_content_hash = null, last_write_wins = false } = body;
336
336
 
337
+ // metadata must be a plain JSON object (or absent). A scalar/array stored in
338
+ // the JSONB column poisons cerefox_list_metadata_keys for the whole dataset
339
+ // (issue #89) — reject at the boundary too, not just in the RPC.
340
+ if (metadata !== null && (typeof metadata !== "object" || Array.isArray(metadata))) {
341
+ return new Response(
342
+ JSON.stringify({ error: 'metadata must be a JSON object of key/value pairs, e.g. {"type":"note"} — not a string, number, or array' }),
343
+ { status: 400, headers: { "Content-Type": "application/json" } },
344
+ );
345
+ }
346
+
337
347
  // Validate + normalize project_names if provided (full-set destructive form)
338
348
  let project_names: string[] | null = null;
339
349
  if (body.project_names !== undefined && body.project_names !== null) {
package/docs/guides/configuration.md CHANGED
@@ -55,9 +55,14 @@ Full rule documented in [`docs/specs/polish-and-distribution-design.md` §7](../
55
55
 
56
56
  Cerefox uses cloud-based embedding APIs. Local models (mpnet, Ollama) are not supported — they require large downloads, fail on some hardware, and add installation complexity.
57
57
 
58
- > **OpenAI only (today).** The TypeScript runtime implements the OpenAI embedder.
59
- > `CEREFOX_EMBEDDER` and the `CEREFOX_FIREWORKS_*` variables are documented but **not yet
60
- > wired in TS** — they're currently no-ops, tracked for a future release.
58
+ > **Two embedders.** `CEREFOX_EMBEDDER` selects the backend: `openai` (default —
59
+ > `text-embedding-3-small` via API) or `local` (the in-container `nomic-embed-text-v1.5`
60
+ > ONNX model — **Cerefox Local only**; the cloud/Supabase deployment embeds inside the
61
+ > Edge Functions, which a local model can't serve, so `local` has no effect there).
62
+ > Set it via the Cerefox Local installer (`--local-embedder`) or `cerefox-local init` —
63
+ > not by hand on existing data (switching requires `server reindex`; see
64
+ > [setup-local.md](setup-local.md#choose-your-embedder-openai-vs-fully-local)).
65
+ > The `CEREFOX_FIREWORKS_*` variables remain documented but **not wired** (no-ops).
61
66
 
62
67
  ### OpenAI (default, recommended)
63
68
 
package/docs/guides/setup-local.md CHANGED
@@ -27,10 +27,35 @@ both, your cloud `~/.cerefox/.env` is never touched by the local installer.
27
27
  ## Prerequisites
28
28
 
29
29
  - **Docker** (Docker Desktop, or [Colima](https://github.com/abiosoft/colima): `colima start`).
30
- - An **OpenAI API key** — [platform.openai.com/api-keys](https://platform.openai.com/api-keys).
30
+ - An **OpenAI API key** — [platform.openai.com/api-keys](https://platform.openai.com/api-keys) —
31
+ **unless** you choose the local embedder (next section), which needs no key at all.
31
32
 
32
33
  That's it. No Node, Bun, Postgres, or repo clone needed.
33
34
 
35
+ ## Choose your embedder (OpenAI vs fully local)
36
+
37
+ Cerefox Local turns text into search vectors with one of two embedders. Pick at
38
+ install; switching later requires a re-index (see below).
39
+
40
+ | | **OpenAI** (default) | **Local** (fully offline) |
41
+ |---|---|---|
42
+ | Model | `text-embedding-3-small` (cloud API) | `nomic-embed-text-v1.5` (ONNX, runs in the container) |
43
+ | Needs | `OPENAI_API_KEY` | nothing — no key, no network after the one-time model download |
44
+ | Privacy | document/query text is sent to OpenAI for embedding | **text never leaves your machine** |
45
+ | Cost | pennies/month typical ([operational-cost.md](operational-cost.md)) | zero |
46
+ | Quality / speed | best retrieval quality; fast API | very good quality; CPU inference (fine at personal scale) |
47
+ | Setup | provide the key (Step 1 or `cerefox-local init`) | `--local-embedder` at install, or `[2] Local` in `cerefox-local init` |
48
+
49
+ The local model (~130 MB) downloads once — at install/init when selected — into the
50
+ data volume, so it survives `cerefox-local upgrade`.
51
+
52
+ > **Switching embedders on existing data is breaking**: the two models produce
53
+ > incompatible vector spaces, so documents embedded with one are invisible to
54
+ > semantic search under the other. `cerefox-local init` warns and requires
55
+ > confirmation (`--force` non-interactively); after switching, run
56
+ > `cerefox-local server reindex` to re-embed everything. `cerefox-local doctor`
57
+ > flags any mismatch.
58
+
34
59
  ---
35
60
 
36
61
  ## Step 1 — Install
@@ -40,7 +65,14 @@ curl -fsSL https://github.com/fstamatelopoulos/cerefox/releases/latest/download/
40
65
  ```
41
66
 
42
67
  This pulls the published multi-arch image (`amd64` + `arm64`), starts the container, and
43
- installs a `cerefox-local` command (symlinked into `~/.local/bin`). To set your OpenAI key
68
+ installs a `cerefox-local` command (symlinked into `~/.local/bin`).
69
+
70
+ **Fully-offline variant** — select the local embedder (no OpenAI key needed; downloads the
71
+ ~130 MB model during install):
72
+
73
+ ```bash
74
+ curl -fsSL https://github.com/fstamatelopoulos/cerefox/releases/latest/download/install-local.sh | sh -s -- --local-embedder
75
+ ``` To set your OpenAI key
44
76
  inline at install instead of via `cerefox-local init` (Step 2), use the command-substitution
45
77
  form: `OPENAI_API_KEY=sk-... sh -c "$(curl -fsSL …/install-local.sh)"`. Pick a specific port
46
78
  with `PORT=8017 …`.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cerefox/memory",
3
- "version": "1.0.0-beta.3",
3
+ "version": "1.0.0-beta.4",
4
4
  "description": "Cerefox — user-owned shared memory for AI agents. The local TypeScript runtime: stdio MCP server in v0.4; CLI binary added in v0.5; in-process web server in v0.6; ingestion pipeline in v0.7.",
5
5
  "license": "Apache-2.0",
6
6
  "homepage": "https://github.com/fstamatelopoulos/cerefox",
@@ -53,6 +53,10 @@
53
53
  "smol-toml": "^1.6.1",
54
54
  "zod": "^3.23.0"
55
55
  },
56
+ "optionalDependencies": {
57
+ "@huggingface/transformers": "^3.5.0",
58
+ "onnxruntime-node": "^1.21.0"
59
+ },
56
60
  "devDependencies": {
57
61
  "@types/bun": "^1.3.14",
58
62
  "@types/cli-progress": "^3.11.5",
@@ -61,7 +65,7 @@
61
65
  "typescript": "^6.0.3"
62
66
  },
63
67
  "scripts": {
64
- "build": "bun build src/bin/cerefox.ts --outdir dist/bin --target node --format esm",
68
+ "build": "bun build src/bin/cerefox.ts --outdir dist/bin --target node --format esm --external @huggingface/transformers --external onnxruntime-node",
65
69
  "clean": "rm -rf dist docs AGENT_GUIDE.md AGENT_QUICK_REFERENCE.md",
66
70
  "bundle-docs": "bun run ../../scripts/bundle_package_docs.ts",
67
71
  "bundle-server-assets": "bun run ../../scripts/bundle_server_assets.ts",