@cerefox/memory 1.0.3 → 1.0.5

package/dist/bin/cerefox.js

@@ -7184,7 +7184,7 @@ var exports_meta = {};
 __export(exports_meta, {
   PKG_VERSION: () => PKG_VERSION
 });
-var PKG_VERSION = "1.0.3";
+var PKG_VERSION = "1.0.5";
 var init_meta = () => {};
 
 // ../../node_modules/.bun/tslib@2.8.1/node_modules/tslib/tslib.js
@@ -25225,6 +25225,13 @@ function getMinSearchScore() {
   const n = Number.parseFloat(raw);
   return Number.isNaN(n) || n < 0 || n > 1 ? fallback : n;
 }
+function getMinTermCoverage() {
+  const raw = globalThis.process?.env?.CEREFOX_MIN_TERM_COVERAGE;
+  if (raw === undefined || raw === "")
+    return;
+  const n = Number.parseFloat(raw);
+  return Number.isNaN(n) || n < 0 || n > 1 ? undefined : n;
+}
 function applyByteBudget(rows, maxBytes) {
   const accepted = [];
   let usedBytes = 0;
@@ -55572,6 +55579,8 @@ async function handler9(supabase, args, ctx) {
   const mode = args.mode ?? "docs";
   const alpha = args.alpha ?? 0.7;
   const min_score = args.min_score ?? getMinSearchScore();
+  const min_term_coverage = args.min_term_coverage ?? getMinTermCoverage();
+  const coverageParam = min_term_coverage !== undefined ? { p_min_term_coverage: min_term_coverage } : {};
   const metadata_filter = args.metadata_filter ?? null;
   const requested_max_bytes = args.max_bytes;
   const ceiling = getMaxResponseBytes();
@@ -55603,7 +55612,8 @@ async function handler9(supabase, args, ctx) {
       p_query_text: query,
       p_match_count: match_count,
       p_project_id: projectId,
-      ...metaFilterParam
+      ...metaFilterParam,
+      ...coverageParam
     };
   } else if (mode === "hybrid") {
     rpcName = "cerefox_hybrid_search";
@@ -55615,7 +55625,8 @@ async function handler9(supabase, args, ctx) {
       p_use_upgrade: false,
       p_project_id: projectId,
       p_min_score: min_score,
-      ...metaFilterParam
+      ...metaFilterParam,
+      ...coverageParam
     };
   } else {
     rpcName = "cerefox_search_docs";
@@ -55626,7 +55637,8 @@ async function handler9(supabase, args, ctx) {
       p_alpha: alpha,
       p_project_id: projectId,
       p_min_score: min_score,
-      ...metaFilterParam
+      ...metaFilterParam,
+      ...coverageParam
     };
   }
   const { data, error: error2 } = await supabase.rpc(rpcName, rpcParams);
@@ -55699,6 +55711,23 @@ var init_search = __esm(() => {
           description: 'Optional JSONB containment filter. Only documents whose metadata contains ALL specified key-value pairs are returned. Example: {"type": "decision", "status": "active"}. Call cerefox_list_metadata_keys first to discover available keys and values. Omit to search all documents.',
           additionalProperties: { type: "string" }
         },
+        mode: {
+          type: "string",
+          enum: ["docs", "hybrid", "fts", "semantic"],
+          description: "Search mode (default: docs — full reconstructed documents). hybrid: ranked chunks; fts: keyword-only (no embedding); semantic: vector-only."
+        },
+        alpha: {
+          type: "number",
+          description: "Hybrid fusion weight 0–1 (default 0.7): 1 = pure semantic, 0 = pure keyword."
+        },
+        min_score: {
+          type: "number",
+          description: "Minimum cosine similarity for vector-side results (default: server-configured, 0.5 OpenAI / 0.6 local embedder)."
+        },
+        min_term_coverage: {
+          type: "number",
+          description: "Keyword OR-fallback confidence bar 0–1 (default 0.5): fraction of the query's meaningful terms a result must match to count as a confident hit; weaker matches return flagged below-confidence. 0 = any matching term. Needs schema ≥ 0.9.1."
+        },
         max_bytes: {
           type: "integer",
           description: "Optional response size budget in bytes. Results are dropped whole until the budget is satisfied; a truncated flag is set when results are dropped. Defaults to the server maximum (200000). Pass a smaller value if your context window is limited. Values above the server maximum are silently capped."
@@ -69032,10 +69061,35 @@ init_cli_core();
 
 // src/cli/commands/backup.ts
 init_cli_core();
-init_client();
 import { existsSync as existsSync2, mkdirSync, writeFileSync } from "node:fs";
 import { homedir as homedir2 } from "node:os";
 import { join as join2, resolve } from "node:path";
+
+// ../../_shared/db-client/paginate.ts
+async function fetchAllPages(makeQuery, batchSize = 200) {
+  const results = [];
+  let offset = 0;
+  let serverTotal = null;
+  for (;; ) {
+    const { data, error, count } = await makeQuery(offset, offset + batchSize - 1);
+    if (error)
+      throw new Error(error.message ?? JSON.stringify(error));
+    if (typeof count === "number")
+      serverTotal = count;
+    const page = data ?? [];
+    results.push(...page);
+    if (page.length < batchSize)
+      break;
+    offset += batchSize;
+  }
+  if (serverTotal !== null && results.length !== serverTotal) {
+    throw new Error(`Paginated read returned ${results.length} row(s) but the server reports ` + `${serverTotal}. Refusing to return a partial result — retry, and if this ` + `persists please report it (the rows may be changing under the read).`);
+  }
+  return results;
+}
+
+// src/cli/commands/backup.ts
+init_client();
 function expandHome(path) {
   if (path === "~")
     return homedir2();
@@ -69056,21 +69110,25 @@ async function action(options) {
   const filename = `cerefox-${stamp}${options.label ? "-" + options.label : ""}.json`;
   const dest = join2(outDir, filename);
   const client = getClient();
-  const { data: docsData, error: docsErr } = await client.raw.from("cerefox_documents").select("id, title, content_hash, source, metadata, total_chars, chunk_count, " + "review_status, created_at, updated_at, deleted_at").is("deleted_at", null).order("created_at", { ascending: true });
-  if (docsErr)
-    throw systemError(`Document fetch failed: ${docsErr.message}`);
-  const docs = docsData ?? [];
+  let docs;
+  try {
+    docs = await fetchAllPages((from, to) => client.raw.from("cerefox_documents").select("id, title, content_hash, source, metadata, total_chars, chunk_count, " + "review_status, created_at, updated_at, deleted_at").is("deleted_at", null).order("created_at", { ascending: true }).order("id", { ascending: true }).range(from, to));
+  } catch (err) {
+    throw systemError(`Document fetch failed: ${err instanceof Error ? err.message : String(err)}`);
+  }
   let chunkTotal = 0;
   const enriched = [];
   for (let i = 0;i < docs.length; i++) {
     const doc = docs[i];
     const docId = doc.id;
-    const { data: chunks, error: chunkErr } = await client.raw.from("cerefox_chunks").select("*").eq("document_id", docId).is("version_id", null).order("chunk_index", { ascending: true });
-    if (chunkErr) {
-      throw systemError(`Chunk fetch failed for ${docId}: ${chunkErr.message}`);
+    let chunks;
+    try {
+      chunks = await fetchAllPages((from, to) => client.raw.from("cerefox_chunks").select("*").eq("document_id", docId).is("version_id", null).order("chunk_index", { ascending: true }).range(from, to));
+    } catch (err) {
+      throw systemError(`Chunk fetch failed for ${docId}: ${err instanceof Error ? err.message : String(err)}`);
     }
-    chunkTotal += (chunks ?? []).length;
-    enriched.push({ ...doc, chunks: chunks ?? [] });
+    chunkTotal += chunks.length;
+    enriched.push({ ...doc, chunks });
     if (process.stdout.isTTY) {
       process.stderr.write(`\r  Dumping documents: ${i + 1}/${docs.length} (${chunkTotal} chunks so far)…`);
     }
@@ -74948,8 +75006,8 @@ import { homedir as homedir6 } from "node:os";
 import { join as join9 } from "node:path";
 
 // ../../_shared/ef-meta/index.ts
-var EF_VERSION = "1.0.3";
-var EF_LAST_CHANGED = "1.0.3";
+var EF_VERSION = "1.0.5";
+var EF_LAST_CHANGED = "1.0.4";
 
 // src/cli/util/checks.ts
 init_config();
@@ -75475,9 +75533,9 @@ async function checkEdgeFunctionsCompat() {
       }
       return {
         name: "edge functions",
-        status: "skipped",
-        detail: `Deployed Edge Functions (v${deployed}) are older than the ones bundled with this client (v${EF_VERSION}). Compatible — nothing is broken — but redeploying ` + `picks up the latest server-side fixes.`,
-        hint: "Redeploy with: cerefox server deploy --functions-only"
+        status: "warn",
+        detail: `Deployed Edge Functions (v${deployed}) are older than this client's bundled v${EF_VERSION} and predate server-side fixes shipped in v${EF_LAST_CHANGED}. Still compatible, but you are missing those fixes.`,
+        hint: "Update the Edge Functions (see remediation below)."
       };
     default:
       return {
@@ -75576,11 +75634,11 @@ async function action15(options) {
     const needsEf = stale("edge functions");
     let remediation = null;
     if (needsSchema && needsEf) {
-      remediation = "Update the server (schema + RPCs + Edge Functions): cerefox server deploy";
+      remediation = "This release needs a server update (schema + RPCs + Edge Functions): cerefox server deploy";
     } else if (needsSchema) {
-      remediation = "Update the schema + RPCs: cerefox server deploy --schema-only";
+      remediation = "This release needs a schema + RPC update: cerefox server deploy --schema-only";
     } else if (needsEf) {
-      remediation = "Update the Edge Functions: cerefox server deploy --functions-only";
+      remediation = "This release needs an Edge Function update: cerefox server deploy --functions-only";
     }
     if (remediation) {
       println(cErr.yellow("→ " + remediation));
@@ -75807,7 +75865,7 @@ class IngestionDbBridge {
     return rows.length > 0 ? rows[0] : null;
   }
   async listChunksForDocument(documentId) {
-    const { data } = await this.supabase.from("cerefox_chunks").select("id, document_id, chunk_index, heading_path, heading_level, title, content, char_count").eq("document_id", documentId).is("version_id", null).order("chunk_index");
+    const data = await fetchAllPages((from, to) => this.supabase.from("cerefox_chunks").select("id, document_id, chunk_index, heading_path, heading_level, title, content, char_count").eq("document_id", documentId).is("version_id", null).order("chunk_index").range(from, to));
     return data ?? [];
   }
   async getDocumentProjectIds(documentId) {
@@ -77239,18 +77297,22 @@ async function action27(options) {
   }
   const reindexAll = Boolean(options.all);
   const dryRun = Boolean(options.dryRun);
-  let query = supabase.from("cerefox_chunks").select("id, document_id, content, embedder_primary, cerefox_documents(title)").is("version_id", null);
-  if (options.documentId) {
-    query = query.eq("document_id", options.documentId);
-  }
   const targetModel = activeEmbedderName();
-  if (!reindexAll) {
-    query = query.neq("embedder_primary", targetModel);
+  let chunks;
+  try {
+    chunks = await fetchAllPages((from, to) => {
+      let query = supabase.from("cerefox_chunks").select("id, document_id, content, embedder_primary, cerefox_documents(title)").is("version_id", null);
+      if (options.documentId) {
+        query = query.eq("document_id", options.documentId);
+      }
+      if (!reindexAll) {
+        query = query.neq("embedder_primary", targetModel);
+      }
+      return query.order("id", { ascending: true }).range(from, to);
+    }, 1000);
+  } catch (err) {
+    throw systemError(`Failed to list chunks: ${err instanceof Error ? err.message : String(err)}`);
   }
-  const { data, error: error3 } = await query;
-  if (error3)
-    throw systemError(`Failed to list chunks: ${error3.message}`);
-  const chunks = data ?? [];
   if (chunks.length === 0) {
     println(c.dim("(nothing to reindex)"));
     return;
@@ -77420,6 +77482,8 @@ async function action29(query, options) {
   const matchCount = parsePositiveInt(options.matchCount, "--match-count", 5);
   const alpha = parseFloat01(options.alpha, "--alpha", 0.7);
   const minScore = parseFloat01(options.minScore, "--min-score", getMinSearchScore());
+  const envCoverage = getMinTermCoverage();
+  const coverageParam = options.minTermCoverage !== undefined ? { p_min_term_coverage: parseFloat01(options.minTermCoverage, "--min-term-coverage", envCoverage ?? 0.5) } : envCoverage !== undefined ? { p_min_term_coverage: envCoverage } : {};
   const maxBytes = parseNonNegativeInt(options.maxBytes, "--max-bytes", getMaxResponseBytes());
   const mode = options.mode ?? "docs";
   if (!["docs", "hybrid", "fts"].includes(mode)) {
@@ -77450,7 +77514,8 @@ async function action29(query, options) {
       p_query_text: query,
       p_match_count: matchCount,
       p_project_id: projectId,
-      ...metaFilterParam
+      ...metaFilterParam,
+      ...coverageParam
     };
   } else if (mode === "hybrid") {
     rpcName = "cerefox_hybrid_search";
@@ -77462,7 +77527,8 @@ async function action29(query, options) {
       p_use_upgrade: false,
       p_project_id: projectId,
       p_min_score: minScore,
-      ...metaFilterParam
+      ...metaFilterParam,
+      ...coverageParam
     };
   } else {
     rpcName = "cerefox_search_docs";
@@ -77473,7 +77539,8 @@ async function action29(query, options) {
       p_alpha: alpha,
       p_project_id: projectId,
       p_min_score: minScore,
-      ...metaFilterParam
+      ...metaFilterParam,
+      ...coverageParam
     };
   }
   const results = await client.rpc(rpcName, rpcParams);
@@ -77572,7 +77639,7 @@ async function action29(query, options) {
   }
 }
 function registerSearch(program2) {
-  program2.command("search").description("Search the knowledge base (hybrid FTS + semantic).").argument("<query>", "Natural-language search query.").option("-c, --match-count <n>", "Maximum number of documents to return.", "5").option("-p, --project-name <name>", "Filter results to a specific project.").option("-f, --metadata-filter <json>", "JSON containment filter; only docs whose metadata contains ALL pairs are returned.").option("--mode <mode>", "Search mode: docs (default), hybrid, fts.", "docs").option("--alpha <float>", "Semantic weight 0..1 (default: 0.7).", "0.7").option("--min-score <float>", "Minimum cosine similarity threshold (default: CEREFOX_MIN_SEARCH_SCORE; else 0.5, or 0.6 with the local embedder).").option("--max-bytes <n>", "Response size budget in bytes (default: CEREFOX_MAX_RESPONSE_BYTES or 200000).").option("-r, --requestor <name>", "Agent / user name (recorded in usage log).").option("--json", "Emit machine-readable JSON instead of the default text.").option("--only-metadata", "List matching docs (id, score, chunks, chars, partial/full) WITHOUT their content — like the web UI's collapsed result list. Grab a [id:…] then `cerefox document get <id>`.").action(action29);
+  program2.command("search").description("Search the knowledge base (hybrid FTS + semantic).").argument("<query>", "Natural-language search query.").option("-c, --match-count <n>", "Maximum number of documents to return.", "5").option("-p, --project-name <name>", "Filter results to a specific project.").option("-f, --metadata-filter <json>", "JSON containment filter; only docs whose metadata contains ALL pairs are returned.").option("--mode <mode>", "Search mode: docs (default), hybrid, fts.", "docs").option("--alpha <float>", "Semantic weight 0..1 (default: 0.7).", "0.7").option("--min-score <float>", "Minimum cosine similarity threshold (default: CEREFOX_MIN_SEARCH_SCORE; else 0.5, or 0.6 with the local embedder).").option("--min-term-coverage <float>", "OR-fallback keyword matches must cover at least this fraction of the query's meaningful terms to count as confident hits (default: CEREFOX_MIN_TERM_COVERAGE; else the server default 0.5; needs schema ≥ 0.9.1).").option("--max-bytes <n>", "Response size budget in bytes (default: CEREFOX_MAX_RESPONSE_BYTES or 200000).").option("-r, --requestor <name>", "Agent / user name (recorded in usage log).").option("--json", "Emit machine-readable JSON instead of the default text.").option("--only-metadata", "List matching docs (id, score, chunks, chars, partial/full) WITHOUT their content — like the web UI's collapsed result list. Grab a [id:…] then `cerefox document get <id>`.").action(action29);
 }
 
 // src/cli/commands/self-update.ts
@@ -81390,23 +81457,18 @@ function parseMetadataFilter(raw2) {
 }
 async function listDocuments(ctx, opts) {
   const { projectId, limit, offset = 0 } = opts;
-  let docIds = null;
-  if (projectId) {
-    const { data: data2, error: error4 } = await ctx.supabase.from("cerefox_document_projects").select("document_id").eq("project_id", projectId);
-    if (error4)
-      throw error4;
-    docIds = (data2 ?? []).map((r) => r.document_id);
-    if (docIds.length === 0)
-      return [];
-  }
-  let q = ctx.supabase.from("cerefox_documents").select(DOC_COLS).is("deleted_at", null).order("updated_at", { ascending: false });
-  if (docIds)
-    q = q.in("id", docIds);
+  const selectCols = projectId ? `${DOC_COLS}, cerefox_document_projects!inner(project_id)` : DOC_COLS;
+  let q = ctx.supabase.from("cerefox_documents").select(selectCols).is("deleted_at", null).order("updated_at", { ascending: false });
+  if (projectId)
+    q = q.eq("cerefox_document_projects.project_id", projectId);
   q = q.range(offset, offset + limit - 1);
   const { data, error: error3 } = await q;
   if (error3)
     throw error3;
-  return data ?? [];
+  return (data ?? []).map((row) => {
+    const { cerefox_document_projects: _j, ...doc2 } = row;
+    return doc2;
+  });
 }
 async function listAllProjects(ctx) {
   const { data, error: error3 } = await ctx.supabase.from("cerefox_projects").select("*").order("name");
@@ -81444,10 +81506,8 @@ async function getProjectDocCounts(ctx, projectIds) {
   if (projectIds.length === 0)
     return { active, deleted };
   try {
-    const { data, error: error3 } = await ctx.supabase.from("cerefox_document_projects").select("project_id, cerefox_documents(deleted_at)").in("project_id", projectIds);
-    if (error3)
-      throw error3;
-    for (const row of data ?? []) {
+    const rows = await fetchAllPages((from, to) => ctx.supabase.from("cerefox_document_projects").select("project_id, cerefox_documents(deleted_at)").in("project_id", projectIds).order("document_id", { ascending: true }).order("project_id", { ascending: true }).range(from, to));
+    for (const row of rows) {
       const pid = row.project_id;
       if (!(pid in active))
         continue;
@@ -81491,16 +81551,13 @@ async function getRecentDocAuthors(ctx, docIds) {
   return out;
 }
 async function countDocumentsForProject(ctx, projectId) {
-  const { data, error: error3 } = await ctx.supabase.from("cerefox_document_projects").select("document_id, cerefox_documents(deleted_at)").eq("project_id", projectId);
+  const { count, error: error3 } = await ctx.supabase.from("cerefox_document_projects").select("document_id, cerefox_documents!inner(deleted_at)", {
+    count: "exact",
+    head: true
+  }).eq("project_id", projectId).is("cerefox_documents.deleted_at", null);
   if (error3)
     throw error3;
-  let n = 0;
-  for (const row of data ?? []) {
-    if (row.cerefox_documents && row.cerefox_documents.deleted_at === null) {
-      n += 1;
-    }
-  }
-  return n;
+  return count ?? 0;
 }
 function dashboardDocFromRow(row, projectIds) {
   return {
@@ -81527,7 +81584,8 @@ function projectDocResult(r) {
     chunk_count: r.chunk_count ?? 0,
     total_chars: r.total_chars ?? 0,
     doc_updated_at: r.doc_updated_at ?? null,
-    is_partial: r.is_partial ?? false
+    is_partial: r.is_partial ?? false,
+    below_confidence: r.below_confidence ?? false
   };
 }
 function projectChunkResult(r) {
@@ -81543,7 +81601,8 @@ function projectChunkResult(r) {
     doc_title: r.doc_title ?? "",
     doc_source: r.doc_source ?? null,
     doc_project_ids: r.doc_project_ids ?? [],
-    doc_metadata: r.doc_metadata ?? {}
+    doc_metadata: r.doc_metadata ?? {},
+    below_confidence: r.below_confidence ?? false
   };
 }
 async function runSearch(ctx, opts) {
@@ -82023,7 +82082,7 @@ function registerDocumentReadRoutes(app, ctx) {
   });
   app.get("/api/v1/documents/:document_id/chunks", async (c2) => {
     const documentId = c2.req.param("document_id");
-    const { data, error: error3 } = await ctx.supabase.from("cerefox_chunks").select("id, document_id, chunk_index, heading_path, heading_level, title, content, char_count").eq("document_id", documentId).is("version_id", null).order("chunk_index");
+    const { data, error: error3 } = await ctx.supabase.from("cerefox_chunks").select("id, document_id, chunk_index, heading_path, heading_level, title, content, char_count").eq("document_id", documentId).is("version_id", null).order("chunk_index").range(0, 4999);
     if (error3)
       return c2.json({ detail: error3.message }, 500);
     const rows = data ?? [];