@cerefox/memory 1.0.3 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7184,7 +7184,7 @@ var exports_meta = {};
7184
7184
  __export(exports_meta, {
7185
7185
  PKG_VERSION: () => PKG_VERSION
7186
7186
  });
7187
- var PKG_VERSION = "1.0.3";
7187
+ var PKG_VERSION = "1.0.4";
7188
7188
  var init_meta = () => {};
7189
7189
 
7190
7190
  // ../../node_modules/.bun/tslib@2.8.1/node_modules/tslib/tslib.js
@@ -25225,6 +25225,13 @@ function getMinSearchScore() {
25225
25225
  const n = Number.parseFloat(raw);
25226
25226
  return Number.isNaN(n) || n < 0 || n > 1 ? fallback : n;
25227
25227
  }
25228
+ function getMinTermCoverage() {
25229
+ const raw = globalThis.process?.env?.CEREFOX_MIN_TERM_COVERAGE;
25230
+ if (raw === undefined || raw === "")
25231
+ return;
25232
+ const n = Number.parseFloat(raw);
25233
+ return Number.isNaN(n) || n < 0 || n > 1 ? undefined : n;
25234
+ }
25228
25235
  function applyByteBudget(rows, maxBytes) {
25229
25236
  const accepted = [];
25230
25237
  let usedBytes = 0;
@@ -55572,6 +55579,8 @@ async function handler9(supabase, args, ctx) {
55572
55579
  const mode = args.mode ?? "docs";
55573
55580
  const alpha = args.alpha ?? 0.7;
55574
55581
  const min_score = args.min_score ?? getMinSearchScore();
55582
+ const min_term_coverage = args.min_term_coverage ?? getMinTermCoverage();
55583
+ const coverageParam = min_term_coverage !== undefined ? { p_min_term_coverage: min_term_coverage } : {};
55575
55584
  const metadata_filter = args.metadata_filter ?? null;
55576
55585
  const requested_max_bytes = args.max_bytes;
55577
55586
  const ceiling = getMaxResponseBytes();
@@ -55603,7 +55612,8 @@ async function handler9(supabase, args, ctx) {
55603
55612
  p_query_text: query,
55604
55613
  p_match_count: match_count,
55605
55614
  p_project_id: projectId,
55606
- ...metaFilterParam
55615
+ ...metaFilterParam,
55616
+ ...coverageParam
55607
55617
  };
55608
55618
  } else if (mode === "hybrid") {
55609
55619
  rpcName = "cerefox_hybrid_search";
@@ -55615,7 +55625,8 @@ async function handler9(supabase, args, ctx) {
55615
55625
  p_use_upgrade: false,
55616
55626
  p_project_id: projectId,
55617
55627
  p_min_score: min_score,
55618
- ...metaFilterParam
55628
+ ...metaFilterParam,
55629
+ ...coverageParam
55619
55630
  };
55620
55631
  } else {
55621
55632
  rpcName = "cerefox_search_docs";
@@ -55626,7 +55637,8 @@ async function handler9(supabase, args, ctx) {
55626
55637
  p_alpha: alpha,
55627
55638
  p_project_id: projectId,
55628
55639
  p_min_score: min_score,
55629
- ...metaFilterParam
55640
+ ...metaFilterParam,
55641
+ ...coverageParam
55630
55642
  };
55631
55643
  }
55632
55644
  const { data, error: error2 } = await supabase.rpc(rpcName, rpcParams);
@@ -55699,6 +55711,23 @@ var init_search = __esm(() => {
55699
55711
  description: 'Optional JSONB containment filter. Only documents whose metadata contains ALL specified key-value pairs are returned. Example: {"type": "decision", "status": "active"}. Call cerefox_list_metadata_keys first to discover available keys and values. Omit to search all documents.',
55700
55712
  additionalProperties: { type: "string" }
55701
55713
  },
55714
+ mode: {
55715
+ type: "string",
55716
+ enum: ["docs", "hybrid", "fts", "semantic"],
55717
+ description: "Search mode (default: docs — full reconstructed documents). hybrid: ranked chunks; fts: keyword-only (no embedding); semantic: vector-only."
55718
+ },
55719
+ alpha: {
55720
+ type: "number",
55721
+ description: "Hybrid fusion weight 0–1 (default 0.7): 1 = pure semantic, 0 = pure keyword."
55722
+ },
55723
+ min_score: {
55724
+ type: "number",
55725
+ description: "Minimum cosine similarity for vector-side results (default: server-configured, 0.5 OpenAI / 0.6 local embedder)."
55726
+ },
55727
+ min_term_coverage: {
55728
+ type: "number",
55729
+ description: "Keyword OR-fallback confidence bar 0–1 (default 0.5): fraction of the query's meaningful terms a result must match to count as a confident hit; weaker matches return flagged below-confidence. 0 = any matching term. Needs schema ≥ 0.9.1."
55730
+ },
55702
55731
  max_bytes: {
55703
55732
  type: "integer",
55704
55733
  description: "Optional response size budget in bytes. Results are dropped whole until the budget is satisfied; a truncated flag is set when results are dropped. Defaults to the server maximum (200000). Pass a smaller value if your context window is limited. Values above the server maximum are silently capped."
@@ -69032,10 +69061,29 @@ init_cli_core();
69032
69061
 
69033
69062
  // src/cli/commands/backup.ts
69034
69063
  init_cli_core();
69035
- init_client();
69036
69064
  import { existsSync as existsSync2, mkdirSync, writeFileSync } from "node:fs";
69037
69065
  import { homedir as homedir2 } from "node:os";
69038
69066
  import { join as join2, resolve } from "node:path";
69067
+
69068
+ // ../../_shared/db-client/paginate.ts
69069
+ async function fetchAllPages(makeQuery, batchSize = 200) {
69070
+ const results = [];
69071
+ let offset = 0;
69072
+ for (;; ) {
69073
+ const { data, error } = await makeQuery(offset, offset + batchSize - 1);
69074
+ if (error)
69075
+ throw new Error(error.message ?? JSON.stringify(error));
69076
+ const page = data ?? [];
69077
+ results.push(...page);
69078
+ if (page.length < batchSize)
69079
+ break;
69080
+ offset += batchSize;
69081
+ }
69082
+ return results;
69083
+ }
69084
+
69085
+ // src/cli/commands/backup.ts
69086
+ init_client();
69039
69087
  function expandHome(path) {
69040
69088
  if (path === "~")
69041
69089
  return homedir2();
@@ -69056,21 +69104,25 @@ async function action(options) {
69056
69104
  const filename = `cerefox-${stamp}${options.label ? "-" + options.label : ""}.json`;
69057
69105
  const dest = join2(outDir, filename);
69058
69106
  const client = getClient();
69059
- const { data: docsData, error: docsErr } = await client.raw.from("cerefox_documents").select("id, title, content_hash, source, metadata, total_chars, chunk_count, " + "review_status, created_at, updated_at, deleted_at").is("deleted_at", null).order("created_at", { ascending: true });
69060
- if (docsErr)
69061
- throw systemError(`Document fetch failed: ${docsErr.message}`);
69062
- const docs = docsData ?? [];
69107
+ let docs;
69108
+ try {
69109
+ docs = await fetchAllPages((from, to) => client.raw.from("cerefox_documents").select("id, title, content_hash, source, metadata, total_chars, chunk_count, " + "review_status, created_at, updated_at, deleted_at").is("deleted_at", null).order("created_at", { ascending: true }).order("id", { ascending: true }).range(from, to));
69110
+ } catch (err) {
69111
+ throw systemError(`Document fetch failed: ${err instanceof Error ? err.message : String(err)}`);
69112
+ }
69063
69113
  let chunkTotal = 0;
69064
69114
  const enriched = [];
69065
69115
  for (let i = 0;i < docs.length; i++) {
69066
69116
  const doc = docs[i];
69067
69117
  const docId = doc.id;
69068
- const { data: chunks, error: chunkErr } = await client.raw.from("cerefox_chunks").select("*").eq("document_id", docId).is("version_id", null).order("chunk_index", { ascending: true });
69069
- if (chunkErr) {
69070
- throw systemError(`Chunk fetch failed for ${docId}: ${chunkErr.message}`);
69118
+ let chunks;
69119
+ try {
69120
+ chunks = await fetchAllPages((from, to) => client.raw.from("cerefox_chunks").select("*").eq("document_id", docId).is("version_id", null).order("chunk_index", { ascending: true }).range(from, to));
69121
+ } catch (err) {
69122
+ throw systemError(`Chunk fetch failed for ${docId}: ${err instanceof Error ? err.message : String(err)}`);
69071
69123
  }
69072
- chunkTotal += (chunks ?? []).length;
69073
- enriched.push({ ...doc, chunks: chunks ?? [] });
69124
+ chunkTotal += chunks.length;
69125
+ enriched.push({ ...doc, chunks });
69074
69126
  if (process.stdout.isTTY) {
69075
69127
  process.stderr.write(`\r Dumping documents: ${i + 1}/${docs.length} (${chunkTotal} chunks so far)…`);
69076
69128
  }
@@ -74948,8 +75000,8 @@ import { homedir as homedir6 } from "node:os";
74948
75000
  import { join as join9 } from "node:path";
74949
75001
 
74950
75002
  // ../../_shared/ef-meta/index.ts
74951
- var EF_VERSION = "1.0.3";
74952
- var EF_LAST_CHANGED = "1.0.3";
75003
+ var EF_VERSION = "1.0.4";
75004
+ var EF_LAST_CHANGED = "1.0.4";
74953
75005
 
74954
75006
  // src/cli/util/checks.ts
74955
75007
  init_config();
@@ -77239,18 +77291,22 @@ async function action27(options) {
77239
77291
  }
77240
77292
  const reindexAll = Boolean(options.all);
77241
77293
  const dryRun = Boolean(options.dryRun);
77242
- let query = supabase.from("cerefox_chunks").select("id, document_id, content, embedder_primary, cerefox_documents(title)").is("version_id", null);
77243
- if (options.documentId) {
77244
- query = query.eq("document_id", options.documentId);
77245
- }
77246
77294
  const targetModel = activeEmbedderName();
77247
- if (!reindexAll) {
77248
- query = query.neq("embedder_primary", targetModel);
77295
+ let chunks;
77296
+ try {
77297
+ chunks = await fetchAllPages((from, to) => {
77298
+ let query = supabase.from("cerefox_chunks").select("id, document_id, content, embedder_primary, cerefox_documents(title)").is("version_id", null);
77299
+ if (options.documentId) {
77300
+ query = query.eq("document_id", options.documentId);
77301
+ }
77302
+ if (!reindexAll) {
77303
+ query = query.neq("embedder_primary", targetModel);
77304
+ }
77305
+ return query.order("id", { ascending: true }).range(from, to);
77306
+ }, 1000);
77307
+ } catch (err) {
77308
+ throw systemError(`Failed to list chunks: ${err instanceof Error ? err.message : String(err)}`);
77249
77309
  }
77250
- const { data, error: error3 } = await query;
77251
- if (error3)
77252
- throw systemError(`Failed to list chunks: ${error3.message}`);
77253
- const chunks = data ?? [];
77254
77310
  if (chunks.length === 0) {
77255
77311
  println(c.dim("(nothing to reindex)"));
77256
77312
  return;
@@ -77420,6 +77476,8 @@ async function action29(query, options) {
77420
77476
  const matchCount = parsePositiveInt(options.matchCount, "--match-count", 5);
77421
77477
  const alpha = parseFloat01(options.alpha, "--alpha", 0.7);
77422
77478
  const minScore = parseFloat01(options.minScore, "--min-score", getMinSearchScore());
77479
+ const envCoverage = getMinTermCoverage();
77480
+ const coverageParam = options.minTermCoverage !== undefined ? { p_min_term_coverage: parseFloat01(options.minTermCoverage, "--min-term-coverage", envCoverage ?? 0.5) } : envCoverage !== undefined ? { p_min_term_coverage: envCoverage } : {};
77423
77481
  const maxBytes = parseNonNegativeInt(options.maxBytes, "--max-bytes", getMaxResponseBytes());
77424
77482
  const mode = options.mode ?? "docs";
77425
77483
  if (!["docs", "hybrid", "fts"].includes(mode)) {
@@ -77450,7 +77508,8 @@ async function action29(query, options) {
77450
77508
  p_query_text: query,
77451
77509
  p_match_count: matchCount,
77452
77510
  p_project_id: projectId,
77453
- ...metaFilterParam
77511
+ ...metaFilterParam,
77512
+ ...coverageParam
77454
77513
  };
77455
77514
  } else if (mode === "hybrid") {
77456
77515
  rpcName = "cerefox_hybrid_search";
@@ -77462,7 +77521,8 @@ async function action29(query, options) {
77462
77521
  p_use_upgrade: false,
77463
77522
  p_project_id: projectId,
77464
77523
  p_min_score: minScore,
77465
- ...metaFilterParam
77524
+ ...metaFilterParam,
77525
+ ...coverageParam
77466
77526
  };
77467
77527
  } else {
77468
77528
  rpcName = "cerefox_search_docs";
@@ -77473,7 +77533,8 @@ async function action29(query, options) {
77473
77533
  p_alpha: alpha,
77474
77534
  p_project_id: projectId,
77475
77535
  p_min_score: minScore,
77476
- ...metaFilterParam
77536
+ ...metaFilterParam,
77537
+ ...coverageParam
77477
77538
  };
77478
77539
  }
77479
77540
  const results = await client.rpc(rpcName, rpcParams);
@@ -77572,7 +77633,7 @@ async function action29(query, options) {
77572
77633
  }
77573
77634
  }
77574
77635
  function registerSearch(program2) {
77575
- program2.command("search").description("Search the knowledge base (hybrid FTS + semantic).").argument("<query>", "Natural-language search query.").option("-c, --match-count <n>", "Maximum number of documents to return.", "5").option("-p, --project-name <name>", "Filter results to a specific project.").option("-f, --metadata-filter <json>", "JSON containment filter; only docs whose metadata contains ALL pairs are returned.").option("--mode <mode>", "Search mode: docs (default), hybrid, fts.", "docs").option("--alpha <float>", "Semantic weight 0..1 (default: 0.7).", "0.7").option("--min-score <float>", "Minimum cosine similarity threshold (default: CEREFOX_MIN_SEARCH_SCORE; else 0.5, or 0.6 with the local embedder).").option("--max-bytes <n>", "Response size budget in bytes (default: CEREFOX_MAX_RESPONSE_BYTES or 200000).").option("-r, --requestor <name>", "Agent / user name (recorded in usage log).").option("--json", "Emit machine-readable JSON instead of the default text.").option("--only-metadata", "List matching docs (id, score, chunks, chars, partial/full) WITHOUT their content — like the web UI's collapsed result list. Grab a [id:…] then `cerefox document get <id>`.").action(action29);
77636
+ program2.command("search").description("Search the knowledge base (hybrid FTS + semantic).").argument("<query>", "Natural-language search query.").option("-c, --match-count <n>", "Maximum number of documents to return.", "5").option("-p, --project-name <name>", "Filter results to a specific project.").option("-f, --metadata-filter <json>", "JSON containment filter; only docs whose metadata contains ALL pairs are returned.").option("--mode <mode>", "Search mode: docs (default), hybrid, fts.", "docs").option("--alpha <float>", "Semantic weight 0..1 (default: 0.7).", "0.7").option("--min-score <float>", "Minimum cosine similarity threshold (default: CEREFOX_MIN_SEARCH_SCORE; else 0.5, or 0.6 with the local embedder).").option("--min-term-coverage <float>", "OR-fallback keyword matches must cover at least this fraction of the query's meaningful terms to count as confident hits (default: CEREFOX_MIN_TERM_COVERAGE; else the server default 0.5; needs schema ≥ 0.9.1).").option("--max-bytes <n>", "Response size budget in bytes (default: CEREFOX_MAX_RESPONSE_BYTES or 200000).").option("-r, --requestor <name>", "Agent / user name (recorded in usage log).").option("--json", "Emit machine-readable JSON instead of the default text.").option("--only-metadata", "List matching docs (id, score, chunks, chars, partial/full) WITHOUT their content — like the web UI's collapsed result list. Grab a [id:…] then `cerefox document get <id>`.").action(action29);
77576
77637
  }
77577
77638
 
77578
77639
  // src/cli/commands/self-update.ts
@@ -81444,10 +81505,8 @@ async function getProjectDocCounts(ctx, projectIds) {
81444
81505
  if (projectIds.length === 0)
81445
81506
  return { active, deleted };
81446
81507
  try {
81447
- const { data, error: error3 } = await ctx.supabase.from("cerefox_document_projects").select("project_id, cerefox_documents(deleted_at)").in("project_id", projectIds);
81448
- if (error3)
81449
- throw error3;
81450
- for (const row of data ?? []) {
81508
+ const rows = await fetchAllPages((from, to) => ctx.supabase.from("cerefox_document_projects").select("project_id, cerefox_documents(deleted_at)").in("project_id", projectIds).order("document_id", { ascending: true }).order("project_id", { ascending: true }).range(from, to));
81509
+ for (const row of rows) {
81451
81510
  const pid = row.project_id;
81452
81511
  if (!(pid in active))
81453
81512
  continue;
@@ -81491,16 +81550,13 @@ async function getRecentDocAuthors(ctx, docIds) {
81491
81550
  return out;
81492
81551
  }
81493
81552
  async function countDocumentsForProject(ctx, projectId) {
81494
- const { data, error: error3 } = await ctx.supabase.from("cerefox_document_projects").select("document_id, cerefox_documents(deleted_at)").eq("project_id", projectId);
81553
+ const { count, error: error3 } = await ctx.supabase.from("cerefox_document_projects").select("document_id, cerefox_documents!inner(deleted_at)", {
81554
+ count: "exact",
81555
+ head: true
81556
+ }).eq("project_id", projectId).is("cerefox_documents.deleted_at", null);
81495
81557
  if (error3)
81496
81558
  throw error3;
81497
- let n = 0;
81498
- for (const row of data ?? []) {
81499
- if (row.cerefox_documents && row.cerefox_documents.deleted_at === null) {
81500
- n += 1;
81501
- }
81502
- }
81503
- return n;
81559
+ return count ?? 0;
81504
81560
  }
81505
81561
  function dashboardDocFromRow(row, projectIds) {
81506
81562
  return {
@@ -18,7 +18,7 @@
18
18
  * doesn't touch `supabase/functions/` leaves it alone).
19
19
  */
20
20
 
21
- export const EF_VERSION = "1.0.3";
21
+ export const EF_VERSION = "1.0.4";
22
22
 
23
23
  /**
24
24
  * The most recent version whose EF-side SOURCE actually changed (#127).
@@ -28,7 +28,7 @@ export const EF_VERSION = "1.0.3";
28
28
  * `cut_release.ts` ONLY when EF source changed since the last tag; doctor
29
29
  * uses it to stay silent on label-only drift.
30
30
  */
31
- export const EF_LAST_CHANGED = "1.0.3";
31
+ export const EF_LAST_CHANGED = "1.0.4";
32
32
 
33
33
  /**
34
34
  * The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp
@@ -68,6 +68,21 @@ export function getMinSearchScore(): number {
68
68
  return Number.isNaN(n) || n < 0 || n > 1 ? fallback : n;
69
69
  }
70
70
 
71
+ /**
72
+ * CEREFOX_MIN_TERM_COVERAGE (v1.0.4): user-configurable default for the
73
+ * OR-fallback term-coverage gate. Returns undefined when unset/invalid —
74
+ * callers then OMIT p_min_term_coverage from the RPC call, deferring to the
75
+ * server default (0.5) and staying compatible with pre-0.9.1 servers
76
+ * (an unknown named argument fails the PostgREST function match).
77
+ */
78
+ export function getMinTermCoverage(): number | undefined {
79
+ const raw = (globalThis as { process?: { env?: Record<string, string | undefined> } })
80
+ .process?.env?.CEREFOX_MIN_TERM_COVERAGE;
81
+ if (raw === undefined || raw === "") return undefined;
82
+ const n = Number.parseFloat(raw);
83
+ return Number.isNaN(n) || n < 0 || n > 1 ? undefined : n;
84
+ }
85
+
71
86
  export function applyByteBudget(
72
87
  rows: unknown[],
73
88
  maxBytes: number,
@@ -18,7 +18,8 @@
18
18
  import type { MCPSupabaseClient } from "./types.ts";
19
19
 
20
20
  import { getEmbedding, resolveEmbedderKind } from "../embeddings/index.ts";
21
- import { applyByteBudget, getMaxResponseBytes, getMinSearchScore, logUsage } from "./_utils.ts";
21
+ import { applyByteBudget, getMaxResponseBytes, getMinSearchScore,
22
+ getMinTermCoverage, logUsage } from "./_utils.ts";
22
23
  import { lookupProjectId } from "./_projects.ts";
23
24
  import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
24
25
 
@@ -33,6 +34,12 @@ async function handler(
33
34
  const mode = (args.mode as string | undefined) ?? "docs";
34
35
  const alpha = (args.alpha as number | undefined) ?? 0.7;
35
36
  const min_score = (args.min_score as number | undefined) ?? getMinSearchScore();
37
+ // v1.0.4: coverage gate default from CEREFOX_MIN_TERM_COVERAGE; only sent
38
+ // when configured (see getMinTermCoverage — keeps pre-0.9.1 servers working).
39
+ const min_term_coverage =
40
+ (args.min_term_coverage as number | undefined) ?? getMinTermCoverage();
41
+ const coverageParam =
42
+ min_term_coverage !== undefined ? { p_min_term_coverage: min_term_coverage } : {};
36
43
  const metadata_filter =
37
44
  (args.metadata_filter as Record<string, string> | null | undefined) ?? null;
38
45
  const requested_max_bytes = args.max_bytes as number | undefined;
@@ -84,6 +91,7 @@ async function handler(
84
91
  p_match_count: match_count,
85
92
  p_project_id: projectId,
86
93
  ...metaFilterParam,
94
+ ...coverageParam,
87
95
  };
88
96
  } else if (mode === "hybrid") {
89
97
  rpcName = "cerefox_hybrid_search";
@@ -96,6 +104,7 @@ async function handler(
96
104
  p_project_id: projectId,
97
105
  p_min_score: min_score,
98
106
  ...metaFilterParam,
107
+ ...coverageParam,
99
108
  };
100
109
  } else {
101
110
  rpcName = "cerefox_search_docs";
@@ -107,6 +116,7 @@ async function handler(
107
116
  p_project_id: projectId,
108
117
  p_min_score: min_score,
109
118
  ...metaFilterParam,
119
+ ...coverageParam,
110
120
  };
111
121
  }
112
122
 
@@ -195,6 +205,27 @@ export const searchTool: ToolDefinition = {
195
205
  'Optional JSONB containment filter. Only documents whose metadata contains ALL specified key-value pairs are returned. Example: {"type": "decision", "status": "active"}. Call cerefox_list_metadata_keys first to discover available keys and values. Omit to search all documents.',
196
206
  additionalProperties: { type: "string" },
197
207
  },
208
+ mode: {
209
+ type: "string",
210
+ enum: ["docs", "hybrid", "fts", "semantic"],
211
+ description:
212
+ "Search mode (default: docs — full reconstructed documents). hybrid: ranked chunks; fts: keyword-only (no embedding); semantic: vector-only.",
213
+ },
214
+ alpha: {
215
+ type: "number",
216
+ description:
217
+ "Hybrid fusion weight 0–1 (default 0.7): 1 = pure semantic, 0 = pure keyword.",
218
+ },
219
+ min_score: {
220
+ type: "number",
221
+ description:
222
+ "Minimum cosine similarity for vector-side results (default: server-configured, 0.5 OpenAI / 0.6 local embedder).",
223
+ },
224
+ min_term_coverage: {
225
+ type: "number",
226
+ description:
227
+ "Keyword OR-fallback confidence bar 0–1 (default 0.5): fraction of the query's meaningful terms a result must match to count as a confident hit; weaker matches return flagged below-confidence. 0 = any matching term. Needs schema ≥ 0.9.1.",
228
+ },
198
229
  max_bytes: {
199
230
  type: "integer",
200
231
  description:
@@ -69,6 +69,13 @@ DROP FUNCTION IF EXISTS cerefox_get_document(UUID, UUID);
69
69
  DROP FUNCTION IF EXISTS cerefox_hybrid_search(TEXT, VECTOR(768), INT, FLOAT, BOOLEAN, UUID, FLOAT, JSONB);
70
70
  DROP FUNCTION IF EXISTS cerefox_search_docs(TEXT, VECTOR(768), INT, FLOAT, UUID, FLOAT, INT, INT, JSONB);
71
71
 
72
+ -- Iteration 28I follow-up (v1.0.4, term-coverage gate): p_min_term_coverage
73
+ -- added to the search RPC signatures (new arg count = new function; the old
74
+ -- overloads must go or PostgREST calls become ambiguous).
75
+ DROP FUNCTION IF EXISTS cerefox_hybrid_search(TEXT, VECTOR(768), INT, FLOAT, BOOLEAN, UUID, FLOAT, JSONB);
76
+ DROP FUNCTION IF EXISTS cerefox_fts_search(TEXT, INT, UUID, JSONB);
77
+ DROP FUNCTION IF EXISTS cerefox_search_docs(TEXT, VECTOR(768), INT, FLOAT, UUID, FLOAT, INT, INT, JSONB);
78
+
72
79
  -- ── Shared return type note ────────────────────────────────────────────────────
73
80
  -- All chunk-level search RPCs return the same shape for consistency:
74
81
  -- chunk_id, document_id, chunk_index, title, content, heading_path,
@@ -96,7 +103,14 @@ CREATE OR REPLACE FUNCTION cerefox_hybrid_search(
96
103
  p_use_upgrade BOOLEAN DEFAULT FALSE,
97
104
  p_project_id UUID DEFAULT NULL,
98
105
  p_min_score FLOAT DEFAULT 0.0,
99
- p_metadata_filter JSONB DEFAULT NULL
106
+ p_metadata_filter JSONB DEFAULT NULL,
107
+ -- 28I follow-up (v1.0.4): in OR-fallback mode, the unconditional FTS pass
108
+ -- requires at least this fraction of the query's meaningful (non-stopword,
109
+ -- deduplicated) terms to match the chunk. Under AND semantics a match
110
+ -- meant 100% of terms — the pass this gate generalizes. Chunks below the
111
+ -- bar can still pass via the vector threshold, else they are
112
+ -- below-confidence material. 0 restores the pre-gate OR behavior.
113
+ p_min_term_coverage FLOAT DEFAULT 0.5
100
114
  )
101
115
  RETURNS TABLE (
102
116
  chunk_id UUID,
@@ -144,18 +158,26 @@ DECLARE
144
158
  tok TEXT;
145
159
  tok_q tsquery;
146
160
  and_matches BOOLEAN := FALSE;
161
+ -- v1.0.4 coverage gate: the per-token queries (deduplicated by normalized
162
+ -- lexeme text, so "run running" counts once) and their count.
163
+ tok_queries tsquery[] := '{}';
164
+ seen_tokens TEXT[] := '{}';
165
+ total_tokens INT;
147
166
  candidate_count INT := p_match_count * 5;
148
167
  BEGIN
149
168
  -- Build the OR-composed query: plainto each whitespace token (so tokens get
150
169
  -- the same normalization/stemming as the AND path), skip stopword-only
151
- -- tokens, fold with the tsquery OR operator (||).
170
+ -- tokens, dedupe by normalized form, fold with the tsquery OR operator (||).
152
171
  FOR tok IN SELECT unnest(regexp_split_to_array(trim(p_query_text), '\s+')) LOOP
153
172
  tok_q := plainto_tsquery('english', tok);
154
- IF numnode(tok_q) > 0 THEN
173
+ IF numnode(tok_q) > 0 AND NOT (tok_q::TEXT = ANY(seen_tokens)) THEN
174
+ seen_tokens := seen_tokens || tok_q::TEXT;
175
+ tok_queries := tok_queries || tok_q;
155
176
  query_fts_or := CASE WHEN query_fts_or IS NULL
156
177
  THEN tok_q ELSE query_fts_or || tok_q END;
157
178
  END IF;
158
179
  END LOOP;
180
+ total_tokens := COALESCE(array_length(tok_queries, 1), 0);
159
181
 
160
182
  -- Does the strict AND query match anything at all (under the caller's
161
183
  -- filters)? Cheap probe against the partial FTS index.
@@ -183,7 +205,20 @@ BEGIN
183
205
  fts_results AS (
184
206
  SELECT
185
207
  c.id,
186
- ts_rank_cd(c.fts, query_fts)::FLOAT AS fts_score
208
+ ts_rank_cd(c.fts, query_fts)::FLOAT AS fts_score,
209
+ -- v1.0.4 coverage gate: in AND mode a match means 100% of the
210
+ -- query's terms are present, so the unconditional pass is
211
+ -- earned by construction. In OR-fallback mode, earn it only
212
+ -- when at least p_min_term_coverage of the meaningful terms
213
+ -- match this chunk; weaker matches keep contributing their
214
+ -- fts_score to the fusion but must pass via the vector
215
+ -- threshold (or surface as below-confidence candidates).
216
+ CASE
217
+ WHEN and_matches OR total_tokens = 0 THEN TRUE
218
+ ELSE (SELECT COUNT(*) FROM unnest(tok_queries) tq
219
+ WHERE c.fts @@ tq)::FLOAT
220
+ >= p_min_term_coverage * total_tokens
221
+ END AS coverage_ok
187
222
  FROM cerefox_chunks c
188
223
  JOIN cerefox_documents d ON c.document_id = d.id
189
224
  WHERE c.version_id IS NULL
@@ -230,12 +265,14 @@ BEGIN
230
265
  (1.0 - p_alpha) * COALESCE(f.fts_score, 0.0)
231
266
  ) AS score,
232
267
  COALESCE(v.vec_score, 0.0) AS vec_score,
233
- -- TRUE when the chunk matched the @@ FTS operator.
234
- -- We use this flag rather than vec_score to decide whether a chunk
235
- -- passes the threshold, because in small corpora every chunk appears
236
- -- in vec_results (LIMIT candidate_count covers all rows), so
237
- -- vec_score is never NULL even for FTS-only matches.
238
- f.id IS NOT NULL AS has_fts_match
268
+ -- TRUE when the chunk matched the @@ FTS operator WITH enough
269
+ -- term coverage to earn the unconditional pass (v1.0.4; always
270
+ -- true for AND-mode matches). We use this flag rather than
271
+ -- vec_score to decide whether a chunk passes the threshold,
272
+ -- because in small corpora every chunk appears in vec_results
273
+ -- (LIMIT candidate_count covers all rows), so vec_score is
274
+ -- never NULL even for FTS-only matches.
275
+ (f.id IS NOT NULL AND f.coverage_ok) AS has_fts_match
239
276
  FROM fts_results f
240
277
  FULL OUTER JOIN vec_results v ON f.id = v.id
241
278
  ),
@@ -293,7 +330,10 @@ CREATE OR REPLACE FUNCTION cerefox_fts_search(
293
330
  p_query_text TEXT,
294
331
  p_match_count INT DEFAULT 10,
295
332
  p_project_id UUID DEFAULT NULL,
296
- p_metadata_filter JSONB DEFAULT NULL
333
+ p_metadata_filter JSONB DEFAULT NULL,
334
+ -- v1.0.4: see cerefox_hybrid_search. In OR-fallback mode results must
335
+ -- match at least this fraction of the query's meaningful terms.
336
+ p_min_term_coverage FLOAT DEFAULT 0.5
297
337
  )
298
338
  RETURNS TABLE (
299
339
  chunk_id UUID,
@@ -324,14 +364,20 @@ DECLARE
324
364
  tok TEXT;
325
365
  tok_q tsquery;
326
366
  and_matches BOOLEAN := FALSE;
367
+ tok_queries tsquery[] := '{}';
368
+ seen_tokens TEXT[] := '{}';
369
+ total_tokens INT;
327
370
  BEGIN
328
371
  FOR tok IN SELECT unnest(regexp_split_to_array(trim(p_query_text), '\s+')) LOOP
329
372
  tok_q := plainto_tsquery('english', tok);
330
- IF numnode(tok_q) > 0 THEN
373
+ IF numnode(tok_q) > 0 AND NOT (tok_q::TEXT = ANY(seen_tokens)) THEN
374
+ seen_tokens := seen_tokens || tok_q::TEXT;
375
+ tok_queries := tok_queries || tok_q;
331
376
  query_fts_or := CASE WHEN query_fts_or IS NULL
332
377
  THEN tok_q ELSE query_fts_or || tok_q END;
333
378
  END IF;
334
379
  END LOOP;
380
+ total_tokens := COALESCE(array_length(tok_queries, 1), 0);
335
381
 
336
382
  IF numnode(query_fts_and) > 0 THEN
337
383
  SELECT EXISTS (
@@ -377,6 +423,11 @@ BEGIN
377
423
  WHERE c.version_id IS NULL
378
424
  AND d.deleted_at IS NULL
379
425
  AND c.fts @@ query_fts
426
+ -- v1.0.4 coverage gate (OR-fallback mode only): pure keyword search
427
+ -- returns only chunks matching enough of the query's terms.
428
+ AND (and_matches OR total_tokens = 0
429
+ OR (SELECT COUNT(*) FROM unnest(tok_queries) tq
430
+ WHERE c.fts @@ tq)::FLOAT >= p_min_term_coverage * total_tokens)
380
431
  AND (p_project_id IS NULL OR EXISTS (
381
432
  SELECT 1 FROM cerefox_document_projects dp
382
433
  WHERE dp.document_id = d.id AND dp.project_id = p_project_id
@@ -687,7 +738,8 @@ CREATE OR REPLACE FUNCTION cerefox_search_docs(
687
738
  p_min_score FLOAT DEFAULT 0.0,
688
739
  p_small_to_big_threshold INT DEFAULT 20000,
689
740
  p_context_window INT DEFAULT 1,
690
- p_metadata_filter JSONB DEFAULT NULL
741
+ p_metadata_filter JSONB DEFAULT NULL,
742
+ p_min_term_coverage FLOAT DEFAULT 0.5
691
743
  )
692
744
  RETURNS TABLE (
693
745
  document_id UUID,
@@ -727,7 +779,8 @@ AS $$
727
779
  p_use_upgrade := FALSE,
728
780
  p_project_id := p_project_id,
729
781
  p_min_score := p_min_score,
730
- p_metadata_filter := p_metadata_filter
782
+ p_metadata_filter := p_metadata_filter,
783
+ p_min_term_coverage := p_min_term_coverage
731
784
  )
732
785
  ),
733
786
  best_per_doc AS (
@@ -1885,7 +1938,7 @@ SET search_path = public, pg_catalog
1885
1938
  AS $$
1886
1939
  -- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
1887
1940
  -- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
1888
- SELECT '0.9.0'::TEXT;
1941
+ SELECT '0.9.1'::TEXT;
1889
1942
  $$;
1890
1943
 
1891
1944
  -- ── cerefox_content_format_stats ─────────────────────────────────────────────
@@ -5,7 +5,7 @@
5
5
  -- Requires extensions: vector (pgvector), uuid-ossp
6
6
  -- These are enabled at the top of db_deploy.py before this file is applied.
7
7
  --
8
- -- @version: 0.9.0
8
+ -- @version: 0.9.1
9
9
  -- The `@version` marker above is read by the schema-version-mismatch banner
10
10
  -- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
11
11
  -- changes in a way that requires `cerefox server deploy` to be re-run —
@@ -117,10 +117,19 @@ This handles intermittent OpenAI API errors (500s) that would otherwise cause se
117
117
 
118
118
  ## Retrieval
119
119
 
120
+ > **Which paths read these?** Client-side tunables in this section are read
121
+ > from *your* `.env` by the **CLI**, the **local MCP server**, and `cerefox
122
+ > web`. The **remote MCP / Edge Function path** runs on Supabase and does not
123
+ > see your `.env` — it uses the server defaults unless the caller passes the
124
+ > per-call parameter (e.g. `min_score`, `min_term_coverage` on
125
+ > `cerefox_search`). Setting them as Supabase **Function secrets** may also
126
+ > work but is not a tested configuration.
127
+
120
128
  | Variable | Default | Description |
121
129
  |----------|---------|-------------|
122
130
  | `CEREFOX_MAX_RESPONSE_BYTES` | `200000` | Maximum bytes in a single search response (local MCP path). See explanation below. |
123
131
  | `CEREFOX_MIN_SEARCH_SCORE` | `0.50` (`0.60` with the local embedder) | Minimum cosine similarity for hybrid and semantic search results (0.0–1.0). The default is embedder-aware: nomic scores unrelated text higher than OpenAI, so `CEREFOX_EMBEDDER=local` raises the floor to 0.60. In **hybrid search**, chunks that matched the FTS keyword operator (`@@`) always pass through regardless of their vector score — the threshold only filters vector-only results. In **semantic search**, all results are filtered. The pure **FTS search** mode is unaffected. Increase for stricter precision; decrease for wider recall. |
132
+ | `CEREFOX_MIN_TERM_COVERAGE` | *(unset — server default `0.5`)* | Confidence bar for the keyword OR-fallback (v1.0.4, schema ≥ 0.9.1): when a strict all-terms match fails and search relaxes to any-term matching, a result counts as a confident hit only if it matches at least this fraction of the query's meaningful terms; weaker matches surface as below-confidence candidates. `0` restores pre-gate behavior (any matching term passes); `1` requires every term. Per-call override: `cerefox search --min-term-coverage`. Leave unset against pre-0.9.1 servers. |
124
133
  | `CEREFOX_EMBED_MAX_INPUT_CHARS` | `20000` | Safety cap on the characters sent to the embedding model per input. The full chunk content is always stored and reconstructed untouched; only the embedding uses the (rare) truncated prefix, so an oversized chunk can never fail an ingest. |
125
134
  | `CEREFOX_MODELS_DIR` | `~/.cerefox/models` (in-container: inside the data volume) | Where the local embedder caches downloaded model weights (Cerefox Local; `CEREFOX_EMBEDDER=local`). |
126
135
  | `CEREFOX_ONNX_BATCH` | `4` | Texts per local-embedder inference call. Peak memory scales with this; the small default keeps ingest/reindex safe on small Docker VMs. |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cerefox/memory",
3
- "version": "1.0.3",
3
+ "version": "1.0.4",
4
4
  "description": "Cerefox — user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
5
5
  "license": "Apache-2.0",
6
6
  "homepage": "https://github.com/fstamatelopoulos/cerefox",