@cerefox/memory 1.0.3 → 1.0.5

package/dist/server-assets/_shared/ef-meta/index.ts

@@ -18,7 +18,7 @@
  * doesn't touch `supabase/functions/` leaves it alone).
  */
 
-export const EF_VERSION = "1.0.3";
+export const EF_VERSION = "1.0.5";
 
 /**
  * The most recent version whose EF-side SOURCE actually changed (#127).
@@ -28,7 +28,7 @@ export const EF_VERSION = "1.0.3";
  * `cut_release.ts` ONLY when EF source changed since the last tag; doctor
  * uses it to stay silent on label-only drift.
  */
-export const EF_LAST_CHANGED = "1.0.3";
+export const EF_LAST_CHANGED = "1.0.4";
 
 /**
  * The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp

package/dist/server-assets/_shared/mcp-tools/_utils.ts

@@ -68,6 +68,21 @@ export function getMinSearchScore(): number {
   return Number.isNaN(n) || n < 0 || n > 1 ? fallback : n;
 }
 
+/**
+ * CEREFOX_MIN_TERM_COVERAGE (v1.0.4): user-configurable default for the
+ * OR-fallback term-coverage gate. Returns undefined when unset/invalid —
+ * callers then OMIT p_min_term_coverage from the RPC call, deferring to the
+ * server default (0.5) and staying compatible with pre-0.9.1 servers
+ * (an unknown named argument fails the PostgREST function match).
+ */
+export function getMinTermCoverage(): number | undefined {
+  const raw = (globalThis as { process?: { env?: Record<string, string | undefined> } })
+    .process?.env?.CEREFOX_MIN_TERM_COVERAGE;
+  if (raw === undefined || raw === "") return undefined;
+  const n = Number.parseFloat(raw);
+  return Number.isNaN(n) || n < 0 || n > 1 ? undefined : n;
+}
+
 export function applyByteBudget(
   rows: unknown[],
   maxBytes: number,

package/dist/server-assets/_shared/mcp-tools/search.ts

@@ -18,7 +18,8 @@
 import type { MCPSupabaseClient } from "./types.ts";
 
 import { getEmbedding, resolveEmbedderKind } from "../embeddings/index.ts";
-import { applyByteBudget, getMaxResponseBytes, getMinSearchScore, logUsage } from "./_utils.ts";
+import { applyByteBudget, getMaxResponseBytes, getMinSearchScore,
+  getMinTermCoverage, logUsage } from "./_utils.ts";
 import { lookupProjectId } from "./_projects.ts";
 import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
 
@@ -33,6 +34,12 @@ async function handler(
   const mode = (args.mode as string | undefined) ?? "docs";
   const alpha = (args.alpha as number | undefined) ?? 0.7;
   const min_score = (args.min_score as number | undefined) ?? getMinSearchScore();
+  // v1.0.4: coverage gate default from CEREFOX_MIN_TERM_COVERAGE; only sent
+  // when configured (see getMinTermCoverage — keeps pre-0.9.1 servers working).
+  const min_term_coverage =
+    (args.min_term_coverage as number | undefined) ?? getMinTermCoverage();
+  const coverageParam =
+    min_term_coverage !== undefined ? { p_min_term_coverage: min_term_coverage } : {};
   const metadata_filter =
     (args.metadata_filter as Record<string, string> | null | undefined) ?? null;
   const requested_max_bytes = args.max_bytes as number | undefined;
@@ -84,6 +91,7 @@ async function handler(
       p_match_count: match_count,
       p_project_id: projectId,
       ...metaFilterParam,
+      ...coverageParam,
     };
   } else if (mode === "hybrid") {
     rpcName = "cerefox_hybrid_search";
@@ -96,6 +104,7 @@ async function handler(
       p_project_id: projectId,
       p_min_score: min_score,
       ...metaFilterParam,
+      ...coverageParam,
     };
   } else {
     rpcName = "cerefox_search_docs";
@@ -107,6 +116,7 @@ async function handler(
       p_project_id: projectId,
       p_min_score: min_score,
       ...metaFilterParam,
+      ...coverageParam,
     };
   }
 
@@ -195,6 +205,27 @@ export const searchTool: ToolDefinition = {
           'Optional JSONB containment filter. Only documents whose metadata contains ALL specified key-value pairs are returned. Example: {"type": "decision", "status": "active"}. Call cerefox_list_metadata_keys first to discover available keys and values. Omit to search all documents.',
         additionalProperties: { type: "string" },
       },
+      mode: {
+        type: "string",
+        enum: ["docs", "hybrid", "fts", "semantic"],
+        description:
+          "Search mode (default: docs — full reconstructed documents). hybrid: ranked chunks; fts: keyword-only (no embedding); semantic: vector-only.",
+      },
+      alpha: {
+        type: "number",
+        description:
+          "Hybrid fusion weight 0–1 (default 0.7): 1 = pure semantic, 0 = pure keyword.",
+      },
+      min_score: {
+        type: "number",
+        description:
+          "Minimum cosine similarity for vector-side results (default: server-configured, 0.5 OpenAI / 0.6 local embedder).",
+      },
+      min_term_coverage: {
+        type: "number",
+        description:
+          "Keyword OR-fallback confidence bar 0–1 (default 0.5): fraction of the query's meaningful terms a result must match to count as a confident hit; weaker matches return flagged below-confidence. 0 = any matching term. Needs schema ≥ 0.9.1.",
+      },
       max_bytes: {
         type: "integer",
         description:

package/dist/server-assets/db/rpcs.sql

@@ -69,6 +69,13 @@ DROP FUNCTION IF EXISTS cerefox_get_document(UUID, UUID);
 DROP FUNCTION IF EXISTS cerefox_hybrid_search(TEXT, VECTOR(768), INT, FLOAT, BOOLEAN, UUID, FLOAT, JSONB);
 DROP FUNCTION IF EXISTS cerefox_search_docs(TEXT, VECTOR(768), INT, FLOAT, UUID, FLOAT, INT, INT, JSONB);
 
+-- Iteration 28I follow-up (v1.0.4, term-coverage gate): p_min_term_coverage
+-- added to the search RPC signatures (new arg count = new function; the old
+-- overloads must go or PostgREST calls become ambiguous).
+DROP FUNCTION IF EXISTS cerefox_hybrid_search(TEXT, VECTOR(768), INT, FLOAT, BOOLEAN, UUID, FLOAT, JSONB);
+DROP FUNCTION IF EXISTS cerefox_fts_search(TEXT, INT, UUID, JSONB);
+DROP FUNCTION IF EXISTS cerefox_search_docs(TEXT, VECTOR(768), INT, FLOAT, UUID, FLOAT, INT, INT, JSONB);
+
 -- ── Shared return type note ────────────────────────────────────────────────────
 -- All chunk-level search RPCs return the same shape for consistency:
 --   chunk_id, document_id, chunk_index, title, content, heading_path,
@@ -96,7 +103,14 @@ CREATE OR REPLACE FUNCTION cerefox_hybrid_search(
     p_use_upgrade     BOOLEAN DEFAULT FALSE,
     p_project_id      UUID    DEFAULT NULL,
     p_min_score       FLOAT   DEFAULT 0.0,
-    p_metadata_filter JSONB   DEFAULT NULL
+    p_metadata_filter JSONB   DEFAULT NULL,
+    -- 28I follow-up (v1.0.4): in OR-fallback mode, the unconditional FTS pass
+    -- requires at least this fraction of the query's meaningful (non-stopword,
+    -- deduplicated) terms to match the chunk. Under AND semantics a match
+    -- meant 100% of terms — the pass this gate generalizes. Chunks below the
+    -- bar can still pass via the vector threshold, else they are
+    -- below-confidence material. 0 restores the pre-gate OR behavior.
+    p_min_term_coverage FLOAT DEFAULT 0.5
 )
 RETURNS TABLE (
     chunk_id        UUID,
@@ -144,18 +158,26 @@ DECLARE
     tok           TEXT;
     tok_q         tsquery;
     and_matches   BOOLEAN := FALSE;
+    -- v1.0.4 coverage gate: the per-token queries (deduplicated by normalized
+    -- lexeme text, so "run running" counts once) and their count.
+    tok_queries   tsquery[] := '{}';
+    seen_tokens   TEXT[]    := '{}';
+    total_tokens  INT;
     candidate_count INT := p_match_count * 5;
 BEGIN
     -- Build the OR-composed query: plainto each whitespace token (so tokens get
     -- the same normalization/stemming as the AND path), skip stopword-only
-    -- tokens, fold with the tsquery OR operator (||).
+    -- tokens, dedupe by normalized form, fold with the tsquery OR operator (||).
     FOR tok IN SELECT unnest(regexp_split_to_array(trim(p_query_text), '\s+')) LOOP
         tok_q := plainto_tsquery('english', tok);
-        IF numnode(tok_q) > 0 THEN
+        IF numnode(tok_q) > 0 AND NOT (tok_q::TEXT = ANY(seen_tokens)) THEN
+            seen_tokens := seen_tokens || tok_q::TEXT;
+            tok_queries := tok_queries || tok_q;
             query_fts_or := CASE WHEN query_fts_or IS NULL
                                  THEN tok_q ELSE query_fts_or || tok_q END;
         END IF;
     END LOOP;
+    total_tokens := COALESCE(array_length(tok_queries, 1), 0);
 
     -- Does the strict AND query match anything at all (under the caller's
     -- filters)? Cheap probe against the partial FTS index.
@@ -183,7 +205,20 @@ BEGIN
         fts_results AS (
             SELECT
                 c.id,
-                ts_rank_cd(c.fts, query_fts)::FLOAT AS fts_score
+                ts_rank_cd(c.fts, query_fts)::FLOAT AS fts_score,
+                -- v1.0.4 coverage gate: in AND mode a match means 100% of the
+                -- query's terms are present, so the unconditional pass is
+                -- earned by construction. In OR-fallback mode, earn it only
+                -- when at least p_min_term_coverage of the meaningful terms
+                -- match this chunk; weaker matches keep contributing their
+                -- fts_score to the fusion but must pass via the vector
+                -- threshold (or surface as below-confidence candidates).
+                CASE
+                    WHEN and_matches OR total_tokens = 0 THEN TRUE
+                    ELSE (SELECT COUNT(*) FROM unnest(tok_queries) tq
+                          WHERE c.fts @@ tq)::FLOAT
+                         >= p_min_term_coverage * total_tokens
+                END AS coverage_ok
             FROM cerefox_chunks c
             JOIN cerefox_documents d ON c.document_id = d.id
             WHERE c.version_id IS NULL
@@ -230,12 +265,14 @@ BEGIN
                     (1.0 - p_alpha) * COALESCE(f.fts_score, 0.0)
                 ) AS score,
                 COALESCE(v.vec_score, 0.0) AS vec_score,
-                -- TRUE when the chunk matched the @@ FTS operator.
-                -- We use this flag rather than vec_score to decide whether a chunk
-                -- passes the threshold, because in small corpora every chunk appears
-                -- in vec_results (LIMIT candidate_count covers all rows), so
-                -- vec_score is never NULL even for FTS-only matches.
-                f.id IS NOT NULL AS has_fts_match
+                -- TRUE when the chunk matched the @@ FTS operator WITH enough
+                -- term coverage to earn the unconditional pass (v1.0.4; always
+                -- true for AND-mode matches). We use this flag rather than
+                -- vec_score to decide whether a chunk passes the threshold,
+                -- because in small corpora every chunk appears in vec_results
+                -- (LIMIT candidate_count covers all rows), so vec_score is
+                -- never NULL even for FTS-only matches.
+                (f.id IS NOT NULL AND f.coverage_ok) AS has_fts_match
             FROM fts_results f
             FULL OUTER JOIN vec_results v ON f.id = v.id
         ),
@@ -293,7 +330,10 @@ CREATE OR REPLACE FUNCTION cerefox_fts_search(
     p_query_text      TEXT,
     p_match_count     INT  DEFAULT 10,
     p_project_id      UUID DEFAULT NULL,
-    p_metadata_filter JSONB DEFAULT NULL
+    p_metadata_filter JSONB DEFAULT NULL,
+    -- v1.0.4: see cerefox_hybrid_search. In OR-fallback mode results must
+    -- match at least this fraction of the query's meaningful terms.
+    p_min_term_coverage FLOAT DEFAULT 0.5
 )
 RETURNS TABLE (
     chunk_id        UUID,
@@ -324,14 +364,20 @@ DECLARE
     tok           TEXT;
     tok_q         tsquery;
     and_matches   BOOLEAN := FALSE;
+    tok_queries   tsquery[] := '{}';
+    seen_tokens   TEXT[]    := '{}';
+    total_tokens  INT;
 BEGIN
     FOR tok IN SELECT unnest(regexp_split_to_array(trim(p_query_text), '\s+')) LOOP
         tok_q := plainto_tsquery('english', tok);
-        IF numnode(tok_q) > 0 THEN
+        IF numnode(tok_q) > 0 AND NOT (tok_q::TEXT = ANY(seen_tokens)) THEN
+            seen_tokens := seen_tokens || tok_q::TEXT;
+            tok_queries := tok_queries || tok_q;
             query_fts_or := CASE WHEN query_fts_or IS NULL
                                  THEN tok_q ELSE query_fts_or || tok_q END;
         END IF;
     END LOOP;
+    total_tokens := COALESCE(array_length(tok_queries, 1), 0);
 
     IF numnode(query_fts_and) > 0 THEN
         SELECT EXISTS (
@@ -377,6 +423,11 @@ BEGIN
     WHERE c.version_id IS NULL
               AND d.deleted_at IS NULL
       AND c.fts @@ query_fts
+      -- v1.0.4 coverage gate (OR-fallback mode only): pure keyword search
+      -- returns only chunks matching enough of the query's terms.
+      AND (and_matches OR total_tokens = 0
+           OR (SELECT COUNT(*) FROM unnest(tok_queries) tq
+               WHERE c.fts @@ tq)::FLOAT >= p_min_term_coverage * total_tokens)
       AND (p_project_id IS NULL OR EXISTS (
               SELECT 1 FROM cerefox_document_projects dp
               WHERE dp.document_id = d.id AND dp.project_id = p_project_id
@@ -687,7 +738,8 @@ CREATE OR REPLACE FUNCTION cerefox_search_docs(
     p_min_score              FLOAT DEFAULT 0.0,
     p_small_to_big_threshold INT   DEFAULT 20000,
     p_context_window         INT   DEFAULT 1,
-    p_metadata_filter        JSONB DEFAULT NULL
+    p_metadata_filter        JSONB DEFAULT NULL,
+    p_min_term_coverage      FLOAT DEFAULT 0.5
 )
 RETURNS TABLE (
     document_id              UUID,
@@ -727,7 +779,8 @@ AS $$
             p_use_upgrade     := FALSE,
             p_project_id      := p_project_id,
             p_min_score       := p_min_score,
-            p_metadata_filter := p_metadata_filter
+            p_metadata_filter := p_metadata_filter,
+            p_min_term_coverage := p_min_term_coverage
         )
     ),
     best_per_doc AS (
@@ -1885,7 +1938,7 @@ SET search_path = public, pg_catalog
 AS $$
     -- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
     -- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
-    SELECT '0.9.0'::TEXT;
+    SELECT '0.9.1'::TEXT;
 $$;
 
 -- ── cerefox_content_format_stats ─────────────────────────────────────────────

package/dist/server-assets/db/schema.sql

@@ -5,7 +5,7 @@
 -- Requires extensions: vector (pgvector), uuid-ossp
 -- These are enabled at the top of db_deploy.py before this file is applied.
 --
--- @version: 0.9.0
+-- @version: 0.9.1
 -- The `@version` marker above is read by the schema-version-mismatch banner
 -- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
 -- changes in a way that requires `cerefox server deploy` to be re-run —

package/docs/guides/configuration.md

@@ -117,10 +117,19 @@ This handles intermittent OpenAI API errors (500s) that would otherwise cause se
 
 ## Retrieval
 
+> **Which paths read these?** Client-side tunables in this section are read
+> from *your* `.env` by the **CLI**, the **local MCP server**, and `cerefox
+> web`. The **remote MCP / Edge Function path** runs on Supabase and does not
+> see your `.env` — it uses the server defaults unless the caller passes the
+> per-call parameter (e.g. `min_score`, `min_term_coverage` on
+> `cerefox_search`). Setting them as Supabase **Function secrets** may also
+> work but is not a tested configuration.
+
 | Variable | Default | Description |
 |----------|---------|-------------|
 | `CEREFOX_MAX_RESPONSE_BYTES` | `200000` | Maximum bytes in a single search response (local MCP path). See explanation below. |
 | `CEREFOX_MIN_SEARCH_SCORE` | `0.50` (`0.60` with the local embedder) | Minimum cosine similarity for hybrid and semantic search results (0.0–1.0). The default is embedder-aware: nomic scores unrelated text higher than OpenAI, so `CEREFOX_EMBEDDER=local` raises the floor to 0.60. In **hybrid search**, chunks that matched the FTS keyword operator (`@@`) always pass through regardless of their vector score — the threshold only filters vector-only results. In **semantic search**, all results are filtered. The pure **FTS search** mode is unaffected. Increase for stricter precision; decrease for wider recall. |
+| `CEREFOX_MIN_TERM_COVERAGE` | *(unset — server default `0.5`)* | Confidence bar for the keyword OR-fallback (v1.0.4, schema ≥ 0.9.1): when a strict all-terms match fails and search relaxes to any-term matching, a result counts as a confident hit only if it matches at least this fraction of the query's meaningful terms; weaker matches surface as below-confidence candidates. `0` restores pre-gate behavior (any matching term passes); `1` requires every term. Per-call override: `cerefox search --min-term-coverage`. Leave unset against pre-0.9.1 servers. |
 | `CEREFOX_EMBED_MAX_INPUT_CHARS` | `20000` | Safety cap on the characters sent to the embedding model per input. The full chunk content is always stored and reconstructed untouched; only the embedding uses the (rare) truncated prefix, so an oversized chunk can never fail an ingest. |
 | `CEREFOX_MODELS_DIR` | `~/.cerefox/models` (in-container: inside the data volume) | Where the local embedder caches downloaded model weights (Cerefox Local; `CEREFOX_EMBEDDER=local`). |
 | `CEREFOX_ONNX_BATCH` | `4` | Texts per local-embedder inference call. Peak memory scales with this; the small default keeps ingest/reindex safe on small Docker VMs. |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerefox/memory",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "Cerefox — user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
   "license": "Apache-2.0",
   "homepage": "https://github.com/fstamatelopoulos/cerefox",