@cerefox/memory 1.16.1 → 1.17.1

package/dist/server-assets/db/rpcs.sql

@@ -152,7 +152,7 @@ DECLARE
     -- plainto_tsquery: ANDs all terms, treats every token as a literal word.
     -- We deliberately avoid websearch_to_tsquery here because it interprets `-` as
     -- a negation operator, which traps natural queries against dashed titles
-    -- (e.g. `Job Hunting - Opportunity Index`). Agent queries don't use the
+    -- (e.g. `Release Notes - Q3 Index`). Agent queries don't use the
     -- websearch operators (phrase, OR, NOT); semantic ranking is the soft-match
     -- layer for "broadly related". If operator support is ever needed, gate it
     -- behind an opt-in flag rather than changing the default.
@@ -177,12 +177,17 @@ DECLARE
     total_tokens  INT;
     candidate_count INT := p_match_count * 5;
     -- #133 resolution: caller value, else deployment config, else built-in.
+    -- 0.18.0: the built-in floor follows the store's embedder (0.5 OpenAI,
+    -- 0.6 nomic), so a store with no config row gets the right one.
     v_min_score     FLOAT := COALESCE(p_min_score,
-                                      cerefox_config_float('min_search_score', 0.5));
+                                      cerefox_config_float('min_search_score',
+                                                           cerefox_default_min_search_score()));
     v_alpha         FLOAT := COALESCE(p_alpha,
                                       cerefox_config_float('search_alpha', 0.7));
+    -- 0.18.0: 0.67 (was 0.5), so a confident OR-fallback match needs two of
+    -- three terms, not one of two (docs/specs/search-calibration.md).
     v_min_coverage  FLOAT := COALESCE(p_min_term_coverage,
-                                      cerefox_config_float('min_term_coverage', 0.5));
+                                      cerefox_config_float('min_term_coverage', 0.67));
 BEGIN
     -- Build the OR-composed query: plainto each whitespace token (so tokens get
     -- the same normalization/stemming as the AND path), skip stopword-only
@@ -225,7 +230,22 @@ BEGIN
         fts_results AS (
             SELECT
                 c.id,
-                ts_rank_cd(c.fts, query_fts)::FLOAT AS fts_score,
+                -- 0.18.0: normalisation 32 bounds the keyword score to
+                -- rank/(rank+1), in [0, 1) like the cosine it is fused with.
+                -- Unbounded (up to ~4 when every term matches), one keyword
+                -- hit outweighed any semantic evidence, and the same document
+                -- scored 1.2 or 0.3 depending on which phrasing found it
+                -- (docs/specs/search-calibration.md).
+                ts_rank_cd(c.fts, query_fts, 32)::FLOAT AS fts_score,
+                -- 0.18.1: the exact cosine of every keyword match, so a chunk
+                -- that matched by keyword is never fused as if its meaning
+                -- scored 0 merely because it fell outside vec_results.
+                CASE
+                    WHEN p_use_upgrade AND c.embedding_upgrade IS NOT NULL
+                        THEN (1.0 - (c.embedding_upgrade <=> p_query_embedding))::FLOAT
+                    ELSE
+                        (1.0 - (c.embedding_primary <=> p_query_embedding))::FLOAT
+                END AS fts_vec_score,
                 -- v1.0.4 coverage gate: in AND mode a match means 100% of the
                 -- query's terms are present, so the unconditional pass is
                 -- earned by construction. In OR-fallback mode, earn it only
@@ -272,21 +292,24 @@ BEGIN
                   ))
               AND (p_metadata_filter IS NULL OR d.metadata @> p_metadata_filter)
               AND (p_review_status IS NULL OR d.review_status = p_review_status)
-            ORDER BY
-                CASE
-                    WHEN p_use_upgrade AND c.embedding_upgrade IS NOT NULL
-                        THEN c.embedding_upgrade <=> p_query_embedding
-                    ELSE c.embedding_primary <=> p_query_embedding
-                END
+            -- 0.18.1: ordered by the computed similarity, NOT by the bare
+            -- `embedding <=> query` distance. The bare form (which the CASE
+            -- folds to once p_use_upgrade is known) lets the planner serve this
+            -- from the HNSW index, which is approximate and returns at most
+            -- hnsw.ef_search (40) rows: on a store with heavy version churn it
+            -- returned ~38 of the 250 candidates asked for, mostly not the
+            -- nearest ones. An exact scan is also faster at knowledge-base
+            -- scale (thousands of chunks).
+            ORDER BY vec_score DESC
             LIMIT candidate_count
         ),
         combined AS (
             SELECT
                 COALESCE(f.id, v.id) AS id,
-                (   v_alpha * COALESCE(v.vec_score, 0.0) +
+                (   v_alpha * COALESCE(v.vec_score, f.fts_vec_score, 0.0) +
                     (1.0 - v_alpha) * COALESCE(f.fts_score, 0.0)
                 ) AS score,
-                COALESCE(v.vec_score, 0.0) AS vec_score,
+                COALESCE(v.vec_score, f.fts_vec_score, 0.0) AS vec_score,
                 -- TRUE when the chunk matched the @@ FTS operator WITH enough
                 -- term coverage to earn the unconditional pass (v1.0.4; always
                 -- true for AND-mode matches). We use this flag rather than
@@ -406,7 +429,7 @@ DECLARE
     seen_tokens   TEXT[]    := '{}';
     total_tokens  INT;
     v_min_coverage FLOAT := COALESCE(p_min_term_coverage,
-                                     cerefox_config_float('min_term_coverage', 0.5));
+                                     cerefox_config_float('min_term_coverage', 0.67));
 BEGIN
     FOR tok IN SELECT unnest(regexp_split_to_array(trim(p_query_text), '\s+')) LOOP
         tok_q := plainto_tsquery('english', tok);
@@ -554,12 +577,10 @@ BEGIN
                   THEN (1.0 - (c.embedding_upgrade <=> p_query_embedding))::FLOAT
               ELSE (1.0 - (c.embedding_primary <=> p_query_embedding))::FLOAT
           END >= p_min_score
-    ORDER BY
-        CASE
-            WHEN p_use_upgrade AND c.embedding_upgrade IS NOT NULL
-                THEN c.embedding_upgrade <=> p_query_embedding
-            ELSE c.embedding_primary <=> p_query_embedding
-        END
+    -- 0.18.1: by the computed similarity, so the approximate HNSW index cannot
+    -- serve (and truncate) it; see vec_results in cerefox_hybrid_search.
+    -- By position: `score` would be ambiguous with the OUT column here.
+    ORDER BY 8 DESC
     LIMIT p_match_count;
 END;
 $$;
@@ -2755,6 +2776,28 @@ BEGIN
 END;
 $$;
 
+-- Built-in vector-similarity floor for this store (0.18.0), used when
+-- `min_search_score` has no config row. It depends on the embedder: nomic's
+-- cosine for unrelated text sits around 0.4-0.55 (OpenAI's around 0.1-0.3), so
+-- 0.5 lets unrelated documents through as confident results. Cerefox Local
+-- used to get its 0.6 from a config row seeded at container start; the s6
+-- image never ran that seed, so fresh Local stores searched at 0.5. Deriving it
+-- here covers every store whatever created it. A store holds one embedder
+-- (`cerefox doctor` enforces it), so one current chunk decides.
+CREATE OR REPLACE FUNCTION cerefox_default_min_search_score()
+RETURNS FLOAT
+LANGUAGE sql
+SECURITY DEFINER
+STABLE
+SET search_path = public, pg_catalog
+AS $$
+    SELECT CASE
+        WHEN (SELECT embedder_primary FROM cerefox_chunks
+              WHERE version_id IS NULL LIMIT 1) LIKE 'nomic-%'
+        THEN 0.6 ELSE 0.5
+    END::FLOAT;
+$$;
+
 -- Integer/boolean companions to cerefox_config_float. Same contract: fall back
 -- to the caller's default when the key is unset or unparseable, so a malformed
 -- row can never break a write path.
@@ -3186,6 +3229,15 @@ SET search_path = public, pg_catalog
 AS $$
     -- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
     -- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
+    -- 0.18.1: exact vector candidates. Hybrid and semantic search ordered by
+    -- the bare distance, which let the planner use the approximate HNSW index
+    -- (at most ef_search = 40 rows, poor recall after heavy version churn);
+    -- they now order by the computed similarity (exact scan). Keyword matches
+    -- carry their exact cosine into fusion. RPC-only.
+    -- 0.18.0 (iteration 48): search calibration, RPC-only. Bounded keyword
+    -- score in hybrid fusion (ts_rank_cd normalisation 32); min_term_coverage
+    -- default 0.67; built-in min_search_score follows the store's embedder
+    -- (new cerefox_default_min_search_score). docs/specs/search-calibration.md.
     -- 0.17.0 (#251): trash auto-purge. New cerefox_purge_expired_trash, called
     -- by cerefox_delete_document after a real soft delete (which now returns
     -- `auto_purged`); two config keys, off by default. Migration 0033.
@@ -3227,7 +3279,7 @@ AS $$
     -- 0.11.0 supersedes 0.10.6 (v1.2.1, #191): this branch carries that fix plus
     -- the partial-edit surface, and both migrations (0019, 0020) are in the
     -- sequence, so a store deploying this gets everything from both lines.
-    SELECT '0.17.0'::TEXT;
+    SELECT '0.18.1'::TEXT;
 $$;
 
 -- ── cerefox_find_dead_links ──────────────────────────────────────────────────

package/dist/server-assets/db/schema.sql

@@ -5,7 +5,7 @@
 -- Requires extensions: vector (pgvector), uuid-ossp
 -- These are enabled at the top of db_deploy.py before this file is applied.
 --
--- @version: 0.17.0
+-- @version: 0.18.1
 -- The `@version` marker above is read by the schema-version-mismatch banner
 -- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
 -- changes in a way that requires `cerefox server deploy` to be re-run —

package/docs/guides/configuration.md

@@ -144,9 +144,26 @@ This handles intermittent OpenAI API errors (500s) that would otherwise cause se
 > ```
 >
 > Resolution order, highest first: **per-call argument** (`--min-score`, the
-> `min_score` MCP parameter) → **client env var** below → **`cerefox_config`**
-> → built-in default. A malformed stored value is ignored in favour of the
-> built-in, so a bad setting can never break search.
+> `min_score` MCP parameter) → **`cerefox_config`** → built-in default. A
+> malformed stored value is ignored in favour of the built-in, so a bad setting
+> can never break search. (Before v1.17.0 the web UI and `/api/v1` search sent
+> their own `min_search_score` and `search_alpha`, so they did not obey a stored
+> value; now every path does.)
+>
+> **Built-in defaults (v1.17.0, schema 0.18.0)**, chosen by the search
+> calibration benchmark (`docs/specs/search-calibration.md`):
+>
+> | Key | Default | Meaning |
+> |---|---|---|
+> | `min_search_score` | **0.5** on an OpenAI store, **0.6** on a local-embedder (nomic) store, derived from the store's own embeddings | A result with no confident keyword match must be at least this similar to the query. Nomic scores unrelated text around 0.4 to 0.55, so 0.5 would let it through. |
+> | `min_term_coverage` | **0.67** (was 0.5) | When no document contains every query word, a keyword match counts as confident only if it covers this share of the words: two of three, not one of two. |
+> | `search_alpha` | **0.7** | Weight of meaning against keywords. Both sides are on a 0 to 1 scale since v1.17.0, so the weight now means what it says. |
+>
+> Tune only with evidence: `bun scripts/search_benchmark.ts` measures any
+> setting against a labelled query vocabulary on a staging store
+> ([ops-scripts.md](ops-scripts.md#search_benchmarkts--search-calibration-benchmark)).
+> A setting that suits one store's content can cost another kind of query
+> without anything looking wrong.
 
 > **Which paths read these?** Client-side tunables in this section are read
 > from *your* `.env` by the **CLI**, the **local MCP server**, and `cerefox
@@ -162,8 +179,8 @@ This handles intermittent OpenAI API errors (500s) that would otherwise cause se
 |----------|---------|-------------|
 | `CEREFOX_MAX_RESPONSE_BYTES` | `200000` | Maximum bytes in a single search response (local MCP path). See explanation below. |
 | `CEREFOX_MIN_SEARCH_SCORE` | **Retired in v1.1.0 — no longer read.** | Now a store setting: `cerefox config set min_search_score <value>`, or the **Settings** page. The right floor depends on which embedder produced the vectors, and the embedder belongs to the store — every client querying one database must use the same one. A per-call override still works (`cerefox search --min-score`, the MCP `min_score` param). `cerefox doctor` and the Settings page report the variable if it is still set. |
-| `CEREFOX_SEARCH_ALPHA` | **Retired in v1.1.0 — no longer read.** | Now a store setting: `cerefox config set search_alpha <value>`, or the **Settings** page.  A per-call override still works (`cerefox search --min-score`, the MCP `min_score` param). `cerefox doctor` and the Settings page report the variable if it is still set. |
-| `CEREFOX_MIN_TERM_COVERAGE` | **Retired in v1.1.0 — no longer read.** | Now a store setting: `cerefox config set min_term_coverage <value>`, or the **Settings** page.  A per-call override still works (`cerefox search --min-score`, the MCP `min_score` param). `cerefox doctor` and the Settings page report the variable if it is still set. |
+| `CEREFOX_SEARCH_ALPHA` | **Retired in v1.1.0 — no longer read.** | Now a store setting: `cerefox config set search_alpha <value>`, or the **Settings** page. A per-call override still works (`cerefox search --alpha`). `cerefox doctor` and the Settings page report the variable if it is still set. |
+| `CEREFOX_MIN_TERM_COVERAGE` | **Retired in v1.1.0 — no longer read.** | Now a store setting: `cerefox config set min_term_coverage <value>`, or the **Settings** page. A per-call override still works (`cerefox search --min-term-coverage`, the MCP `min_term_coverage` param). `cerefox doctor` and the Settings page report the variable if it is still set. |
 | `CEREFOX_EMBED_MAX_INPUT_CHARS` | `20000` | Safety cap on the characters sent to the embedding model per input. The full chunk content is always stored and reconstructed untouched; only the embedding uses the (rare) truncated prefix, so an oversized chunk can never fail an ingest. |
 | `CEREFOX_MODELS_DIR` | `~/.cerefox/models` (in-container: inside the data volume) | Where the local embedder caches downloaded model weights (Cerefox Local; `CEREFOX_EMBEDDER=local`). |
 | `CEREFOX_ONNX_BATCH` | `4` | Texts per local-embedder inference call. Peak memory scales with this; the small default keeps ingest/reindex safe on small Docker VMs. |

package/docs/guides/linking.md

@@ -10,7 +10,7 @@ corrupting a long document id while rewriting content.
 Write a normal markdown link whose target is the document's UUID:
 
 ```markdown
-See the [Opportunity Index](c937b70f-77af-43d3-b9bc-9f31e0d2041d) for details.
+See the [Q3 Goals](3f2b9c1e-8a47-4d2e-9b6a-1c5e7d0a4f82) for details.
 ```
 
 The Cerefox web UI resolves these at click time — the stored markdown is
@@ -22,7 +22,7 @@ Four link forms are accepted; one is recommended:
 
 | Form | Example target | Use |
 |---|---|---|
-| **Document UUID** | `c937b70f-…` | **Always, for cross-references.** Stable, unambiguous, encoding-safe, validated (below). |
+| **Document UUID** | `3f2b9c1e-…` | **Always, for cross-references.** Stable, unambiguous, encoding-safe, validated (below). |
 | Repo-relative path | `docs/guides/quickstart.md` | Repo-ingested files only (their markdown naturally uses paths). |
 | Basename | `quickstart.md` | Best-effort fallback for the same. |
 | `[[Wikilink]]` | `[[Future Doc Name]]` | A deliberate **dangling** reference — "this document should exist someday." Never validated. |

package/docs/guides/ops-scripts.md

@@ -31,6 +31,7 @@ The canonical scripts are **TypeScript**, run with [Bun](https://bun.sh) (instal
 | `reindex_all.ts` | `bun scripts/reindex_all.ts` |
 | `cerefox_export.ts` | `bun scripts/cerefox_export.ts` |
 | `gen_openapi.ts` | `bun scripts/gen_openapi.ts` |
+| `search_benchmark.ts` | `bun scripts/search_benchmark.ts --api … --db … --embed …` (staging only) |
 | `cut_release.ts` | `bun scripts/cut_release.ts` (see [RELEASING.md](../../RELEASING.md)) |
 
 The remaining files under `scripts/` are not run by hand: `bundle_help.ts`,
@@ -326,6 +327,50 @@ Two things worth knowing about what it does *not* claim:
   every release cut — which broke the v1.15.1 npm publish — and churned every
   generated client on patch releases that did not touch the API.
 
+## search_benchmark.ts — Search-calibration benchmark
+
+Measures how search ranks against a labelled query vocabulary
+(`_shared/search-benchmark/vocabulary/`: a synthetic corpus, about 150 queries
+in 13 categories, variant groups, no-answer queries) and compares alternative
+scoring formulas and settings. It is how the v1.17.0 defaults were chosen. Design
+and results: [`docs/specs/search-calibration.md`](../specs/search-calibration.md).
+
+**It writes**, so it refuses a target without a label (`--label` or
+`CEREFOX_ENV_LABEL`): the corpus goes into a project named
+`Search calibration benchmark`, and a temporary read-only probe function
+(`cerefox_bench_signals`) is created for the run and dropped after it. Re-runs
+write only documents whose content changed.
+
+```bash
+# Staging (OpenAI): the staging web server ingests, the database URL probes
+set -a; . ~/.cerefox/staging/.env; set +a
+bun scripts/search_benchmark.ts --api http://127.0.0.1:8030 \
+    --db "$CEREFOX_DATABASE_URL" --embed openai --out /tmp/bench.json > /tmp/bench.md
+
+# A throwaway Cerefox Local (nomic), postgres published on 55432
+bun scripts/search_benchmark.ts --api http://127.0.0.1:8099 \
+    --db postgresql://cerefox:cerefox@127.0.0.1:55432/cerefox \
+    --embed container:<container-name> --label bench-throwaway
+```
+
+| Flag | Description |
+|------|-------------|
+| `--api URL` | The target's web server; the corpus is ingested through `/api/v1/ingest` |
+| `--db URL` | Direct Postgres URL, for the probe and the live search calls |
+| `--embed openai \| container:<name>` | How queries are embedded: OpenAI (needs `OPENAI_API_KEY`), or the repo's own local embedder run inside a Cerefox Local container |
+| `--label TEXT` | Required unless `CEREFOX_ENV_LABEL` is set |
+| `--corpus DIR` | Use another vocabulary directory (for drafting one) |
+| `--out FILE` | JSON report (the markdown summary goes to stdout) |
+| `--write-floors` | Record the per-category floors of the built-in defaults for this store's embedder in `vocabulary/floors.json` |
+| `--check-floors` | Fail (exit 1) if the built-in defaults on this store fall below any recorded floor for its embedder. The release check for search changes (RELEASING.md) |
+| `--cleanup` | Delete the benchmark documents and project afterwards |
+
+Before reporting any number it checks that its reproduction of the live formula
+matches `cerefox_search_docs` exactly for every query, and stops if it does not.
+A mismatch means the harness no longer models the RPC; fix the harness first.
+
+---
+
 ## sync_docs.ts — Sync project documentation into Cerefox
 
 Ingests `README.md`, `AGENT_GUIDE.md`, `AGENT_QUICK_REFERENCE.md`, and every Markdown file under `docs/` into your Cerefox knowledge base, updating existing documents in-place. Run this any time after editing documentation so AI agents always have access to the current state of the project.

package/docs/guides/setup-local.md

@@ -62,8 +62,10 @@ data volume, so it survives `cerefox-local upgrade`.
 
 > Scores are calibrated per embedder: with the local model the default semantic
 > threshold is **0.6** (vs 0.5 for OpenAI) because nomic scores unrelated text
-> higher. The container seeds `min_search_score = 0.6` into its own config at
-> first boot; change it with `cerefox config set min_search_score <value>` or the
+> higher. Since v1.17.0 the store derives that default from its own embedder,
+> so no setting is needed (before it, containers created from the current image
+> never got the 0.6 they were meant to seed, and searched at 0.5 through MCP and
+> the CLI). Change it with `cerefox config set min_search_score <value>` or the
 > Settings page, or override a single query with `--min-score`.
 
 ---

package/docs/guides/upgrading.md

@@ -197,6 +197,13 @@ knowing about:
 - **v1.13.1 — the flag hides, it does not rewrite.** Agent writes are
   recorded `pending_review` whether or not the workflow is shown (v1.13.0
   stored `approved` while off). Redeploy to pick it up.
+- **v1.17.0 — search scoring recalibrated (schema 0.18.0).** Run
+  `cerefox server deploy` (Cerefox Local does it on container start). Scores in
+  `docs` and `hybrid` search are now between 0 and 1, and rankings shift: mostly
+  for the better on paraphrased and short queries, and fewer unrelated results
+  come back as confident. If you compare scores against a fixed number, or set
+  `min_search_score` / `min_term_coverage` yourself, recheck those values; see
+  [configuration.md](configuration.md) for the new defaults.
 
 ## Notable: v1.8.0 storage reclaim (migration 0027)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerefox/memory",
-  "version": "1.16.1",
+  "version": "1.17.1",
   "description": "Cerefox \u2014 user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
   "license": "Apache-2.0",
   "homepage": "https://github.com/fstamatelopoulos/cerefox",