@cerefox/memory 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_QUICK_REFERENCE.md +2 -1
- package/dist/bin/cerefox.js +120 -44
- package/dist/frontend/assets/{index-CLD16BC4.js → index-Csj-6UHY.js} +2 -2
- package/dist/frontend/assets/{index-CLD16BC4.js.map → index-Csj-6UHY.js.map} +1 -1
- package/dist/frontend/index.html +1 -1
- package/dist/server-assets/_shared/ef-meta/index.ts +11 -1
- package/dist/server-assets/_shared/mcp-tools/_utils.ts +15 -0
- package/dist/server-assets/_shared/mcp-tools/get-help-content.ts +2 -2
- package/dist/server-assets/_shared/mcp-tools/search.ts +47 -2
- package/dist/server-assets/db/rpcs.sql +188 -26
- package/dist/server-assets/db/schema.sql +1 -1
- package/docs/guides/configuration.md +9 -0
- package/package.json +1 -1
package/dist/frontend/index.html
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
href="https://fonts.googleapis.com/css2?family=Geist:wght@300;400;500;600;700&display=swap"
|
|
16
16
|
/>
|
|
17
17
|
<title>Cerefox</title>
|
|
18
|
-
<script type="module" crossorigin src="/app/assets/index-
|
|
18
|
+
<script type="module" crossorigin src="/app/assets/index-Csj-6UHY.js"></script>
|
|
19
19
|
<link rel="stylesheet" crossorigin href="/app/assets/index-Asx5wD7g.css">
|
|
20
20
|
</head>
|
|
21
21
|
<body>
|
|
@@ -18,7 +18,17 @@
|
|
|
18
18
|
* doesn't touch `supabase/functions/` leaves it alone).
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
export const EF_VERSION = "1.0.
|
|
21
|
+
export const EF_VERSION = "1.0.4";
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The most recent version whose EF-side SOURCE actually changed (#127).
|
|
25
|
+
* `EF_VERSION` bumps unconditionally at stable cuts (so stable deployments
|
|
26
|
+
* never display a pre-release label), which means a version delta no longer
|
|
27
|
+
* implies the deployed behaviour differs. This constant is bumped by
|
|
28
|
+
* `cut_release.ts` ONLY when EF source changed since the last tag; doctor
|
|
29
|
+
* uses it to stay silent on label-only drift.
|
|
30
|
+
*/
|
|
31
|
+
export const EF_LAST_CHANGED = "1.0.4";
|
|
22
32
|
|
|
23
33
|
/**
|
|
24
34
|
* The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp
|
|
@@ -68,6 +68,21 @@ export function getMinSearchScore(): number {
|
|
|
68
68
|
return Number.isNaN(n) || n < 0 || n > 1 ? fallback : n;
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
+
/**
|
|
72
|
+
* CEREFOX_MIN_TERM_COVERAGE (v1.0.4): user-configurable default for the
|
|
73
|
+
* OR-fallback term-coverage gate. Returns undefined when unset/invalid —
|
|
74
|
+
* callers then OMIT p_min_term_coverage from the RPC call, deferring to the
|
|
75
|
+
* server default (0.5) and staying compatible with pre-0.9.1 servers
|
|
76
|
+
* (an unknown named argument fails the PostgREST function match).
|
|
77
|
+
*/
|
|
78
|
+
export function getMinTermCoverage(): number | undefined {
|
|
79
|
+
const raw = (globalThis as { process?: { env?: Record<string, string | undefined> } })
|
|
80
|
+
.process?.env?.CEREFOX_MIN_TERM_COVERAGE;
|
|
81
|
+
if (raw === undefined || raw === "") return undefined;
|
|
82
|
+
const n = Number.parseFloat(raw);
|
|
83
|
+
return Number.isNaN(n) || n < 0 || n > 1 ? undefined : n;
|
|
84
|
+
}
|
|
85
|
+
|
|
71
86
|
export function applyByteBudget(
|
|
72
87
|
rows: unknown[],
|
|
73
88
|
maxBytes: number,
|
|
@@ -11,12 +11,12 @@
|
|
|
11
11
|
* docs/specs/polish-and-distribution-design.md §10d.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
-
export const HELP_FULL = "# Cerefox Knowledge Base -- Agent Quick Reference\n\nCerefox is a persistent, shared knowledge base. You have **10 MCP tools** (9 of them have CLI equivalents — `cerefox_get_help` is MCP-only). For the full guide, search Cerefox for \"How AI Agents Use Cerefox\" or call `cerefox_get_help` to retrieve this content over MCP.\n\n## Tools\n\n| Tool | Purpose | Key params |\n|------|---------|------------|\n| `cerefox_search` | Find documents (hybrid FTS + semantic) | `query` (required), `project_name`, `metadata_filter`, `requestor` |\n| `cerefox_ingest` | Save or update a document | `title`, `content` (required), `document_id` (update by ID), `expected_content_hash` (**required on content updates** — see rule 9), `last_write_wins`, `update_if_exists`, `project_name` (single, non-destructive add on update), `project_names` (list, destructive replace on update), `metadata` (omit on update to keep existing tags; `{}` clears), `author` |\n| `cerefox_get_document` | Get full document by ID (header includes `content_hash` — the update token) | `document_id` (required) |\n| `cerefox_list_versions` | Version history of a document | `document_id` (required) |\n| `cerefox_metadata_search` | Find or list docs by metadata, project, or time (no text query) | `metadata_filter`, `project_name` (list a project's docs), `updated_since`, `include_content` — **at least one** of metadata_filter/project_name/updated_since/created_since |\n| `cerefox_list_metadata_keys` | Discover available metadata keys | (none required) |\n| `cerefox_list_projects` | List all projects | (none required) |\n| `cerefox_set_document_projects` | Set doc's project memberships to exactly the given list (destructive replace; metadata-only, no content change) | `document_id`, `project_names` (required) |\n| `cerefox_get_audit_log` | Query write operation history | `document_id`, `author`, `operation`, `since` |\n| `cerefox_get_help` | Retrieve Cerefox conventions (this reference) over MCP. **Call this whenever uncertain.** | `topic` (optional, case-insensitive H2 substring match) |\n\n## Essential Rules\n\n1. **Search before ingesting** -- check if the document exists first.\n2. **Prefer ID-based updates** -- pass `document_id` from search results for deterministic updates. Falls back to title-matching with `update_if_exists: true`.\n3. **Set `author`/`requestor`** to your name on every call (e.g., \"Claude Code\", \"archiver\"). On MCP, pass as parameters. On CLI, pass `--author`/`--author-type`/`--requestor` flags, or rely on `CEREFOX_AUTHOR_NAME`/`CEREFOX_AUTHOR_TYPE`/`CEREFOX_REQUESTOR_NAME` env vars set in the user's `.env`.\n4. **Use `document_id` from search results** `[id: uuid]` for get_document and list_versions.\n5. **Add metadata** -- at minimum `type` (\"decision-log\", \"research\", \"design-doc\") and `status` (\"active\", \"draft\").\n6. **Write structured Markdown** with H1/H2/H3 headings for good chunking and search.\n7. **Deletes are soft (recoverable); purge is web-UI-only.** If you decide to delete, surface it to the user (`I soft-deleted X — recoverable from the Cerefox web UI trash`). You cannot un-do your own delete from agent code by design.\n8. **Cross-doc links inside content**: **always use `[Text](document-uuid)`.** UUIDs are the only fully reliable link form — stable across title changes, never ambiguous, no encoding gotchas. Every `cerefox_search` result shows `[id: <uuid>]` after the title; grab it and use it. Title-based linking (`[Text](<Title With Spaces>)`) is fragile (breaks on colons, parens, ampersands, brackets — silently navigates to wrong page) — **don't write title-based links**; do an extra search to get the UUID instead. Repo-path forms (`[Text](docs/path.md)`) exist for repo-ingested files; don't construct manually. See `AGENT_GUIDE.md → Writing linkable content` for the full rule.\n9. **Concurrency: content updates require `expected_content_hash`.** Pass the `content_hash` you read (shown by `cerefox_get_document`, `cerefox_search`, and `cerefox_metadata_search`) when updating a document. If it's stale you get a **conflict** — re-read the document, merge your changes into the latest content, retry with the new hash. **Never resolve a conflict by overwriting blindly** — the current content includes another writer's work. `last_write_wins: true` skips the check; use it ONLY when an external source of truth makes conflicts meaningless (file re-sync), never to silence a conflict.\n10. **Project memberships — non-destructive by default**: on `cerefox_ingest` updates, **`project_name` (singular) is a non-destructive add** (ensures membership, preserves others). Use **`project_names` (list)** when you want to set the doc's full project set in one call (destructive replace). For metadata-only project changes without writing content, use **`cerefox_set_document_projects(document_id, project_names)`** — that tool is the destructive-replace contract made explicit. Never call `cerefox_set_document_projects` with a single name when you mean \"add\" — that would REMOVE the doc from all other projects. When in doubt, use `cerefox_ingest` with singular `project_name`.\n\n## Update Workflow (ID-based -- preferred)\n\n```\nsearch(\"topic\") -> find doc [id: abc123] -> get_document(abc123) -> note its content_hash -> modify ->\ningest(title=\"Same Title\", content=\"...\", document_id=\"abc123\",\n expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```\n\nOn a **conflict** error: get_document again (fresh content + fresh hash) -> merge your changes -> retry with the new hash.\n\n## Update Workflow (title-based -- fallback)\n\n```\nsearch(\"topic\") -> find doc (note its hash) -> modify ->\ningest(title=\"Same Title\", content=\"...\", update_if_exists=true,\n expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```\n\n## Catch-Up Workflow\n\n```\nmetadata_search(metadata_filter={\"type\": \"decision-log\"}, updated_since=\"2026-03-28T00:00:00Z\")\n```\n\n## CLI fallback (when MCP is unavailable)\n\nIf `cerefox_search` is not in your tool list, your user has likely installed the Cerefox CLI. The canonical invocation is plain **`cerefox <subcommand>`** (the TypeScript CLI, installed via `npm install -g @cerefox/memory`). It uses a resource-verb shape (`cerefox document get`, `cerefox project list`, …).\n\nSame operations, same conventions. Full reference: [`docs/guides/cli.md`](docs/guides/cli.md). CLI flag names match MCP parameter names exactly (e.g. `metadata_filter` ↔ `--metadata-filter`); common flags also have single-letter short forms (`-f`, `-p`, `-c`, `-m`, `-u`, `-a`, `-r`). Use the canonical long name (what `--help` shows) or its short form — there are no long-form aliases like `--filter` or `--count`.\n\n| MCP tool | CLI |\n|---|---|\n| `cerefox_search` | `cerefox search \"<q>\" --requestor \"<your-name>\"` |\n| `cerefox_ingest` (paste) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_ingest` (update by ID) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --document-id \"<uuid>\" --expected-content-hash \"<hash>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_get_document` | `cerefox document get <id> --version-id <vid> --requestor \"<your-name>\"` |\n| `cerefox_list_versions` | `cerefox document version list <id> --requestor \"<your-name>\"` |\n| `cerefox_list_projects` | `cerefox project list --requestor \"<your-name>\"` |\n| `cerefox_list_metadata_keys` | `cerefox metadata keys` |\n| `cerefox_metadata_search` | `cerefox metadata search --metadata-filter '<json>' --requestor \"<your-name>\"` (list a project: `cerefox document list --project <name>`) |\n| `cerefox_set_document_projects` | `cerefox document set-projects <id> <name...> --author \"<your-name>\" --author-type agent` (or `--clear` to remove all) |\n| `cerefox_get_audit_log` | `cerefox audit list --requestor \"<your-name>\"` (add `--json` for scripted access) |\n| `cerefox_get_help` | `cerefox guides show agent-quick-reference` (or `cerefox guides list` for the full bundled-docs index) |\n\n**Set identity on every call**, exactly as you would on MCP:\n- Writes (`document ingest`, `document ingest-dir`): `--author \"<your-name>\" --author-type agent`\n- Reads: `--requestor \"<your-name>\"`\n\nOr have your user set `CEREFOX_AUTHOR_NAME` / `CEREFOX_AUTHOR_TYPE` / `CEREFOX_REQUESTOR_NAME` in their `.env` to apply defaults once.\n";
|
|
14
|
+
export const HELP_FULL = "# Cerefox Knowledge Base -- Agent Quick Reference\n\nCerefox is a persistent, shared knowledge base. You have **10 MCP tools** (9 of them have CLI equivalents — `cerefox_get_help` is MCP-only). For the full guide, search Cerefox for \"How AI Agents Use Cerefox\" or call `cerefox_get_help` to retrieve this content over MCP.\n\n## Tools\n\n| Tool | Purpose | Key params |\n|------|---------|------------|\n| `cerefox_search` | Find documents (hybrid FTS + semantic) | `query` (required), `project_name`, `metadata_filter`, `requestor` |\n| `cerefox_ingest` | Save or update a document | `title`, `content` (required), `document_id` (update by ID), `expected_content_hash` (**required on content updates** — see rule 9), `last_write_wins`, `update_if_exists`, `project_name` (single, non-destructive add on update), `project_names` (list, destructive replace on update), `metadata` (omit on update to keep existing tags; `{}` clears), `author` |\n| `cerefox_get_document` | Get full document by ID (header includes `content_hash` — the update token) | `document_id` (required) |\n| `cerefox_list_versions` | Version history of a document | `document_id` (required) |\n| `cerefox_metadata_search` | Find or list docs by metadata, project, or time (no text query) | `metadata_filter`, `project_name` (list a project's docs), `updated_since`, `include_content` — **at least one** of metadata_filter/project_name/updated_since/created_since |\n| `cerefox_list_metadata_keys` | Discover available metadata keys | (none required) |\n| `cerefox_list_projects` | List all projects | (none required) |\n| `cerefox_set_document_projects` | Set doc's project memberships to exactly the given list (destructive replace; metadata-only, no content change) | `document_id`, `project_names` (required) |\n| `cerefox_get_audit_log` | Query write operation history | `document_id`, `author`, `operation`, `since` |\n| `cerefox_get_help` | Retrieve Cerefox conventions (this reference) over MCP. **Call this whenever uncertain.** | `topic` (optional, case-insensitive H2 substring match) |\n\n## Essential Rules\n\n1. **Search before ingesting** -- check if the document exists first.\n2. **Prefer ID-based updates** -- pass `document_id` from search results for deterministic updates. Falls back to title-matching with `update_if_exists: true`.\n3. **Set `author`/`requestor`** to your name on every call (e.g., \"Claude Code\", \"archiver\"). On MCP, pass as parameters. On CLI, pass `--author`/`--author-type`/`--requestor` flags, or rely on `CEREFOX_AUTHOR_NAME`/`CEREFOX_AUTHOR_TYPE`/`CEREFOX_REQUESTOR_NAME` env vars set in the user's `.env`.\n4. **Use `document_id` from search results** `[id: uuid]` for get_document and list_versions.\n5. **Add metadata** -- at minimum `type` (\"decision-log\", \"research\", \"design-doc\") and `status` (\"active\", \"draft\").\n6. **Write structured Markdown** with H1/H2/H3 headings for good chunking and search.\n7. **Deletes are soft (recoverable); purge is web-UI-only.** If you decide to delete, surface it to the user (`I soft-deleted X — recoverable from the Cerefox web UI trash`). You cannot un-do your own delete from agent code by design.\n8. **Cross-doc links inside content**: **always use `[Text](document-uuid)`.** UUIDs are the only fully reliable link form — stable across title changes, never ambiguous, no encoding gotchas. Every `cerefox_search` result shows `[id: <uuid>]` after the title; grab it and use it. Title-based linking (`[Text](<Title With Spaces>)`) is fragile (breaks on colons, parens, ampersands, brackets — silently navigates to wrong page) — **don't write title-based links**; do an extra search to get the UUID instead. Repo-path forms (`[Text](docs/path.md)`) exist for repo-ingested files; don't construct manually. See `AGENT_GUIDE.md → Writing linkable content` for the full rule.\n9. **Concurrency: content updates require `expected_content_hash`.** Pass the `content_hash` you read (shown by `cerefox_get_document`, `cerefox_search`, and `cerefox_metadata_search`) when updating a document. If it's stale you get a **conflict** — re-read the document, merge your changes into the latest content, retry with the new hash. **Never resolve a conflict by overwriting blindly** — the current content includes another writer's work. `last_write_wins: true` skips the check; use it ONLY when an external source of truth makes conflicts meaningless (file re-sync), never to silence a conflict.\n10. **Search: prefer a few distinctive terms; heed `below confidence`.** When nothing clears the relevance threshold, `cerefox_search` returns the closest candidates prefixed with a `below confidence` warning instead of an empty set — that flag means **weak signal, not absent knowledge**: check the candidates' scores and titles before concluding the KB lacks the content. A truly empty response means nothing even weakly related exists.\n11. **Project memberships — non-destructive by default**: on `cerefox_ingest` updates, **`project_name` (singular) is a non-destructive add** (ensures membership, preserves others). Use **`project_names` (list)** when you want to set the doc's full project set in one call (destructive replace). For metadata-only project changes without writing content, use **`cerefox_set_document_projects(document_id, project_names)`** — that tool is the destructive-replace contract made explicit. Never call `cerefox_set_document_projects` with a single name when you mean \"add\" — that would REMOVE the doc from all other projects. When in doubt, use `cerefox_ingest` with singular `project_name`.\n\n## Update Workflow (ID-based -- preferred)\n\n```\nsearch(\"topic\") -> find doc [id: abc123] -> get_document(abc123) -> note its content_hash -> modify ->\ningest(title=\"Same Title\", content=\"...\", document_id=\"abc123\",\n expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```\n\nOn a **conflict** error: get_document again (fresh content + fresh hash) -> merge your changes -> retry with the new hash.\n\n## Update Workflow (title-based -- fallback)\n\n```\nsearch(\"topic\") -> find doc (note its hash) -> modify ->\ningest(title=\"Same Title\", content=\"...\", update_if_exists=true,\n expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```\n\n## Catch-Up Workflow\n\n```\nmetadata_search(metadata_filter={\"type\": \"decision-log\"}, updated_since=\"2026-03-28T00:00:00Z\")\n```\n\n## CLI fallback (when MCP is unavailable)\n\nIf `cerefox_search` is not in your tool list, your user has likely installed the Cerefox CLI. The canonical invocation is plain **`cerefox <subcommand>`** (the TypeScript CLI, installed via `npm install -g @cerefox/memory`). It uses a resource-verb shape (`cerefox document get`, `cerefox project list`, …).\n\nSame operations, same conventions. Full reference: [`docs/guides/cli.md`](docs/guides/cli.md). CLI flag names match MCP parameter names exactly (e.g. `metadata_filter` ↔ `--metadata-filter`); common flags also have single-letter short forms (`-f`, `-p`, `-c`, `-m`, `-u`, `-a`, `-r`). Use the canonical long name (what `--help` shows) or its short form — there are no long-form aliases like `--filter` or `--count`.\n\n| MCP tool | CLI |\n|---|---|\n| `cerefox_search` | `cerefox search \"<q>\" --requestor \"<your-name>\"` |\n| `cerefox_ingest` (paste) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_ingest` (update by ID) | `printf '...' \\| cerefox document ingest --paste --title \"<t>\" --document-id \"<uuid>\" --expected-content-hash \"<hash>\" --author \"<your-name>\" --author-type agent` |\n| `cerefox_get_document` | `cerefox document get <id> --version-id <vid> --requestor \"<your-name>\"` |\n| `cerefox_list_versions` | `cerefox document version list <id> --requestor \"<your-name>\"` |\n| `cerefox_list_projects` | `cerefox project list --requestor \"<your-name>\"` |\n| `cerefox_list_metadata_keys` | `cerefox metadata keys` |\n| `cerefox_metadata_search` | `cerefox metadata search --metadata-filter '<json>' --requestor \"<your-name>\"` (list a project: `cerefox document list --project <name>`) |\n| `cerefox_set_document_projects` | `cerefox document set-projects <id> <name...> --author \"<your-name>\" --author-type agent` (or `--clear` to remove all) |\n| `cerefox_get_audit_log` | `cerefox audit list --requestor \"<your-name>\"` (add `--json` for scripted access) |\n| `cerefox_get_help` | `cerefox guides show agent-quick-reference` (or `cerefox guides list` for the full bundled-docs index) |\n\n**Set identity on every call**, exactly as you would on MCP:\n- Writes (`document ingest`, `document ingest-dir`): `--author \"<your-name>\" --author-type agent`\n- Reads: `--requestor \"<your-name>\"`\n\nOr have your user set `CEREFOX_AUTHOR_NAME` / `CEREFOX_AUTHOR_TYPE` / `CEREFOX_REQUESTOR_NAME` in their `.env` to apply defaults once.\n";
|
|
15
15
|
|
|
16
16
|
/** Sections keyed by their H2 heading text (lower-cased for matching). */
|
|
17
17
|
export const HELP_SECTIONS: Record<string, string> = {
|
|
18
18
|
"Tools": "## Tools\n\n| Tool | Purpose | Key params |\n|------|---------|------------|\n| `cerefox_search` | Find documents (hybrid FTS + semantic) | `query` (required), `project_name`, `metadata_filter`, `requestor` |\n| `cerefox_ingest` | Save or update a document | `title`, `content` (required), `document_id` (update by ID), `expected_content_hash` (**required on content updates** — see rule 9), `last_write_wins`, `update_if_exists`, `project_name` (single, non-destructive add on update), `project_names` (list, destructive replace on update), `metadata` (omit on update to keep existing tags; `{}` clears), `author` |\n| `cerefox_get_document` | Get full document by ID (header includes `content_hash` — the update token) | `document_id` (required) |\n| `cerefox_list_versions` | Version history of a document | `document_id` (required) |\n| `cerefox_metadata_search` | Find or list docs by metadata, project, or time (no text query) | `metadata_filter`, `project_name` (list a project's docs), `updated_since`, `include_content` — **at least one** of metadata_filter/project_name/updated_since/created_since |\n| `cerefox_list_metadata_keys` | Discover available metadata keys | (none required) |\n| `cerefox_list_projects` | List all projects | (none required) |\n| `cerefox_set_document_projects` | Set doc's project memberships to exactly the given list (destructive replace; metadata-only, no content change) | `document_id`, `project_names` (required) |\n| `cerefox_get_audit_log` | Query write operation history | `document_id`, `author`, `operation`, `since` |\n| `cerefox_get_help` | Retrieve Cerefox conventions (this reference) over MCP. **Call this whenever uncertain.** | `topic` (optional, case-insensitive H2 substring match) |",
|
|
19
|
-
"Essential Rules": "## Essential Rules\n\n1. **Search before ingesting** -- check if the document exists first.\n2. **Prefer ID-based updates** -- pass `document_id` from search results for deterministic updates. Falls back to title-matching with `update_if_exists: true`.\n3. **Set `author`/`requestor`** to your name on every call (e.g., \"Claude Code\", \"archiver\"). On MCP, pass as parameters. On CLI, pass `--author`/`--author-type`/`--requestor` flags, or rely on `CEREFOX_AUTHOR_NAME`/`CEREFOX_AUTHOR_TYPE`/`CEREFOX_REQUESTOR_NAME` env vars set in the user's `.env`.\n4. **Use `document_id` from search results** `[id: uuid]` for get_document and list_versions.\n5. **Add metadata** -- at minimum `type` (\"decision-log\", \"research\", \"design-doc\") and `status` (\"active\", \"draft\").\n6. **Write structured Markdown** with H1/H2/H3 headings for good chunking and search.\n7. **Deletes are soft (recoverable); purge is web-UI-only.** If you decide to delete, surface it to the user (`I soft-deleted X — recoverable from the Cerefox web UI trash`). You cannot un-do your own delete from agent code by design.\n8. **Cross-doc links inside content**: **always use `[Text](document-uuid)`.** UUIDs are the only fully reliable link form — stable across title changes, never ambiguous, no encoding gotchas. Every `cerefox_search` result shows `[id: <uuid>]` after the title; grab it and use it. Title-based linking (`[Text](<Title With Spaces>)`) is fragile (breaks on colons, parens, ampersands, brackets — silently navigates to wrong page) — **don't write title-based links**; do an extra search to get the UUID instead. Repo-path forms (`[Text](docs/path.md)`) exist for repo-ingested files; don't construct manually. See `AGENT_GUIDE.md → Writing linkable content` for the full rule.\n9. **Concurrency: content updates require `expected_content_hash`.** Pass the `content_hash` you read (shown by `cerefox_get_document`, `cerefox_search`, and `cerefox_metadata_search`) when updating a document. If it's stale you get a **conflict** — re-read the document, merge your changes into the latest content, retry with the new hash. **Never resolve a conflict by overwriting blindly** — the current content includes another writer's work. `last_write_wins: true` skips the check; use it ONLY when an external source of truth makes conflicts meaningless (file re-sync), never to silence a conflict.\n10. **Project memberships — non-destructive by default**: on `cerefox_ingest` updates, **`project_name` (singular) is a non-destructive add** (ensures membership, preserves others). Use **`project_names` (list)** when you want to set the doc's full project set in one call (destructive replace). For metadata-only project changes without writing content, use **`cerefox_set_document_projects(document_id, project_names)`** — that tool is the destructive-replace contract made explicit. Never call `cerefox_set_document_projects` with a single name when you mean \"add\" — that would REMOVE the doc from all other projects. When in doubt, use `cerefox_ingest` with singular `project_name`.",
|
|
19
|
+
"Essential Rules": "## Essential Rules\n\n1. **Search before ingesting** -- check if the document exists first.\n2. **Prefer ID-based updates** -- pass `document_id` from search results for deterministic updates. Falls back to title-matching with `update_if_exists: true`.\n3. **Set `author`/`requestor`** to your name on every call (e.g., \"Claude Code\", \"archiver\"). On MCP, pass as parameters. On CLI, pass `--author`/`--author-type`/`--requestor` flags, or rely on `CEREFOX_AUTHOR_NAME`/`CEREFOX_AUTHOR_TYPE`/`CEREFOX_REQUESTOR_NAME` env vars set in the user's `.env`.\n4. **Use `document_id` from search results** `[id: uuid]` for get_document and list_versions.\n5. **Add metadata** -- at minimum `type` (\"decision-log\", \"research\", \"design-doc\") and `status` (\"active\", \"draft\").\n6. **Write structured Markdown** with H1/H2/H3 headings for good chunking and search.\n7. **Deletes are soft (recoverable); purge is web-UI-only.** If you decide to delete, surface it to the user (`I soft-deleted X — recoverable from the Cerefox web UI trash`). You cannot un-do your own delete from agent code by design.\n8. **Cross-doc links inside content**: **always use `[Text](document-uuid)`.** UUIDs are the only fully reliable link form — stable across title changes, never ambiguous, no encoding gotchas. Every `cerefox_search` result shows `[id: <uuid>]` after the title; grab it and use it. Title-based linking (`[Text](<Title With Spaces>)`) is fragile (breaks on colons, parens, ampersands, brackets — silently navigates to wrong page) — **don't write title-based links**; do an extra search to get the UUID instead. Repo-path forms (`[Text](docs/path.md)`) exist for repo-ingested files; don't construct manually. See `AGENT_GUIDE.md → Writing linkable content` for the full rule.\n9. **Concurrency: content updates require `expected_content_hash`.** Pass the `content_hash` you read (shown by `cerefox_get_document`, `cerefox_search`, and `cerefox_metadata_search`) when updating a document. If it's stale you get a **conflict** — re-read the document, merge your changes into the latest content, retry with the new hash. **Never resolve a conflict by overwriting blindly** — the current content includes another writer's work. `last_write_wins: true` skips the check; use it ONLY when an external source of truth makes conflicts meaningless (file re-sync), never to silence a conflict.\n10. **Search: prefer a few distinctive terms; heed `below confidence`.** When nothing clears the relevance threshold, `cerefox_search` returns the closest candidates prefixed with a `below confidence` warning instead of an empty set — that flag means **weak signal, not absent knowledge**: check the candidates' scores and titles before concluding the KB lacks the content. A truly empty response means nothing even weakly related exists.\n11. **Project memberships — non-destructive by default**: on `cerefox_ingest` updates, **`project_name` (singular) is a non-destructive add** (ensures membership, preserves others). Use **`project_names` (list)** when you want to set the doc's full project set in one call (destructive replace). For metadata-only project changes without writing content, use **`cerefox_set_document_projects(document_id, project_names)`** — that tool is the destructive-replace contract made explicit. Never call `cerefox_set_document_projects` with a single name when you mean \"add\" — that would REMOVE the doc from all other projects. When in doubt, use `cerefox_ingest` with singular `project_name`.",
|
|
20
20
|
"Update Workflow (ID-based -- preferred)": "## Update Workflow (ID-based -- preferred)\n\n```\nsearch(\"topic\") -> find doc [id: abc123] -> get_document(abc123) -> note its content_hash -> modify ->\ningest(title=\"Same Title\", content=\"...\", document_id=\"abc123\",\n expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```\n\nOn a **conflict** error: get_document again (fresh content + fresh hash) -> merge your changes -> retry with the new hash.",
|
|
21
21
|
"Update Workflow (title-based -- fallback)": "## Update Workflow (title-based -- fallback)\n\n```\nsearch(\"topic\") -> find doc (note its hash) -> modify ->\ningest(title=\"Same Title\", content=\"...\", update_if_exists=true,\n expected_content_hash=\"<the hash you read>\", author=\"my-agent\")\n```",
|
|
22
22
|
"Catch-Up Workflow": "## Catch-Up Workflow\n\n```\nmetadata_search(metadata_filter={\"type\": \"decision-log\"}, updated_since=\"2026-03-28T00:00:00Z\")\n```",
|
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
import type { MCPSupabaseClient } from "./types.ts";
|
|
19
19
|
|
|
20
20
|
import { getEmbedding, resolveEmbedderKind } from "../embeddings/index.ts";
|
|
21
|
-
import { applyByteBudget, getMaxResponseBytes, getMinSearchScore,
|
|
21
|
+
import { applyByteBudget, getMaxResponseBytes, getMinSearchScore,
|
|
22
|
+
getMinTermCoverage, logUsage } from "./_utils.ts";
|
|
22
23
|
import { lookupProjectId } from "./_projects.ts";
|
|
23
24
|
import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
|
|
24
25
|
|
|
@@ -33,6 +34,12 @@ async function handler(
|
|
|
33
34
|
const mode = (args.mode as string | undefined) ?? "docs";
|
|
34
35
|
const alpha = (args.alpha as number | undefined) ?? 0.7;
|
|
35
36
|
const min_score = (args.min_score as number | undefined) ?? getMinSearchScore();
|
|
37
|
+
// v1.0.4: coverage gate default from CEREFOX_MIN_TERM_COVERAGE; only sent
|
|
38
|
+
// when configured (see getMinTermCoverage — keeps pre-0.9.1 servers working).
|
|
39
|
+
const min_term_coverage =
|
|
40
|
+
(args.min_term_coverage as number | undefined) ?? getMinTermCoverage();
|
|
41
|
+
const coverageParam =
|
|
42
|
+
min_term_coverage !== undefined ? { p_min_term_coverage: min_term_coverage } : {};
|
|
36
43
|
const metadata_filter =
|
|
37
44
|
(args.metadata_filter as Record<string, string> | null | undefined) ?? null;
|
|
38
45
|
const requested_max_bytes = args.max_bytes as number | undefined;
|
|
@@ -84,6 +91,7 @@ async function handler(
|
|
|
84
91
|
p_match_count: match_count,
|
|
85
92
|
p_project_id: projectId,
|
|
86
93
|
...metaFilterParam,
|
|
94
|
+
...coverageParam,
|
|
87
95
|
};
|
|
88
96
|
} else if (mode === "hybrid") {
|
|
89
97
|
rpcName = "cerefox_hybrid_search";
|
|
@@ -96,6 +104,7 @@ async function handler(
|
|
|
96
104
|
p_project_id: projectId,
|
|
97
105
|
p_min_score: min_score,
|
|
98
106
|
...metaFilterParam,
|
|
107
|
+
...coverageParam,
|
|
99
108
|
};
|
|
100
109
|
} else {
|
|
101
110
|
rpcName = "cerefox_search_docs";
|
|
@@ -107,6 +116,7 @@ async function handler(
|
|
|
107
116
|
p_project_id: projectId,
|
|
108
117
|
p_min_score: min_score,
|
|
109
118
|
...metaFilterParam,
|
|
119
|
+
...coverageParam,
|
|
110
120
|
};
|
|
111
121
|
}
|
|
112
122
|
|
|
@@ -132,16 +142,24 @@ async function handler(
|
|
|
132
142
|
doc_title?: string;
|
|
133
143
|
full_content?: string;
|
|
134
144
|
best_score?: number;
|
|
145
|
+
score?: number;
|
|
135
146
|
is_partial?: boolean;
|
|
136
147
|
chunk_count?: number;
|
|
137
148
|
total_chars?: number;
|
|
138
149
|
content_hash?: string;
|
|
150
|
+
below_confidence?: boolean;
|
|
139
151
|
}>;
|
|
140
152
|
|
|
153
|
+
// 28I: nothing cleared the relevance threshold, so the server returned its
|
|
154
|
+
// best-effort top candidates flagged below_confidence instead of an empty
|
|
155
|
+
// set (which agents misread as "this knowledge does not exist").
|
|
156
|
+
const belowConfidence = rows.length > 0 && rows.every((r) => r.below_confidence === true);
|
|
157
|
+
|
|
141
158
|
const parts: string[] = rows.map((row) => {
|
|
142
159
|
const title = row.doc_title ?? "Untitled";
|
|
143
160
|
const docId = row.document_id ? ` [id: ${row.document_id}]` : "";
|
|
144
|
-
const
|
|
161
|
+
const rawScore = row.best_score ?? row.score;
|
|
162
|
+
const score = rawScore != null ? ` (score: ${rawScore.toFixed(3)})` : "";
|
|
145
163
|
const partial = row.is_partial
|
|
146
164
|
? ` -- partial (${row.chunk_count} of ${(row.total_chars ?? 0).toLocaleString()} chars)`
|
|
147
165
|
: "";
|
|
@@ -151,6 +169,12 @@ async function handler(
|
|
|
151
169
|
});
|
|
152
170
|
|
|
153
171
|
let output = parts.join("\n\n---\n\n");
|
|
172
|
+
if (belowConfidence) {
|
|
173
|
+
output =
|
|
174
|
+
`⚠ No results cleared the confidence threshold. Showing the closest ${rows.length} ` +
|
|
175
|
+
`candidate(s) with scores — judge relevance yourself; a low score means weak signal, ` +
|
|
176
|
+
`not necessarily absent knowledge.\n\n` + output;
|
|
177
|
+
}
|
|
154
178
|
if (truncated) {
|
|
155
179
|
output +=
|
|
156
180
|
`\n\n[Results truncated at ${usedBytes} bytes. Use a more specific query or a smaller match_count to see more.]`;
|
|
@@ -181,6 +205,27 @@ export const searchTool: ToolDefinition = {
|
|
|
181
205
|
'Optional JSONB containment filter. Only documents whose metadata contains ALL specified key-value pairs are returned. Example: {"type": "decision", "status": "active"}. Call cerefox_list_metadata_keys first to discover available keys and values. Omit to search all documents.',
|
|
182
206
|
additionalProperties: { type: "string" },
|
|
183
207
|
},
|
|
208
|
+
mode: {
|
|
209
|
+
type: "string",
|
|
210
|
+
enum: ["docs", "hybrid", "fts", "semantic"],
|
|
211
|
+
description:
|
|
212
|
+
"Search mode (default: docs — full reconstructed documents). hybrid: ranked chunks; fts: keyword-only (no embedding); semantic: vector-only.",
|
|
213
|
+
},
|
|
214
|
+
alpha: {
|
|
215
|
+
type: "number",
|
|
216
|
+
description:
|
|
217
|
+
"Hybrid fusion weight 0–1 (default 0.7): 1 = pure semantic, 0 = pure keyword.",
|
|
218
|
+
},
|
|
219
|
+
min_score: {
|
|
220
|
+
type: "number",
|
|
221
|
+
description:
|
|
222
|
+
"Minimum cosine similarity for vector-side results (default: server-configured, 0.5 OpenAI / 0.6 local embedder).",
|
|
223
|
+
},
|
|
224
|
+
min_term_coverage: {
|
|
225
|
+
type: "number",
|
|
226
|
+
description:
|
|
227
|
+
"Keyword OR-fallback confidence bar 0–1 (default 0.5): fraction of the query's meaningful terms a result must match to count as a confident hit; weaker matches return flagged below-confidence. 0 = any matching term. Needs schema ≥ 0.9.1.",
|
|
228
|
+
},
|
|
184
229
|
max_bytes: {
|
|
185
230
|
type: "integer",
|
|
186
231
|
description:
|
|
@@ -63,6 +63,19 @@ DROP FUNCTION IF EXISTS cerefox_search_docs(TEXT, VECTOR(768), INT, FLOAT, UUID,
|
|
|
63
63
|
DROP FUNCTION IF EXISTS cerefox_reconstruct_doc(UUID);
|
|
64
64
|
DROP FUNCTION IF EXISTS cerefox_get_document(UUID, UUID);
|
|
65
65
|
|
|
66
|
+
-- Iteration 28I (v1.0.3, search recall): below_confidence BOOLEAN added to the
|
|
67
|
+
-- return types of cerefox_hybrid_search and cerefox_search_docs (never-silently-
|
|
68
|
+
-- empty fallback). Drop the pre-change signatures first.
|
|
69
|
+
DROP FUNCTION IF EXISTS cerefox_hybrid_search(TEXT, VECTOR(768), INT, FLOAT, BOOLEAN, UUID, FLOAT, JSONB);
|
|
70
|
+
DROP FUNCTION IF EXISTS cerefox_search_docs(TEXT, VECTOR(768), INT, FLOAT, UUID, FLOAT, INT, INT, JSONB);
|
|
71
|
+
|
|
72
|
+
-- Iteration 28I follow-up (v1.0.4, term-coverage gate): p_min_term_coverage
|
|
73
|
+
-- added to the search RPC signatures (new arg count = new function; the old
|
|
74
|
+
-- overloads must go or PostgREST calls become ambiguous).
|
|
75
|
+
DROP FUNCTION IF EXISTS cerefox_hybrid_search(TEXT, VECTOR(768), INT, FLOAT, BOOLEAN, UUID, FLOAT, JSONB);
|
|
76
|
+
DROP FUNCTION IF EXISTS cerefox_fts_search(TEXT, INT, UUID, JSONB);
|
|
77
|
+
DROP FUNCTION IF EXISTS cerefox_search_docs(TEXT, VECTOR(768), INT, FLOAT, UUID, FLOAT, INT, INT, JSONB);
|
|
78
|
+
|
|
66
79
|
-- ── Shared return type note ────────────────────────────────────────────────────
|
|
67
80
|
-- All chunk-level search RPCs return the same shape for consistency:
|
|
68
81
|
-- chunk_id, document_id, chunk_index, title, content, heading_path,
|
|
@@ -90,7 +103,14 @@ CREATE OR REPLACE FUNCTION cerefox_hybrid_search(
|
|
|
90
103
|
p_use_upgrade BOOLEAN DEFAULT FALSE,
|
|
91
104
|
p_project_id UUID DEFAULT NULL,
|
|
92
105
|
p_min_score FLOAT DEFAULT 0.0,
|
|
93
|
-
p_metadata_filter JSONB DEFAULT NULL
|
|
106
|
+
p_metadata_filter JSONB DEFAULT NULL,
|
|
107
|
+
-- 28I follow-up (v1.0.4): in OR-fallback mode, the unconditional FTS pass
|
|
108
|
+
-- requires at least this fraction of the query's meaningful (non-stopword,
|
|
109
|
+
-- deduplicated) terms to match the chunk. Under AND semantics a match
|
|
110
|
+
-- meant 100% of terms — the pass this gate generalizes. Chunks below the
|
|
111
|
+
-- bar can still pass via the vector threshold, else they are
|
|
112
|
+
-- below-confidence material. 0 restores the pre-gate OR behavior.
|
|
113
|
+
p_min_term_coverage FLOAT DEFAULT 0.5
|
|
94
114
|
)
|
|
95
115
|
RETURNS TABLE (
|
|
96
116
|
chunk_id UUID,
|
|
@@ -106,7 +126,11 @@ RETURNS TABLE (
|
|
|
106
126
|
doc_project_ids UUID[],
|
|
107
127
|
doc_project_names TEXT[],
|
|
108
128
|
doc_metadata JSONB,
|
|
109
|
-
version_count INT
|
|
129
|
+
version_count INT,
|
|
130
|
+
-- 28I: TRUE on every row of a below-confidence fallback response — nothing
|
|
131
|
+
-- cleared the pass-filter, so these are the best-effort top candidates for
|
|
132
|
+
-- the caller to judge (scores included). FALSE on all normal results.
|
|
133
|
+
below_confidence BOOLEAN
|
|
110
134
|
)
|
|
111
135
|
LANGUAGE plpgsql
|
|
112
136
|
SECURITY DEFINER
|
|
@@ -120,15 +144,81 @@ DECLARE
|
|
|
120
144
|
-- websearch operators (phrase, OR, NOT); semantic ranking is the soft-match
|
|
121
145
|
-- layer for "broadly related". If operator support is ever needed, gate it
|
|
122
146
|
-- behind an opt-in flag rather than changing the default.
|
|
123
|
-
|
|
147
|
+
--
|
|
148
|
+
-- 28I progressive relaxation: AND-first, OR-fallback. The AND query stays
|
|
149
|
+
-- the primary match (byte-identical behavior whenever it matches anything),
|
|
150
|
+
-- but when it matches ZERO chunks (under the same project/metadata filters),
|
|
151
|
+
-- we retry with an OR-composition of the same tokens: one absent term then
|
|
152
|
+
-- no longer vetoes the terms that DO occur, and ts_rank_cd naturally ranks
|
|
153
|
+
-- chunks matching more terms higher. Multi-term evidence accumulates
|
|
154
|
+
-- instead of vetoing.
|
|
155
|
+
query_fts_and tsquery := plainto_tsquery('english', p_query_text);
|
|
156
|
+
query_fts_or tsquery := NULL;
|
|
157
|
+
query_fts tsquery;
|
|
158
|
+
tok TEXT;
|
|
159
|
+
tok_q tsquery;
|
|
160
|
+
and_matches BOOLEAN := FALSE;
|
|
161
|
+
-- v1.0.4 coverage gate: the per-token queries (deduplicated by normalized
|
|
162
|
+
-- lexeme text, so "run running" counts once) and their count.
|
|
163
|
+
tok_queries tsquery[] := '{}';
|
|
164
|
+
seen_tokens TEXT[] := '{}';
|
|
165
|
+
total_tokens INT;
|
|
124
166
|
candidate_count INT := p_match_count * 5;
|
|
125
167
|
BEGIN
|
|
168
|
+
-- Build the OR-composed query: plainto each whitespace token (so tokens get
|
|
169
|
+
-- the same normalization/stemming as the AND path), skip stopword-only
|
|
170
|
+
-- tokens, dedupe by normalized form, fold with the tsquery OR operator (||).
|
|
171
|
+
FOR tok IN SELECT unnest(regexp_split_to_array(trim(p_query_text), '\s+')) LOOP
|
|
172
|
+
tok_q := plainto_tsquery('english', tok);
|
|
173
|
+
IF numnode(tok_q) > 0 AND NOT (tok_q::TEXT = ANY(seen_tokens)) THEN
|
|
174
|
+
seen_tokens := seen_tokens || tok_q::TEXT;
|
|
175
|
+
tok_queries := tok_queries || tok_q;
|
|
176
|
+
query_fts_or := CASE WHEN query_fts_or IS NULL
|
|
177
|
+
THEN tok_q ELSE query_fts_or || tok_q END;
|
|
178
|
+
END IF;
|
|
179
|
+
END LOOP;
|
|
180
|
+
total_tokens := COALESCE(array_length(tok_queries, 1), 0);
|
|
181
|
+
|
|
182
|
+
-- Does the strict AND query match anything at all (under the caller's
|
|
183
|
+
-- filters)? Cheap probe against the partial FTS index.
|
|
184
|
+
IF numnode(query_fts_and) > 0 THEN
|
|
185
|
+
SELECT EXISTS (
|
|
186
|
+
SELECT 1
|
|
187
|
+
FROM cerefox_chunks c
|
|
188
|
+
JOIN cerefox_documents d ON c.document_id = d.id
|
|
189
|
+
WHERE c.version_id IS NULL
|
|
190
|
+
AND d.deleted_at IS NULL
|
|
191
|
+
AND c.fts @@ query_fts_and
|
|
192
|
+
AND (p_project_id IS NULL OR EXISTS (
|
|
193
|
+
SELECT 1 FROM cerefox_document_projects dp
|
|
194
|
+
WHERE dp.document_id = d.id AND dp.project_id = p_project_id
|
|
195
|
+
))
|
|
196
|
+
AND (p_metadata_filter IS NULL OR d.metadata @> p_metadata_filter)
|
|
197
|
+
) INTO and_matches;
|
|
198
|
+
END IF;
|
|
199
|
+
|
|
200
|
+
query_fts := CASE WHEN and_matches THEN query_fts_and
|
|
201
|
+
ELSE COALESCE(query_fts_or, query_fts_and) END;
|
|
202
|
+
|
|
126
203
|
RETURN QUERY
|
|
127
204
|
WITH
|
|
128
205
|
fts_results AS (
|
|
129
206
|
SELECT
|
|
130
207
|
c.id,
|
|
131
|
-
ts_rank_cd(c.fts, query_fts)::FLOAT AS fts_score
|
|
208
|
+
ts_rank_cd(c.fts, query_fts)::FLOAT AS fts_score,
|
|
209
|
+
-- v1.0.4 coverage gate: in AND mode a match means 100% of the
|
|
210
|
+
-- query's terms are present, so the unconditional pass is
|
|
211
|
+
-- earned by construction. In OR-fallback mode, earn it only
|
|
212
|
+
-- when at least p_min_term_coverage of the meaningful terms
|
|
213
|
+
-- match this chunk; weaker matches keep contributing their
|
|
214
|
+
-- fts_score to the fusion but must pass via the vector
|
|
215
|
+
-- threshold (or surface as below-confidence candidates).
|
|
216
|
+
CASE
|
|
217
|
+
WHEN and_matches OR total_tokens = 0 THEN TRUE
|
|
218
|
+
ELSE (SELECT COUNT(*) FROM unnest(tok_queries) tq
|
|
219
|
+
WHERE c.fts @@ tq)::FLOAT
|
|
220
|
+
>= p_min_term_coverage * total_tokens
|
|
221
|
+
END AS coverage_ok
|
|
132
222
|
FROM cerefox_chunks c
|
|
133
223
|
JOIN cerefox_documents d ON c.document_id = d.id
|
|
134
224
|
WHERE c.version_id IS NULL
|
|
@@ -175,15 +265,27 @@ BEGIN
|
|
|
175
265
|
(1.0 - p_alpha) * COALESCE(f.fts_score, 0.0)
|
|
176
266
|
) AS score,
|
|
177
267
|
COALESCE(v.vec_score, 0.0) AS vec_score,
|
|
178
|
-
-- TRUE when the chunk matched the @@ FTS operator
|
|
179
|
-
--
|
|
180
|
-
--
|
|
181
|
-
--
|
|
182
|
-
--
|
|
183
|
-
|
|
268
|
+
-- TRUE when the chunk matched the @@ FTS operator WITH enough
|
|
269
|
+
-- term coverage to earn the unconditional pass (v1.0.4; always
|
|
270
|
+
-- true for AND-mode matches). We use this flag rather than
|
|
271
|
+
-- vec_score to decide whether a chunk passes the threshold,
|
|
272
|
+
-- because in small corpora every chunk appears in vec_results
|
|
273
|
+
-- (LIMIT candidate_count covers all rows), so vec_score is
|
|
274
|
+
-- never NULL even for FTS-only matches.
|
|
275
|
+
(f.id IS NOT NULL AND f.coverage_ok) AS has_fts_match
|
|
184
276
|
FROM fts_results f
|
|
185
277
|
FULL OUTER JOIN vec_results v ON f.id = v.id
|
|
186
|
-
)
|
|
278
|
+
),
|
|
279
|
+
-- 28I: pass-filter as a flag rather than a WHERE, so we can fall back.
|
|
280
|
+
-- FTS matches pass through unconditionally: the @@ operator is a hard
|
|
281
|
+
-- gate and guarantees the query terms appear in the chunk. Vector-only
|
|
282
|
+
-- results (no FTS match) are filtered by the cosine threshold.
|
|
283
|
+
flagged AS (
|
|
284
|
+
SELECT *,
|
|
285
|
+
(combined.has_fts_match OR combined.vec_score >= p_min_score) AS passes
|
|
286
|
+
FROM combined
|
|
287
|
+
),
|
|
288
|
+
any_pass AS (SELECT bool_or(fl.passes) AS ok FROM flagged fl)
|
|
187
289
|
SELECT
|
|
188
290
|
c.id AS chunk_id,
|
|
189
291
|
c.document_id,
|
|
@@ -202,16 +304,22 @@ BEGIN
|
|
|
202
304
|
WHERE dp.document_id = d.id) AS doc_project_names,
|
|
203
305
|
d.metadata AS doc_metadata,
|
|
204
306
|
(SELECT COUNT(*)::INT FROM cerefox_document_versions dv
|
|
205
|
-
WHERE dv.document_id = d.id) AS version_count
|
|
206
|
-
|
|
307
|
+
WHERE dv.document_id = d.id) AS version_count,
|
|
308
|
+
-- 28I: when NOTHING clears the pass-filter, return the top candidates
|
|
309
|
+
-- anyway, flagged — an empty response reads to agent callers as "this
|
|
310
|
+
-- knowledge does not exist", the most expensive wrong conclusion a
|
|
311
|
+
-- memory layer can produce. "Truly nothing" (no candidates at all)
|
|
312
|
+
-- still returns zero rows.
|
|
313
|
+
NOT ap.ok AS below_confidence
|
|
314
|
+
FROM flagged cm
|
|
315
|
+
CROSS JOIN any_pass ap
|
|
207
316
|
JOIN cerefox_chunks c ON c.id = cm.id
|
|
208
317
|
JOIN cerefox_documents d ON c.document_id = d.id
|
|
209
|
-
|
|
210
|
-
-- and guarantees the query terms appear in the chunk.
|
|
211
|
-
-- Vector-only results (no FTS match) are filtered by the cosine threshold.
|
|
212
|
-
WHERE cm.has_fts_match OR cm.vec_score >= p_min_score
|
|
318
|
+
WHERE cm.passes OR NOT ap.ok
|
|
213
319
|
ORDER BY cm.score DESC
|
|
214
|
-
LIMIT p_match_count
|
|
320
|
+
LIMIT (SELECT CASE WHEN ap2.ok THEN p_match_count
|
|
321
|
+
ELSE LEAST(p_match_count, 3) END
|
|
322
|
+
FROM any_pass ap2);
|
|
215
323
|
END;
|
|
216
324
|
$$;
|
|
217
325
|
|
|
@@ -222,7 +330,10 @@ CREATE OR REPLACE FUNCTION cerefox_fts_search(
|
|
|
222
330
|
p_query_text TEXT,
|
|
223
331
|
p_match_count INT DEFAULT 10,
|
|
224
332
|
p_project_id UUID DEFAULT NULL,
|
|
225
|
-
p_metadata_filter JSONB DEFAULT NULL
|
|
333
|
+
p_metadata_filter JSONB DEFAULT NULL,
|
|
334
|
+
-- v1.0.4: see cerefox_hybrid_search. In OR-fallback mode results must
|
|
335
|
+
-- match at least this fraction of the query's meaningful terms.
|
|
336
|
+
p_min_term_coverage FLOAT DEFAULT 0.5
|
|
226
337
|
)
|
|
227
338
|
RETURNS TABLE (
|
|
228
339
|
chunk_id UUID,
|
|
@@ -245,9 +356,48 @@ SECURITY DEFINER
|
|
|
245
356
|
SET search_path = public, pg_catalog
|
|
246
357
|
AS $$
|
|
247
358
|
DECLARE
|
|
248
|
-
-- plainto_tsquery: see rationale
|
|
249
|
-
|
|
359
|
+
-- plainto_tsquery + 28I AND-first/OR-fallback: see the rationale comments
|
|
360
|
+
-- in cerefox_hybrid_search above.
|
|
361
|
+
query_fts_and tsquery := plainto_tsquery('english', p_query_text);
|
|
362
|
+
query_fts_or tsquery := NULL;
|
|
363
|
+
query_fts tsquery;
|
|
364
|
+
tok TEXT;
|
|
365
|
+
tok_q tsquery;
|
|
366
|
+
and_matches BOOLEAN := FALSE;
|
|
367
|
+
tok_queries tsquery[] := '{}';
|
|
368
|
+
seen_tokens TEXT[] := '{}';
|
|
369
|
+
total_tokens INT;
|
|
250
370
|
BEGIN
|
|
371
|
+
FOR tok IN SELECT unnest(regexp_split_to_array(trim(p_query_text), '\s+')) LOOP
|
|
372
|
+
tok_q := plainto_tsquery('english', tok);
|
|
373
|
+
IF numnode(tok_q) > 0 AND NOT (tok_q::TEXT = ANY(seen_tokens)) THEN
|
|
374
|
+
seen_tokens := seen_tokens || tok_q::TEXT;
|
|
375
|
+
tok_queries := tok_queries || tok_q;
|
|
376
|
+
query_fts_or := CASE WHEN query_fts_or IS NULL
|
|
377
|
+
THEN tok_q ELSE query_fts_or || tok_q END;
|
|
378
|
+
END IF;
|
|
379
|
+
END LOOP;
|
|
380
|
+
total_tokens := COALESCE(array_length(tok_queries, 1), 0);
|
|
381
|
+
|
|
382
|
+
IF numnode(query_fts_and) > 0 THEN
|
|
383
|
+
SELECT EXISTS (
|
|
384
|
+
SELECT 1
|
|
385
|
+
FROM cerefox_chunks c
|
|
386
|
+
JOIN cerefox_documents d ON c.document_id = d.id
|
|
387
|
+
WHERE c.version_id IS NULL
|
|
388
|
+
AND d.deleted_at IS NULL
|
|
389
|
+
AND c.fts @@ query_fts_and
|
|
390
|
+
AND (p_project_id IS NULL OR EXISTS (
|
|
391
|
+
SELECT 1 FROM cerefox_document_projects dp
|
|
392
|
+
WHERE dp.document_id = d.id AND dp.project_id = p_project_id
|
|
393
|
+
))
|
|
394
|
+
AND (p_metadata_filter IS NULL OR d.metadata @> p_metadata_filter)
|
|
395
|
+
) INTO and_matches;
|
|
396
|
+
END IF;
|
|
397
|
+
|
|
398
|
+
query_fts := CASE WHEN and_matches THEN query_fts_and
|
|
399
|
+
ELSE COALESCE(query_fts_or, query_fts_and) END;
|
|
400
|
+
|
|
251
401
|
RETURN QUERY
|
|
252
402
|
SELECT
|
|
253
403
|
c.id AS chunk_id,
|
|
@@ -273,6 +423,11 @@ BEGIN
|
|
|
273
423
|
WHERE c.version_id IS NULL
|
|
274
424
|
AND d.deleted_at IS NULL
|
|
275
425
|
AND c.fts @@ query_fts
|
|
426
|
+
-- v1.0.4 coverage gate (OR-fallback mode only): pure keyword search
|
|
427
|
+
-- returns only chunks matching enough of the query's terms.
|
|
428
|
+
AND (and_matches OR total_tokens = 0
|
|
429
|
+
OR (SELECT COUNT(*) FROM unnest(tok_queries) tq
|
|
430
|
+
WHERE c.fts @@ tq)::FLOAT >= p_min_term_coverage * total_tokens)
|
|
276
431
|
AND (p_project_id IS NULL OR EXISTS (
|
|
277
432
|
SELECT 1 FROM cerefox_document_projects dp
|
|
278
433
|
WHERE dp.document_id = d.id AND dp.project_id = p_project_id
|
|
@@ -583,7 +738,8 @@ CREATE OR REPLACE FUNCTION cerefox_search_docs(
|
|
|
583
738
|
p_min_score FLOAT DEFAULT 0.0,
|
|
584
739
|
p_small_to_big_threshold INT DEFAULT 20000,
|
|
585
740
|
p_context_window INT DEFAULT 1,
|
|
586
|
-
p_metadata_filter JSONB DEFAULT NULL
|
|
741
|
+
p_metadata_filter JSONB DEFAULT NULL,
|
|
742
|
+
p_min_term_coverage FLOAT DEFAULT 0.5
|
|
587
743
|
)
|
|
588
744
|
RETURNS TABLE (
|
|
589
745
|
document_id UUID,
|
|
@@ -602,7 +758,10 @@ RETURNS TABLE (
|
|
|
602
758
|
is_partial BOOL,
|
|
603
759
|
-- Optimistic-concurrency token (iter-32): the document's current
|
|
604
760
|
-- content_hash, to pass back as expected_content_hash on update.
|
|
605
|
-
content_hash TEXT
|
|
761
|
+
content_hash TEXT,
|
|
762
|
+
-- 28I: TRUE when this is a below-confidence fallback response (nothing
|
|
763
|
+
-- cleared the hybrid pass-filter). See cerefox_hybrid_search.
|
|
764
|
+
below_confidence BOOLEAN
|
|
606
765
|
)
|
|
607
766
|
LANGUAGE sql
|
|
608
767
|
SECURITY DEFINER
|
|
@@ -620,7 +779,8 @@ AS $$
|
|
|
620
779
|
p_use_upgrade := FALSE,
|
|
621
780
|
p_project_id := p_project_id,
|
|
622
781
|
p_min_score := p_min_score,
|
|
623
|
-
p_metadata_filter := p_metadata_filter
|
|
782
|
+
p_metadata_filter := p_metadata_filter,
|
|
783
|
+
p_min_term_coverage := p_min_term_coverage
|
|
624
784
|
)
|
|
625
785
|
),
|
|
626
786
|
best_per_doc AS (
|
|
@@ -635,6 +795,7 @@ AS $$
|
|
|
635
795
|
cr.doc_project_ids,
|
|
636
796
|
cr.doc_project_names,
|
|
637
797
|
cr.version_count,
|
|
798
|
+
cr.below_confidence,
|
|
638
799
|
d.updated_at AS doc_updated_at,
|
|
639
800
|
d.content_hash
|
|
640
801
|
FROM chunk_results cr
|
|
@@ -720,7 +881,8 @@ AS $$
|
|
|
720
881
|
td.doc_updated_at,
|
|
721
882
|
td.version_count,
|
|
722
883
|
ac.is_partial,
|
|
723
|
-
td.content_hash
|
|
884
|
+
td.content_hash,
|
|
885
|
+
td.below_confidence
|
|
724
886
|
FROM top_docs td
|
|
725
887
|
JOIN doc_sizes ds ON ds.document_id = td.document_id
|
|
726
888
|
JOIN all_content ac ON ac.document_id = td.document_id
|
|
@@ -1776,7 +1938,7 @@ SET search_path = public, pg_catalog
|
|
|
1776
1938
|
AS $$
|
|
1777
1939
|
-- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
|
|
1778
1940
|
-- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
|
|
1779
|
-
SELECT '0.
|
|
1941
|
+
SELECT '0.9.1'::TEXT;
|
|
1780
1942
|
$$;
|
|
1781
1943
|
|
|
1782
1944
|
-- ── cerefox_content_format_stats ─────────────────────────────────────────────
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
-- Requires extensions: vector (pgvector), uuid-ossp
|
|
6
6
|
-- These are enabled at the top of db_deploy.py before this file is applied.
|
|
7
7
|
--
|
|
8
|
-
-- @version: 0.
|
|
8
|
+
-- @version: 0.9.1
|
|
9
9
|
-- The `@version` marker above is read by the schema-version-mismatch banner
|
|
10
10
|
-- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
|
|
11
11
|
-- changes in a way that requires `cerefox server deploy` to be re-run —
|
|
@@ -117,10 +117,19 @@ This handles intermittent OpenAI API errors (500s) that would otherwise cause se
|
|
|
117
117
|
|
|
118
118
|
## Retrieval
|
|
119
119
|
|
|
120
|
+
> **Which paths read these?** Client-side tunables in this section are read
|
|
121
|
+
> from *your* `.env` by the **CLI**, the **local MCP server**, and `cerefox
|
|
122
|
+
> web`. The **remote MCP / Edge Function path** runs on Supabase and does not
|
|
123
|
+
> see your `.env` — it uses the server defaults unless the caller passes the
|
|
124
|
+
> per-call parameter (e.g. `min_score`, `min_term_coverage` on
|
|
125
|
+
> `cerefox_search`). Setting them as Supabase **Function secrets** may also
|
|
126
|
+
> work but is not a tested configuration.
|
|
127
|
+
|
|
120
128
|
| Variable | Default | Description |
|
|
121
129
|
|----------|---------|-------------|
|
|
122
130
|
| `CEREFOX_MAX_RESPONSE_BYTES` | `200000` | Maximum bytes in a single search response (local MCP path). See explanation below. |
|
|
123
131
|
| `CEREFOX_MIN_SEARCH_SCORE` | `0.50` (`0.60` with the local embedder) | Minimum cosine similarity for hybrid and semantic search results (0.0–1.0). The default is embedder-aware: nomic scores unrelated text higher than OpenAI, so `CEREFOX_EMBEDDER=local` raises the floor to 0.60. In **hybrid search**, chunks that matched the FTS keyword operator (`@@`) always pass through regardless of their vector score — the threshold only filters vector-only results. In **semantic search**, all results are filtered. The pure **FTS search** mode is unaffected. Increase for stricter precision; decrease for wider recall. |
|
|
132
|
+
| `CEREFOX_MIN_TERM_COVERAGE` | *(unset — server default `0.5`)* | Confidence bar for the keyword OR-fallback (v1.0.4, schema ≥ 0.9.1): when a strict all-terms match fails and search relaxes to any-term matching, a result counts as a confident hit only if it matches at least this fraction of the query's meaningful terms; weaker matches surface as below-confidence candidates. `0` restores pre-gate behavior (any matching term passes); `1` requires every term. Per-call override: `cerefox search --min-term-coverage`. Leave unset against pre-0.9.1 servers. |
|
|
124
133
|
| `CEREFOX_EMBED_MAX_INPUT_CHARS` | `20000` | Safety cap on the characters sent to the embedding model per input. The full chunk content is always stored and reconstructed untouched; only the embedding uses the (rare) truncated prefix, so an oversized chunk can never fail an ingest. |
|
|
125
134
|
| `CEREFOX_MODELS_DIR` | `~/.cerefox/models` (in-container: inside the data volume) | Where the local embedder caches downloaded model weights (Cerefox Local; `CEREFOX_EMBEDDER=local`). |
|
|
126
135
|
| `CEREFOX_ONNX_BATCH` | `4` | Texts per local-embedder inference call. Peak memory scales with this; the small default keeps ingest/reindex safe on small Docker VMs. |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cerefox/memory",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.4",
|
|
4
4
|
"description": "Cerefox — user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"homepage": "https://github.com/fstamatelopoulos/cerefox",
|