@cerefox/memory 1.14.2 → 1.14.4

package/dist/server-assets/supabase/functions/cerefox-metadata-search/index.ts

@@ -4,6 +4,7 @@ import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/inde
 import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
 import { callerIdentity } from "../../../_shared/mcp-tools/identity.ts";
 import { reviewWorkflowEnabled } from "../../../_shared/mcp-tools/feature-flags.ts";
+import { resolveByteBudget } from "../../../_shared/mcp-tools/_utils.ts";
 
 /**
  * cerefox-metadata-search -- Supabase Edge Function
@@ -103,8 +104,12 @@ Deno.serve(async (req: Request): Promise<Response> => {
     const include_content = body.include_content ?? false;
     const requested_max_bytes = body.max_bytes;
 
+    // One implementation of this arithmetic, shared with every other surface
+    // that takes a budget (#268): non-numeric and null both mean "unset" and
+    // fall back to the ceiling, while a real number of zero or less is
+    // honoured as "almost no budget".
     const max_bytes = include_content
-      ? Math.min(requested_max_bytes ?? MAX_BYTES, MAX_BYTES)
+      ? resolveByteBudget(requested_max_bytes, MAX_BYTES)
       : null;
 
     const supabaseUrl = Deno.env.get("SUPABASE_URL")!;
@@ -154,18 +159,90 @@ Deno.serve(async (req: Request): Promise<Response> => {
       });
     }
 
-    // Fire-and-forget usage logging
+    let rows = (data ?? []) as Array<Record<string, unknown>>;
+
+    // Never answer with a shorter list than what matched (#268).
+    //
+    // The RPC applies `p_max_bytes` server-side by stopping at the first row
+    // whose content does not fit, so this list has two indistinguishable
+    // causes: fewer documents matched, or the budget cut it short. When the
+    // first row is the oversized one the array comes back EMPTY, and a caller
+    // reads `[]` as "this knowledge does not exist" and stops looking — the
+    // false negative #254 exists to prevent, reached here through a sibling
+    // that never got the guard.
+    //
+    // The response shape stays a bare array, because Custom GPTs are
+    // configured against it: every matching document is listed, and only
+    // CONTENT is negotiable. Rows the budget could not afford come back
+    // content-free and marked, so `results.length` is always the true count
+    // and nothing is held back silently.
+    //
+    // Cost: a second RPC round-trip whenever a content-bearing search returns
+    // less than a full page, which is the common case rather than a rare one.
+    // The RPC signals no total, so the only alternative to asking is guessing
+    // whether a short list was cut — and guessing wrong is the bug.
+    if (include_content && max_bytes !== null && rows.length < limit) {
+      const { data: headerData, error: probeError } = await supabase.rpc(
+        "cerefox_metadata_search",
+        { ...params, p_include_content: false, p_max_bytes: null },
+      );
+      // supabase-js RESOLVES with `{ data: null, error }` for PostgREST and
+      // network failures rather than throwing, so a probe failure must be read
+      // from the error, not inferred from an empty list — reading it as "no
+      // documents" is the very false empty this branch prevents (#261).
+      if (probeError && rows.length === 0) {
+        // Falling through here would ship exactly the false empty this block
+        // exists to prevent — `200 []`, which a caller reads as "no such
+        // knowledge". An error is the honest answer: it says the question was
+        // not resolved, rather than answering it wrongly.
+        return new Response(
+          JSON.stringify({
+            error:
+              `Nothing fit max_bytes=${max_bytes} with include_content, and the follow-up ` +
+              `query that lists what matched failed: ${probeError.message}. This is NOT a ` +
+              `confirmed empty result — retry with a larger max_bytes, or include_content: false.`,
+          }),
+          { status: 502, headers: { ...CORS_HEADERS, "Content-Type": "application/json" } },
+        );
+      }
+      if (!probeError) {
+        const headers = (headerData ?? []) as Array<Record<string, unknown>>;
+        if (headers.length > rows.length) {
+          const withContent = new Map(rows.map((r) => [r.document_id as string, r]));
+          // The probe carries the full, correctly ordered match set; the
+          // content-bearing rows are folded into it by id so ordering is the
+          // RPC's, not an artefact of which rows happened to fit.
+          const merged = headers.map(
+            (h) => withContent.get(h.document_id as string) ?? { ...h, content_omitted: true },
+          );
+          // Two queries, two chances to disagree: the RPC orders by
+          // `updated_at DESC` with no tiebreaker under a LIMIT, and a
+          // concurrent write between the calls shifts the window. Anything the
+          // content query returned that the probe did not is APPENDED rather
+          // than dropped — losing a document we already hold, while fixing a
+          // bug about losing documents, would be its own joke.
+          const seen = new Set(merged.map((r) => r.document_id as string));
+          for (const r of rows) {
+            if (!seen.has(r.document_id as string)) merged.push(r);
+          }
+          rows = merged;
+        }
+      }
+    }
+
+    // Fire-and-forget usage logging. Counts what MATCHED, not what the budget
+    // allowed through: a budget-wiped search logging `0` misreports the store
+    // as empty in analytics (#259).
     Promise.resolve(supabase.rpc("cerefox_log_usage", {
       p_operation: "metadata_search",
       p_access_path: "edge-function",
       p_requestor: identityValue ?? null,
       p_query_text: JSON.stringify(metadata_filter),
-      p_result_count: (data ?? []).length,
+      p_result_count: rows.length,
       p_project_id: project_id,
     })).catch(() => {});
 
     // Presentation only: the same shared reader every other surface uses.
-    const rows = (data ?? []) as Array<Record<string, unknown>>;
     const showReview = await reviewWorkflowEnabled(supabase);
     const out = showReview
       ? rows

package/dist/server-assets/supabase/functions/cerefox-search/index.ts

@@ -5,7 +5,7 @@ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
 import { callerIdentity } from "../../../_shared/mcp-tools/identity.ts";
 import { capEmbeddingInput } from "../../../_shared/embeddings/index.ts";
 // One implementation of the byte budget, shared with the MCP tools (#254).
-import { applyByteBudget } from "../../../_shared/mcp-tools/_utils.ts";
+import { applyByteBudget, resolveByteBudget } from "../../../_shared/mcp-tools/_utils.ts";
 
 /**
  * cerefox-search — Supabase Edge Function
@@ -36,7 +36,16 @@ import { applyByteBudget } from "../../../_shared/mcp-tools/_utils.ts";
  *                                      `truncated` flag when results were dropped.
  *
  * Response: { results: [...], query, mode, match_count, project_name?,
- *             truncated: boolean, response_bytes: number }
+ *             truncated: boolean, response_bytes: number,
+ *             matched: number, degraded: boolean, note?: string }
+ *
+ * `matched` is how many results the query found, before the byte budget.
+ * `degraded` is true when everything that matched was larger than max_bytes:
+ * the items then carry NO content, they name what exists so the caller can
+ * re-ask with a larger budget. That header list is capped to max_bytes and so
+ * may be a SUBSET of `matched`, except that one item is always returned even
+ * if it exceeds the budget — an empty `results` would read as "nothing was
+ * found", which is `matched: 0` and nothing else (#254, #257, #261).
  *
  * Example agent prompt:
  *   "Invoke the cerefox-search edge function with query='knowledge management'
@@ -150,14 +159,6 @@ async function lookupProjectId(
   return data[0].id;
 }
 
-/**
- * Apply a byte budget to an array of result rows.
- *
- * Each row is serialised to JSON to measure its size. Rows are included in
- * order until the next row would push the running total over `maxBytes`.
- * Rows are always kept or dropped whole — content is never truncated
- * mid-document. Returns the accepted rows and a `truncated` flag.
- */
 const headers = {
   "Content-Type": "application/json",
   "Access-Control-Allow-Origin": "*",
@@ -214,7 +215,11 @@ Deno.serve(async (req: Request) => {
   } = body;
 
   // Enforce ceiling: agents may request less but never more than MAX_BYTES.
-  const max_bytes = Math.min(requested_max_bytes ?? MAX_BYTES, MAX_BYTES);
+  // Sanitised before clamping: `NaN` compares false against every budget
+  // check, so a non-numeric value bypassed the ceiling entirely (#266).
+  // A number of 0 or less still means "almost no budget"; only a missing or
+  // non-numeric value falls back to the ceiling (#267).
+  const max_bytes = resolveByteBudget(requested_max_bytes, MAX_BYTES);
   // Clamp match_count to [1, MAX_MATCH_COUNT] (bounds query work; see MAX_MATCH_COUNT).
   const match_count = Math.min(Math.max(1, Math.floor(Number(raw_match_count)) || 5), MAX_MATCH_COUNT);
 
@@ -362,10 +367,33 @@ Deno.serve(async (req: Request) => {
   // knowledge does not exist" to whatever is on the other end, so send the
   // rows WITHOUT their content instead: the caller learns what matched, how
   // big it is, and can re-ask with a larger budget or fetch one document.
+  //
+  // Both content columns have to go: `cerefox_search_docs` (mode "docs")
+  // returns `full_content`, while `cerefox_hybrid_search` and the FTS RPC
+  // return `content`. Stripping one of them shipped in v1.14.2 and returned
+  // 83 KB of chunk text against a 3 KB budget while the response claimed the
+  // content had been omitted (#257).
   const degraded = accepted.length === 0 && matched.length > 0;
-  const results = degraded
-    ? matched.map(({ full_content: _omitted, ...header }) => header)
-    : accepted;
+  // And the headers themselves are held to the budget the caller asked for:
+  // a `match_count` of 200 makes even a content-free list large. At least one
+  // survives regardless: an empty `results` is the "nothing was found" shape
+  // #254 exists to prevent, and a header row can exceed a very small budget
+  // on its own (#259).
+  const listed = degraded
+    ? (() => {
+        const headerRows = matched.map(
+          ({ full_content: _full, content: _chunk, ...header }) => header,
+        );
+        const fitted = applyByteBudget(headerRows, max_bytes).accepted;
+        return fitted.length > 0 ? fitted : headerRows.slice(0, 1);
+      })()
+    : [];
+  const results = degraded ? listed : accepted;
+  // What is actually being returned, which on a degraded response is the
+  // header list rather than the (empty) content-bearing set.
+  const responseBytes = degraded
+    ? new TextEncoder().encode(JSON.stringify(results)).length
+    : usedBytes;
 
   // Fire-and-forget usage logging (never blocks the response)
   Promise.resolve(supabase.rpc("cerefox_log_usage", {
@@ -388,14 +416,20 @@ Deno.serve(async (req: Request) => {
       project_name: project_name ?? null,
       metadata_filter: metadata_filter ?? null,
       truncated,
-      response_bytes: usedBytes,
+      response_bytes: responseBytes,
       matched: matched.length,
       degraded,
       ...(degraded
         ? {
             note:
               `${matched.length} result(s) matched but none fit max_bytes=${max_bytes}; ` +
-              `content omitted. Raise max_bytes, or fetch one document with cerefox-get-document.`,
+              `content omitted${
+                listed.length < matched.length
+                  ? `, and only ${listed.length} of them are listed here` +
+                    " (the rest did not fit either)"
+                  : ""
+              }. This is NOT an empty result. Raise max_bytes, or fetch one ` +
+              `document with cerefox-get-document.`,
           }
         : {}),
     }),

package/docs/guides/connect-agents.md

@@ -638,7 +638,7 @@ In the action editor, paste this schema (replace `<your-project-ref>`):
 openapi: 3.1.0
 info:
   title: Cerefox Knowledge Base
-  version: 4.1.0
+  version: 4.3.0
 servers:
   - url: https://<your-project-ref>.supabase.co/functions/v1
 paths:
@@ -697,7 +697,10 @@ paths:
                     Response size budget in bytes (server hard ceiling 200000).
                     Whole results are dropped (never truncated mid-document) until
                     the budget is met; the response sets `truncated: true` when this
-                    happens. Advanced; leave unset for the default.
+                    happens. One exception: when nothing fits at all, a single
+                    content-free header is returned even if it exceeds the budget,
+                    because an empty results array would read as "nothing was found"
+                    (see degraded). Advanced; leave unset for the default.
                 author:
                   type: string
                   description: >
@@ -713,9 +716,11 @@ paths:
             chunk_count, total_chars, best_score, is_partial.
             matched is how many results the query found, before the byte budget.
             When degraded is true, everything that matched was larger than max_bytes, so the
-            items carry NO full_content — they name what exists so you can re-ask with a
-            larger max_bytes or fetch one document. An empty results array with degraded
-            true is never "nothing was found"; results is empty only when matched is 0.
+            items carry NO content of any kind — they name what exists so you can re-ask with
+            a larger max_bytes or fetch one document. That header list is itself capped to
+            max_bytes, so it may be a SUBSET of matched (at least one item is always
+            returned). Read matched, never results.length, to know how much the query found:
+            "nothing was found" is matched == 0.
             is_partial is true when the document exceeded the small-to-big threshold — in that
             case full_content contains matched chunks plus their neighbours rather than the
             complete document, and total_chars still reflects the full document size.
@@ -1046,8 +1051,10 @@ paths:
                   type: integer
                   default: 200000
                   description: >
-                    Response size budget in bytes when include_content is true
-                    (whole results dropped to fit). Advanced; leave unset for the default.
+                    Response size budget in bytes when include_content is true.
+                    Content is dropped whole, never truncated mid-document, and a
+                    document whose content is dropped is still listed with
+                    content_omitted: true. Advanced; leave unset for the default.
                 author:
                   type: string
                   description: >
@@ -1062,6 +1069,12 @@ paths:
                version_count, content_hash, content }], plus review_status
             only while the store's review workflow is on (the key is absent
             when it is off).
+            The array lists EVERY matching document, so its length is the true
+            match count. When include_content is true and max_bytes cannot
+            carry a document's text, that document is still listed, with its
+            content omitted and "content_omitted": true set on the item — the
+            list is never silently shortened, and an empty array always means
+            nothing matched.
             content_hash is the concurrency token — pass it back as
             expected_content_hash when updating via ingestNote.
 ```
@@ -1130,7 +1143,7 @@ If the same content was already ingested (SHA-256 hash match), returns `"skipped
 | `mode` | string | `"docs"` | `"docs"` = full document results (recommended) |
 | `alpha` | number | 0.7 | Semantic weight (0 = FTS only, 1 = semantic only) |
 | `min_score` | number | 0.5 | Minimum cosine similarity threshold |
-| `max_bytes` | number | 200000 | Response size budget in bytes. Results are dropped whole (never truncated mid-document) once the budget is reached. The response includes `truncated: true` and `response_bytes` when the limit was hit. See "Response size limit" below. |
+| `max_bytes` | number | 200000 | Response size budget in bytes. Results are dropped whole (never truncated mid-document) once the budget is reached. The response includes `truncated: true` and `response_bytes` when the limit was hit. When nothing fits, `degraded: true` and a content-free header list come back instead of an empty `results` — always at least one item, even if it exceeds the budget. See "Response size limit" below. |
 
 **Response envelope fields:**
 

package/docs/guides/response-limits.md

@@ -8,11 +8,18 @@ explains how response size limits work and how to tune them.
 
 ## The key principle: opt-in limits, never truncate the web UI
 
-The web UI and CLI never truncate results. They have no size limit — the browser or terminal
-can handle arbitrarily large responses and there is no LLM context window to worry about.
+The web UI never truncates results. It has no size limit — the browser can handle
+arbitrarily large responses and there is no LLM context window to worry about.
 
-Limits are **opt-in per call**, used only on the MCP and Edge Function paths where an AI
-agent's context window matters. Callers always choose whether to apply a limit.
+Limits apply on the MCP, Edge Function **and CLI** paths. On MCP and the Edge Functions
+they exist because an AI agent's context window matters; the CLI applies the same default
+so that one setting (`CEREFOX_MAX_RESPONSE_BYTES`) governs every non-browser path, and
+raises or lowers it per call with `--max-bytes`.
+
+> **Changed in v0.10.2.** The CLI originally returned everything, like the web UI. It now
+> honours `CEREFOX_MAX_RESPONSE_BYTES` (200 000 default) and prints
+> `(results truncated at N bytes; use --max-bytes to raise)` when results are dropped.
+> This guide described the pre-v0.10.2 behaviour until v1.14.4.
 
 ---
 
@@ -21,7 +28,7 @@ agent's context window matters. Callers always choose whether to apply a limit.
 | Path | Limit behaviour |
 |------|----------------|
 | Web UI (`/search`) | **No limit** — all results returned |
-| CLI (`cerefox search`) | **No limit** — all results returned |
+| CLI (`cerefox search`) | Defaults to `CEREFOX_MAX_RESPONSE_BYTES` (200 000); raise or lower per call with `--max-bytes`. Announces truncation. |
 | Local MCP server (`cerefox mcp`) | Defaults to `CEREFOX_MAX_RESPONSE_BYTES` (200 000); agent can request less |
 | Edge Function (`cerefox-search`) | Defaults to 200 000 bytes; agent can request less via `max_bytes` body param |
 | Remote MCP (`cerefox-mcp` Edge Function) | Defaults to 200 000 bytes; agent can request less via `max_bytes` tool param |
@@ -30,13 +37,61 @@ agent's context window matters. Callers always choose whether to apply a limit.
 
 ## How limits are applied
 
-Truncation is always **whole-document**: results are dropped in full once adding the next
-document would exceed the budget. Cerefox never cuts a document mid-content.
+Truncation is always **whole-document**: a result is returned in full or not at all.
+Cerefox never cuts a document mid-content.
+
+A result that does not fit is **skipped**, not treated as the end of the list, so the
+returned set is not necessarily the top N by rank: one oversized document ranked first
+does not hide the smaller results behind it (v1.14.3 on the MCP tool; v1.14.4 on the
+`cerefox-search` Edge Function and the CLI, which both still stopped at the first
+oversized row). Anything skipped is named in the footer, so what is missing is always
+visible.
 
 When truncation occurs:
-- The local MCP server appends `[Results truncated at N bytes — ...]` to the response text.
+- The MCP tool appends a footer naming what was held back:
+  `[3 of 12 result(s) shown; 9 did not fit max_bytes=8000: Plan › Rollout (chunk 4) [id: …] and 8 more. Raise max_bytes, narrow the query, or lower match_count.]`
 - The Edge Function includes `"truncated": true` and `"response_bytes": N` in the JSON response.
 
+**The reply as a whole stays inside the budget**, footer and warnings included:
+they are measured in the same rendered bytes the caller receives (v1.14.3).
+When the budget is tight the framing gives way before the results do.
+
+**When nothing fits at all** — the smallest matching document is larger than
+the whole budget — the reply is NOT "no results found", which an agent acts on
+as "this knowledge does not exist". It is a header list naming what matched,
+its size and its id, prefixed with a warning and the remedy. The same case on
+the Edge Function sets `"degraded": true`, returns items with no content, and
+reports `"matched"`: read that, not `results.length`, to know what the query
+found. A reply may exceed `max_bytes` only by the framing that cannot be dropped
+without misleading you: the notice that results were held back, or the
+below-confidence advisory. Both are a few dozen bytes, and neither is ever
+traded for content. Returning 1 of 5 results without saying so, or presenting
+weak candidates as confident ones, would be worse than a small overrun.
+
+### Metadata search follows the same rules (v1.14.4)
+
+`cerefox_metadata_search` and the `cerefox-metadata-search` Edge Function apply
+`max_bytes` only when `include_content: true`, and the budget is applied by the
+database, which stops at the first document whose content does not fit. That
+made two silent failures possible until v1.14.4, and both are now closed:
+
+- **The reply is never empty when documents matched.** If the first document is
+  the oversized one, the budget used to empty the result set — the MCP tool
+  returned "No documents match", the Edge Function returned `[]`. Both now say
+  what matched: the tool with a warning and a header list, the Edge Function by
+  listing every matching document with content omitted.
+- **Documents are never held back silently.** The MCP tool appends
+  `[2 of 7 document(s) shown; 5 did not fit max_bytes=20000. …]`. The Edge
+  Function keeps its array shape and lists **every** matching document, marking
+  the ones whose content did not fit with `"content_omitted": true` — so
+  `results.length` is always the true match count and only content is dropped.
+
+`max_bytes` is resolved the same way on every path: `null`, absent, empty or
+non-numeric all mean **unset** and fall back to the server ceiling, and none of
+them disables the limit. A real number of zero or less means "almost no
+budget" and is honoured as such — a caller whose allowance has run out is not
+handed the largest possible reply.
+
 ---
 
 ## The server ceiling — agents can request less, never more
@@ -142,7 +197,7 @@ threshold (it is a SQL DEFAULT in `rpcs.sql`, changed via `cerefox server deploy
 | Question | Answer |
 |----------|--------|
 | Does the web UI truncate results? | No — unlimited |
-| Does the CLI truncate results? | No — unlimited |
+| Does the CLI truncate results? | Yes — at `CEREFOX_MAX_RESPONSE_BYTES`, or `--max-bytes`. It says so when it does. |
 | What is the default MCP response limit? | 200 000 bytes |
 | Can an agent request a smaller limit? | Yes — `max_bytes` tool parameter |
 | Can an agent exceed the server ceiling? | No — always capped |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerefox/memory",
-  "version": "1.14.2",
+  "version": "1.14.4",
   "description": "Cerefox — user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
   "license": "Apache-2.0",
   "homepage": "https://github.com/fstamatelopoulos/cerefox",
@@ -39,12 +39,12 @@
     "CHANGELOG.md"
   ],
   "dependencies": {
-    "@hono/node-server": "^2.0.12",
+    "@hono/node-server": "^2.1.1",
     "@modelcontextprotocol/sdk": "^1.30.0",
     "@supabase/supabase-js": "^2.45.0",
     "cli-progress": "^3.12.0",
     "commander": "^14.0.3",
-    "hono": "^4.12.34",
+    "hono": "^4.13.7",
     "mammoth": "^1.9.0",
     "ora": "^9.4.0",
     "picocolors": "^1.0.0",