@cerefox/memory 1.14.3 → 1.14.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cerefox.js +55 -10
- package/dist/server-assets/_shared/ef-meta/index.ts +3 -3
- package/dist/server-assets/_shared/mcp-tools/_utils.ts +38 -6
- package/dist/server-assets/_shared/mcp-tools/metadata-search.ts +47 -14
- package/dist/server-assets/_shared/mcp-tools/search.ts +2 -6
- package/dist/server-assets/supabase/functions/cerefox-metadata-search/index.ts +81 -4
- package/dist/server-assets/supabase/functions/cerefox-search/index.ts +2 -6
- package/docs/guides/connect-agents.md +11 -3
- package/docs/guides/response-limits.md +41 -8
- package/package.json +1 -1
package/dist/bin/cerefox.js
CHANGED
|
@@ -7400,7 +7400,7 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
7400
7400
|
});
|
|
7401
7401
|
|
|
7402
7402
|
// src/meta.ts
|
|
7403
|
-
var PKG_VERSION = "1.14.
|
|
7403
|
+
var PKG_VERSION = "1.14.4";
|
|
7404
7404
|
var init_meta = () => {};
|
|
7405
7405
|
|
|
7406
7406
|
// ../../_shared/config/paths.ts
|
|
@@ -23317,6 +23317,14 @@ function logUsage(supabase, params) {
|
|
|
23317
23317
|
p_extra: params.extra ?? {}
|
|
23318
23318
|
})).catch(() => {});
|
|
23319
23319
|
}
|
|
23320
|
+
function resolveByteBudget(requested, ceiling) {
|
|
23321
|
+
if (requested === null || requested === undefined || requested === "")
|
|
23322
|
+
return ceiling;
|
|
23323
|
+
const n = Math.floor(Number(requested));
|
|
23324
|
+
if (!Number.isFinite(n))
|
|
23325
|
+
return ceiling;
|
|
23326
|
+
return Math.min(Math.max(n, 1), ceiling);
|
|
23327
|
+
}
|
|
23320
23328
|
var MAX_RESPONSE_BYTES = 200000, DEFAULT_MIN_SEARCH_SCORE = 0.5, DEFAULT_MIN_SEARCH_SCORE_LOCAL = 0.6, DEFAULT_SEARCH_ALPHA = 0.7;
|
|
23321
23329
|
var init__utils = __esm(() => {
|
|
23322
23330
|
init_audit_ops();
|
|
@@ -25759,7 +25767,7 @@ var init_bundled_docs = __esm(() => {
|
|
|
25759
25767
|
});
|
|
25760
25768
|
|
|
25761
25769
|
// ../../_shared/ef-meta/index.ts
|
|
25762
|
-
var EF_VERSION = "1.14.
|
|
25770
|
+
var EF_VERSION = "1.14.4", CEREFOX_VERSION = "1.14.4", EF_LAST_CHANGED = "1.14.4";
|
|
25763
25771
|
var init_ef_meta = () => {};
|
|
25764
25772
|
|
|
25765
25773
|
// ../../_shared/compatibility/index.ts
|
|
@@ -56411,8 +56419,7 @@ async function handler10(supabase, args, ctx) {
|
|
|
56411
56419
|
throw new Error(`Project not found: ${project_name}`);
|
|
56412
56420
|
}
|
|
56413
56421
|
const ceiling = getMaxResponseBytes();
|
|
56414
|
-
const
|
|
56415
|
-
const max_bytes = include_content ? Math.min(Number.isFinite(requestedBytes) ? Math.max(requestedBytes, 1) : ceiling, ceiling) : null;
|
|
56422
|
+
const max_bytes = include_content ? resolveByteBudget(requested_max_bytes, ceiling) : null;
|
|
56416
56423
|
const params = {
|
|
56417
56424
|
p_metadata_filter: metadata_filter ?? {},
|
|
56418
56425
|
p_project_id: projectId,
|
|
@@ -56465,7 +56472,13 @@ ${lines.join(`
|
|
|
56465
56472
|
log(0);
|
|
56466
56473
|
return "No documents match the given criteria.";
|
|
56467
56474
|
}
|
|
56468
|
-
|
|
56475
|
+
let matched = rows.length;
|
|
56476
|
+
if (include_content && max_bytes !== null && rows.length < limit) {
|
|
56477
|
+
const { data: headers, error: probeError } = await supabase.rpc("cerefox_metadata_search", { ...params, p_include_content: false, p_max_bytes: null });
|
|
56478
|
+
if (!probeError)
|
|
56479
|
+
matched = Math.max(rows.length, (headers ?? []).length);
|
|
56480
|
+
}
|
|
56481
|
+
log(matched, matched > rows.length ? { returned: rows.length, truncated: true } : undefined);
|
|
56469
56482
|
const showReview = await reviewWorkflowEnabled(supabase);
|
|
56470
56483
|
const parts = rows.map((row) => {
|
|
56471
56484
|
const projects = row.project_names?.length ? ` | projects: ${row.project_names.join(", ")}` : "";
|
|
@@ -56481,6 +56494,16 @@ ${row.content}`;
|
|
|
56481
56494
|
}
|
|
56482
56495
|
return header;
|
|
56483
56496
|
});
|
|
56497
|
+
if (matched > rows.length) {
|
|
56498
|
+
const held = matched - rows.length;
|
|
56499
|
+
return `${parts.join(`
|
|
56500
|
+
|
|
56501
|
+
---
|
|
56502
|
+
|
|
56503
|
+
`)}
|
|
56504
|
+
|
|
56505
|
+
` + `[${rows.length} of ${matched} document(s) shown; ${held} did not fit ` + `max_bytes=${max_bytes}. Raise max_bytes, lower limit, or use ` + `include_content: false to list them all.]`;
|
|
56506
|
+
}
|
|
56484
56507
|
return parts.join(`
|
|
56485
56508
|
|
|
56486
56509
|
---
|
|
@@ -56612,8 +56635,7 @@ async function handler11(supabase, args, ctx) {
|
|
|
56612
56635
|
const metadata_filter = args.metadata_filter ?? null;
|
|
56613
56636
|
const requested_max_bytes = args.max_bytes;
|
|
56614
56637
|
const ceiling = getMaxResponseBytes();
|
|
56615
|
-
const
|
|
56616
|
-
const max_bytes = Math.min(Number.isFinite(requestedBytes) ? Math.max(requestedBytes, 1) : ceiling, ceiling);
|
|
56638
|
+
const max_bytes = resolveByteBudget(requested_max_bytes, ceiling);
|
|
56617
56639
|
if (metadata_filter !== null && metadata_filter !== undefined && (typeof metadata_filter !== "object" || Array.isArray(metadata_filter))) {
|
|
56618
56640
|
throw new McpInvalidParams("metadata_filter must be a JSON object or null");
|
|
56619
56641
|
}
|
|
@@ -80608,10 +80630,28 @@ async function action28(options) {
|
|
|
80608
80630
|
};
|
|
80609
80631
|
if (options.includeContent)
|
|
80610
80632
|
params.p_max_bytes = maxBytes;
|
|
80611
|
-
|
|
80633
|
+
let rows = await client.rpc("cerefox_metadata_search", params);
|
|
80612
80634
|
if (rows === null) {
|
|
80613
80635
|
throw systemError("cerefox_metadata_search: RPC returned no data.");
|
|
80614
80636
|
}
|
|
80637
|
+
let heldBack = 0;
|
|
80638
|
+
if (options.includeContent && rows.length < limit) {
|
|
80639
|
+
const all = await client.rpc("cerefox_metadata_search", {
|
|
80640
|
+
...params,
|
|
80641
|
+
p_include_content: false,
|
|
80642
|
+
p_max_bytes: null
|
|
80643
|
+
});
|
|
80644
|
+
if (all !== null && all.length > rows.length) {
|
|
80645
|
+
const withContent = new Map(rows.map((r) => [r.document_id, r]));
|
|
80646
|
+
heldBack = all.length - rows.length;
|
|
80647
|
+
const merged = all.map((h) => withContent.get(h.document_id) ?? { ...h, content: null });
|
|
80648
|
+
const seen = new Set(merged.map((r) => r.document_id));
|
|
80649
|
+
for (const r of rows)
|
|
80650
|
+
if (!seen.has(r.document_id))
|
|
80651
|
+
merged.push(r);
|
|
80652
|
+
rows = merged;
|
|
80653
|
+
}
|
|
80654
|
+
}
|
|
80615
80655
|
const requestor = resolveRequestor(options.author ?? options.requestor);
|
|
80616
80656
|
client.raw.rpc("cerefox_log_usage", {
|
|
80617
80657
|
p_operation: "metadata_search",
|
|
@@ -80633,6 +80673,10 @@ async function action28(options) {
|
|
|
80633
80673
|
println("No documents match the metadata filter.");
|
|
80634
80674
|
return;
|
|
80635
80675
|
}
|
|
80676
|
+
if (heldBack > 0) {
|
|
80677
|
+
println(c.dim(`(${rows.length - heldBack} of ${rows.length} document(s) have content here; ` + `${heldBack} did not fit --max-bytes ${maxBytes} and are listed without it)`));
|
|
80678
|
+
println("");
|
|
80679
|
+
}
|
|
80636
80680
|
for (const row of rows) {
|
|
80637
80681
|
const projects = row.project_names?.length ? ` | projects: ${row.project_names.join(", ")}` : "";
|
|
80638
80682
|
const meta = Object.entries(row.doc_metadata ?? {}).map(([k, v]) => `${k}=${v}`).join(", ");
|
|
@@ -81171,7 +81215,7 @@ async function action32(query, options) {
|
|
|
81171
81215
|
const rowBytes = Buffer.byteLength(JSON.stringify(row), "utf8");
|
|
81172
81216
|
if (usedBytes + rowBytes > maxBytes && accepted.length > 0) {
|
|
81173
81217
|
truncated = true;
|
|
81174
|
-
|
|
81218
|
+
continue;
|
|
81175
81219
|
}
|
|
81176
81220
|
accepted.push(row);
|
|
81177
81221
|
usedBytes += rowBytes;
|
|
@@ -81252,7 +81296,8 @@ async function action32(query, options) {
|
|
|
81252
81296
|
}
|
|
81253
81297
|
}
|
|
81254
81298
|
if (truncated) {
|
|
81255
|
-
|
|
81299
|
+
const held = results.length - accepted.length;
|
|
81300
|
+
println(c.dim(`(${accepted.length} of ${results.length} result(s) shown; ${held} did not fit ` + `${usedBytes} bytes used of --max-bytes ${maxBytes} — raise it to see the rest)`));
|
|
81256
81301
|
}
|
|
81257
81302
|
}
|
|
81258
81303
|
function registerSearch(program) {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* doesn't touch `supabase/functions/` leaves it alone).
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
export const EF_VERSION = "1.14.
|
|
21
|
+
export const EF_VERSION = "1.14.4";
|
|
22
22
|
|
|
23
23
|
/**
|
|
24
24
|
* The Cerefox RELEASE version — what `cerefox --version` reports and what npm
|
|
@@ -36,7 +36,7 @@ export const EF_VERSION = "1.14.3";
|
|
|
36
36
|
* is imported by the Deno Edge Functions, which cannot reach into the npm
|
|
37
37
|
* package.
|
|
38
38
|
*/
|
|
39
|
-
export const CEREFOX_VERSION = "1.14.
|
|
39
|
+
export const CEREFOX_VERSION = "1.14.4";
|
|
40
40
|
|
|
41
41
|
/**
|
|
42
42
|
* The most recent version whose EF-side SOURCE actually changed (#127).
|
|
@@ -46,7 +46,7 @@ export const CEREFOX_VERSION = "1.14.3";
|
|
|
46
46
|
* `cut_release.ts` ONLY when EF source changed since the last tag; doctor
|
|
47
47
|
* uses it to stay silent on label-only drift.
|
|
48
48
|
*/
|
|
49
|
-
export const EF_LAST_CHANGED = "1.14.
|
|
49
|
+
export const EF_LAST_CHANGED = "1.14.4";
|
|
50
50
|
|
|
51
51
|
/**
|
|
52
52
|
* The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp
|
|
@@ -147,25 +147,32 @@ export function applyByteBudget(
|
|
|
147
147
|
maxBytes: number,
|
|
148
148
|
): { accepted: unknown[]; dropped: unknown[]; truncated: boolean; usedBytes: number } {
|
|
149
149
|
const accepted: unknown[] = [];
|
|
150
|
+
const dropped: unknown[] = [];
|
|
150
151
|
let usedBytes = 0;
|
|
151
152
|
let truncated = false;
|
|
152
|
-
let cut = rows.length;
|
|
153
153
|
|
|
154
|
-
for (const
|
|
154
|
+
for (const row of rows) {
|
|
155
155
|
const rowBytes = new TextEncoder().encode(JSON.stringify(row)).length;
|
|
156
|
+
// Skipped, not the end of the list (#266, #268). This used to `break`, so
|
|
157
|
+
// one oversized top hit suppressed every smaller result behind it: the
|
|
158
|
+
// Edge Function answered with an empty `accepted`, flipped to the degraded
|
|
159
|
+
// shape and stripped content from results 2 and 3 that would have fitted
|
|
160
|
+
// comfortably. The MCP tool has skipped since #266; this is the same rule
|
|
161
|
+
// reaching the surface GPT Actions and direct HTTP callers use.
|
|
156
162
|
if (usedBytes + rowBytes > maxBytes) {
|
|
157
163
|
truncated = true;
|
|
158
|
-
|
|
159
|
-
|
|
164
|
+
dropped.push(row);
|
|
165
|
+
continue;
|
|
160
166
|
}
|
|
161
167
|
accepted.push(row);
|
|
162
168
|
usedBytes += rowBytes;
|
|
163
169
|
}
|
|
164
170
|
|
|
165
171
|
// What did not fit, so a caller can say so instead of reporting nothing
|
|
166
|
-
// (#254):
|
|
172
|
+
// (#254): every row larger than the budget lands here — they are no longer
|
|
173
|
+
// a contiguous tail, because the scan no longer stops at the first one —
|
|
167
174
|
// and "no results" is the one answer an agent acts on irreversibly.
|
|
168
|
-
return { accepted, dropped
|
|
175
|
+
return { accepted, dropped, truncated, usedBytes };
|
|
169
176
|
}
|
|
170
177
|
|
|
171
178
|
import type { AccessPath } from "./types.ts";
|
|
@@ -293,3 +300,28 @@ export function logUsage(supabase: MCPSupabaseClient, params: LogUsageParams): v
|
|
|
293
300
|
}),
|
|
294
301
|
).catch(() => {});
|
|
295
302
|
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Resolve a caller-supplied byte budget against the server ceiling.
|
|
306
|
+
*
|
|
307
|
+
* One implementation, because this arithmetic was written out by hand on four
|
|
308
|
+
* surfaces and each hand-written copy was wrong in its own way (#267, #268):
|
|
309
|
+
*
|
|
310
|
+
* - **Non-numeric means unset, not unbounded.** `Math.min("lots", CEILING)` is
|
|
311
|
+
* `NaN`; `NaN` compares false against every `>` check and serialises to JSON
|
|
312
|
+
* `null`, and `p_max_bytes NULL` means NO limit in Postgres. So the one
|
|
313
|
+
* parameter that exists to bound a reply, handed a word, removed the bound.
|
|
314
|
+
* - **`null` and `undefined` mean unset too.** `Number(null)` is `0`, which is
|
|
315
|
+
* finite, so a clamp to `>= 1` turned an explicitly-null budget into a
|
|
316
|
+
* ONE-BYTE budget — a client that serialises optional fields as `null` asked
|
|
317
|
+
* for content and got none.
|
|
318
|
+
* - **A real number of zero or less means "almost nothing", and is honoured.**
|
|
319
|
+
* Falling back to the ceiling there would hand a caller whose allowance had
|
|
320
|
+
* run out the largest possible reply.
|
|
321
|
+
*/
|
|
322
|
+
export function resolveByteBudget(requested: unknown, ceiling: number): number {
|
|
323
|
+
if (requested === null || requested === undefined || requested === "") return ceiling;
|
|
324
|
+
const n = Math.floor(Number(requested));
|
|
325
|
+
if (!Number.isFinite(n)) return ceiling;
|
|
326
|
+
return Math.min(Math.max(n, 1), ceiling);
|
|
327
|
+
}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import type { MCPSupabaseClient } from "./types.ts";
|
|
9
9
|
|
|
10
|
-
import { getMaxResponseBytes, logUsage } from "./_utils.ts";
|
|
10
|
+
import { getMaxResponseBytes, logUsage, resolveByteBudget } from "./_utils.ts";
|
|
11
11
|
import { lookupProjectId } from "./_projects.ts";
|
|
12
12
|
import { reviewWorkflowEnabled } from "./feature-flags.ts";
|
|
13
13
|
import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
|
|
@@ -51,20 +51,14 @@ async function handler(
|
|
|
51
51
|
if (!projectId) throw new Error(`Project not found: ${project_name}`);
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
// Enforce byte ceiling for content mode
|
|
55
|
-
//
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
// the in-process guard below disabled too (#267). Same hole as the search
|
|
60
|
-
// tool's, in the sibling that shares its transport.
|
|
54
|
+
// Enforce byte ceiling for content mode, via the one shared resolver every
|
|
55
|
+
// budget-taking surface uses (#268). It was written out by hand here and on
|
|
56
|
+
// three other surfaces, and each copy was wrong differently: a non-numeric
|
|
57
|
+
// value became `NaN` and disabled the limit entirely (#267), and an explicit
|
|
58
|
+
// `null` coerced to 0 and became a ONE-BYTE budget.
|
|
61
59
|
const ceiling = getMaxResponseBytes();
|
|
62
|
-
const requestedBytes = Math.floor(Number(requested_max_bytes));
|
|
63
60
|
const max_bytes = include_content
|
|
64
|
-
?
|
|
65
|
-
Number.isFinite(requestedBytes) ? Math.max(requestedBytes, 1) : ceiling,
|
|
66
|
-
ceiling,
|
|
67
|
-
)
|
|
61
|
+
? resolveByteBudget(requested_max_bytes, ceiling)
|
|
68
62
|
: null;
|
|
69
63
|
|
|
70
64
|
const params: Record<string, unknown> = {
|
|
@@ -166,7 +160,33 @@ async function handler(
|
|
|
166
160
|
log(0);
|
|
167
161
|
return "No documents match the given criteria.";
|
|
168
162
|
}
|
|
169
|
-
|
|
163
|
+
// How many documents actually matched, as opposed to how many the budget
|
|
164
|
+
// let through (#268). The RPC applies `p_max_bytes` by stopping at the first
|
|
165
|
+
// row that does not fit, so a short list has two indistinguishable causes:
|
|
166
|
+
// fewer documents matched, or the budget cut the list. Only the caller's
|
|
167
|
+
// side knows which, and only after asking — so ask, with the same
|
|
168
|
+
// content-free probe the empty branch above uses.
|
|
169
|
+
//
|
|
170
|
+
// Cost, stated honestly: this is a second RPC round-trip, and it fires
|
|
171
|
+
// whenever a content-bearing search returns less than a full page — which is
|
|
172
|
+
// the COMMON case, not a rare one, since `limit` defaults to 10. A full page
|
|
173
|
+
// and a budget-free call both skip it, but nothing else does. The RPC gives
|
|
174
|
+
// no "there was more" signal, and the alternative to asking is guessing:
|
|
175
|
+
// a short list is indistinguishable from a cut list from here, and guessing
|
|
176
|
+
// wrong is the bug (#268). Worth revisiting if the RPC ever returns a total.
|
|
177
|
+
let matched = rows.length;
|
|
178
|
+
if (include_content && max_bytes !== null && rows.length < limit) {
|
|
179
|
+
const { data: headers, error: probeError } = await supabase.rpc(
|
|
180
|
+
"cerefox_metadata_search",
|
|
181
|
+
{ ...params, p_include_content: false, p_max_bytes: null },
|
|
182
|
+
);
|
|
183
|
+
// A failed probe must not invent a count. supabase-js resolves with
|
|
184
|
+
// `{ data: null, error }` rather than throwing (#261), so read the error:
|
|
185
|
+
// leaving `matched` at `rows.length` states only what is known.
|
|
186
|
+
if (!probeError) matched = Math.max(rows.length, ((headers ?? []) as unknown[]).length);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
log(matched, matched > rows.length ? { returned: rows.length, truncated: true } : undefined);
|
|
170
190
|
|
|
171
191
|
// The review status is a column of a feature that may be off (#241); when
|
|
172
192
|
// it is, an agent should not see "approved" and wonder what it means.
|
|
@@ -194,6 +214,19 @@ async function handler(
|
|
|
194
214
|
return header;
|
|
195
215
|
});
|
|
196
216
|
|
|
217
|
+
// Never hold results back silently (#268). A caller who receives 1 of 5 and
|
|
218
|
+
// is told nothing believes they saw everything — the same failure as a false
|
|
219
|
+
// empty, in a quieter form. This notice is framing that is never dropped.
|
|
220
|
+
if (matched > rows.length) {
|
|
221
|
+
const held = matched - rows.length;
|
|
222
|
+
return (
|
|
223
|
+
`${parts.join("\n\n---\n\n")}\n\n` +
|
|
224
|
+
`[${rows.length} of ${matched} document(s) shown; ${held} did not fit ` +
|
|
225
|
+
`max_bytes=${max_bytes}. Raise max_bytes, lower limit, or use ` +
|
|
226
|
+
`include_content: false to list them all.]`
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
|
|
197
230
|
return parts.join("\n\n---\n\n");
|
|
198
231
|
}
|
|
199
232
|
|
|
@@ -19,7 +19,7 @@ import type { MCPSupabaseClient } from "./types.ts";
|
|
|
19
19
|
|
|
20
20
|
import { getEmbedding, resolveEmbedderKind } from "../embeddings/index.ts";
|
|
21
21
|
import { getConfiguredMinSearchScore, getConfiguredSearchAlpha,
|
|
22
|
-
getMaxResponseBytes, getMinTermCoverage, logUsage } from "./_utils.ts";
|
|
22
|
+
getMaxResponseBytes, getMinTermCoverage, logUsage , resolveByteBudget } from "./_utils.ts";
|
|
23
23
|
import { lookupProjectId } from "./_projects.ts";
|
|
24
24
|
import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
|
|
25
25
|
import { AUTHOR_PARAM_READ, callerIdentity } from "./identity.ts";
|
|
@@ -214,11 +214,7 @@ async function handler(
|
|
|
214
214
|
// falling back to the ceiling there would hand a caller whose remaining
|
|
215
215
|
// allowance ran out the largest possible reply (#267). Only a missing or
|
|
216
216
|
// non-numeric value defaults to the ceiling.
|
|
217
|
-
const
|
|
218
|
-
const max_bytes = Math.min(
|
|
219
|
-
Number.isFinite(requestedBytes) ? Math.max(requestedBytes, 1) : ceiling,
|
|
220
|
-
ceiling,
|
|
221
|
-
);
|
|
217
|
+
const max_bytes = resolveByteBudget(requested_max_bytes, ceiling);
|
|
222
218
|
|
|
223
219
|
if (
|
|
224
220
|
metadata_filter !== null &&
|
|
@@ -4,6 +4,7 @@ import { isVersionRequest, versionResponse } from "../../../_shared/ef-meta/inde
|
|
|
4
4
|
import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
5
5
|
import { callerIdentity } from "../../../_shared/mcp-tools/identity.ts";
|
|
6
6
|
import { reviewWorkflowEnabled } from "../../../_shared/mcp-tools/feature-flags.ts";
|
|
7
|
+
import { resolveByteBudget } from "../../../_shared/mcp-tools/_utils.ts";
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* cerefox-metadata-search -- Supabase Edge Function
|
|
@@ -103,8 +104,12 @@ Deno.serve(async (req: Request): Promise<Response> => {
|
|
|
103
104
|
const include_content = body.include_content ?? false;
|
|
104
105
|
const requested_max_bytes = body.max_bytes;
|
|
105
106
|
|
|
107
|
+
// One implementation of this arithmetic, shared with every other surface
|
|
108
|
+
// that takes a budget (#268): non-numeric and null both mean "unset" and
|
|
109
|
+
// fall back to the ceiling, while a real number of zero or less is
|
|
110
|
+
// honoured as "almost no budget".
|
|
106
111
|
const max_bytes = include_content
|
|
107
|
-
?
|
|
112
|
+
? resolveByteBudget(requested_max_bytes, MAX_BYTES)
|
|
108
113
|
: null;
|
|
109
114
|
|
|
110
115
|
const supabaseUrl = Deno.env.get("SUPABASE_URL")!;
|
|
@@ -154,18 +159,90 @@ Deno.serve(async (req: Request): Promise<Response> => {
|
|
|
154
159
|
});
|
|
155
160
|
}
|
|
156
161
|
|
|
157
|
-
|
|
162
|
+
let rows = (data ?? []) as Array<Record<string, unknown>>;
|
|
163
|
+
|
|
164
|
+
// Never answer with a shorter list than what matched (#268).
|
|
165
|
+
//
|
|
166
|
+
// The RPC applies `p_max_bytes` server-side by stopping at the first row
|
|
167
|
+
// whose content does not fit, so this list has two indistinguishable
|
|
168
|
+
// causes: fewer documents matched, or the budget cut it short. When the
|
|
169
|
+
// first row is the oversized one the array comes back EMPTY, and a caller
|
|
170
|
+
// reads `[]` as "this knowledge does not exist" and stops looking — the
|
|
171
|
+
// false negative #254 exists to prevent, reached here through a sibling
|
|
172
|
+
// that never got the guard.
|
|
173
|
+
//
|
|
174
|
+
// The response shape stays a bare array, because Custom GPTs are
|
|
175
|
+
// configured against it: every matching document is listed, and only
|
|
176
|
+
// CONTENT is negotiable. Rows the budget could not afford come back
|
|
177
|
+
// content-free and marked, so `results.length` is always the true count
|
|
178
|
+
// and nothing is held back silently.
|
|
179
|
+
//
|
|
180
|
+
// Cost: a second RPC round-trip whenever a content-bearing search returns
|
|
181
|
+
// less than a full page, which is the common case rather than a rare one.
|
|
182
|
+
// The RPC signals no total, so the only alternative to asking is guessing
|
|
183
|
+
// whether a short list was cut — and guessing wrong is the bug.
|
|
184
|
+
if (include_content && max_bytes !== null && rows.length < limit) {
|
|
185
|
+
const { data: headerData, error: probeError } = await supabase.rpc(
|
|
186
|
+
"cerefox_metadata_search",
|
|
187
|
+
{ ...params, p_include_content: false, p_max_bytes: null },
|
|
188
|
+
);
|
|
189
|
+
// supabase-js RESOLVES with `{ data: null, error }` for PostgREST and
|
|
190
|
+
// network failures rather than throwing, so a probe failure must be read
|
|
191
|
+
// from the error, not inferred from an empty list — reading it as "no
|
|
192
|
+
// documents" is the very false empty this branch prevents (#261).
|
|
193
|
+
if (probeError && rows.length === 0) {
|
|
194
|
+
// Falling through here would ship exactly the false empty this block
|
|
195
|
+
// exists to prevent — `200 []`, which a caller reads as "no such
|
|
196
|
+
// knowledge". An error is the honest answer: it says the question was
|
|
197
|
+
// not resolved, rather than answering it wrongly.
|
|
198
|
+
return new Response(
|
|
199
|
+
JSON.stringify({
|
|
200
|
+
error:
|
|
201
|
+
`Nothing fit max_bytes=${max_bytes} with include_content, and the follow-up ` +
|
|
202
|
+
`query that lists what matched failed: ${probeError.message}. This is NOT a ` +
|
|
203
|
+
`confirmed empty result — retry with a larger max_bytes, or include_content: false.`,
|
|
204
|
+
}),
|
|
205
|
+
{ status: 502, headers: { ...CORS_HEADERS, "Content-Type": "application/json" } },
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
if (!probeError) {
|
|
209
|
+
const headers = (headerData ?? []) as Array<Record<string, unknown>>;
|
|
210
|
+
if (headers.length > rows.length) {
|
|
211
|
+
const withContent = new Map(rows.map((r) => [r.document_id as string, r]));
|
|
212
|
+
// The probe carries the full, correctly ordered match set; the
|
|
213
|
+
// content-bearing rows are folded into it by id so ordering is the
|
|
214
|
+
// RPC's, not an artefact of which rows happened to fit.
|
|
215
|
+
const merged = headers.map(
|
|
216
|
+
(h) => withContent.get(h.document_id as string) ?? { ...h, content_omitted: true },
|
|
217
|
+
);
|
|
218
|
+
// Two queries, two chances to disagree: the RPC orders by
|
|
219
|
+
// `updated_at DESC` with no tiebreaker under a LIMIT, and a
|
|
220
|
+
// concurrent write between the calls shifts the window. Anything the
|
|
221
|
+
// content query returned that the probe did not is APPENDED rather
|
|
222
|
+
// than dropped — losing a document we already hold, while fixing a
|
|
223
|
+
// bug about losing documents, would be its own joke.
|
|
224
|
+
const seen = new Set(merged.map((r) => r.document_id as string));
|
|
225
|
+
for (const r of rows) {
|
|
226
|
+
if (!seen.has(r.document_id as string)) merged.push(r);
|
|
227
|
+
}
|
|
228
|
+
rows = merged;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// Fire-and-forget usage logging. Counts what MATCHED, not what the budget
|
|
234
|
+
// allowed through: a budget-wiped search logging `0` misreports the store
|
|
235
|
+
// as empty in analytics (#259).
|
|
158
236
|
Promise.resolve(supabase.rpc("cerefox_log_usage", {
|
|
159
237
|
p_operation: "metadata_search",
|
|
160
238
|
p_access_path: "edge-function",
|
|
161
239
|
p_requestor: identityValue ?? null,
|
|
162
240
|
p_query_text: JSON.stringify(metadata_filter),
|
|
163
|
-
p_result_count:
|
|
241
|
+
p_result_count: rows.length,
|
|
164
242
|
p_project_id: project_id,
|
|
165
243
|
})).catch(() => {});
|
|
166
244
|
|
|
167
245
|
// Presentation only: the same shared reader every other surface uses.
|
|
168
|
-
const rows = (data ?? []) as Array<Record<string, unknown>>;
|
|
169
246
|
const showReview = await reviewWorkflowEnabled(supabase);
|
|
170
247
|
const out = showReview
|
|
171
248
|
? rows
|
|
@@ -5,7 +5,7 @@ import { efAuthGate } from "../../../_shared/ef-auth/index.ts";
|
|
|
5
5
|
import { callerIdentity } from "../../../_shared/mcp-tools/identity.ts";
|
|
6
6
|
import { capEmbeddingInput } from "../../../_shared/embeddings/index.ts";
|
|
7
7
|
// One implementation of the byte budget, shared with the MCP tools (#254).
|
|
8
|
-
import { applyByteBudget } from "../../../_shared/mcp-tools/_utils.ts";
|
|
8
|
+
import { applyByteBudget, resolveByteBudget } from "../../../_shared/mcp-tools/_utils.ts";
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
11
|
* cerefox-search — Supabase Edge Function
|
|
@@ -219,11 +219,7 @@ Deno.serve(async (req: Request) => {
|
|
|
219
219
|
// check, so a non-numeric value bypassed the ceiling entirely (#266).
|
|
220
220
|
// A number of 0 or less still means "almost no budget"; only a missing or
|
|
221
221
|
// non-numeric value falls back to the ceiling (#267).
|
|
222
|
-
const
|
|
223
|
-
const max_bytes = Math.min(
|
|
224
|
-
Number.isFinite(requestedBytes) ? Math.max(requestedBytes, 1) : MAX_BYTES,
|
|
225
|
-
MAX_BYTES,
|
|
226
|
-
);
|
|
222
|
+
const max_bytes = resolveByteBudget(requested_max_bytes, MAX_BYTES);
|
|
227
223
|
// Clamp match_count to [1, MAX_MATCH_COUNT] (bounds query work; see MAX_MATCH_COUNT).
|
|
228
224
|
const match_count = Math.min(Math.max(1, Math.floor(Number(raw_match_count)) || 5), MAX_MATCH_COUNT);
|
|
229
225
|
|
|
@@ -638,7 +638,7 @@ In the action editor, paste this schema (replace `<your-project-ref>`):
|
|
|
638
638
|
openapi: 3.1.0
|
|
639
639
|
info:
|
|
640
640
|
title: Cerefox Knowledge Base
|
|
641
|
-
version: 4.
|
|
641
|
+
version: 4.3.0
|
|
642
642
|
servers:
|
|
643
643
|
- url: https://<your-project-ref>.supabase.co/functions/v1
|
|
644
644
|
paths:
|
|
@@ -1051,8 +1051,10 @@ paths:
|
|
|
1051
1051
|
type: integer
|
|
1052
1052
|
default: 200000
|
|
1053
1053
|
description: >
|
|
1054
|
-
Response size budget in bytes when include_content is true
|
|
1055
|
-
|
|
1054
|
+
Response size budget in bytes when include_content is true.
|
|
1055
|
+
Content is dropped whole, never truncated mid-document, and a
|
|
1056
|
+
document whose content is dropped is still listed with
|
|
1057
|
+
content_omitted: true. Advanced; leave unset for the default.
|
|
1056
1058
|
author:
|
|
1057
1059
|
type: string
|
|
1058
1060
|
description: >
|
|
@@ -1067,6 +1069,12 @@ paths:
|
|
|
1067
1069
|
version_count, content_hash, content }], plus review_status
|
|
1068
1070
|
only while the store's review workflow is on (the key is absent
|
|
1069
1071
|
when it is off).
|
|
1072
|
+
The array lists EVERY matching document, so its length is the true
|
|
1073
|
+
match count. When include_content is true and max_bytes cannot
|
|
1074
|
+
carry a document's text, that document is still listed, with its
|
|
1075
|
+
content omitted and "content_omitted": true set on the item — the
|
|
1076
|
+
list is never silently shortened, and an empty array always means
|
|
1077
|
+
nothing matched.
|
|
1070
1078
|
content_hash is the concurrency token — pass it back as
|
|
1071
1079
|
expected_content_hash when updating via ingestNote.
|
|
1072
1080
|
```
|
|
@@ -8,11 +8,18 @@ explains how response size limits work and how to tune them.
|
|
|
8
8
|
|
|
9
9
|
## The key principle: opt-in limits, never truncate the web UI
|
|
10
10
|
|
|
11
|
-
The web UI
|
|
12
|
-
|
|
11
|
+
The web UI never truncates results. It has no size limit — the browser can handle
|
|
12
|
+
arbitrarily large responses and there is no LLM context window to worry about.
|
|
13
13
|
|
|
14
|
-
Limits
|
|
15
|
-
agent's context window matters
|
|
14
|
+
Limits apply on the MCP, Edge Function **and CLI** paths. On MCP and the Edge Functions
|
|
15
|
+
they exist because an AI agent's context window matters; the CLI applies the same default
|
|
16
|
+
so that one setting (`CEREFOX_MAX_RESPONSE_BYTES`) governs every non-browser path, and
|
|
17
|
+
raises or lowers it per call with `--max-bytes`.
|
|
18
|
+
|
|
19
|
+
> **Changed in v0.10.2.** The CLI originally returned everything, like the web UI. It now
|
|
20
|
+
> honours `CEREFOX_MAX_RESPONSE_BYTES` (200 000 default) and prints
|
|
21
|
+
> `(results truncated at N bytes; use --max-bytes to raise)` when results are dropped.
|
|
22
|
+
> This guide described the pre-v0.10.2 behaviour until v1.14.4.
|
|
16
23
|
|
|
17
24
|
---
|
|
18
25
|
|
|
@@ -21,7 +28,7 @@ agent's context window matters. Callers always choose whether to apply a limit.
|
|
|
21
28
|
| Path | Limit behaviour |
|
|
22
29
|
|------|----------------|
|
|
23
30
|
| Web UI (`/search`) | **No limit** — all results returned |
|
|
24
|
-
| CLI (`cerefox search`) |
|
|
31
|
+
| CLI (`cerefox search`) | Defaults to `CEREFOX_MAX_RESPONSE_BYTES` (200 000); raise or lower per call with `--max-bytes`. Announces truncation. |
|
|
25
32
|
| Local MCP server (`cerefox mcp`) | Defaults to `CEREFOX_MAX_RESPONSE_BYTES` (200 000); agent can request less |
|
|
26
33
|
| Edge Function (`cerefox-search`) | Defaults to 200 000 bytes; agent can request less via `max_bytes` body param |
|
|
27
34
|
| Remote MCP (`cerefox-mcp` Edge Function) | Defaults to 200 000 bytes; agent can request less via `max_bytes` tool param |
|
|
@@ -35,8 +42,10 @@ Cerefox never cuts a document mid-content.
|
|
|
35
42
|
|
|
36
43
|
A result that does not fit is **skipped**, not treated as the end of the list, so the
|
|
37
44
|
returned set is not necessarily the top N by rank: one oversized document ranked first
|
|
38
|
-
does not hide the smaller results behind it (v1.14.3
|
|
39
|
-
|
|
45
|
+
does not hide the smaller results behind it (v1.14.3 on the MCP tool; v1.14.4 on the
|
|
46
|
+
`cerefox-search` Edge Function and the CLI, which both still stopped at the first
|
|
47
|
+
oversized row). Anything skipped is named in the footer, so what is missing is always
|
|
48
|
+
visible.
|
|
40
49
|
|
|
41
50
|
When truncation occurs:
|
|
42
51
|
- The MCP tool appends a footer naming what was held back:
|
|
@@ -59,6 +68,30 @@ below-confidence advisory. Both are a few dozen bytes, and neither is ever
|
|
|
59
68
|
traded for content. Returning 1 of 5 results without saying so, or presenting
|
|
60
69
|
weak candidates as confident ones, would be worse than a small overrun.
|
|
61
70
|
|
|
71
|
+
### Metadata search follows the same rules (v1.14.4)
|
|
72
|
+
|
|
73
|
+
`cerefox_metadata_search` and the `cerefox-metadata-search` Edge Function apply
|
|
74
|
+
`max_bytes` only when `include_content: true`, and the budget is applied by the
|
|
75
|
+
database, which stops at the first document whose content does not fit. That
|
|
76
|
+
made two silent failures possible until v1.14.4, and both are now closed:
|
|
77
|
+
|
|
78
|
+
- **The reply is never empty when documents matched.** If the first document is
|
|
79
|
+
the oversized one, the budget used to empty the result set — the MCP tool
|
|
80
|
+
returned "No documents match", the Edge Function returned `[]`. Both now say
|
|
81
|
+
what matched: the tool with a warning and a header list, the Edge Function by
|
|
82
|
+
listing every matching document with content omitted.
|
|
83
|
+
- **Documents are never held back silently.** The MCP tool appends
|
|
84
|
+
`[2 of 7 document(s) shown; 5 did not fit max_bytes=20000. …]`. The Edge
|
|
85
|
+
Function keeps its array shape and lists **every** matching document, marking
|
|
86
|
+
the ones whose content did not fit with `"content_omitted": true` — so
|
|
87
|
+
`results.length` is always the true match count and only content is dropped.
|
|
88
|
+
|
|
89
|
+
`max_bytes` is resolved the same way on every path: `null`, absent, empty or
|
|
90
|
+
non-numeric all mean **unset** and fall back to the server ceiling, and none of
|
|
91
|
+
them disables the limit. A real number of zero or less means "almost no
|
|
92
|
+
budget" and is honoured as such — a caller whose allowance has run out is not
|
|
93
|
+
handed the largest possible reply.
|
|
94
|
+
|
|
62
95
|
---
|
|
63
96
|
|
|
64
97
|
## The server ceiling — agents can request less, never more
|
|
@@ -164,7 +197,7 @@ threshold (it is a SQL DEFAULT in `rpcs.sql`, changed via `cerefox server deploy
|
|
|
164
197
|
| Question | Answer |
|
|
165
198
|
|----------|--------|
|
|
166
199
|
| Does the web UI truncate results? | No — unlimited |
|
|
167
|
-
| Does the CLI truncate results? |
|
|
200
|
+
| Does the CLI truncate results? | Yes — at `CEREFOX_MAX_RESPONSE_BYTES`, or `--max-bytes`. It says so when it does. |
|
|
168
201
|
| What is the default MCP response limit? | 200 000 bytes |
|
|
169
202
|
| Can an agent request a smaller limit? | Yes — `max_bytes` tool parameter |
|
|
170
203
|
| Can an agent exceed the server ceiling? | No — always capped |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cerefox/memory",
|
|
3
|
-
"version": "1.14.
|
|
3
|
+
"version": "1.14.4",
|
|
4
4
|
"description": "Cerefox — user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"homepage": "https://github.com/fstamatelopoulos/cerefox",
|