@cerefox/memory 1.14.2 → 1.14.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_GUIDE.md +1 -1
- package/dist/bin/cerefox.js +657 -418
- package/dist/server-assets/_shared/ef-meta/index.ts +3 -3
- package/dist/server-assets/_shared/mcp-tools/_utils.ts +38 -6
- package/dist/server-assets/_shared/mcp-tools/metadata-search.ts +103 -21
- package/dist/server-assets/_shared/mcp-tools/search.ts +256 -66
- package/dist/server-assets/supabase/functions/cerefox-metadata-search/index.ts +81 -4
- package/dist/server-assets/supabase/functions/cerefox-search/index.ts +50 -16
- package/docs/guides/connect-agents.md +21 -8
- package/docs/guides/response-limits.md +64 -9
- package/package.json +3 -3
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* doesn't touch `supabase/functions/` leaves it alone).
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
export const EF_VERSION = "1.14.
|
|
21
|
+
export const EF_VERSION = "1.14.4";
|
|
22
22
|
|
|
23
23
|
/**
|
|
24
24
|
* The Cerefox RELEASE version — what `cerefox --version` reports and what npm
|
|
@@ -36,7 +36,7 @@ export const EF_VERSION = "1.14.2";
|
|
|
36
36
|
* is imported by the Deno Edge Functions, which cannot reach into the npm
|
|
37
37
|
* package.
|
|
38
38
|
*/
|
|
39
|
-
export const CEREFOX_VERSION = "1.14.
|
|
39
|
+
export const CEREFOX_VERSION = "1.14.4";
|
|
40
40
|
|
|
41
41
|
/**
|
|
42
42
|
* The most recent version whose EF-side SOURCE actually changed (#127).
|
|
@@ -46,7 +46,7 @@ export const CEREFOX_VERSION = "1.14.2";
|
|
|
46
46
|
* `cut_release.ts` ONLY when EF source changed since the last tag; doctor
|
|
47
47
|
* uses it to stay silent on label-only drift.
|
|
48
48
|
*/
|
|
49
|
-
export const EF_LAST_CHANGED = "1.14.
|
|
49
|
+
export const EF_LAST_CHANGED = "1.14.4";
|
|
50
50
|
|
|
51
51
|
/**
|
|
52
52
|
* The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp
|
|
@@ -147,25 +147,32 @@ export function applyByteBudget(
|
|
|
147
147
|
maxBytes: number,
|
|
148
148
|
): { accepted: unknown[]; dropped: unknown[]; truncated: boolean; usedBytes: number } {
|
|
149
149
|
const accepted: unknown[] = [];
|
|
150
|
+
const dropped: unknown[] = [];
|
|
150
151
|
let usedBytes = 0;
|
|
151
152
|
let truncated = false;
|
|
152
|
-
let cut = rows.length;
|
|
153
153
|
|
|
154
|
-
for (const
|
|
154
|
+
for (const row of rows) {
|
|
155
155
|
const rowBytes = new TextEncoder().encode(JSON.stringify(row)).length;
|
|
156
|
+
// Skipped, not the end of the list (#266, #268). This used to `break`, so
|
|
157
|
+
// one oversized top hit suppressed every smaller result behind it: the
|
|
158
|
+
// Edge Function answered with an empty `accepted`, flipped to the degraded
|
|
159
|
+
// shape and stripped content from results 2 and 3 that would have fitted
|
|
160
|
+
// comfortably. The MCP tool has skipped since #266; this is the same rule
|
|
161
|
+
// reaching the surface GPT Actions and direct HTTP callers use.
|
|
156
162
|
if (usedBytes + rowBytes > maxBytes) {
|
|
157
163
|
truncated = true;
|
|
158
|
-
|
|
159
|
-
|
|
164
|
+
dropped.push(row);
|
|
165
|
+
continue;
|
|
160
166
|
}
|
|
161
167
|
accepted.push(row);
|
|
162
168
|
usedBytes += rowBytes;
|
|
163
169
|
}
|
|
164
170
|
|
|
165
171
|
// What did not fit, so a caller can say so instead of reporting nothing
|
|
166
|
-
// (#254):
|
|
172
|
+
// (#254): every row larger than the budget lands here — they are no longer
|
|
173
|
+
// a contiguous tail, because the scan no longer stops at the first one —
|
|
167
174
|
// and "no results" is the one answer an agent acts on irreversibly.
|
|
168
|
-
return { accepted, dropped
|
|
175
|
+
return { accepted, dropped, truncated, usedBytes };
|
|
169
176
|
}
|
|
170
177
|
|
|
171
178
|
import type { AccessPath } from "./types.ts";
|
|
@@ -293,3 +300,28 @@ export function logUsage(supabase: MCPSupabaseClient, params: LogUsageParams): v
|
|
|
293
300
|
}),
|
|
294
301
|
).catch(() => {});
|
|
295
302
|
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Resolve a caller-supplied byte budget against the server ceiling.
|
|
306
|
+
*
|
|
307
|
+
* One implementation, because this arithmetic was written out by hand on four
|
|
308
|
+
* surfaces and each hand-written copy was wrong in its own way (#267, #268):
|
|
309
|
+
*
|
|
310
|
+
* - **Non-numeric means unset, not unbounded.** `Math.min("lots", CEILING)` is
|
|
311
|
+
* `NaN`; `NaN` compares false against every `>` check and serialises to JSON
|
|
312
|
+
* `null`, and `p_max_bytes NULL` means NO limit in Postgres. So the one
|
|
313
|
+
* parameter that exists to bound a reply, handed a word, removed the bound.
|
|
314
|
+
* - **`null` and `undefined` mean unset too.** `Number(null)` is `0`, which is
|
|
315
|
+
* finite, so a clamp to `>= 1` turned an explicitly-null budget into a
|
|
316
|
+
* ONE-BYTE budget — a client that serialises optional fields as `null` asked
|
|
317
|
+
* for content and got none.
|
|
318
|
+
* - **A real number of zero or less means "almost nothing", and is honoured.**
|
|
319
|
+
* Falling back to the ceiling there would hand a caller whose allowance had
|
|
320
|
+
* run out the largest possible reply.
|
|
321
|
+
*/
|
|
322
|
+
export function resolveByteBudget(requested: unknown, ceiling: number): number {
|
|
323
|
+
if (requested === null || requested === undefined || requested === "") return ceiling;
|
|
324
|
+
const n = Math.floor(Number(requested));
|
|
325
|
+
if (!Number.isFinite(n)) return ceiling;
|
|
326
|
+
return Math.min(Math.max(n, 1), ceiling);
|
|
327
|
+
}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import type { MCPSupabaseClient } from "./types.ts";
|
|
9
9
|
|
|
10
|
-
import { getMaxResponseBytes, logUsage } from "./_utils.ts";
|
|
10
|
+
import { getMaxResponseBytes, logUsage, resolveByteBudget } from "./_utils.ts";
|
|
11
11
|
import { lookupProjectId } from "./_projects.ts";
|
|
12
12
|
import { reviewWorkflowEnabled } from "./feature-flags.ts";
|
|
13
13
|
import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
|
|
@@ -22,7 +22,11 @@ async function handler(
|
|
|
22
22
|
const project_name = args.project_name as string | undefined;
|
|
23
23
|
const updated_since = args.updated_since as string | undefined;
|
|
24
24
|
const created_since = args.created_since as string | undefined;
|
|
25
|
-
|
|
25
|
+
// Sanitised and clamped, like `match_count` on the search tool: the local
|
|
26
|
+
// stdio server passes tool arguments through unvalidated, so a caller could
|
|
27
|
+
// ask for a million rows (#267). 500 matches the largest listing the web
|
|
28
|
+
// API serves, so no legitimate enumeration is cut short.
|
|
29
|
+
const limit = Math.min(Math.max(1, Math.floor(Number(args.limit)) || 10), 500);
|
|
26
30
|
const include_content = (args.include_content as boolean | undefined) ?? false;
|
|
27
31
|
const requested_max_bytes = args.max_bytes as number | undefined;
|
|
28
32
|
|
|
@@ -47,10 +51,14 @@ async function handler(
|
|
|
47
51
|
if (!projectId) throw new Error(`Project not found: ${project_name}`);
|
|
48
52
|
}
|
|
49
53
|
|
|
50
|
-
// Enforce byte ceiling for content mode
|
|
54
|
+
// Enforce byte ceiling for content mode, via the one shared resolver every
|
|
55
|
+
// budget-taking surface uses (#268). It was written out by hand here and on
|
|
56
|
+
// three other surfaces, and each copy was wrong differently: a non-numeric
|
|
57
|
+
// value became `NaN` and disabled the limit entirely (#267), and an explicit
|
|
58
|
+
// `null` coerced to 0 and became a ONE-BYTE budget.
|
|
51
59
|
const ceiling = getMaxResponseBytes();
|
|
52
60
|
const max_bytes = include_content
|
|
53
|
-
?
|
|
61
|
+
? resolveByteBudget(requested_max_bytes, ceiling)
|
|
54
62
|
: null;
|
|
55
63
|
|
|
56
64
|
const params: Record<string, unknown> = {
|
|
@@ -84,14 +92,20 @@ async function handler(
|
|
|
84
92
|
content: string | null;
|
|
85
93
|
}>;
|
|
86
94
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
+
// Logged once, AFTER the degraded branch below may have found the real
|
|
96
|
+
// count: firing here unconditionally wrote a `result_count: 0` row for
|
|
97
|
+
// every budget-wiped search, and adding a second row in the branch made
|
|
98
|
+
// analytics double-count them (#259).
|
|
99
|
+
const log = (result_count: number, extra?: Record<string, unknown>) =>
|
|
100
|
+
logUsage(supabase, {
|
|
101
|
+
operation: "metadata_search",
|
|
102
|
+
accessPath: ctx.accessPath,
|
|
103
|
+
requestor: callerIdentity(args),
|
|
104
|
+
query_text: JSON.stringify(metadata_filter ?? {}),
|
|
105
|
+
project_id: projectId,
|
|
106
|
+
result_count,
|
|
107
|
+
...(extra ? { extra } : {}),
|
|
108
|
+
});
|
|
95
109
|
|
|
96
110
|
if (rows.length === 0) {
|
|
97
111
|
// The RPC applies the byte budget server-side by stopping at the first row
|
|
@@ -100,24 +114,79 @@ async function handler(
|
|
|
100
114
|
// without content before reporting nothing: an agent that is told "no
|
|
101
115
|
// documents" stops looking, and here that would be false.
|
|
102
116
|
if (include_content && max_bytes !== null) {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
117
|
+
// supabase-js RESOLVES with `{ data: null, error }` for PostgREST
|
|
118
|
+
// failures and for network errors; it does not throw. A try/catch here
|
|
119
|
+
// was dead code, and worse: `headers` came back null, the branch below
|
|
120
|
+
// was skipped, and the handler answered "No documents match the given
|
|
121
|
+
// criteria" — the exact false-empty this branch exists to prevent
|
|
122
|
+
// (#261). Read the error, and say what is actually known.
|
|
123
|
+
const { data: headers, error: probeError } = await supabase.rpc(
|
|
124
|
+
"cerefox_metadata_search",
|
|
125
|
+
{ ...params, p_include_content: false, p_max_bytes: null },
|
|
126
|
+
);
|
|
127
|
+
if (probeError) {
|
|
128
|
+
log(0, { degraded_probe_failed: true });
|
|
129
|
+
return (
|
|
130
|
+
`⚠ Nothing fit max_bytes=${max_bytes} with include_content, and the ` +
|
|
131
|
+
`follow-up query that lists what matched failed: ${probeError.message}. ` +
|
|
132
|
+
`This is NOT a confirmed empty result — retry with a larger max_bytes, ` +
|
|
133
|
+
`or with include_content: false.`
|
|
134
|
+
);
|
|
135
|
+
}
|
|
108
136
|
const headerRows = (headers ?? []) as Array<{ document_id: string; title: string }>;
|
|
109
137
|
if (headerRows.length > 0) {
|
|
110
|
-
|
|
138
|
+
// The caller asked for a budget; honour it in the answer that explains
|
|
139
|
+
// the budget. `limit` is caller-supplied, so an uncapped list could be
|
|
140
|
+
// tens of KB in reply to a 2 KB request (#257).
|
|
141
|
+
const lead =
|
|
111
142
|
`⚠ ${headerRows.length} document(s) match, but none fit max_bytes=${max_bytes} ` +
|
|
112
143
|
`with include_content. This is NOT an empty result. Listing them without ` +
|
|
113
144
|
`content — raise max_bytes, or read one with cerefox_get_document ` +
|
|
114
|
-
`(outline: true for structure)
|
|
115
|
-
|
|
116
|
-
);
|
|
145
|
+
`(outline: true for structure).`;
|
|
146
|
+
const lines: string[] = [];
|
|
147
|
+
let used = new TextEncoder().encode(lead).length;
|
|
148
|
+
for (const r of headerRows) {
|
|
149
|
+
const line = `## ${r.title} [id: ${r.document_id}]`;
|
|
150
|
+
const size = new TextEncoder().encode(line).length + 1;
|
|
151
|
+
if (used + size > max_bytes) break;
|
|
152
|
+
lines.push(line);
|
|
153
|
+
used += size;
|
|
154
|
+
}
|
|
155
|
+
// What matched, not what the budget allowed through (#257).
|
|
156
|
+
log(headerRows.length, { returned: lines.length, degraded: true });
|
|
157
|
+
return lines.length > 0 ? `${lead}\n${lines.join("\n")}` : lead;
|
|
117
158
|
}
|
|
118
159
|
}
|
|
160
|
+
log(0);
|
|
119
161
|
return "No documents match the given criteria.";
|
|
120
162
|
}
|
|
163
|
+
// How many documents actually matched, as opposed to how many the budget
|
|
164
|
+
// let through (#268). The RPC applies `p_max_bytes` by stopping at the first
|
|
165
|
+
// row that does not fit, so a short list has two indistinguishable causes:
|
|
166
|
+
// fewer documents matched, or the budget cut the list. Only the caller's
|
|
167
|
+
// side knows which, and only after asking — so ask, with the same
|
|
168
|
+
// content-free probe the empty branch above uses.
|
|
169
|
+
//
|
|
170
|
+
// Cost, stated honestly: this is a second RPC round-trip, and it fires
|
|
171
|
+
// whenever a content-bearing search returns less than a full page — which is
|
|
172
|
+
// the COMMON case, not a rare one, since `limit` defaults to 10. A full page
|
|
173
|
+
// and a budget-free call both skip it, but nothing else does. The RPC gives
|
|
174
|
+
// no "there was more" signal, and the alternative to asking is guessing:
|
|
175
|
+
// a short list is indistinguishable from a cut list from here, and guessing
|
|
176
|
+
// wrong is the bug (#268). Worth revisiting if the RPC ever returns a total.
|
|
177
|
+
let matched = rows.length;
|
|
178
|
+
if (include_content && max_bytes !== null && rows.length < limit) {
|
|
179
|
+
const { data: headers, error: probeError } = await supabase.rpc(
|
|
180
|
+
"cerefox_metadata_search",
|
|
181
|
+
{ ...params, p_include_content: false, p_max_bytes: null },
|
|
182
|
+
);
|
|
183
|
+
// A failed probe must not invent a count. supabase-js resolves with
|
|
184
|
+
// `{ data: null, error }` rather than throwing (#261), so read the error:
|
|
185
|
+
// leaving `matched` at `rows.length` states only what is known.
|
|
186
|
+
if (!probeError) matched = Math.max(rows.length, ((headers ?? []) as unknown[]).length);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
log(matched, matched > rows.length ? { returned: rows.length, truncated: true } : undefined);
|
|
121
190
|
|
|
122
191
|
// The review status is a column of a feature that may be off (#241); when
|
|
123
192
|
// it is, an agent should not see "approved" and wonder what it means.
|
|
@@ -145,6 +214,19 @@ async function handler(
|
|
|
145
214
|
return header;
|
|
146
215
|
});
|
|
147
216
|
|
|
217
|
+
// Never hold results back silently (#268). A caller who receives 1 of 5 and
|
|
218
|
+
// is told nothing believes they saw everything — the same failure as a false
|
|
219
|
+
// empty, in a quieter form. This notice is framing that is never dropped.
|
|
220
|
+
if (matched > rows.length) {
|
|
221
|
+
const held = matched - rows.length;
|
|
222
|
+
return (
|
|
223
|
+
`${parts.join("\n\n---\n\n")}\n\n` +
|
|
224
|
+
`[${rows.length} of ${matched} document(s) shown; ${held} did not fit ` +
|
|
225
|
+
`max_bytes=${max_bytes}. Raise max_bytes, lower limit, or use ` +
|
|
226
|
+
`include_content: false to list them all.]`
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
|
|
148
230
|
return parts.join("\n\n---\n\n");
|
|
149
231
|
}
|
|
150
232
|
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
import type { MCPSupabaseClient } from "./types.ts";
|
|
19
19
|
|
|
20
20
|
import { getEmbedding, resolveEmbedderKind } from "../embeddings/index.ts";
|
|
21
|
-
import {
|
|
22
|
-
getMaxResponseBytes, getMinTermCoverage, logUsage } from "./_utils.ts";
|
|
21
|
+
import { getConfiguredMinSearchScore, getConfiguredSearchAlpha,
|
|
22
|
+
getMaxResponseBytes, getMinTermCoverage, logUsage , resolveByteBudget } from "./_utils.ts";
|
|
23
23
|
import { lookupProjectId } from "./_projects.ts";
|
|
24
24
|
import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
|
|
25
25
|
import { AUTHOR_PARAM_READ, callerIdentity } from "./identity.ts";
|
|
@@ -27,7 +27,16 @@ import { AUTHOR_PARAM_READ, callerIdentity } from "./identity.ts";
|
|
|
27
27
|
interface SearchRow {
|
|
28
28
|
document_id?: string;
|
|
29
29
|
doc_title?: string;
|
|
30
|
+
/** Document-level modes (`docs`) return this… */
|
|
30
31
|
full_content?: string;
|
|
32
|
+
/** …while `hybrid` and `fts` return the chunk text under this name. */
|
|
33
|
+
content?: string;
|
|
34
|
+
/** Chunk-mode identity: the RPCs return several chunks OF THE SAME document. */
|
|
35
|
+
chunk_id?: string;
|
|
36
|
+
chunk_index?: number;
|
|
37
|
+
/** The chunk's own heading, and its full path from the document root. */
|
|
38
|
+
title?: string;
|
|
39
|
+
heading_path?: string[];
|
|
31
40
|
best_score?: number;
|
|
32
41
|
score?: number;
|
|
33
42
|
is_partial?: boolean;
|
|
@@ -37,15 +46,86 @@ interface SearchRow {
|
|
|
37
46
|
below_confidence?: boolean;
|
|
38
47
|
}
|
|
39
48
|
|
|
49
|
+
/** The row's text, under whichever column name this search mode returns. */
|
|
50
|
+
export function rowContent(row: SearchRow): string {
|
|
51
|
+
return row.full_content ?? row.content ?? "";
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* `Document Title › Section` for a chunk row, `Document Title` for a document
|
|
56
|
+
* row, plus the ids that identify it.
|
|
57
|
+
*
|
|
58
|
+
* The chunk RPCs return several chunks OF THE SAME document, so without the
|
|
59
|
+
* section and the index those results are indistinguishable (#261). Shared by
|
|
60
|
+
* the rendered path and the degraded one, where identical headings would be
|
|
61
|
+
* the entire answer.
|
|
62
|
+
*/
|
|
63
|
+
function rowHeading(row: SearchRow): string {
|
|
64
|
+
const doc = row.doc_title ?? "Untitled";
|
|
65
|
+
// `heading_path` normally opens with the document's own H1, so the FIRST
|
|
66
|
+
// element is dropped when it repeats the title. Only the first: a section
|
|
67
|
+
// legitimately named after the document must still appear (#261).
|
|
68
|
+
const path = [...(row.heading_path ?? [])];
|
|
69
|
+
if (path.length > 0 && path[0] === doc) path.shift();
|
|
70
|
+
const section = path.length
|
|
71
|
+
? path.filter(Boolean).join(" › ")
|
|
72
|
+
: row.title && row.title !== doc
|
|
73
|
+
? row.title
|
|
74
|
+
: "";
|
|
75
|
+
const docId = row.document_id ? ` [id: ${row.document_id}]` : "";
|
|
76
|
+
const chunk = row.chunk_index != null ? ` (chunk ${row.chunk_index})` : "";
|
|
77
|
+
return `${doc}${section ? ` › ${section}` : ""}${docId}${chunk}`;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* A short label for a row in the truncation footer: the leaf section, the
|
|
82
|
+
* chunk index, and the document id.
|
|
83
|
+
*
|
|
84
|
+
* Shorter than the rendered heading, which carries the whole breadcrumb — the
|
|
85
|
+
* footer names up to five rows, and full headings pushed the reply over the
|
|
86
|
+
* budget it was reporting on (#263). The id stays, because chunk modes span
|
|
87
|
+
* documents too and two documents sharing a title and a section name are
|
|
88
|
+
* otherwise indistinguishable, with no way to fetch either (#265). The footer
|
|
89
|
+
* is inside the budget and drops names it cannot afford, so it is accounted
|
|
90
|
+
* for either way.
|
|
91
|
+
*/
|
|
92
|
+
function shortLabel(row: SearchRow): string {
|
|
93
|
+
const doc = row.doc_title ?? "Untitled";
|
|
94
|
+
const path = [...(row.heading_path ?? [])];
|
|
95
|
+
if (path.length > 0 && path[0] === doc) path.shift();
|
|
96
|
+
const leaf = path.filter(Boolean).at(-1) ?? (row.title && row.title !== doc ? row.title : "");
|
|
97
|
+
const chunk = row.chunk_index != null ? ` (chunk ${row.chunk_index})` : "";
|
|
98
|
+
// Always the id. Chunk modes are not single-document either, so two
|
|
99
|
+
// documents sharing a title and a section name would otherwise produce
|
|
100
|
+
// byte-identical entries with no way to fetch either (#265). The footer is
|
|
101
|
+
// inside the budget now and drops names it cannot afford, so carrying the
|
|
102
|
+
// id costs nothing that is not accounted for.
|
|
103
|
+
const id = row.document_id ? ` [id: ${row.document_id}]` : "";
|
|
104
|
+
return `${leaf ? `${doc} › ${leaf}` : doc}${chunk}${id}`;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** One rendered result: heading, score, then the row's text. */
|
|
108
|
+
function renderRow(row: SearchRow): string {
|
|
109
|
+
const raw = row.best_score ?? row.score;
|
|
110
|
+
const score = raw != null ? ` (score: ${raw.toFixed(3)})` : "";
|
|
111
|
+
const partial = row.is_partial
|
|
112
|
+
? ` -- partial (${row.chunk_count} of ${(row.total_chars ?? 0).toLocaleString()} chars)`
|
|
113
|
+
: "";
|
|
114
|
+
// content_hash = the concurrency token for cerefox_ingest updates (iter-32).
|
|
115
|
+
const hash = row.content_hash ? `\nhash: ${row.content_hash}` : "";
|
|
116
|
+
// Whichever column this mode returns (#259): `cerefox_search_docs` gives
|
|
117
|
+
// `full_content`, the chunk RPCs give `content`, and reading only the first
|
|
118
|
+
// rendered every hybrid/fts result as a title with an empty body.
|
|
119
|
+
return `## ${rowHeading(row)}${score}${partial}${hash}\n\n${rowContent(row)}`;
|
|
120
|
+
}
|
|
121
|
+
|
|
40
122
|
/** `## Title [id: …] (score: …) -- 20,297 chars` — everything but the content. */
|
|
41
123
|
function headerLine(row: SearchRow): string {
|
|
42
|
-
const title = row.doc_title ?? "Untitled";
|
|
43
|
-
const docId = row.document_id ? ` [id: ${row.document_id}]` : "";
|
|
44
124
|
const raw = row.best_score ?? row.score;
|
|
45
125
|
const score = raw != null ? ` (score: ${raw.toFixed(3)})` : "";
|
|
46
126
|
const size = row.total_chars != null ? ` -- ${row.total_chars.toLocaleString()} chars` : "";
|
|
47
127
|
const hash = row.content_hash ? `\nhash: ${row.content_hash}` : "";
|
|
48
|
-
return `## ${
|
|
128
|
+
return `## ${rowHeading(row)}${score}${size}${hash}`;
|
|
49
129
|
}
|
|
50
130
|
|
|
51
131
|
/**
|
|
@@ -56,15 +136,30 @@ function headerLine(row: SearchRow): string {
|
|
|
56
136
|
* still honours the limit the caller asked for; if even one header does not
|
|
57
137
|
* fit, the count and the remedy alone still beat silence.
|
|
58
138
|
*/
|
|
59
|
-
function degradedToHeaders(
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
139
|
+
function degradedToHeaders(
|
|
140
|
+
matched: SearchRow[],
|
|
141
|
+
rendered: string[],
|
|
142
|
+
maxBytes: number,
|
|
143
|
+
belowConfidence: boolean,
|
|
144
|
+
): string {
|
|
145
|
+
// Rendered bytes, not JSON: the fit decision that reaches here is made in
|
|
146
|
+
// rendered bytes, and quoting a JSON size produced messages saying the
|
|
147
|
+
// largest result was 490 bytes and did not fit a 600-byte budget (#265).
|
|
148
|
+
// Sizes come from the render the caller already paid for — re-rendering
|
|
149
|
+
// whole documents here just to measure them tripled peak memory.
|
|
150
|
+
const biggest = Math.max(...rendered.map((t) => new TextEncoder().encode(t).length));
|
|
151
|
+
// A degraded response must not silently promote 28I's weak-signal
|
|
152
|
+
// candidates into real matches: an agent told "N result(s) matched" about
|
|
153
|
+
// rows that cleared no threshold would trust them.
|
|
154
|
+
const confidence = belowConfidence
|
|
155
|
+
? "None of these cleared the confidence threshold — they are the closest " +
|
|
156
|
+
"candidates, so judge relevance from the scores. "
|
|
157
|
+
: "";
|
|
63
158
|
const lead =
|
|
64
159
|
`⚠ ${matched.length} result(s) matched, but none fit max_bytes=${maxBytes} ` +
|
|
65
|
-
`(the largest is ${biggest.toLocaleString()} bytes). This is NOT an
|
|
66
|
-
`knowledge base. Listing what matched, without content — raise max_bytes
|
|
67
|
-
`read it, or read one document with cerefox_get_document (outline: true for ` +
|
|
160
|
+
`(the largest is ${biggest.toLocaleString()} bytes). ${confidence}This is NOT an ` +
|
|
161
|
+
`empty knowledge base. Listing what matched, without content — raise max_bytes ` +
|
|
162
|
+
`to read it, or read one document with cerefox_get_document (outline: true for ` +
|
|
68
163
|
`structure, or section: "## Heading" for one part).`;
|
|
69
164
|
|
|
70
165
|
const lines: string[] = [];
|
|
@@ -86,7 +181,12 @@ async function handler(
|
|
|
86
181
|
): Promise<string> {
|
|
87
182
|
const query = args.query as string;
|
|
88
183
|
const project_name = args.project_name as string | undefined;
|
|
89
|
-
|
|
184
|
+
// Sanitised, then clamped, exactly as the Edge Function does it. Clamping
|
|
185
|
+
// alone left `NaN` for a non-numeric value, which serialises to JSON null,
|
|
186
|
+
// and `LIMIT NULL` in Postgres means NO limit — the unbounded work the
|
|
187
|
+
// clamp exists to prevent, reachable because the local MCP server passes
|
|
188
|
+
// tool arguments through unvalidated (#265).
|
|
189
|
+
const match_count = Math.min(Math.max(1, Math.floor(Number(args.match_count)) || 5), 200);
|
|
90
190
|
const mode = (args.mode as string | undefined) ?? "docs";
|
|
91
191
|
// #133: omit unconfigured tunables so the server resolves them from
|
|
92
192
|
// cerefox_config (one setting governs every access path).
|
|
@@ -107,7 +207,14 @@ async function handler(
|
|
|
107
207
|
const requested_max_bytes = args.max_bytes as number | undefined;
|
|
108
208
|
|
|
109
209
|
const ceiling = getMaxResponseBytes();
|
|
110
|
-
|
|
210
|
+
// Sanitised before clamping, for the reason `match_count` is: `NaN` makes
|
|
211
|
+
// every `> max_bytes` comparison below false, so a non-numeric value emitted
|
|
212
|
+
// every row unbounded — the ceiling bypassed by passing it a word (#266).
|
|
213
|
+
// A NUMBER of 0 or less still means "almost no budget", as it always did:
|
|
214
|
+
// falling back to the ceiling there would hand a caller whose remaining
|
|
215
|
+
// allowance ran out the largest possible reply (#267). Only a missing or
|
|
216
|
+
// non-numeric value defaults to the ceiling.
|
|
217
|
+
const max_bytes = resolveByteBudget(requested_max_bytes, ceiling);
|
|
111
218
|
|
|
112
219
|
if (
|
|
113
220
|
metadata_filter !== null &&
|
|
@@ -187,7 +294,137 @@ async function handler(
|
|
|
187
294
|
if (error) throw new Error(`RPC error: ${error.message}`);
|
|
188
295
|
|
|
189
296
|
const matched = (data ?? []) as SearchRow[];
|
|
190
|
-
|
|
297
|
+
|
|
298
|
+
// 28I: nothing cleared the relevance threshold, so the server returned its
|
|
299
|
+
// best-effort candidates flagged below_confidence rather than an empty set
|
|
300
|
+
// (which agents misread as "this knowledge does not exist"). The flag is
|
|
301
|
+
// all-or-nothing per response, so it is read from what MATCHED.
|
|
302
|
+
const belowConfidence = matched.length > 0 && matched.every((r) => r.below_confidence === true);
|
|
303
|
+
|
|
304
|
+
if (matched.length === 0) {
|
|
305
|
+
logUsage(supabase, {
|
|
306
|
+
operation: "search",
|
|
307
|
+
accessPath: ctx.accessPath,
|
|
308
|
+
requestor: callerIdentity(args),
|
|
309
|
+
query_text: query,
|
|
310
|
+
project_id: projectId,
|
|
311
|
+
result_count: 0,
|
|
312
|
+
});
|
|
313
|
+
return "No results found.";
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// ── Fit the reply to max_bytes, in the units the caller receives ──────────
|
|
317
|
+
//
|
|
318
|
+
// The budget is spent on RENDERED text, not on the JSON the RPC returned:
|
|
319
|
+
// `applyByteBudget` measures `JSON.stringify(row)`, which is the right unit
|
|
320
|
+
// for the Edge Function (it ships JSON) and the wrong one here (this returns
|
|
321
|
+
// markdown). Reserving rendered bytes out of a JSON-measured budget
|
|
322
|
+
// guaranteed nothing, and neither the below-confidence preamble (~185 bytes)
|
|
323
|
+
// nor the footer was counted by anything (#265).
|
|
324
|
+
//
|
|
325
|
+
// So: render, then take as many rows as the whole assembled reply can carry.
|
|
326
|
+
// Assembling and measuring is the only way to be sure, because the preamble
|
|
327
|
+
// and the footer both depend on how many rows were kept.
|
|
328
|
+
const rendered = matched.map(renderRow);
|
|
329
|
+
const size = (t: string) => new TextEncoder().encode(t).length;
|
|
330
|
+
const SEP = "\n\n---\n\n";
|
|
331
|
+
|
|
332
|
+
// The 28I advisory, long and short. It is charged to the budget like
|
|
333
|
+
// everything else, but it must never displace the answer it is advising
|
|
334
|
+
// about: a ~190-byte banner that pushes the only result out of a small reply
|
|
335
|
+
// leaves the caller with a warning and no content (#265). The short form is
|
|
336
|
+
// the floor — the contract is that weak candidates are never presented as
|
|
337
|
+
// confident ones, and that survives in six words.
|
|
338
|
+
const banner = (take: number, short: boolean) =>
|
|
339
|
+
!belowConfidence
|
|
340
|
+
? ""
|
|
341
|
+
: short
|
|
342
|
+
? `⚠ Below the confidence threshold — judge relevance from the scores.\n\n`
|
|
343
|
+
: `⚠ No results cleared the confidence threshold. Showing the closest ${take} ` +
|
|
344
|
+
`candidate(s) with scores — judge relevance yourself; a low score means weak ` +
|
|
345
|
+
`signal, not necessarily absent knowledge.\n\n`;
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* The whole reply for a set of kept rows.
|
|
349
|
+
*
|
|
350
|
+
* Everything but the results is optional, in this order: the long advisory
|
|
351
|
+
* gives way first, then the named list of what was dropped, then the bare
|
|
352
|
+
* footer. What is NOT optional is saying that results were dropped — a
|
|
353
|
+
* reply that quietly returns 1 of 5 hits leaves the caller believing they
|
|
354
|
+
* saw everything, which is the failure this whole file is about (#266).
|
|
355
|
+
*/
|
|
356
|
+
const assemble = (keptCount: number, short: boolean): string => {
|
|
357
|
+
const kept = keptIdx.slice(0, keptCount);
|
|
358
|
+
const body = banner(keptCount, short) + kept.map((i) => rendered[i]!).join(SEP);
|
|
359
|
+
if (kept.length === matched.length) return body;
|
|
360
|
+
|
|
361
|
+
const droppedRows = matched.filter((_, i) => !kept.includes(i));
|
|
362
|
+
const room = max_bytes - size(body);
|
|
363
|
+
const footer = (named: number) => {
|
|
364
|
+
const labels = droppedRows.slice(0, named).map(shortLabel);
|
|
365
|
+
const rest = droppedRows.length - labels.length;
|
|
366
|
+
const naming = labels.length
|
|
367
|
+
? `: ${labels.join(", ")}${rest > 0 ? ` and ${rest} more` : ""}`
|
|
368
|
+
: "";
|
|
369
|
+
return (
|
|
370
|
+
`\n\n[${kept.length} of ${matched.length} result(s) shown; ${droppedRows.length} did ` +
|
|
371
|
+
`not fit max_bytes=${max_bytes}${naming}. Raise max_bytes, narrow the query, or ` +
|
|
372
|
+
`lower match_count.]`
|
|
373
|
+
);
|
|
374
|
+
};
|
|
375
|
+
for (let named = Math.min(5, droppedRows.length); named >= 1; named--) {
|
|
376
|
+
const candidate = footer(named);
|
|
377
|
+
if (size(candidate) <= room) return body + candidate;
|
|
378
|
+
}
|
|
379
|
+
const bare = footer(0);
|
|
380
|
+
if (size(bare) <= room) return body + bare;
|
|
381
|
+
// The floor, ~25 bytes, never dropped: the caller must know there is more.
|
|
382
|
+
return body + `\n\n[${kept.length} of ${matched.length} shown; raise max_bytes]`;
|
|
383
|
+
};
|
|
384
|
+
|
|
385
|
+
// Which rows the budget can carry, in rank order. Rows that do not fit are
|
|
386
|
+
// SKIPPED rather than ending the scan: one oversized top hit used to suppress
|
|
387
|
+
// every smaller result behind it, and the reply then claimed nothing fit when
|
|
388
|
+
// rows two and three would have fitted comfortably (#266).
|
|
389
|
+
const sepBytes = size(SEP);
|
|
390
|
+
const keptIdx: number[] = [];
|
|
391
|
+
let acc = 0;
|
|
392
|
+
for (let i = 0; i < rendered.length; i++) {
|
|
393
|
+
const add = size(rendered[i]!) + (keptIdx.length > 0 ? sepBytes : 0);
|
|
394
|
+
if (acc + add > max_bytes) continue;
|
|
395
|
+
acc += add;
|
|
396
|
+
keptIdx.push(i);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// Not one result fits (#254) — the only case that answers with no content,
|
|
400
|
+
// and now the only case that can truthfully say so.
|
|
401
|
+
if (keptIdx.length === 0) {
|
|
402
|
+
logUsage(supabase, {
|
|
403
|
+
operation: "search",
|
|
404
|
+
accessPath: ctx.accessPath,
|
|
405
|
+
requestor: callerIdentity(args),
|
|
406
|
+
query_text: query,
|
|
407
|
+
project_id: projectId,
|
|
408
|
+
result_count: matched.length,
|
|
409
|
+
extra: { returned: 0, truncated: true, degraded: true },
|
|
410
|
+
});
|
|
411
|
+
return degradedToHeaders(matched, rendered, max_bytes, belowConfidence);
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Give up the framing before the content, but never the fact of truncation:
|
|
415
|
+
// long advisory, then a result, and the "N of M shown" notice always stays.
|
|
416
|
+
let keptCount = keptIdx.length;
|
|
417
|
+
let short = false;
|
|
418
|
+
let output = assemble(keptCount, short);
|
|
419
|
+
while (size(output) > max_bytes) {
|
|
420
|
+
if (belowConfidence && !short) short = true;
|
|
421
|
+
else if (keptCount > 1) {
|
|
422
|
+
keptCount -= 1;
|
|
423
|
+
short = belowConfidence;
|
|
424
|
+
} else break; // one result plus the shortest possible framing: the floor
|
|
425
|
+
output = assemble(keptCount, short);
|
|
426
|
+
}
|
|
427
|
+
const take = keptCount;
|
|
191
428
|
|
|
192
429
|
logUsage(supabase, {
|
|
193
430
|
operation: "search",
|
|
@@ -195,59 +432,12 @@ async function handler(
|
|
|
195
432
|
requestor: callerIdentity(args),
|
|
196
433
|
query_text: query,
|
|
197
434
|
project_id: projectId,
|
|
198
|
-
// What the QUERY matched, not what survived the
|
|
199
|
-
//
|
|
200
|
-
// budget-wiped search look like an empty knowledge base in analytics too.
|
|
435
|
+
// What the QUERY matched, not what survived the budget: recording the
|
|
436
|
+
// latter made a budget-wiped search look like an empty store in analytics.
|
|
201
437
|
result_count: matched.length,
|
|
202
|
-
...(
|
|
438
|
+
...(take < matched.length ? { extra: { returned: take, truncated: true } } : {}),
|
|
203
439
|
});
|
|
204
440
|
|
|
205
|
-
// Nothing matched: the honest empty answer.
|
|
206
|
-
if (matched.length === 0) return "No results found.";
|
|
207
|
-
|
|
208
|
-
// Something matched but none of it fit the budget (#254). Reporting "no
|
|
209
|
-
// results" here is the most damaging answer the tool can give: an agent
|
|
210
|
-
// stops looking and often recreates the document it failed to find. Show
|
|
211
|
-
// the headers instead — a few hundred bytes that name what exists and how
|
|
212
|
-
// to read it.
|
|
213
|
-
if (accepted.length === 0) {
|
|
214
|
-
return degradedToHeaders(matched, max_bytes);
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
const rows = accepted as SearchRow[];
|
|
218
|
-
|
|
219
|
-
// 28I: nothing cleared the relevance threshold, so the server returned its
|
|
220
|
-
// best-effort top candidates flagged below_confidence instead of an empty
|
|
221
|
-
// set (which agents misread as "this knowledge does not exist").
|
|
222
|
-
const belowConfidence = rows.length > 0 && rows.every((r) => r.below_confidence === true);
|
|
223
|
-
|
|
224
|
-
const parts: string[] = rows.map((row) => {
|
|
225
|
-
const title = row.doc_title ?? "Untitled";
|
|
226
|
-
const docId = row.document_id ? ` [id: ${row.document_id}]` : "";
|
|
227
|
-
const rawScore = row.best_score ?? row.score;
|
|
228
|
-
const score = rawScore != null ? ` (score: ${rawScore.toFixed(3)})` : "";
|
|
229
|
-
const partial = row.is_partial
|
|
230
|
-
? ` -- partial (${row.chunk_count} of ${(row.total_chars ?? 0).toLocaleString()} chars)`
|
|
231
|
-
: "";
|
|
232
|
-
// content_hash = the concurrency token for cerefox_ingest updates (iter-32).
|
|
233
|
-
const hash = row.content_hash ? `\nhash: ${row.content_hash}` : "";
|
|
234
|
-
return `## ${title}${docId}${score}${partial}${hash}\n\n${row.full_content ?? ""}`;
|
|
235
|
-
});
|
|
236
|
-
|
|
237
|
-
let output = parts.join("\n\n---\n\n");
|
|
238
|
-
if (belowConfidence) {
|
|
239
|
-
output =
|
|
240
|
-
`⚠ No results cleared the confidence threshold. Showing the closest ${rows.length} ` +
|
|
241
|
-
`candidate(s) with scores — judge relevance yourself; a low score means weak signal, ` +
|
|
242
|
-
`not necessarily absent knowledge.\n\n` + output;
|
|
243
|
-
}
|
|
244
|
-
if (truncated) {
|
|
245
|
-
output +=
|
|
246
|
-
`\n\n[${accepted.length} of ${matched.length} result(s) shown; truncated at ` +
|
|
247
|
-
`${usedBytes} bytes. ${dropped.length} did not fit: ` +
|
|
248
|
-
`${dropped.map((r) => (r as SearchRow).doc_title ?? "Untitled").join(", ")}. ` +
|
|
249
|
-
`Raise max_bytes, narrow the query, or lower match_count.]`;
|
|
250
|
-
}
|
|
251
441
|
return output;
|
|
252
442
|
}
|
|
253
443
|
|
|
@@ -269,7 +459,7 @@ export const searchTool: ToolDefinition = {
|
|
|
269
459
|
query: { type: "string", description: "Natural-language search query" },
|
|
270
460
|
match_count: {
|
|
271
461
|
type: "integer",
|
|
272
|
-
description: "Maximum number of documents to return (default: 5)",
|
|
462
|
+
description: "Maximum number of documents to return (default: 5, maximum: 200)",
|
|
273
463
|
},
|
|
274
464
|
project_name: {
|
|
275
465
|
type: "string",
|