@cerefox/memory 1.14.2 → 1.14.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_GUIDE.md +1 -1
- package/dist/bin/cerefox.js +609 -415
- package/dist/server-assets/_shared/ef-meta/index.ts +3 -3
- package/dist/server-assets/_shared/mcp-tools/metadata-search.ts +69 -20
- package/dist/server-assets/_shared/mcp-tools/search.ts +259 -65
- package/dist/server-assets/supabase/functions/cerefox-search/index.ts +53 -15
- package/docs/guides/connect-agents.md +11 -6
- package/docs/guides/response-limits.md +25 -3
- package/package.json +3 -3
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* doesn't touch `supabase/functions/` leaves it alone).
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
export const EF_VERSION = "1.14.
|
|
21
|
+
export const EF_VERSION = "1.14.3";
|
|
22
22
|
|
|
23
23
|
/**
|
|
24
24
|
* The Cerefox RELEASE version — what `cerefox --version` reports and what npm
|
|
@@ -36,7 +36,7 @@ export const EF_VERSION = "1.14.2";
|
|
|
36
36
|
* is imported by the Deno Edge Functions, which cannot reach into the npm
|
|
37
37
|
* package.
|
|
38
38
|
*/
|
|
39
|
-
export const CEREFOX_VERSION = "1.14.
|
|
39
|
+
export const CEREFOX_VERSION = "1.14.3";
|
|
40
40
|
|
|
41
41
|
/**
|
|
42
42
|
* The most recent version whose EF-side SOURCE actually changed (#127).
|
|
@@ -46,7 +46,7 @@ export const CEREFOX_VERSION = "1.14.2";
|
|
|
46
46
|
* `cut_release.ts` ONLY when EF source changed since the last tag; doctor
|
|
47
47
|
* uses it to stay silent on label-only drift.
|
|
48
48
|
*/
|
|
49
|
-
export const EF_LAST_CHANGED = "1.14.
|
|
49
|
+
export const EF_LAST_CHANGED = "1.14.3";
|
|
50
50
|
|
|
51
51
|
/**
|
|
52
52
|
* The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp
|
|
@@ -22,7 +22,11 @@ async function handler(
|
|
|
22
22
|
const project_name = args.project_name as string | undefined;
|
|
23
23
|
const updated_since = args.updated_since as string | undefined;
|
|
24
24
|
const created_since = args.created_since as string | undefined;
|
|
25
|
-
|
|
25
|
+
// Sanitised and clamped, like `match_count` on the search tool: the local
|
|
26
|
+
// stdio server passes tool arguments through unvalidated, so a caller could
|
|
27
|
+
// ask for a million rows (#267). 500 matches the largest listing the web
|
|
28
|
+
// API serves, so no legitimate enumeration is cut short.
|
|
29
|
+
const limit = Math.min(Math.max(1, Math.floor(Number(args.limit)) || 10), 500);
|
|
26
30
|
const include_content = (args.include_content as boolean | undefined) ?? false;
|
|
27
31
|
const requested_max_bytes = args.max_bytes as number | undefined;
|
|
28
32
|
|
|
@@ -47,10 +51,20 @@ async function handler(
|
|
|
47
51
|
if (!projectId) throw new Error(`Project not found: ${project_name}`);
|
|
48
52
|
}
|
|
49
53
|
|
|
50
|
-
// Enforce byte ceiling for content mode
|
|
54
|
+
// Enforce byte ceiling for content mode.
|
|
55
|
+
//
|
|
56
|
+
// Sanitised first: a non-numeric `max_bytes` became `NaN`, which reaches the
|
|
57
|
+
// RPC as JSON null, and `p_max_bytes NULL` means NO limit — so one word
|
|
58
|
+
// instead of a number returned every matching document's full content, with
|
|
59
|
+
// the in-process guard below disabled too (#267). Same hole as the search
|
|
60
|
+
// tool's, in the sibling that shares its transport.
|
|
51
61
|
const ceiling = getMaxResponseBytes();
|
|
62
|
+
const requestedBytes = Math.floor(Number(requested_max_bytes));
|
|
52
63
|
const max_bytes = include_content
|
|
53
|
-
? Math.min(
|
|
64
|
+
? Math.min(
|
|
65
|
+
Number.isFinite(requestedBytes) ? Math.max(requestedBytes, 1) : ceiling,
|
|
66
|
+
ceiling,
|
|
67
|
+
)
|
|
54
68
|
: null;
|
|
55
69
|
|
|
56
70
|
const params: Record<string, unknown> = {
|
|
@@ -84,14 +98,20 @@ async function handler(
|
|
|
84
98
|
content: string | null;
|
|
85
99
|
}>;
|
|
86
100
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
101
|
+
// Logged once, AFTER the degraded branch below may have found the real
|
|
102
|
+
// count: firing here unconditionally wrote a `result_count: 0` row for
|
|
103
|
+
// every budget-wiped search, and adding a second row in the branch made
|
|
104
|
+
// analytics double-count them (#259).
|
|
105
|
+
const log = (result_count: number, extra?: Record<string, unknown>) =>
|
|
106
|
+
logUsage(supabase, {
|
|
107
|
+
operation: "metadata_search",
|
|
108
|
+
accessPath: ctx.accessPath,
|
|
109
|
+
requestor: callerIdentity(args),
|
|
110
|
+
query_text: JSON.stringify(metadata_filter ?? {}),
|
|
111
|
+
project_id: projectId,
|
|
112
|
+
result_count,
|
|
113
|
+
...(extra ? { extra } : {}),
|
|
114
|
+
});
|
|
95
115
|
|
|
96
116
|
if (rows.length === 0) {
|
|
97
117
|
// The RPC applies the byte budget server-side by stopping at the first row
|
|
@@ -100,24 +120,53 @@ async function handler(
|
|
|
100
120
|
// without content before reporting nothing: an agent that is told "no
|
|
101
121
|
// documents" stops looking, and here that would be false.
|
|
102
122
|
if (include_content && max_bytes !== null) {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
123
|
+
// supabase-js RESOLVES with `{ data: null, error }` for PostgREST
|
|
124
|
+
// failures and for network errors; it does not throw. A try/catch here
|
|
125
|
+
// was dead code, and worse: `headers` came back null, the branch below
|
|
126
|
+
// was skipped, and the handler answered "No documents match the given
|
|
127
|
+
// criteria" — the exact false-empty this branch exists to prevent
|
|
128
|
+
// (#261). Read the error, and say what is actually known.
|
|
129
|
+
const { data: headers, error: probeError } = await supabase.rpc(
|
|
130
|
+
"cerefox_metadata_search",
|
|
131
|
+
{ ...params, p_include_content: false, p_max_bytes: null },
|
|
132
|
+
);
|
|
133
|
+
if (probeError) {
|
|
134
|
+
log(0, { degraded_probe_failed: true });
|
|
135
|
+
return (
|
|
136
|
+
`⚠ Nothing fit max_bytes=${max_bytes} with include_content, and the ` +
|
|
137
|
+
`follow-up query that lists what matched failed: ${probeError.message}. ` +
|
|
138
|
+
`This is NOT a confirmed empty result — retry with a larger max_bytes, ` +
|
|
139
|
+
`or with include_content: false.`
|
|
140
|
+
);
|
|
141
|
+
}
|
|
108
142
|
const headerRows = (headers ?? []) as Array<{ document_id: string; title: string }>;
|
|
109
143
|
if (headerRows.length > 0) {
|
|
110
|
-
|
|
144
|
+
// The caller asked for a budget; honour it in the answer that explains
|
|
145
|
+
// the budget. `limit` is caller-supplied, so an uncapped list could be
|
|
146
|
+
// tens of KB in reply to a 2 KB request (#257).
|
|
147
|
+
const lead =
|
|
111
148
|
`⚠ ${headerRows.length} document(s) match, but none fit max_bytes=${max_bytes} ` +
|
|
112
149
|
`with include_content. This is NOT an empty result. Listing them without ` +
|
|
113
150
|
`content — raise max_bytes, or read one with cerefox_get_document ` +
|
|
114
|
-
`(outline: true for structure)
|
|
115
|
-
|
|
116
|
-
);
|
|
151
|
+
`(outline: true for structure).`;
|
|
152
|
+
const lines: string[] = [];
|
|
153
|
+
let used = new TextEncoder().encode(lead).length;
|
|
154
|
+
for (const r of headerRows) {
|
|
155
|
+
const line = `## ${r.title} [id: ${r.document_id}]`;
|
|
156
|
+
const size = new TextEncoder().encode(line).length + 1;
|
|
157
|
+
if (used + size > max_bytes) break;
|
|
158
|
+
lines.push(line);
|
|
159
|
+
used += size;
|
|
160
|
+
}
|
|
161
|
+
// What matched, not what the budget allowed through (#257).
|
|
162
|
+
log(headerRows.length, { returned: lines.length, degraded: true });
|
|
163
|
+
return lines.length > 0 ? `${lead}\n${lines.join("\n")}` : lead;
|
|
117
164
|
}
|
|
118
165
|
}
|
|
166
|
+
log(0);
|
|
119
167
|
return "No documents match the given criteria.";
|
|
120
168
|
}
|
|
169
|
+
log(rows.length);
|
|
121
170
|
|
|
122
171
|
// The review status is a column of a feature that may be off (#241); when
|
|
123
172
|
// it is, an agent should not see "approved" and wonder what it means.
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
import type { MCPSupabaseClient } from "./types.ts";
|
|
19
19
|
|
|
20
20
|
import { getEmbedding, resolveEmbedderKind } from "../embeddings/index.ts";
|
|
21
|
-
import {
|
|
21
|
+
import { getConfiguredMinSearchScore, getConfiguredSearchAlpha,
|
|
22
22
|
getMaxResponseBytes, getMinTermCoverage, logUsage } from "./_utils.ts";
|
|
23
23
|
import { lookupProjectId } from "./_projects.ts";
|
|
24
24
|
import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
|
|
@@ -27,7 +27,16 @@ import { AUTHOR_PARAM_READ, callerIdentity } from "./identity.ts";
|
|
|
27
27
|
interface SearchRow {
|
|
28
28
|
document_id?: string;
|
|
29
29
|
doc_title?: string;
|
|
30
|
+
/** Document-level modes (`docs`) return this… */
|
|
30
31
|
full_content?: string;
|
|
32
|
+
/** …while `hybrid` and `fts` return the chunk text under this name. */
|
|
33
|
+
content?: string;
|
|
34
|
+
/** Chunk-mode identity: the RPCs return several chunks OF THE SAME document. */
|
|
35
|
+
chunk_id?: string;
|
|
36
|
+
chunk_index?: number;
|
|
37
|
+
/** The chunk's own heading, and its full path from the document root. */
|
|
38
|
+
title?: string;
|
|
39
|
+
heading_path?: string[];
|
|
31
40
|
best_score?: number;
|
|
32
41
|
score?: number;
|
|
33
42
|
is_partial?: boolean;
|
|
@@ -37,15 +46,86 @@ interface SearchRow {
|
|
|
37
46
|
below_confidence?: boolean;
|
|
38
47
|
}
|
|
39
48
|
|
|
49
|
+
/** The row's text, under whichever column name this search mode returns. */
|
|
50
|
+
export function rowContent(row: SearchRow): string {
|
|
51
|
+
return row.full_content ?? row.content ?? "";
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* `Document Title › Section` for a chunk row, `Document Title` for a document
|
|
56
|
+
* row, plus the ids that identify it.
|
|
57
|
+
*
|
|
58
|
+
* The chunk RPCs return several chunks OF THE SAME document, so without the
|
|
59
|
+
* section and the index those results are indistinguishable (#261). Shared by
|
|
60
|
+
* the rendered path and the degraded one, where identical headings would be
|
|
61
|
+
* the entire answer.
|
|
62
|
+
*/
|
|
63
|
+
function rowHeading(row: SearchRow): string {
|
|
64
|
+
const doc = row.doc_title ?? "Untitled";
|
|
65
|
+
// `heading_path` normally opens with the document's own H1, so the FIRST
|
|
66
|
+
// element is dropped when it repeats the title. Only the first: a section
|
|
67
|
+
// legitimately named after the document must still appear (#261).
|
|
68
|
+
const path = [...(row.heading_path ?? [])];
|
|
69
|
+
if (path.length > 0 && path[0] === doc) path.shift();
|
|
70
|
+
const section = path.length
|
|
71
|
+
? path.filter(Boolean).join(" › ")
|
|
72
|
+
: row.title && row.title !== doc
|
|
73
|
+
? row.title
|
|
74
|
+
: "";
|
|
75
|
+
const docId = row.document_id ? ` [id: ${row.document_id}]` : "";
|
|
76
|
+
const chunk = row.chunk_index != null ? ` (chunk ${row.chunk_index})` : "";
|
|
77
|
+
return `${doc}${section ? ` › ${section}` : ""}${docId}${chunk}`;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* A short label for a row in the truncation footer: the leaf section, the
|
|
82
|
+
* chunk index, and the document id.
|
|
83
|
+
*
|
|
84
|
+
* Shorter than the rendered heading, which carries the whole breadcrumb — the
|
|
85
|
+
* footer names up to five rows, and full headings pushed the reply over the
|
|
86
|
+
* budget it was reporting on (#263). The id stays, because chunk modes span
|
|
87
|
+
* documents too and two documents sharing a title and a section name are
|
|
88
|
+
* otherwise indistinguishable, with no way to fetch either (#265). The footer
|
|
89
|
+
* is inside the budget and drops names it cannot afford, so it is accounted
|
|
90
|
+
* for either way.
|
|
91
|
+
*/
|
|
92
|
+
function shortLabel(row: SearchRow): string {
|
|
93
|
+
const doc = row.doc_title ?? "Untitled";
|
|
94
|
+
const path = [...(row.heading_path ?? [])];
|
|
95
|
+
if (path.length > 0 && path[0] === doc) path.shift();
|
|
96
|
+
const leaf = path.filter(Boolean).at(-1) ?? (row.title && row.title !== doc ? row.title : "");
|
|
97
|
+
const chunk = row.chunk_index != null ? ` (chunk ${row.chunk_index})` : "";
|
|
98
|
+
// Always the id. Chunk modes are not single-document either, so two
|
|
99
|
+
// documents sharing a title and a section name would otherwise produce
|
|
100
|
+
// byte-identical entries with no way to fetch either (#265). The footer is
|
|
101
|
+
// inside the budget now and drops names it cannot afford, so carrying the
|
|
102
|
+
// id costs nothing that is not accounted for.
|
|
103
|
+
const id = row.document_id ? ` [id: ${row.document_id}]` : "";
|
|
104
|
+
return `${leaf ? `${doc} › ${leaf}` : doc}${chunk}${id}`;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** One rendered result: heading, score, then the row's text. */
|
|
108
|
+
function renderRow(row: SearchRow): string {
|
|
109
|
+
const raw = row.best_score ?? row.score;
|
|
110
|
+
const score = raw != null ? ` (score: ${raw.toFixed(3)})` : "";
|
|
111
|
+
const partial = row.is_partial
|
|
112
|
+
? ` -- partial (${row.chunk_count} of ${(row.total_chars ?? 0).toLocaleString()} chars)`
|
|
113
|
+
: "";
|
|
114
|
+
// content_hash = the concurrency token for cerefox_ingest updates (iter-32).
|
|
115
|
+
const hash = row.content_hash ? `\nhash: ${row.content_hash}` : "";
|
|
116
|
+
// Whichever column this mode returns (#259): `cerefox_search_docs` gives
|
|
117
|
+
// `full_content`, the chunk RPCs give `content`, and reading only the first
|
|
118
|
+
// rendered every hybrid/fts result as a title with an empty body.
|
|
119
|
+
return `## ${rowHeading(row)}${score}${partial}${hash}\n\n${rowContent(row)}`;
|
|
120
|
+
}
|
|
121
|
+
|
|
40
122
|
/** `## Title [id: …] (score: …) -- 20,297 chars` — everything but the content. */
|
|
41
123
|
function headerLine(row: SearchRow): string {
|
|
42
|
-
const title = row.doc_title ?? "Untitled";
|
|
43
|
-
const docId = row.document_id ? ` [id: ${row.document_id}]` : "";
|
|
44
124
|
const raw = row.best_score ?? row.score;
|
|
45
125
|
const score = raw != null ? ` (score: ${raw.toFixed(3)})` : "";
|
|
46
126
|
const size = row.total_chars != null ? ` -- ${row.total_chars.toLocaleString()} chars` : "";
|
|
47
127
|
const hash = row.content_hash ? `\nhash: ${row.content_hash}` : "";
|
|
48
|
-
return `## ${
|
|
128
|
+
return `## ${rowHeading(row)}${score}${size}${hash}`;
|
|
49
129
|
}
|
|
50
130
|
|
|
51
131
|
/**
|
|
@@ -56,15 +136,30 @@ function headerLine(row: SearchRow): string {
|
|
|
56
136
|
* still honours the limit the caller asked for; if even one header does not
|
|
57
137
|
* fit, the count and the remedy alone still beat silence.
|
|
58
138
|
*/
|
|
59
|
-
function degradedToHeaders(
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
139
|
+
function degradedToHeaders(
|
|
140
|
+
matched: SearchRow[],
|
|
141
|
+
rendered: string[],
|
|
142
|
+
maxBytes: number,
|
|
143
|
+
belowConfidence: boolean,
|
|
144
|
+
): string {
|
|
145
|
+
// Rendered bytes, not JSON: the fit decision that reaches here is made in
|
|
146
|
+
// rendered bytes, and quoting a JSON size produced messages saying the
|
|
147
|
+
// largest result was 490 bytes and did not fit a 600-byte budget (#265).
|
|
148
|
+
// Sizes come from the render the caller already paid for — re-rendering
|
|
149
|
+
// whole documents here just to measure them tripled peak memory.
|
|
150
|
+
const biggest = Math.max(...rendered.map((t) => new TextEncoder().encode(t).length));
|
|
151
|
+
// A degraded response must not silently promote 28I's weak-signal
|
|
152
|
+
// candidates into real matches: an agent told "N result(s) matched" about
|
|
153
|
+
// rows that cleared no threshold would trust them.
|
|
154
|
+
const confidence = belowConfidence
|
|
155
|
+
? "None of these cleared the confidence threshold — they are the closest " +
|
|
156
|
+
"candidates, so judge relevance from the scores. "
|
|
157
|
+
: "";
|
|
63
158
|
const lead =
|
|
64
159
|
`⚠ ${matched.length} result(s) matched, but none fit max_bytes=${maxBytes} ` +
|
|
65
|
-
`(the largest is ${biggest.toLocaleString()} bytes). This is NOT an
|
|
66
|
-
`knowledge base. Listing what matched, without content — raise max_bytes
|
|
67
|
-
`read it, or read one document with cerefox_get_document (outline: true for ` +
|
|
160
|
+
`(the largest is ${biggest.toLocaleString()} bytes). ${confidence}This is NOT an ` +
|
|
161
|
+
`empty knowledge base. Listing what matched, without content — raise max_bytes ` +
|
|
162
|
+
`to read it, or read one document with cerefox_get_document (outline: true for ` +
|
|
68
163
|
`structure, or section: "## Heading" for one part).`;
|
|
69
164
|
|
|
70
165
|
const lines: string[] = [];
|
|
@@ -86,7 +181,12 @@ async function handler(
|
|
|
86
181
|
): Promise<string> {
|
|
87
182
|
const query = args.query as string;
|
|
88
183
|
const project_name = args.project_name as string | undefined;
|
|
89
|
-
|
|
184
|
+
// Sanitised, then clamped, exactly as the Edge Function does it. Clamping
|
|
185
|
+
// alone left `NaN` for a non-numeric value, which serialises to JSON null,
|
|
186
|
+
// and `LIMIT NULL` in Postgres means NO limit — the unbounded work the
|
|
187
|
+
// clamp exists to prevent, reachable because the local MCP server passes
|
|
188
|
+
// tool arguments through unvalidated (#265).
|
|
189
|
+
const match_count = Math.min(Math.max(1, Math.floor(Number(args.match_count)) || 5), 200);
|
|
90
190
|
const mode = (args.mode as string | undefined) ?? "docs";
|
|
91
191
|
// #133: omit unconfigured tunables so the server resolves them from
|
|
92
192
|
// cerefox_config (one setting governs every access path).
|
|
@@ -107,7 +207,18 @@ async function handler(
|
|
|
107
207
|
const requested_max_bytes = args.max_bytes as number | undefined;
|
|
108
208
|
|
|
109
209
|
const ceiling = getMaxResponseBytes();
|
|
110
|
-
|
|
210
|
+
// Sanitised before clamping, for the reason `match_count` is: `NaN` makes
|
|
211
|
+
// every `> max_bytes` comparison below false, so a non-numeric value emitted
|
|
212
|
+
// every row unbounded — the ceiling bypassed by passing it a word (#266).
|
|
213
|
+
// A NUMBER of 0 or less still means "almost no budget", as it always did:
|
|
214
|
+
// falling back to the ceiling there would hand a caller whose remaining
|
|
215
|
+
// allowance ran out the largest possible reply (#267). Only a missing or
|
|
216
|
+
// non-numeric value defaults to the ceiling.
|
|
217
|
+
const requestedBytes = Math.floor(Number(requested_max_bytes));
|
|
218
|
+
const max_bytes = Math.min(
|
|
219
|
+
Number.isFinite(requestedBytes) ? Math.max(requestedBytes, 1) : ceiling,
|
|
220
|
+
ceiling,
|
|
221
|
+
);
|
|
111
222
|
|
|
112
223
|
if (
|
|
113
224
|
metadata_filter !== null &&
|
|
@@ -187,7 +298,137 @@ async function handler(
|
|
|
187
298
|
if (error) throw new Error(`RPC error: ${error.message}`);
|
|
188
299
|
|
|
189
300
|
const matched = (data ?? []) as SearchRow[];
|
|
190
|
-
|
|
301
|
+
|
|
302
|
+
// 28I: nothing cleared the relevance threshold, so the server returned its
|
|
303
|
+
// best-effort candidates flagged below_confidence rather than an empty set
|
|
304
|
+
// (which agents misread as "this knowledge does not exist"). The flag is
|
|
305
|
+
// all-or-nothing per response, so it is read from what MATCHED.
|
|
306
|
+
const belowConfidence = matched.length > 0 && matched.every((r) => r.below_confidence === true);
|
|
307
|
+
|
|
308
|
+
if (matched.length === 0) {
|
|
309
|
+
logUsage(supabase, {
|
|
310
|
+
operation: "search",
|
|
311
|
+
accessPath: ctx.accessPath,
|
|
312
|
+
requestor: callerIdentity(args),
|
|
313
|
+
query_text: query,
|
|
314
|
+
project_id: projectId,
|
|
315
|
+
result_count: 0,
|
|
316
|
+
});
|
|
317
|
+
return "No results found.";
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// ── Fit the reply to max_bytes, in the units the caller receives ──────────
|
|
321
|
+
//
|
|
322
|
+
// The budget is spent on RENDERED text, not on the JSON the RPC returned:
|
|
323
|
+
// `applyByteBudget` measures `JSON.stringify(row)`, which is the right unit
|
|
324
|
+
// for the Edge Function (it ships JSON) and the wrong one here (this returns
|
|
325
|
+
// markdown). Reserving rendered bytes out of a JSON-measured budget
|
|
326
|
+
// guaranteed nothing, and neither the below-confidence preamble (~185 bytes)
|
|
327
|
+
// nor the footer was counted by anything (#265).
|
|
328
|
+
//
|
|
329
|
+
// So: render, then take as many rows as the whole assembled reply can carry.
|
|
330
|
+
// Assembling and measuring is the only way to be sure, because the preamble
|
|
331
|
+
// and the footer both depend on how many rows were kept.
|
|
332
|
+
const rendered = matched.map(renderRow);
|
|
333
|
+
const size = (t: string) => new TextEncoder().encode(t).length;
|
|
334
|
+
const SEP = "\n\n---\n\n";
|
|
335
|
+
|
|
336
|
+
// The 28I advisory, long and short. It is charged to the budget like
|
|
337
|
+
// everything else, but it must never displace the answer it is advising
|
|
338
|
+
// about: a ~190-byte banner that pushes the only result out of a small reply
|
|
339
|
+
// leaves the caller with a warning and no content (#265). The short form is
|
|
340
|
+
// the floor — the contract is that weak candidates are never presented as
|
|
341
|
+
// confident ones, and that survives in six words.
|
|
342
|
+
const banner = (take: number, short: boolean) =>
|
|
343
|
+
!belowConfidence
|
|
344
|
+
? ""
|
|
345
|
+
: short
|
|
346
|
+
? `⚠ Below the confidence threshold — judge relevance from the scores.\n\n`
|
|
347
|
+
: `⚠ No results cleared the confidence threshold. Showing the closest ${take} ` +
|
|
348
|
+
`candidate(s) with scores — judge relevance yourself; a low score means weak ` +
|
|
349
|
+
`signal, not necessarily absent knowledge.\n\n`;
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* The whole reply for a set of kept rows.
|
|
353
|
+
*
|
|
354
|
+
* Everything but the results is optional, in this order: the long advisory
|
|
355
|
+
* gives way first, then the named list of what was dropped, then the bare
|
|
356
|
+
* footer. What is NOT optional is saying that results were dropped — a
|
|
357
|
+
* reply that quietly returns 1 of 5 hits leaves the caller believing they
|
|
358
|
+
* saw everything, which is the failure this whole file is about (#266).
|
|
359
|
+
*/
|
|
360
|
+
const assemble = (keptCount: number, short: boolean): string => {
|
|
361
|
+
const kept = keptIdx.slice(0, keptCount);
|
|
362
|
+
const body = banner(keptCount, short) + kept.map((i) => rendered[i]!).join(SEP);
|
|
363
|
+
if (kept.length === matched.length) return body;
|
|
364
|
+
|
|
365
|
+
const droppedRows = matched.filter((_, i) => !kept.includes(i));
|
|
366
|
+
const room = max_bytes - size(body);
|
|
367
|
+
const footer = (named: number) => {
|
|
368
|
+
const labels = droppedRows.slice(0, named).map(shortLabel);
|
|
369
|
+
const rest = droppedRows.length - labels.length;
|
|
370
|
+
const naming = labels.length
|
|
371
|
+
? `: ${labels.join(", ")}${rest > 0 ? ` and ${rest} more` : ""}`
|
|
372
|
+
: "";
|
|
373
|
+
return (
|
|
374
|
+
`\n\n[${kept.length} of ${matched.length} result(s) shown; ${droppedRows.length} did ` +
|
|
375
|
+
`not fit max_bytes=${max_bytes}${naming}. Raise max_bytes, narrow the query, or ` +
|
|
376
|
+
`lower match_count.]`
|
|
377
|
+
);
|
|
378
|
+
};
|
|
379
|
+
for (let named = Math.min(5, droppedRows.length); named >= 1; named--) {
|
|
380
|
+
const candidate = footer(named);
|
|
381
|
+
if (size(candidate) <= room) return body + candidate;
|
|
382
|
+
}
|
|
383
|
+
const bare = footer(0);
|
|
384
|
+
if (size(bare) <= room) return body + bare;
|
|
385
|
+
// The floor, ~25 bytes, never dropped: the caller must know there is more.
|
|
386
|
+
return body + `\n\n[${kept.length} of ${matched.length} shown; raise max_bytes]`;
|
|
387
|
+
};
|
|
388
|
+
|
|
389
|
+
// Which rows the budget can carry, in rank order. Rows that do not fit are
|
|
390
|
+
// SKIPPED rather than ending the scan: one oversized top hit used to suppress
|
|
391
|
+
// every smaller result behind it, and the reply then claimed nothing fit when
|
|
392
|
+
// rows two and three would have fitted comfortably (#266).
|
|
393
|
+
const sepBytes = size(SEP);
|
|
394
|
+
const keptIdx: number[] = [];
|
|
395
|
+
let acc = 0;
|
|
396
|
+
for (let i = 0; i < rendered.length; i++) {
|
|
397
|
+
const add = size(rendered[i]!) + (keptIdx.length > 0 ? sepBytes : 0);
|
|
398
|
+
if (acc + add > max_bytes) continue;
|
|
399
|
+
acc += add;
|
|
400
|
+
keptIdx.push(i);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// Not one result fits (#254) — the only case that answers with no content,
|
|
404
|
+
// and now the only case that can truthfully say so.
|
|
405
|
+
if (keptIdx.length === 0) {
|
|
406
|
+
logUsage(supabase, {
|
|
407
|
+
operation: "search",
|
|
408
|
+
accessPath: ctx.accessPath,
|
|
409
|
+
requestor: callerIdentity(args),
|
|
410
|
+
query_text: query,
|
|
411
|
+
project_id: projectId,
|
|
412
|
+
result_count: matched.length,
|
|
413
|
+
extra: { returned: 0, truncated: true, degraded: true },
|
|
414
|
+
});
|
|
415
|
+
return degradedToHeaders(matched, rendered, max_bytes, belowConfidence);
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// Give up the framing before the content, but never the fact of truncation:
|
|
419
|
+
// long advisory, then a result, and the "N of M shown" notice always stays.
|
|
420
|
+
let keptCount = keptIdx.length;
|
|
421
|
+
let short = false;
|
|
422
|
+
let output = assemble(keptCount, short);
|
|
423
|
+
while (size(output) > max_bytes) {
|
|
424
|
+
if (belowConfidence && !short) short = true;
|
|
425
|
+
else if (keptCount > 1) {
|
|
426
|
+
keptCount -= 1;
|
|
427
|
+
short = belowConfidence;
|
|
428
|
+
} else break; // one result plus the shortest possible framing: the floor
|
|
429
|
+
output = assemble(keptCount, short);
|
|
430
|
+
}
|
|
431
|
+
const take = keptCount;
|
|
191
432
|
|
|
192
433
|
logUsage(supabase, {
|
|
193
434
|
operation: "search",
|
|
@@ -195,59 +436,12 @@ async function handler(
|
|
|
195
436
|
requestor: callerIdentity(args),
|
|
196
437
|
query_text: query,
|
|
197
438
|
project_id: projectId,
|
|
198
|
-
// What the QUERY matched, not what survived the
|
|
199
|
-
//
|
|
200
|
-
// budget-wiped search look like an empty knowledge base in analytics too.
|
|
439
|
+
// What the QUERY matched, not what survived the budget: recording the
|
|
440
|
+
// latter made a budget-wiped search look like an empty store in analytics.
|
|
201
441
|
result_count: matched.length,
|
|
202
|
-
...(
|
|
442
|
+
...(take < matched.length ? { extra: { returned: take, truncated: true } } : {}),
|
|
203
443
|
});
|
|
204
444
|
|
|
205
|
-
// Nothing matched: the honest empty answer.
|
|
206
|
-
if (matched.length === 0) return "No results found.";
|
|
207
|
-
|
|
208
|
-
// Something matched but none of it fit the budget (#254). Reporting "no
|
|
209
|
-
// results" here is the most damaging answer the tool can give: an agent
|
|
210
|
-
// stops looking and often recreates the document it failed to find. Show
|
|
211
|
-
// the headers instead — a few hundred bytes that name what exists and how
|
|
212
|
-
// to read it.
|
|
213
|
-
if (accepted.length === 0) {
|
|
214
|
-
return degradedToHeaders(matched, max_bytes);
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
const rows = accepted as SearchRow[];
|
|
218
|
-
|
|
219
|
-
// 28I: nothing cleared the relevance threshold, so the server returned its
|
|
220
|
-
// best-effort top candidates flagged below_confidence instead of an empty
|
|
221
|
-
// set (which agents misread as "this knowledge does not exist").
|
|
222
|
-
const belowConfidence = rows.length > 0 && rows.every((r) => r.below_confidence === true);
|
|
223
|
-
|
|
224
|
-
const parts: string[] = rows.map((row) => {
|
|
225
|
-
const title = row.doc_title ?? "Untitled";
|
|
226
|
-
const docId = row.document_id ? ` [id: ${row.document_id}]` : "";
|
|
227
|
-
const rawScore = row.best_score ?? row.score;
|
|
228
|
-
const score = rawScore != null ? ` (score: ${rawScore.toFixed(3)})` : "";
|
|
229
|
-
const partial = row.is_partial
|
|
230
|
-
? ` -- partial (${row.chunk_count} of ${(row.total_chars ?? 0).toLocaleString()} chars)`
|
|
231
|
-
: "";
|
|
232
|
-
// content_hash = the concurrency token for cerefox_ingest updates (iter-32).
|
|
233
|
-
const hash = row.content_hash ? `\nhash: ${row.content_hash}` : "";
|
|
234
|
-
return `## ${title}${docId}${score}${partial}${hash}\n\n${row.full_content ?? ""}`;
|
|
235
|
-
});
|
|
236
|
-
|
|
237
|
-
let output = parts.join("\n\n---\n\n");
|
|
238
|
-
if (belowConfidence) {
|
|
239
|
-
output =
|
|
240
|
-
`⚠ No results cleared the confidence threshold. Showing the closest ${rows.length} ` +
|
|
241
|
-
`candidate(s) with scores — judge relevance yourself; a low score means weak signal, ` +
|
|
242
|
-
`not necessarily absent knowledge.\n\n` + output;
|
|
243
|
-
}
|
|
244
|
-
if (truncated) {
|
|
245
|
-
output +=
|
|
246
|
-
`\n\n[${accepted.length} of ${matched.length} result(s) shown; truncated at ` +
|
|
247
|
-
`${usedBytes} bytes. ${dropped.length} did not fit: ` +
|
|
248
|
-
`${dropped.map((r) => (r as SearchRow).doc_title ?? "Untitled").join(", ")}. ` +
|
|
249
|
-
`Raise max_bytes, narrow the query, or lower match_count.]`;
|
|
250
|
-
}
|
|
251
445
|
return output;
|
|
252
446
|
}
|
|
253
447
|
|
|
@@ -269,7 +463,7 @@ export const searchTool: ToolDefinition = {
|
|
|
269
463
|
query: { type: "string", description: "Natural-language search query" },
|
|
270
464
|
match_count: {
|
|
271
465
|
type: "integer",
|
|
272
|
-
description: "Maximum number of documents to return (default: 5)",
|
|
466
|
+
description: "Maximum number of documents to return (default: 5, maximum: 200)",
|
|
273
467
|
},
|
|
274
468
|
project_name: {
|
|
275
469
|
type: "string",
|
|
@@ -36,7 +36,16 @@ import { applyByteBudget } from "../../../_shared/mcp-tools/_utils.ts";
|
|
|
36
36
|
* `truncated` flag when results were dropped.
|
|
37
37
|
*
|
|
38
38
|
* Response: { results: [...], query, mode, match_count, project_name?,
|
|
39
|
-
* truncated: boolean, response_bytes: number
|
|
39
|
+
* truncated: boolean, response_bytes: number,
|
|
40
|
+
* matched: number, degraded: boolean, note?: string }
|
|
41
|
+
*
|
|
42
|
+
* `matched` is how many results the query found, before the byte budget.
|
|
43
|
+
* `degraded` is true when everything that matched was larger than max_bytes:
|
|
44
|
+
* the items then carry NO content, they name what exists so the caller can
|
|
45
|
+
* re-ask with a larger budget. That header list is capped to max_bytes and so
|
|
46
|
+
* may be a SUBSET of `matched`, except that one item is always returned even
|
|
47
|
+
* if it exceeds the budget — an empty `results` would read as "nothing was
|
|
48
|
+
* found", which is `matched: 0` and nothing else (#254, #257, #261).
|
|
40
49
|
*
|
|
41
50
|
* Example agent prompt:
|
|
42
51
|
* "Invoke the cerefox-search edge function with query='knowledge management'
|
|
@@ -150,14 +159,6 @@ async function lookupProjectId(
|
|
|
150
159
|
return data[0].id;
|
|
151
160
|
}
|
|
152
161
|
|
|
153
|
-
/**
|
|
154
|
-
* Apply a byte budget to an array of result rows.
|
|
155
|
-
*
|
|
156
|
-
* Each row is serialised to JSON to measure its size. Rows are included in
|
|
157
|
-
* order until the next row would push the running total over `maxBytes`.
|
|
158
|
-
* Rows are always kept or dropped whole — content is never truncated
|
|
159
|
-
* mid-document. Returns the accepted rows and a `truncated` flag.
|
|
160
|
-
*/
|
|
161
162
|
const headers = {
|
|
162
163
|
"Content-Type": "application/json",
|
|
163
164
|
"Access-Control-Allow-Origin": "*",
|
|
@@ -214,7 +215,15 @@ Deno.serve(async (req: Request) => {
|
|
|
214
215
|
} = body;
|
|
215
216
|
|
|
216
217
|
// Enforce ceiling: agents may request less but never more than MAX_BYTES.
|
|
217
|
-
|
|
218
|
+
// Sanitised before clamping: `NaN` compares false against every budget
|
|
219
|
+
// check, so a non-numeric value bypassed the ceiling entirely (#266).
|
|
220
|
+
// A number of 0 or less still means "almost no budget"; only a missing or
|
|
221
|
+
// non-numeric value falls back to the ceiling (#267).
|
|
222
|
+
const requestedBytes = Math.floor(Number(requested_max_bytes));
|
|
223
|
+
const max_bytes = Math.min(
|
|
224
|
+
Number.isFinite(requestedBytes) ? Math.max(requestedBytes, 1) : MAX_BYTES,
|
|
225
|
+
MAX_BYTES,
|
|
226
|
+
);
|
|
218
227
|
// Clamp match_count to [1, MAX_MATCH_COUNT] (bounds query work; see MAX_MATCH_COUNT).
|
|
219
228
|
const match_count = Math.min(Math.max(1, Math.floor(Number(raw_match_count)) || 5), MAX_MATCH_COUNT);
|
|
220
229
|
|
|
@@ -362,10 +371,33 @@ Deno.serve(async (req: Request) => {
|
|
|
362
371
|
// knowledge does not exist" to whatever is on the other end, so send the
|
|
363
372
|
// rows WITHOUT their content instead: the caller learns what matched, how
|
|
364
373
|
// big it is, and can re-ask with a larger budget or fetch one document.
|
|
374
|
+
//
|
|
375
|
+
// Both content columns have to go: `cerefox_search_docs` (mode "docs")
|
|
376
|
+
// returns `full_content`, while `cerefox_hybrid_search` and the FTS RPC
|
|
377
|
+
// return `content`. Stripping one of them shipped in v1.14.2 and returned
|
|
378
|
+
// 83 KB of chunk text against a 3 KB budget while the response claimed the
|
|
379
|
+
// content had been omitted (#257).
|
|
365
380
|
const degraded = accepted.length === 0 && matched.length > 0;
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
381
|
+
// And the headers themselves are held to the budget the caller asked for:
|
|
382
|
+
// a `match_count` of 200 makes even a content-free list large. At least one
|
|
383
|
+
// survives regardless: an empty `results` is the "nothing was found" shape
|
|
384
|
+
// #254 exists to prevent, and a header row can exceed a very small budget
|
|
385
|
+
// on its own (#259).
|
|
386
|
+
const listed = degraded
|
|
387
|
+
? (() => {
|
|
388
|
+
const headerRows = matched.map(
|
|
389
|
+
({ full_content: _full, content: _chunk, ...header }) => header,
|
|
390
|
+
);
|
|
391
|
+
const fitted = applyByteBudget(headerRows, max_bytes).accepted;
|
|
392
|
+
return fitted.length > 0 ? fitted : headerRows.slice(0, 1);
|
|
393
|
+
})()
|
|
394
|
+
: [];
|
|
395
|
+
const results = degraded ? listed : accepted;
|
|
396
|
+
// What is actually being returned, which on a degraded response is the
|
|
397
|
+
// header list rather than the (empty) content-bearing set.
|
|
398
|
+
const responseBytes = degraded
|
|
399
|
+
? new TextEncoder().encode(JSON.stringify(results)).length
|
|
400
|
+
: usedBytes;
|
|
369
401
|
|
|
370
402
|
// Fire-and-forget usage logging (never blocks the response)
|
|
371
403
|
Promise.resolve(supabase.rpc("cerefox_log_usage", {
|
|
@@ -388,14 +420,20 @@ Deno.serve(async (req: Request) => {
|
|
|
388
420
|
project_name: project_name ?? null,
|
|
389
421
|
metadata_filter: metadata_filter ?? null,
|
|
390
422
|
truncated,
|
|
391
|
-
response_bytes:
|
|
423
|
+
response_bytes: responseBytes,
|
|
392
424
|
matched: matched.length,
|
|
393
425
|
degraded,
|
|
394
426
|
...(degraded
|
|
395
427
|
? {
|
|
396
428
|
note:
|
|
397
429
|
`${matched.length} result(s) matched but none fit max_bytes=${max_bytes}; ` +
|
|
398
|
-
`content omitted
|
|
430
|
+
`content omitted${
|
|
431
|
+
listed.length < matched.length
|
|
432
|
+
? `, and only ${listed.length} of them are listed here` +
|
|
433
|
+
" (the rest did not fit either)"
|
|
434
|
+
: ""
|
|
435
|
+
}. This is NOT an empty result. Raise max_bytes, or fetch one ` +
|
|
436
|
+
`document with cerefox-get-document.`,
|
|
399
437
|
}
|
|
400
438
|
: {}),
|
|
401
439
|
}),
|