@cerefox/memory 1.0.3 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cerefox.js +98 -42
- package/dist/server-assets/_shared/ef-meta/index.ts +2 -2
- package/dist/server-assets/_shared/mcp-tools/_utils.ts +15 -0
- package/dist/server-assets/_shared/mcp-tools/search.ts +32 -1
- package/dist/server-assets/db/rpcs.sql +68 -15
- package/dist/server-assets/db/schema.sql +1 -1
- package/docs/guides/configuration.md +9 -0
- package/package.json +1 -1
package/dist/bin/cerefox.js
CHANGED
|
@@ -7184,7 +7184,7 @@ var exports_meta = {};
|
|
|
7184
7184
|
__export(exports_meta, {
|
|
7185
7185
|
PKG_VERSION: () => PKG_VERSION
|
|
7186
7186
|
});
|
|
7187
|
-
var PKG_VERSION = "1.0.
|
|
7187
|
+
var PKG_VERSION = "1.0.4";
|
|
7188
7188
|
var init_meta = () => {};
|
|
7189
7189
|
|
|
7190
7190
|
// ../../node_modules/.bun/tslib@2.8.1/node_modules/tslib/tslib.js
|
|
@@ -25225,6 +25225,13 @@ function getMinSearchScore() {
|
|
|
25225
25225
|
const n = Number.parseFloat(raw);
|
|
25226
25226
|
return Number.isNaN(n) || n < 0 || n > 1 ? fallback : n;
|
|
25227
25227
|
}
|
|
25228
|
+
function getMinTermCoverage() {
|
|
25229
|
+
const raw = globalThis.process?.env?.CEREFOX_MIN_TERM_COVERAGE;
|
|
25230
|
+
if (raw === undefined || raw === "")
|
|
25231
|
+
return;
|
|
25232
|
+
const n = Number.parseFloat(raw);
|
|
25233
|
+
return Number.isNaN(n) || n < 0 || n > 1 ? undefined : n;
|
|
25234
|
+
}
|
|
25228
25235
|
function applyByteBudget(rows, maxBytes) {
|
|
25229
25236
|
const accepted = [];
|
|
25230
25237
|
let usedBytes = 0;
|
|
@@ -55572,6 +55579,8 @@ async function handler9(supabase, args, ctx) {
|
|
|
55572
55579
|
const mode = args.mode ?? "docs";
|
|
55573
55580
|
const alpha = args.alpha ?? 0.7;
|
|
55574
55581
|
const min_score = args.min_score ?? getMinSearchScore();
|
|
55582
|
+
const min_term_coverage = args.min_term_coverage ?? getMinTermCoverage();
|
|
55583
|
+
const coverageParam = min_term_coverage !== undefined ? { p_min_term_coverage: min_term_coverage } : {};
|
|
55575
55584
|
const metadata_filter = args.metadata_filter ?? null;
|
|
55576
55585
|
const requested_max_bytes = args.max_bytes;
|
|
55577
55586
|
const ceiling = getMaxResponseBytes();
|
|
@@ -55603,7 +55612,8 @@ async function handler9(supabase, args, ctx) {
|
|
|
55603
55612
|
p_query_text: query,
|
|
55604
55613
|
p_match_count: match_count,
|
|
55605
55614
|
p_project_id: projectId,
|
|
55606
|
-
...metaFilterParam
|
|
55615
|
+
...metaFilterParam,
|
|
55616
|
+
...coverageParam
|
|
55607
55617
|
};
|
|
55608
55618
|
} else if (mode === "hybrid") {
|
|
55609
55619
|
rpcName = "cerefox_hybrid_search";
|
|
@@ -55615,7 +55625,8 @@ async function handler9(supabase, args, ctx) {
|
|
|
55615
55625
|
p_use_upgrade: false,
|
|
55616
55626
|
p_project_id: projectId,
|
|
55617
55627
|
p_min_score: min_score,
|
|
55618
|
-
...metaFilterParam
|
|
55628
|
+
...metaFilterParam,
|
|
55629
|
+
...coverageParam
|
|
55619
55630
|
};
|
|
55620
55631
|
} else {
|
|
55621
55632
|
rpcName = "cerefox_search_docs";
|
|
@@ -55626,7 +55637,8 @@ async function handler9(supabase, args, ctx) {
|
|
|
55626
55637
|
p_alpha: alpha,
|
|
55627
55638
|
p_project_id: projectId,
|
|
55628
55639
|
p_min_score: min_score,
|
|
55629
|
-
...metaFilterParam
|
|
55640
|
+
...metaFilterParam,
|
|
55641
|
+
...coverageParam
|
|
55630
55642
|
};
|
|
55631
55643
|
}
|
|
55632
55644
|
const { data, error: error2 } = await supabase.rpc(rpcName, rpcParams);
|
|
@@ -55699,6 +55711,23 @@ var init_search = __esm(() => {
|
|
|
55699
55711
|
description: 'Optional JSONB containment filter. Only documents whose metadata contains ALL specified key-value pairs are returned. Example: {"type": "decision", "status": "active"}. Call cerefox_list_metadata_keys first to discover available keys and values. Omit to search all documents.',
|
|
55700
55712
|
additionalProperties: { type: "string" }
|
|
55701
55713
|
},
|
|
55714
|
+
mode: {
|
|
55715
|
+
type: "string",
|
|
55716
|
+
enum: ["docs", "hybrid", "fts", "semantic"],
|
|
55717
|
+
description: "Search mode (default: docs — full reconstructed documents). hybrid: ranked chunks; fts: keyword-only (no embedding); semantic: vector-only."
|
|
55718
|
+
},
|
|
55719
|
+
alpha: {
|
|
55720
|
+
type: "number",
|
|
55721
|
+
description: "Hybrid fusion weight 0–1 (default 0.7): 1 = pure semantic, 0 = pure keyword."
|
|
55722
|
+
},
|
|
55723
|
+
min_score: {
|
|
55724
|
+
type: "number",
|
|
55725
|
+
description: "Minimum cosine similarity for vector-side results (default: server-configured, 0.5 OpenAI / 0.6 local embedder)."
|
|
55726
|
+
},
|
|
55727
|
+
min_term_coverage: {
|
|
55728
|
+
type: "number",
|
|
55729
|
+
description: "Keyword OR-fallback confidence bar 0–1 (default 0.5): fraction of the query's meaningful terms a result must match to count as a confident hit; weaker matches return flagged below-confidence. 0 = any matching term. Needs schema ≥ 0.9.1."
|
|
55730
|
+
},
|
|
55702
55731
|
max_bytes: {
|
|
55703
55732
|
type: "integer",
|
|
55704
55733
|
description: "Optional response size budget in bytes. Results are dropped whole until the budget is satisfied; a truncated flag is set when results are dropped. Defaults to the server maximum (200000). Pass a smaller value if your context window is limited. Values above the server maximum are silently capped."
|
|
@@ -69032,10 +69061,29 @@ init_cli_core();
|
|
|
69032
69061
|
|
|
69033
69062
|
// src/cli/commands/backup.ts
|
|
69034
69063
|
init_cli_core();
|
|
69035
|
-
init_client();
|
|
69036
69064
|
import { existsSync as existsSync2, mkdirSync, writeFileSync } from "node:fs";
|
|
69037
69065
|
import { homedir as homedir2 } from "node:os";
|
|
69038
69066
|
import { join as join2, resolve } from "node:path";
|
|
69067
|
+
|
|
69068
|
+
// ../../_shared/db-client/paginate.ts
|
|
69069
|
+
async function fetchAllPages(makeQuery, batchSize = 200) {
|
|
69070
|
+
const results = [];
|
|
69071
|
+
let offset = 0;
|
|
69072
|
+
for (;; ) {
|
|
69073
|
+
const { data, error } = await makeQuery(offset, offset + batchSize - 1);
|
|
69074
|
+
if (error)
|
|
69075
|
+
throw new Error(error.message ?? JSON.stringify(error));
|
|
69076
|
+
const page = data ?? [];
|
|
69077
|
+
results.push(...page);
|
|
69078
|
+
if (page.length < batchSize)
|
|
69079
|
+
break;
|
|
69080
|
+
offset += batchSize;
|
|
69081
|
+
}
|
|
69082
|
+
return results;
|
|
69083
|
+
}
|
|
69084
|
+
|
|
69085
|
+
// src/cli/commands/backup.ts
|
|
69086
|
+
init_client();
|
|
69039
69087
|
function expandHome(path) {
|
|
69040
69088
|
if (path === "~")
|
|
69041
69089
|
return homedir2();
|
|
@@ -69056,21 +69104,25 @@ async function action(options) {
|
|
|
69056
69104
|
const filename = `cerefox-${stamp}${options.label ? "-" + options.label : ""}.json`;
|
|
69057
69105
|
const dest = join2(outDir, filename);
|
|
69058
69106
|
const client = getClient();
|
|
69059
|
-
|
|
69060
|
-
|
|
69061
|
-
|
|
69062
|
-
|
|
69107
|
+
let docs;
|
|
69108
|
+
try {
|
|
69109
|
+
docs = await fetchAllPages((from, to) => client.raw.from("cerefox_documents").select("id, title, content_hash, source, metadata, total_chars, chunk_count, " + "review_status, created_at, updated_at, deleted_at").is("deleted_at", null).order("created_at", { ascending: true }).order("id", { ascending: true }).range(from, to));
|
|
69110
|
+
} catch (err) {
|
|
69111
|
+
throw systemError(`Document fetch failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
69112
|
+
}
|
|
69063
69113
|
let chunkTotal = 0;
|
|
69064
69114
|
const enriched = [];
|
|
69065
69115
|
for (let i = 0;i < docs.length; i++) {
|
|
69066
69116
|
const doc = docs[i];
|
|
69067
69117
|
const docId = doc.id;
|
|
69068
|
-
|
|
69069
|
-
|
|
69070
|
-
|
|
69118
|
+
let chunks;
|
|
69119
|
+
try {
|
|
69120
|
+
chunks = await fetchAllPages((from, to) => client.raw.from("cerefox_chunks").select("*").eq("document_id", docId).is("version_id", null).order("chunk_index", { ascending: true }).range(from, to));
|
|
69121
|
+
} catch (err) {
|
|
69122
|
+
throw systemError(`Chunk fetch failed for ${docId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
69071
69123
|
}
|
|
69072
|
-
chunkTotal +=
|
|
69073
|
-
enriched.push({ ...doc, chunks
|
|
69124
|
+
chunkTotal += chunks.length;
|
|
69125
|
+
enriched.push({ ...doc, chunks });
|
|
69074
69126
|
if (process.stdout.isTTY) {
|
|
69075
69127
|
process.stderr.write(`\r Dumping documents: ${i + 1}/${docs.length} (${chunkTotal} chunks so far)…`);
|
|
69076
69128
|
}
|
|
@@ -74948,8 +75000,8 @@ import { homedir as homedir6 } from "node:os";
|
|
|
74948
75000
|
import { join as join9 } from "node:path";
|
|
74949
75001
|
|
|
74950
75002
|
// ../../_shared/ef-meta/index.ts
|
|
74951
|
-
var EF_VERSION = "1.0.
|
|
74952
|
-
var EF_LAST_CHANGED = "1.0.
|
|
75003
|
+
var EF_VERSION = "1.0.4";
|
|
75004
|
+
var EF_LAST_CHANGED = "1.0.4";
|
|
74953
75005
|
|
|
74954
75006
|
// src/cli/util/checks.ts
|
|
74955
75007
|
init_config();
|
|
@@ -77239,18 +77291,22 @@ async function action27(options) {
|
|
|
77239
77291
|
}
|
|
77240
77292
|
const reindexAll = Boolean(options.all);
|
|
77241
77293
|
const dryRun = Boolean(options.dryRun);
|
|
77242
|
-
let query = supabase.from("cerefox_chunks").select("id, document_id, content, embedder_primary, cerefox_documents(title)").is("version_id", null);
|
|
77243
|
-
if (options.documentId) {
|
|
77244
|
-
query = query.eq("document_id", options.documentId);
|
|
77245
|
-
}
|
|
77246
77294
|
const targetModel = activeEmbedderName();
|
|
77247
|
-
|
|
77248
|
-
|
|
77295
|
+
let chunks;
|
|
77296
|
+
try {
|
|
77297
|
+
chunks = await fetchAllPages((from, to) => {
|
|
77298
|
+
let query = supabase.from("cerefox_chunks").select("id, document_id, content, embedder_primary, cerefox_documents(title)").is("version_id", null);
|
|
77299
|
+
if (options.documentId) {
|
|
77300
|
+
query = query.eq("document_id", options.documentId);
|
|
77301
|
+
}
|
|
77302
|
+
if (!reindexAll) {
|
|
77303
|
+
query = query.neq("embedder_primary", targetModel);
|
|
77304
|
+
}
|
|
77305
|
+
return query.order("id", { ascending: true }).range(from, to);
|
|
77306
|
+
}, 1000);
|
|
77307
|
+
} catch (err) {
|
|
77308
|
+
throw systemError(`Failed to list chunks: ${err instanceof Error ? err.message : String(err)}`);
|
|
77249
77309
|
}
|
|
77250
|
-
const { data, error: error3 } = await query;
|
|
77251
|
-
if (error3)
|
|
77252
|
-
throw systemError(`Failed to list chunks: ${error3.message}`);
|
|
77253
|
-
const chunks = data ?? [];
|
|
77254
77310
|
if (chunks.length === 0) {
|
|
77255
77311
|
println(c.dim("(nothing to reindex)"));
|
|
77256
77312
|
return;
|
|
@@ -77420,6 +77476,8 @@ async function action29(query, options) {
|
|
|
77420
77476
|
const matchCount = parsePositiveInt(options.matchCount, "--match-count", 5);
|
|
77421
77477
|
const alpha = parseFloat01(options.alpha, "--alpha", 0.7);
|
|
77422
77478
|
const minScore = parseFloat01(options.minScore, "--min-score", getMinSearchScore());
|
|
77479
|
+
const envCoverage = getMinTermCoverage();
|
|
77480
|
+
const coverageParam = options.minTermCoverage !== undefined ? { p_min_term_coverage: parseFloat01(options.minTermCoverage, "--min-term-coverage", envCoverage ?? 0.5) } : envCoverage !== undefined ? { p_min_term_coverage: envCoverage } : {};
|
|
77423
77481
|
const maxBytes = parseNonNegativeInt(options.maxBytes, "--max-bytes", getMaxResponseBytes());
|
|
77424
77482
|
const mode = options.mode ?? "docs";
|
|
77425
77483
|
if (!["docs", "hybrid", "fts"].includes(mode)) {
|
|
@@ -77450,7 +77508,8 @@ async function action29(query, options) {
|
|
|
77450
77508
|
p_query_text: query,
|
|
77451
77509
|
p_match_count: matchCount,
|
|
77452
77510
|
p_project_id: projectId,
|
|
77453
|
-
...metaFilterParam
|
|
77511
|
+
...metaFilterParam,
|
|
77512
|
+
...coverageParam
|
|
77454
77513
|
};
|
|
77455
77514
|
} else if (mode === "hybrid") {
|
|
77456
77515
|
rpcName = "cerefox_hybrid_search";
|
|
@@ -77462,7 +77521,8 @@ async function action29(query, options) {
|
|
|
77462
77521
|
p_use_upgrade: false,
|
|
77463
77522
|
p_project_id: projectId,
|
|
77464
77523
|
p_min_score: minScore,
|
|
77465
|
-
...metaFilterParam
|
|
77524
|
+
...metaFilterParam,
|
|
77525
|
+
...coverageParam
|
|
77466
77526
|
};
|
|
77467
77527
|
} else {
|
|
77468
77528
|
rpcName = "cerefox_search_docs";
|
|
@@ -77473,7 +77533,8 @@ async function action29(query, options) {
|
|
|
77473
77533
|
p_alpha: alpha,
|
|
77474
77534
|
p_project_id: projectId,
|
|
77475
77535
|
p_min_score: minScore,
|
|
77476
|
-
...metaFilterParam
|
|
77536
|
+
...metaFilterParam,
|
|
77537
|
+
...coverageParam
|
|
77477
77538
|
};
|
|
77478
77539
|
}
|
|
77479
77540
|
const results = await client.rpc(rpcName, rpcParams);
|
|
@@ -77572,7 +77633,7 @@ async function action29(query, options) {
|
|
|
77572
77633
|
}
|
|
77573
77634
|
}
|
|
77574
77635
|
function registerSearch(program2) {
|
|
77575
|
-
program2.command("search").description("Search the knowledge base (hybrid FTS + semantic).").argument("<query>", "Natural-language search query.").option("-c, --match-count <n>", "Maximum number of documents to return.", "5").option("-p, --project-name <name>", "Filter results to a specific project.").option("-f, --metadata-filter <json>", "JSON containment filter; only docs whose metadata contains ALL pairs are returned.").option("--mode <mode>", "Search mode: docs (default), hybrid, fts.", "docs").option("--alpha <float>", "Semantic weight 0..1 (default: 0.7).", "0.7").option("--min-score <float>", "Minimum cosine similarity threshold (default: CEREFOX_MIN_SEARCH_SCORE; else 0.5, or 0.6 with the local embedder).").option("--max-bytes <n>", "Response size budget in bytes (default: CEREFOX_MAX_RESPONSE_BYTES or 200000).").option("-r, --requestor <name>", "Agent / user name (recorded in usage log).").option("--json", "Emit machine-readable JSON instead of the default text.").option("--only-metadata", "List matching docs (id, score, chunks, chars, partial/full) WITHOUT their content — like the web UI's collapsed result list. Grab a [id:…] then `cerefox document get <id>`.").action(action29);
|
|
77636
|
+
program2.command("search").description("Search the knowledge base (hybrid FTS + semantic).").argument("<query>", "Natural-language search query.").option("-c, --match-count <n>", "Maximum number of documents to return.", "5").option("-p, --project-name <name>", "Filter results to a specific project.").option("-f, --metadata-filter <json>", "JSON containment filter; only docs whose metadata contains ALL pairs are returned.").option("--mode <mode>", "Search mode: docs (default), hybrid, fts.", "docs").option("--alpha <float>", "Semantic weight 0..1 (default: 0.7).", "0.7").option("--min-score <float>", "Minimum cosine similarity threshold (default: CEREFOX_MIN_SEARCH_SCORE; else 0.5, or 0.6 with the local embedder).").option("--min-term-coverage <float>", "OR-fallback keyword matches must cover at least this fraction of the query's meaningful terms to count as confident hits (default: CEREFOX_MIN_TERM_COVERAGE; else the server default 0.5; needs schema ≥ 0.9.1).").option("--max-bytes <n>", "Response size budget in bytes (default: CEREFOX_MAX_RESPONSE_BYTES or 200000).").option("-r, --requestor <name>", "Agent / user name (recorded in usage log).").option("--json", "Emit machine-readable JSON instead of the default text.").option("--only-metadata", "List matching docs (id, score, chunks, chars, partial/full) WITHOUT their content — like the web UI's collapsed result list. Grab a [id:…] then `cerefox document get <id>`.").action(action29);
|
|
77576
77637
|
}
|
|
77577
77638
|
|
|
77578
77639
|
// src/cli/commands/self-update.ts
|
|
@@ -81444,10 +81505,8 @@ async function getProjectDocCounts(ctx, projectIds) {
|
|
|
81444
81505
|
if (projectIds.length === 0)
|
|
81445
81506
|
return { active, deleted };
|
|
81446
81507
|
try {
|
|
81447
|
-
const
|
|
81448
|
-
|
|
81449
|
-
throw error3;
|
|
81450
|
-
for (const row of data ?? []) {
|
|
81508
|
+
const rows = await fetchAllPages((from, to) => ctx.supabase.from("cerefox_document_projects").select("project_id, cerefox_documents(deleted_at)").in("project_id", projectIds).order("document_id", { ascending: true }).order("project_id", { ascending: true }).range(from, to));
|
|
81509
|
+
for (const row of rows) {
|
|
81451
81510
|
const pid = row.project_id;
|
|
81452
81511
|
if (!(pid in active))
|
|
81453
81512
|
continue;
|
|
@@ -81491,16 +81550,13 @@ async function getRecentDocAuthors(ctx, docIds) {
|
|
|
81491
81550
|
return out;
|
|
81492
81551
|
}
|
|
81493
81552
|
async function countDocumentsForProject(ctx, projectId) {
|
|
81494
|
-
const {
|
|
81553
|
+
const { count, error: error3 } = await ctx.supabase.from("cerefox_document_projects").select("document_id, cerefox_documents!inner(deleted_at)", {
|
|
81554
|
+
count: "exact",
|
|
81555
|
+
head: true
|
|
81556
|
+
}).eq("project_id", projectId).is("cerefox_documents.deleted_at", null);
|
|
81495
81557
|
if (error3)
|
|
81496
81558
|
throw error3;
|
|
81497
|
-
|
|
81498
|
-
for (const row of data ?? []) {
|
|
81499
|
-
if (row.cerefox_documents && row.cerefox_documents.deleted_at === null) {
|
|
81500
|
-
n += 1;
|
|
81501
|
-
}
|
|
81502
|
-
}
|
|
81503
|
-
return n;
|
|
81559
|
+
return count ?? 0;
|
|
81504
81560
|
}
|
|
81505
81561
|
function dashboardDocFromRow(row, projectIds) {
|
|
81506
81562
|
return {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* doesn't touch `supabase/functions/` leaves it alone).
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
export const EF_VERSION = "1.0.
|
|
21
|
+
export const EF_VERSION = "1.0.4";
|
|
22
22
|
|
|
23
23
|
/**
|
|
24
24
|
* The most recent version whose EF-side SOURCE actually changed (#127).
|
|
@@ -28,7 +28,7 @@ export const EF_VERSION = "1.0.3";
|
|
|
28
28
|
* `cut_release.ts` ONLY when EF source changed since the last tag; doctor
|
|
29
29
|
* uses it to stay silent on label-only drift.
|
|
30
30
|
*/
|
|
31
|
-
export const EF_LAST_CHANGED = "1.0.
|
|
31
|
+
export const EF_LAST_CHANGED = "1.0.4";
|
|
32
32
|
|
|
33
33
|
/**
|
|
34
34
|
* The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp
|
|
@@ -68,6 +68,21 @@ export function getMinSearchScore(): number {
|
|
|
68
68
|
return Number.isNaN(n) || n < 0 || n > 1 ? fallback : n;
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
+
/**
|
|
72
|
+
* CEREFOX_MIN_TERM_COVERAGE (v1.0.4): user-configurable default for the
|
|
73
|
+
* OR-fallback term-coverage gate. Returns undefined when unset/invalid —
|
|
74
|
+
* callers then OMIT p_min_term_coverage from the RPC call, deferring to the
|
|
75
|
+
* server default (0.5) and staying compatible with pre-0.9.1 servers
|
|
76
|
+
* (an unknown named argument fails the PostgREST function match).
|
|
77
|
+
*/
|
|
78
|
+
export function getMinTermCoverage(): number | undefined {
|
|
79
|
+
const raw = (globalThis as { process?: { env?: Record<string, string | undefined> } })
|
|
80
|
+
.process?.env?.CEREFOX_MIN_TERM_COVERAGE;
|
|
81
|
+
if (raw === undefined || raw === "") return undefined;
|
|
82
|
+
const n = Number.parseFloat(raw);
|
|
83
|
+
return Number.isNaN(n) || n < 0 || n > 1 ? undefined : n;
|
|
84
|
+
}
|
|
85
|
+
|
|
71
86
|
export function applyByteBudget(
|
|
72
87
|
rows: unknown[],
|
|
73
88
|
maxBytes: number,
|
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
import type { MCPSupabaseClient } from "./types.ts";
|
|
19
19
|
|
|
20
20
|
import { getEmbedding, resolveEmbedderKind } from "../embeddings/index.ts";
|
|
21
|
-
import { applyByteBudget, getMaxResponseBytes, getMinSearchScore,
|
|
21
|
+
import { applyByteBudget, getMaxResponseBytes, getMinSearchScore,
|
|
22
|
+
getMinTermCoverage, logUsage } from "./_utils.ts";
|
|
22
23
|
import { lookupProjectId } from "./_projects.ts";
|
|
23
24
|
import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
|
|
24
25
|
|
|
@@ -33,6 +34,12 @@ async function handler(
|
|
|
33
34
|
const mode = (args.mode as string | undefined) ?? "docs";
|
|
34
35
|
const alpha = (args.alpha as number | undefined) ?? 0.7;
|
|
35
36
|
const min_score = (args.min_score as number | undefined) ?? getMinSearchScore();
|
|
37
|
+
// v1.0.4: coverage gate default from CEREFOX_MIN_TERM_COVERAGE; only sent
|
|
38
|
+
// when configured (see getMinTermCoverage — keeps pre-0.9.1 servers working).
|
|
39
|
+
const min_term_coverage =
|
|
40
|
+
(args.min_term_coverage as number | undefined) ?? getMinTermCoverage();
|
|
41
|
+
const coverageParam =
|
|
42
|
+
min_term_coverage !== undefined ? { p_min_term_coverage: min_term_coverage } : {};
|
|
36
43
|
const metadata_filter =
|
|
37
44
|
(args.metadata_filter as Record<string, string> | null | undefined) ?? null;
|
|
38
45
|
const requested_max_bytes = args.max_bytes as number | undefined;
|
|
@@ -84,6 +91,7 @@ async function handler(
|
|
|
84
91
|
p_match_count: match_count,
|
|
85
92
|
p_project_id: projectId,
|
|
86
93
|
...metaFilterParam,
|
|
94
|
+
...coverageParam,
|
|
87
95
|
};
|
|
88
96
|
} else if (mode === "hybrid") {
|
|
89
97
|
rpcName = "cerefox_hybrid_search";
|
|
@@ -96,6 +104,7 @@ async function handler(
|
|
|
96
104
|
p_project_id: projectId,
|
|
97
105
|
p_min_score: min_score,
|
|
98
106
|
...metaFilterParam,
|
|
107
|
+
...coverageParam,
|
|
99
108
|
};
|
|
100
109
|
} else {
|
|
101
110
|
rpcName = "cerefox_search_docs";
|
|
@@ -107,6 +116,7 @@ async function handler(
|
|
|
107
116
|
p_project_id: projectId,
|
|
108
117
|
p_min_score: min_score,
|
|
109
118
|
...metaFilterParam,
|
|
119
|
+
...coverageParam,
|
|
110
120
|
};
|
|
111
121
|
}
|
|
112
122
|
|
|
@@ -195,6 +205,27 @@ export const searchTool: ToolDefinition = {
|
|
|
195
205
|
'Optional JSONB containment filter. Only documents whose metadata contains ALL specified key-value pairs are returned. Example: {"type": "decision", "status": "active"}. Call cerefox_list_metadata_keys first to discover available keys and values. Omit to search all documents.',
|
|
196
206
|
additionalProperties: { type: "string" },
|
|
197
207
|
},
|
|
208
|
+
mode: {
|
|
209
|
+
type: "string",
|
|
210
|
+
enum: ["docs", "hybrid", "fts", "semantic"],
|
|
211
|
+
description:
|
|
212
|
+
"Search mode (default: docs — full reconstructed documents). hybrid: ranked chunks; fts: keyword-only (no embedding); semantic: vector-only.",
|
|
213
|
+
},
|
|
214
|
+
alpha: {
|
|
215
|
+
type: "number",
|
|
216
|
+
description:
|
|
217
|
+
"Hybrid fusion weight 0–1 (default 0.7): 1 = pure semantic, 0 = pure keyword.",
|
|
218
|
+
},
|
|
219
|
+
min_score: {
|
|
220
|
+
type: "number",
|
|
221
|
+
description:
|
|
222
|
+
"Minimum cosine similarity for vector-side results (default: server-configured, 0.5 OpenAI / 0.6 local embedder).",
|
|
223
|
+
},
|
|
224
|
+
min_term_coverage: {
|
|
225
|
+
type: "number",
|
|
226
|
+
description:
|
|
227
|
+
"Keyword OR-fallback confidence bar 0–1 (default 0.5): fraction of the query's meaningful terms a result must match to count as a confident hit; weaker matches return flagged below-confidence. 0 = any matching term. Needs schema ≥ 0.9.1.",
|
|
228
|
+
},
|
|
198
229
|
max_bytes: {
|
|
199
230
|
type: "integer",
|
|
200
231
|
description:
|
|
@@ -69,6 +69,13 @@ DROP FUNCTION IF EXISTS cerefox_get_document(UUID, UUID);
|
|
|
69
69
|
DROP FUNCTION IF EXISTS cerefox_hybrid_search(TEXT, VECTOR(768), INT, FLOAT, BOOLEAN, UUID, FLOAT, JSONB);
|
|
70
70
|
DROP FUNCTION IF EXISTS cerefox_search_docs(TEXT, VECTOR(768), INT, FLOAT, UUID, FLOAT, INT, INT, JSONB);
|
|
71
71
|
|
|
72
|
+
-- Iteration 28I follow-up (v1.0.4, term-coverage gate): p_min_term_coverage
|
|
73
|
+
-- added to the search RPC signatures (new arg count = new function; the old
|
|
74
|
+
-- overloads must go or PostgREST calls become ambiguous).
|
|
75
|
+
DROP FUNCTION IF EXISTS cerefox_hybrid_search(TEXT, VECTOR(768), INT, FLOAT, BOOLEAN, UUID, FLOAT, JSONB);
|
|
76
|
+
DROP FUNCTION IF EXISTS cerefox_fts_search(TEXT, INT, UUID, JSONB);
|
|
77
|
+
DROP FUNCTION IF EXISTS cerefox_search_docs(TEXT, VECTOR(768), INT, FLOAT, UUID, FLOAT, INT, INT, JSONB);
|
|
78
|
+
|
|
72
79
|
-- ── Shared return type note ────────────────────────────────────────────────────
|
|
73
80
|
-- All chunk-level search RPCs return the same shape for consistency:
|
|
74
81
|
-- chunk_id, document_id, chunk_index, title, content, heading_path,
|
|
@@ -96,7 +103,14 @@ CREATE OR REPLACE FUNCTION cerefox_hybrid_search(
|
|
|
96
103
|
p_use_upgrade BOOLEAN DEFAULT FALSE,
|
|
97
104
|
p_project_id UUID DEFAULT NULL,
|
|
98
105
|
p_min_score FLOAT DEFAULT 0.0,
|
|
99
|
-
p_metadata_filter JSONB DEFAULT NULL
|
|
106
|
+
p_metadata_filter JSONB DEFAULT NULL,
|
|
107
|
+
-- 28I follow-up (v1.0.4): in OR-fallback mode, the unconditional FTS pass
|
|
108
|
+
-- requires at least this fraction of the query's meaningful (non-stopword,
|
|
109
|
+
-- deduplicated) terms to match the chunk. Under AND semantics a match
|
|
110
|
+
-- meant 100% of terms — the pass this gate generalizes. Chunks below the
|
|
111
|
+
-- bar can still pass via the vector threshold, else they are
|
|
112
|
+
-- below-confidence material. 0 restores the pre-gate OR behavior.
|
|
113
|
+
p_min_term_coverage FLOAT DEFAULT 0.5
|
|
100
114
|
)
|
|
101
115
|
RETURNS TABLE (
|
|
102
116
|
chunk_id UUID,
|
|
@@ -144,18 +158,26 @@ DECLARE
|
|
|
144
158
|
tok TEXT;
|
|
145
159
|
tok_q tsquery;
|
|
146
160
|
and_matches BOOLEAN := FALSE;
|
|
161
|
+
-- v1.0.4 coverage gate: the per-token queries (deduplicated by normalized
|
|
162
|
+
-- lexeme text, so "run running" counts once) and their count.
|
|
163
|
+
tok_queries tsquery[] := '{}';
|
|
164
|
+
seen_tokens TEXT[] := '{}';
|
|
165
|
+
total_tokens INT;
|
|
147
166
|
candidate_count INT := p_match_count * 5;
|
|
148
167
|
BEGIN
|
|
149
168
|
-- Build the OR-composed query: plainto each whitespace token (so tokens get
|
|
150
169
|
-- the same normalization/stemming as the AND path), skip stopword-only
|
|
151
|
-
-- tokens, fold with the tsquery OR operator (||).
|
|
170
|
+
-- tokens, dedupe by normalized form, fold with the tsquery OR operator (||).
|
|
152
171
|
FOR tok IN SELECT unnest(regexp_split_to_array(trim(p_query_text), '\s+')) LOOP
|
|
153
172
|
tok_q := plainto_tsquery('english', tok);
|
|
154
|
-
IF numnode(tok_q) > 0 THEN
|
|
173
|
+
IF numnode(tok_q) > 0 AND NOT (tok_q::TEXT = ANY(seen_tokens)) THEN
|
|
174
|
+
seen_tokens := seen_tokens || tok_q::TEXT;
|
|
175
|
+
tok_queries := tok_queries || tok_q;
|
|
155
176
|
query_fts_or := CASE WHEN query_fts_or IS NULL
|
|
156
177
|
THEN tok_q ELSE query_fts_or || tok_q END;
|
|
157
178
|
END IF;
|
|
158
179
|
END LOOP;
|
|
180
|
+
total_tokens := COALESCE(array_length(tok_queries, 1), 0);
|
|
159
181
|
|
|
160
182
|
-- Does the strict AND query match anything at all (under the caller's
|
|
161
183
|
-- filters)? Cheap probe against the partial FTS index.
|
|
@@ -183,7 +205,20 @@ BEGIN
|
|
|
183
205
|
fts_results AS (
|
|
184
206
|
SELECT
|
|
185
207
|
c.id,
|
|
186
|
-
ts_rank_cd(c.fts, query_fts)::FLOAT AS fts_score
|
|
208
|
+
ts_rank_cd(c.fts, query_fts)::FLOAT AS fts_score,
|
|
209
|
+
-- v1.0.4 coverage gate: in AND mode a match means 100% of the
|
|
210
|
+
-- query's terms are present, so the unconditional pass is
|
|
211
|
+
-- earned by construction. In OR-fallback mode, earn it only
|
|
212
|
+
-- when at least p_min_term_coverage of the meaningful terms
|
|
213
|
+
-- match this chunk; weaker matches keep contributing their
|
|
214
|
+
-- fts_score to the fusion but must pass via the vector
|
|
215
|
+
-- threshold (or surface as below-confidence candidates).
|
|
216
|
+
CASE
|
|
217
|
+
WHEN and_matches OR total_tokens = 0 THEN TRUE
|
|
218
|
+
ELSE (SELECT COUNT(*) FROM unnest(tok_queries) tq
|
|
219
|
+
WHERE c.fts @@ tq)::FLOAT
|
|
220
|
+
>= p_min_term_coverage * total_tokens
|
|
221
|
+
END AS coverage_ok
|
|
187
222
|
FROM cerefox_chunks c
|
|
188
223
|
JOIN cerefox_documents d ON c.document_id = d.id
|
|
189
224
|
WHERE c.version_id IS NULL
|
|
@@ -230,12 +265,14 @@ BEGIN
|
|
|
230
265
|
(1.0 - p_alpha) * COALESCE(f.fts_score, 0.0)
|
|
231
266
|
) AS score,
|
|
232
267
|
COALESCE(v.vec_score, 0.0) AS vec_score,
|
|
233
|
-
-- TRUE when the chunk matched the @@ FTS operator
|
|
234
|
-
--
|
|
235
|
-
--
|
|
236
|
-
--
|
|
237
|
-
--
|
|
238
|
-
|
|
268
|
+
-- TRUE when the chunk matched the @@ FTS operator WITH enough
|
|
269
|
+
-- term coverage to earn the unconditional pass (v1.0.4; always
|
|
270
|
+
-- true for AND-mode matches). We use this flag rather than
|
|
271
|
+
-- vec_score to decide whether a chunk passes the threshold,
|
|
272
|
+
-- because in small corpora every chunk appears in vec_results
|
|
273
|
+
-- (LIMIT candidate_count covers all rows), so vec_score is
|
|
274
|
+
-- never NULL even for FTS-only matches.
|
|
275
|
+
(f.id IS NOT NULL AND f.coverage_ok) AS has_fts_match
|
|
239
276
|
FROM fts_results f
|
|
240
277
|
FULL OUTER JOIN vec_results v ON f.id = v.id
|
|
241
278
|
),
|
|
@@ -293,7 +330,10 @@ CREATE OR REPLACE FUNCTION cerefox_fts_search(
|
|
|
293
330
|
p_query_text TEXT,
|
|
294
331
|
p_match_count INT DEFAULT 10,
|
|
295
332
|
p_project_id UUID DEFAULT NULL,
|
|
296
|
-
p_metadata_filter JSONB DEFAULT NULL
|
|
333
|
+
p_metadata_filter JSONB DEFAULT NULL,
|
|
334
|
+
-- v1.0.4: see cerefox_hybrid_search. In OR-fallback mode results must
|
|
335
|
+
-- match at least this fraction of the query's meaningful terms.
|
|
336
|
+
p_min_term_coverage FLOAT DEFAULT 0.5
|
|
297
337
|
)
|
|
298
338
|
RETURNS TABLE (
|
|
299
339
|
chunk_id UUID,
|
|
@@ -324,14 +364,20 @@ DECLARE
|
|
|
324
364
|
tok TEXT;
|
|
325
365
|
tok_q tsquery;
|
|
326
366
|
and_matches BOOLEAN := FALSE;
|
|
367
|
+
tok_queries tsquery[] := '{}';
|
|
368
|
+
seen_tokens TEXT[] := '{}';
|
|
369
|
+
total_tokens INT;
|
|
327
370
|
BEGIN
|
|
328
371
|
FOR tok IN SELECT unnest(regexp_split_to_array(trim(p_query_text), '\s+')) LOOP
|
|
329
372
|
tok_q := plainto_tsquery('english', tok);
|
|
330
|
-
IF numnode(tok_q) > 0 THEN
|
|
373
|
+
IF numnode(tok_q) > 0 AND NOT (tok_q::TEXT = ANY(seen_tokens)) THEN
|
|
374
|
+
seen_tokens := seen_tokens || tok_q::TEXT;
|
|
375
|
+
tok_queries := tok_queries || tok_q;
|
|
331
376
|
query_fts_or := CASE WHEN query_fts_or IS NULL
|
|
332
377
|
THEN tok_q ELSE query_fts_or || tok_q END;
|
|
333
378
|
END IF;
|
|
334
379
|
END LOOP;
|
|
380
|
+
total_tokens := COALESCE(array_length(tok_queries, 1), 0);
|
|
335
381
|
|
|
336
382
|
IF numnode(query_fts_and) > 0 THEN
|
|
337
383
|
SELECT EXISTS (
|
|
@@ -377,6 +423,11 @@ BEGIN
|
|
|
377
423
|
WHERE c.version_id IS NULL
|
|
378
424
|
AND d.deleted_at IS NULL
|
|
379
425
|
AND c.fts @@ query_fts
|
|
426
|
+
-- v1.0.4 coverage gate (OR-fallback mode only): pure keyword search
|
|
427
|
+
-- returns only chunks matching enough of the query's terms.
|
|
428
|
+
AND (and_matches OR total_tokens = 0
|
|
429
|
+
OR (SELECT COUNT(*) FROM unnest(tok_queries) tq
|
|
430
|
+
WHERE c.fts @@ tq)::FLOAT >= p_min_term_coverage * total_tokens)
|
|
380
431
|
AND (p_project_id IS NULL OR EXISTS (
|
|
381
432
|
SELECT 1 FROM cerefox_document_projects dp
|
|
382
433
|
WHERE dp.document_id = d.id AND dp.project_id = p_project_id
|
|
@@ -687,7 +738,8 @@ CREATE OR REPLACE FUNCTION cerefox_search_docs(
|
|
|
687
738
|
p_min_score FLOAT DEFAULT 0.0,
|
|
688
739
|
p_small_to_big_threshold INT DEFAULT 20000,
|
|
689
740
|
p_context_window INT DEFAULT 1,
|
|
690
|
-
p_metadata_filter JSONB DEFAULT NULL
|
|
741
|
+
p_metadata_filter JSONB DEFAULT NULL,
|
|
742
|
+
p_min_term_coverage FLOAT DEFAULT 0.5
|
|
691
743
|
)
|
|
692
744
|
RETURNS TABLE (
|
|
693
745
|
document_id UUID,
|
|
@@ -727,7 +779,8 @@ AS $$
|
|
|
727
779
|
p_use_upgrade := FALSE,
|
|
728
780
|
p_project_id := p_project_id,
|
|
729
781
|
p_min_score := p_min_score,
|
|
730
|
-
p_metadata_filter := p_metadata_filter
|
|
782
|
+
p_metadata_filter := p_metadata_filter,
|
|
783
|
+
p_min_term_coverage := p_min_term_coverage
|
|
731
784
|
)
|
|
732
785
|
),
|
|
733
786
|
best_per_doc AS (
|
|
@@ -1885,7 +1938,7 @@ SET search_path = public, pg_catalog
|
|
|
1885
1938
|
AS $$
|
|
1886
1939
|
-- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
|
|
1887
1940
|
-- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
|
|
1888
|
-
SELECT '0.9.
|
|
1941
|
+
SELECT '0.9.1'::TEXT;
|
|
1889
1942
|
$$;
|
|
1890
1943
|
|
|
1891
1944
|
-- ── cerefox_content_format_stats ─────────────────────────────────────────────
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
-- Requires extensions: vector (pgvector), uuid-ossp
|
|
6
6
|
-- These are enabled at the top of db_deploy.py before this file is applied.
|
|
7
7
|
--
|
|
8
|
-
-- @version: 0.9.
|
|
8
|
+
-- @version: 0.9.1
|
|
9
9
|
-- The `@version` marker above is read by the schema-version-mismatch banner
|
|
10
10
|
-- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
|
|
11
11
|
-- changes in a way that requires `cerefox server deploy` to be re-run —
|
|
@@ -117,10 +117,19 @@ This handles intermittent OpenAI API errors (500s) that would otherwise cause se
|
|
|
117
117
|
|
|
118
118
|
## Retrieval
|
|
119
119
|
|
|
120
|
+
> **Which paths read these?** Client-side tunables in this section are read
|
|
121
|
+
> from *your* `.env` by the **CLI**, the **local MCP server**, and `cerefox
|
|
122
|
+
> web`. The **remote MCP / Edge Function path** runs on Supabase and does not
|
|
123
|
+
> see your `.env` — it uses the server defaults unless the caller passes the
|
|
124
|
+
> per-call parameter (e.g. `min_score`, `min_term_coverage` on
|
|
125
|
+
> `cerefox_search`). Setting them as Supabase **Function secrets** may also
|
|
126
|
+
> work but is not a tested configuration.
|
|
127
|
+
|
|
120
128
|
| Variable | Default | Description |
|
|
121
129
|
|----------|---------|-------------|
|
|
122
130
|
| `CEREFOX_MAX_RESPONSE_BYTES` | `200000` | Maximum bytes in a single search response (local MCP path). See explanation below. |
|
|
123
131
|
| `CEREFOX_MIN_SEARCH_SCORE` | `0.50` (`0.60` with the local embedder) | Minimum cosine similarity for hybrid and semantic search results (0.0–1.0). The default is embedder-aware: nomic scores unrelated text higher than OpenAI, so `CEREFOX_EMBEDDER=local` raises the floor to 0.60. In **hybrid search**, chunks that matched the FTS keyword operator (`@@`) always pass through regardless of their vector score — the threshold only filters vector-only results. In **semantic search**, all results are filtered. The pure **FTS search** mode is unaffected. Increase for stricter precision; decrease for wider recall. |
|
|
132
|
+
| `CEREFOX_MIN_TERM_COVERAGE` | *(unset — server default `0.5`)* | Confidence bar for the keyword OR-fallback (v1.0.4, schema ≥ 0.9.1): when a strict all-terms match fails and search relaxes to any-term matching, a result counts as a confident hit only if it matches at least this fraction of the query's meaningful terms; weaker matches surface as below-confidence candidates. `0` restores pre-gate behavior (any matching term passes); `1` requires every term. Per-call override: `cerefox search --min-term-coverage`. Leave unset against pre-0.9.1 servers. |
|
|
124
133
|
| `CEREFOX_EMBED_MAX_INPUT_CHARS` | `20000` | Safety cap on the characters sent to the embedding model per input. The full chunk content is always stored and reconstructed untouched; only the embedding uses the (rare) truncated prefix, so an oversized chunk can never fail an ingest. |
|
|
125
134
|
| `CEREFOX_MODELS_DIR` | `~/.cerefox/models` (in-container: inside the data volume) | Where the local embedder caches downloaded model weights (Cerefox Local; `CEREFOX_EMBEDDER=local`). |
|
|
126
135
|
| `CEREFOX_ONNX_BATCH` | `4` | Texts per local-embedder inference call. Peak memory scales with this; the small default keeps ingest/reindex safe on small Docker VMs. |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cerefox/memory",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.4",
|
|
4
4
|
"description": "Cerefox — user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"homepage": "https://github.com/fstamatelopoulos/cerefox",
|