@cerefox/memory 0.10.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cerefox.js +62 -18
- package/dist/server-assets/_shared/ef-meta/index.ts +1 -1
- package/dist/server-assets/_shared/embeddings/index.ts +34 -6
- package/dist/server-assets/_shared/mcp-tools/_utils.ts +38 -2
- package/dist/server-assets/_shared/mcp-tools/metadata-search.ts +3 -2
- package/dist/server-assets/_shared/mcp-tools/search.ts +4 -3
- package/docs/guides/cli.md +2 -2
- package/docs/guides/configuration.md +20 -20
- package/docs/guides/ops-scripts.md +1 -1
- package/docs/guides/quickstart.md +8 -7
- package/docs/guides/setup-supabase.md +1 -1
- package/package.json +1 -1
package/dist/bin/cerefox.js
CHANGED
|
@@ -7184,7 +7184,7 @@ var exports_meta = {};
|
|
|
7184
7184
|
__export(exports_meta, {
|
|
7185
7185
|
PKG_VERSION: () => PKG_VERSION
|
|
7186
7186
|
});
|
|
7187
|
-
var PKG_VERSION = "0.10.
|
|
7187
|
+
var PKG_VERSION = "0.10.1";
|
|
7188
7188
|
var init_meta = () => {};
|
|
7189
7189
|
|
|
7190
7190
|
// ../../node_modules/.bun/tslib@2.8.1/node_modules/tslib/tslib.js
|
|
@@ -24852,20 +24852,31 @@ var init_bundled_docs = __esm(() => {
|
|
|
24852
24852
|
});
|
|
24853
24853
|
|
|
24854
24854
|
// ../../_shared/embeddings/index.ts
|
|
24855
|
+
function openaiEmbeddingConfig() {
|
|
24856
|
+
const env4 = globalThis.process?.env ?? {};
|
|
24857
|
+
const base = env4.CEREFOX_OPENAI_BASE_URL?.replace(/\/+$/, "");
|
|
24858
|
+
const dims = Number.parseInt(env4.CEREFOX_OPENAI_EMBEDDING_DIMENSIONS ?? "", 10);
|
|
24859
|
+
return {
|
|
24860
|
+
url: base ? `${base}/embeddings` : OPENAI_EMBEDDING_URL,
|
|
24861
|
+
model: env4.CEREFOX_OPENAI_EMBEDDING_MODEL || OPENAI_MODEL,
|
|
24862
|
+
dimensions: Number.isNaN(dims) || dims <= 0 ? EMBEDDING_DIMENSIONS : dims
|
|
24863
|
+
};
|
|
24864
|
+
}
|
|
24855
24865
|
async function getEmbedding(text, apiKey) {
|
|
24856
24866
|
let lastError = null;
|
|
24867
|
+
const cfg = openaiEmbeddingConfig();
|
|
24857
24868
|
for (let attempt = 0;attempt < EMBEDDING_MAX_RETRIES; attempt++) {
|
|
24858
24869
|
try {
|
|
24859
|
-
const response = await fetch(
|
|
24870
|
+
const response = await fetch(cfg.url, {
|
|
24860
24871
|
method: "POST",
|
|
24861
24872
|
headers: {
|
|
24862
24873
|
Authorization: `Bearer ${apiKey}`,
|
|
24863
24874
|
"Content-Type": "application/json"
|
|
24864
24875
|
},
|
|
24865
24876
|
body: JSON.stringify({
|
|
24866
|
-
model:
|
|
24877
|
+
model: cfg.model,
|
|
24867
24878
|
input: text,
|
|
24868
|
-
dimensions:
|
|
24879
|
+
dimensions: cfg.dimensions
|
|
24869
24880
|
})
|
|
24870
24881
|
});
|
|
24871
24882
|
if (!response.ok) {
|
|
@@ -24896,18 +24907,19 @@ async function getEmbedding(text, apiKey) {
|
|
|
24896
24907
|
}
|
|
24897
24908
|
async function embedBatchSingleCall(texts, apiKey) {
|
|
24898
24909
|
let lastError = null;
|
|
24910
|
+
const cfg = openaiEmbeddingConfig();
|
|
24899
24911
|
for (let attempt = 0;attempt < EMBEDDING_MAX_RETRIES; attempt++) {
|
|
24900
24912
|
try {
|
|
24901
|
-
const response = await fetch(
|
|
24913
|
+
const response = await fetch(cfg.url, {
|
|
24902
24914
|
method: "POST",
|
|
24903
24915
|
headers: {
|
|
24904
24916
|
Authorization: `Bearer ${apiKey}`,
|
|
24905
24917
|
"Content-Type": "application/json"
|
|
24906
24918
|
},
|
|
24907
24919
|
body: JSON.stringify({
|
|
24908
|
-
model:
|
|
24920
|
+
model: cfg.model,
|
|
24909
24921
|
input: texts,
|
|
24910
|
-
dimensions:
|
|
24922
|
+
dimensions: cfg.dimensions
|
|
24911
24923
|
})
|
|
24912
24924
|
});
|
|
24913
24925
|
if (!response.ok) {
|
|
@@ -53721,6 +53733,20 @@ var require_cli_progress = __commonJS((exports, module) => {
|
|
|
53721
53733
|
});
|
|
53722
53734
|
|
|
53723
53735
|
// ../../_shared/mcp-tools/_utils.ts
|
|
53736
|
+
function getMaxResponseBytes() {
|
|
53737
|
+
const raw = globalThis.process?.env?.CEREFOX_MAX_RESPONSE_BYTES;
|
|
53738
|
+
if (raw === undefined || raw === "")
|
|
53739
|
+
return MAX_RESPONSE_BYTES;
|
|
53740
|
+
const n = Number.parseInt(raw, 10);
|
|
53741
|
+
return Number.isNaN(n) || n <= 0 ? MAX_RESPONSE_BYTES : n;
|
|
53742
|
+
}
|
|
53743
|
+
function getMinSearchScore() {
|
|
53744
|
+
const raw = globalThis.process?.env?.CEREFOX_MIN_SEARCH_SCORE;
|
|
53745
|
+
if (raw === undefined || raw === "")
|
|
53746
|
+
return DEFAULT_MIN_SEARCH_SCORE;
|
|
53747
|
+
const n = Number.parseFloat(raw);
|
|
53748
|
+
return Number.isNaN(n) || n < 0 || n > 1 ? DEFAULT_MIN_SEARCH_SCORE : n;
|
|
53749
|
+
}
|
|
53724
53750
|
function applyByteBudget(rows, maxBytes) {
|
|
53725
53751
|
const accepted = [];
|
|
53726
53752
|
let usedBytes = 0;
|
|
@@ -53748,7 +53774,7 @@ function logUsage(supabase, params) {
|
|
|
53748
53774
|
p_extra: params.extra ?? {}
|
|
53749
53775
|
})).catch(() => {});
|
|
53750
53776
|
}
|
|
53751
|
-
var MAX_RESPONSE_BYTES = 200000;
|
|
53777
|
+
var MAX_RESPONSE_BYTES = 200000, DEFAULT_MIN_SEARCH_SCORE = 0.5;
|
|
53752
53778
|
|
|
53753
53779
|
// ../../_shared/mcp-tools/audit-log.ts
|
|
53754
53780
|
async function handler(supabase, args, ctx) {
|
|
@@ -54531,7 +54557,8 @@ async function handler8(supabase, args, ctx) {
|
|
|
54531
54557
|
if (!projectId)
|
|
54532
54558
|
throw new Error(`Project not found: ${project_name}`);
|
|
54533
54559
|
}
|
|
54534
|
-
const
|
|
54560
|
+
const ceiling = getMaxResponseBytes();
|
|
54561
|
+
const max_bytes = include_content ? Math.min(requested_max_bytes ?? ceiling, ceiling) : null;
|
|
54535
54562
|
const params = {
|
|
54536
54563
|
p_metadata_filter: metadata_filter,
|
|
54537
54564
|
p_project_id: projectId,
|
|
@@ -54624,10 +54651,11 @@ async function handler9(supabase, args, ctx) {
|
|
|
54624
54651
|
const match_count = args.match_count ?? 5;
|
|
54625
54652
|
const mode = args.mode ?? "docs";
|
|
54626
54653
|
const alpha = args.alpha ?? 0.7;
|
|
54627
|
-
const min_score = args.min_score ??
|
|
54654
|
+
const min_score = args.min_score ?? getMinSearchScore();
|
|
54628
54655
|
const metadata_filter = args.metadata_filter ?? null;
|
|
54629
54656
|
const requested_max_bytes = args.max_bytes;
|
|
54630
|
-
const
|
|
54657
|
+
const ceiling = getMaxResponseBytes();
|
|
54658
|
+
const max_bytes = Math.min(requested_max_bytes ?? ceiling, ceiling);
|
|
54631
54659
|
if (metadata_filter !== null && metadata_filter !== undefined && (typeof metadata_filter !== "object" || Array.isArray(metadata_filter))) {
|
|
54632
54660
|
throw new McpInvalidParams("metadata_filter must be a JSON object or null");
|
|
54633
54661
|
}
|
|
@@ -68087,7 +68115,7 @@ function utcStamp() {
|
|
|
68087
68115
|
return d.getUTCFullYear().toString() + pad(d.getUTCMonth() + 1) + pad(d.getUTCDate()) + "T" + pad(d.getUTCHours()) + pad(d.getUTCMinutes()) + pad(d.getUTCSeconds()) + "Z";
|
|
68088
68116
|
}
|
|
68089
68117
|
async function action(options) {
|
|
68090
|
-
const outDir = resolve(expandHome(options.outputDir ?? "~/.cerefox/backups"));
|
|
68118
|
+
const outDir = resolve(expandHome(options.outputDir ?? process.env.CEREFOX_BACKUP_DIR ?? "~/.cerefox/backups"));
|
|
68091
68119
|
if (!existsSync2(outDir))
|
|
68092
68120
|
mkdirSync(outDir, { recursive: true });
|
|
68093
68121
|
const stamp = utcStamp();
|
|
@@ -68133,7 +68161,7 @@ async function action(options) {
|
|
|
68133
68161
|
}
|
|
68134
68162
|
}
|
|
68135
68163
|
function registerBackup(program2) {
|
|
68136
|
-
program2.command("backup").description("Write a JSON snapshot of the knowledge base.").option("-o, --output-dir <dir>", "Snapshot output directory
|
|
68164
|
+
program2.command("backup").description("Write a JSON snapshot of the knowledge base.").option("-o, --output-dir <dir>", "Snapshot output directory (default: CEREFOX_BACKUP_DIR or ~/.cerefox/backups).").option("-l, --label <label>", "Optional suffix added to the filename.").option("--include-versions", "Include archived versions in the snapshot. (v0.5: ignored — current chunks only.)").option("--git", "Commit the snapshot to the output dir as a git checkpoint. (v0.5: ignored.)").action(action);
|
|
68137
68165
|
}
|
|
68138
68166
|
|
|
68139
68167
|
// src/cli/commands/completion.ts
|
|
@@ -73925,7 +73953,7 @@ import { homedir as homedir5 } from "node:os";
|
|
|
73925
73953
|
import { join as join8 } from "node:path";
|
|
73926
73954
|
|
|
73927
73955
|
// ../../_shared/ef-meta/index.ts
|
|
73928
|
-
var EF_VERSION = "0.
|
|
73956
|
+
var EF_VERSION = "0.10.1";
|
|
73929
73957
|
|
|
73930
73958
|
// src/cli/util/checks.ts
|
|
73931
73959
|
init_config();
|
|
@@ -75049,6 +75077,22 @@ var DEFAULT_PIPELINE_SETTINGS = {
|
|
|
75049
75077
|
versionRetentionHours: 48,
|
|
75050
75078
|
versionCleanupEnabled: true
|
|
75051
75079
|
};
|
|
75080
|
+
function loadPipelineSettings() {
|
|
75081
|
+
const env4 = globalThis.process?.env ?? {};
|
|
75082
|
+
const intMin = (raw, def, min) => {
|
|
75083
|
+
if (raw === undefined || raw === "")
|
|
75084
|
+
return def;
|
|
75085
|
+
const n = Number.parseInt(raw, 10);
|
|
75086
|
+
return Number.isNaN(n) || n < min ? def : n;
|
|
75087
|
+
};
|
|
75088
|
+
const bool = (raw, def) => raw === undefined || raw === "" ? def : !/^(false|0|no|off)$/i.test(raw.trim());
|
|
75089
|
+
return {
|
|
75090
|
+
maxChunkChars: intMin(env4.CEREFOX_MAX_CHUNK_CHARS, DEFAULT_PIPELINE_SETTINGS.maxChunkChars, 1),
|
|
75091
|
+
minChunkChars: intMin(env4.CEREFOX_MIN_CHUNK_CHARS, DEFAULT_PIPELINE_SETTINGS.minChunkChars, 0),
|
|
75092
|
+
versionRetentionHours: intMin(env4.CEREFOX_VERSION_RETENTION_HOURS, DEFAULT_PIPELINE_SETTINGS.versionRetentionHours, 0),
|
|
75093
|
+
versionCleanupEnabled: bool(env4.CEREFOX_VERSION_CLEANUP_ENABLED, DEFAULT_PIPELINE_SETTINGS.versionCleanupEnabled)
|
|
75094
|
+
};
|
|
75095
|
+
}
|
|
75052
75096
|
|
|
75053
75097
|
// src/ingestion/pipeline.ts
|
|
75054
75098
|
class IngestionPipeline {
|
|
@@ -75060,7 +75104,7 @@ class IngestionPipeline {
|
|
|
75060
75104
|
this.db = new IngestionDbBridge(deps.supabase);
|
|
75061
75105
|
this.apiKey = deps.openAiApiKey;
|
|
75062
75106
|
this.embedderModel = deps.embedderModel ?? "text-embedding-3-small";
|
|
75063
|
-
this.settings = { ...
|
|
75107
|
+
this.settings = { ...loadPipelineSettings(), ...deps.settings ?? {} };
|
|
75064
75108
|
}
|
|
75065
75109
|
async ingestText(opts) {
|
|
75066
75110
|
const {
|
|
@@ -76474,7 +76518,7 @@ async function action28(query, options) {
|
|
|
76474
76518
|
}
|
|
76475
76519
|
const matchCount = parsePositiveInt(options.matchCount, "--match-count", 5);
|
|
76476
76520
|
const alpha = parseFloat01(options.alpha, "--alpha", 0.7);
|
|
76477
|
-
const minScore = parseFloat01(options.minScore, "--min-score",
|
|
76521
|
+
const minScore = parseFloat01(options.minScore, "--min-score", getMinSearchScore());
|
|
76478
76522
|
const maxBytes = parseNonNegativeInt(options.maxBytes, "--max-bytes", 200000);
|
|
76479
76523
|
const mode = options.mode ?? "docs";
|
|
76480
76524
|
if (!["docs", "hybrid", "fts"].includes(mode)) {
|
|
@@ -76620,7 +76664,7 @@ async function action28(query, options) {
|
|
|
76620
76664
|
}
|
|
76621
76665
|
}
|
|
76622
76666
|
function registerSearch(program2) {
|
|
76623
|
-
program2.command("search").description("Search the knowledge base (hybrid FTS + semantic).").argument("<query>", "Natural-language search query.").option("-c, --match-count <n>", "Maximum number of documents to return.", "5").option("-p, --project-name <name>", "Filter results to a specific project.").option("-f, --metadata-filter <json>", "JSON containment filter; only docs whose metadata contains ALL pairs are returned.").option("--mode <mode>", "Search mode: docs (default), hybrid, fts.", "docs").option("--alpha <float>", "Semantic weight 0..1 (default: 0.7).", "0.7").option("--min-score <float>", "Minimum cosine similarity threshold
|
|
76667
|
+
program2.command("search").description("Search the knowledge base (hybrid FTS + semantic).").argument("<query>", "Natural-language search query.").option("-c, --match-count <n>", "Maximum number of documents to return.", "5").option("-p, --project-name <name>", "Filter results to a specific project.").option("-f, --metadata-filter <json>", "JSON containment filter; only docs whose metadata contains ALL pairs are returned.").option("--mode <mode>", "Search mode: docs (default), hybrid, fts.", "docs").option("--alpha <float>", "Semantic weight 0..1 (default: 0.7).", "0.7").option("--min-score <float>", "Minimum cosine similarity threshold (default: CEREFOX_MIN_SEARCH_SCORE or 0.5).").option("--max-bytes <n>", "Response size budget in bytes.", "200000").option("-r, --requestor <name>", "Agent / user name (recorded in usage log).").option("--json", "Emit machine-readable JSON instead of the default text.").option("--only-metadata", "List matching docs (id, score, chunks, chars, partial/full) WITHOUT their content — like the web UI's collapsed result list. Grab a [id:…] then `cerefox document get <id>`.").action(action28);
|
|
76624
76668
|
}
|
|
76625
76669
|
|
|
76626
76670
|
// src/cli/commands/self-update.ts
|
|
@@ -80241,7 +80285,7 @@ async function runSearch(ctx, opts) {
|
|
|
80241
80285
|
p_alpha: 0.7,
|
|
80242
80286
|
p_use_upgrade: false,
|
|
80243
80287
|
p_project_id: projectId,
|
|
80244
|
-
p_min_score:
|
|
80288
|
+
p_min_score: getMinSearchScore()
|
|
80245
80289
|
};
|
|
80246
80290
|
if (metadataFilter)
|
|
80247
80291
|
params2.p_metadata_filter = metadataFilter;
|
|
@@ -17,25 +17,52 @@ export const OPENAI_EMBEDDING_URL = "https://api.openai.com/v1/embeddings";
|
|
|
17
17
|
export const OPENAI_MODEL = "text-embedding-3-small";
|
|
18
18
|
export const EMBEDDING_DIMENSIONS = 768;
|
|
19
19
|
|
|
20
|
+
/**
|
|
21
|
+
* Resolve the OpenAI embedding endpoint/model/dimensions, applying `.env`
|
|
22
|
+
* overrides over the built-in defaults. These were configurable in the Python
|
|
23
|
+
* runtime; the TS migration hardcoded them.
|
|
24
|
+
*
|
|
25
|
+
* ⚠ Overriding the MODEL or DIMENSIONS is a BREAKING change: query vectors must
|
|
26
|
+
* match the stored vectors and the DB column is `vector(768)`. Changing either
|
|
27
|
+
* requires re-embedding the whole corpus (`cerefox server reindex`) and, for a
|
|
28
|
+
* non-768 model, a schema change. `CEREFOX_OPENAI_BASE_URL` (proxy/gateway) is
|
|
29
|
+
* the only safe one to flip on an existing KB.
|
|
30
|
+
*
|
|
31
|
+
* Runtime-agnostic env read; the Deno Edge Function (no host env) keeps the
|
|
32
|
+
* constants — matching the EF's "model config is a constant" design.
|
|
33
|
+
*/
|
|
34
|
+
export function openaiEmbeddingConfig(): { url: string; model: string; dimensions: number } {
|
|
35
|
+
const env =
|
|
36
|
+
(globalThis as { process?: { env?: Record<string, string | undefined> } }).process?.env ?? {};
|
|
37
|
+
const base = env.CEREFOX_OPENAI_BASE_URL?.replace(/\/+$/, "");
|
|
38
|
+
const dims = Number.parseInt(env.CEREFOX_OPENAI_EMBEDDING_DIMENSIONS ?? "", 10);
|
|
39
|
+
return {
|
|
40
|
+
url: base ? `${base}/embeddings` : OPENAI_EMBEDDING_URL,
|
|
41
|
+
model: env.CEREFOX_OPENAI_EMBEDDING_MODEL || OPENAI_MODEL,
|
|
42
|
+
dimensions: Number.isNaN(dims) || dims <= 0 ? EMBEDDING_DIMENSIONS : dims,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
20
46
|
const EMBEDDING_MAX_RETRIES = 3;
|
|
21
47
|
const EMBEDDING_INITIAL_BACKOFF_MS = 500; // 500ms → 1s → 2s
|
|
22
48
|
|
|
23
49
|
/** Embed a single string. Used for the query vector in `cerefox_search`. */
|
|
24
50
|
export async function getEmbedding(text: string, apiKey: string): Promise<number[]> {
|
|
25
51
|
let lastError: Error | null = null;
|
|
52
|
+
const cfg = openaiEmbeddingConfig();
|
|
26
53
|
|
|
27
54
|
for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
|
|
28
55
|
try {
|
|
29
|
-
const response = await fetch(
|
|
56
|
+
const response = await fetch(cfg.url, {
|
|
30
57
|
method: "POST",
|
|
31
58
|
headers: {
|
|
32
59
|
"Authorization": `Bearer ${apiKey}`,
|
|
33
60
|
"Content-Type": "application/json",
|
|
34
61
|
},
|
|
35
62
|
body: JSON.stringify({
|
|
36
|
-
model:
|
|
63
|
+
model: cfg.model,
|
|
37
64
|
input: text,
|
|
38
|
-
dimensions:
|
|
65
|
+
dimensions: cfg.dimensions,
|
|
39
66
|
}),
|
|
40
67
|
});
|
|
41
68
|
|
|
@@ -93,19 +120,20 @@ async function embedBatchSingleCall(
|
|
|
93
120
|
apiKey: string,
|
|
94
121
|
): Promise<number[][]> {
|
|
95
122
|
let lastError: Error | null = null;
|
|
123
|
+
const cfg = openaiEmbeddingConfig();
|
|
96
124
|
|
|
97
125
|
for (let attempt = 0; attempt < EMBEDDING_MAX_RETRIES; attempt++) {
|
|
98
126
|
try {
|
|
99
|
-
const response = await fetch(
|
|
127
|
+
const response = await fetch(cfg.url, {
|
|
100
128
|
method: "POST",
|
|
101
129
|
headers: {
|
|
102
130
|
"Authorization": `Bearer ${apiKey}`,
|
|
103
131
|
"Content-Type": "application/json",
|
|
104
132
|
},
|
|
105
133
|
body: JSON.stringify({
|
|
106
|
-
model:
|
|
134
|
+
model: cfg.model,
|
|
107
135
|
input: texts,
|
|
108
|
-
dimensions:
|
|
136
|
+
dimensions: cfg.dimensions,
|
|
109
137
|
}),
|
|
110
138
|
});
|
|
111
139
|
|
|
@@ -14,10 +14,46 @@
|
|
|
14
14
|
|
|
15
15
|
import type { MCPSupabaseClient } from "./types.ts";
|
|
16
16
|
|
|
17
|
-
/**
|
|
18
|
-
* smaller budgets via `max_bytes`; values above this are capped. */
|
|
17
|
+
/** Built-in default response-size ceiling for MCP/EF results. */
|
|
19
18
|
export const MAX_RESPONSE_BYTES = 200_000;
|
|
20
19
|
|
|
20
|
+
/**
|
|
21
|
+
* Server-enforced response-size ceiling for MCP/Edge-Function results (agents
|
|
22
|
+
* can request smaller via `max_bytes`; larger is capped). Overridable via
|
|
23
|
+
* `CEREFOX_MAX_RESPONSE_BYTES`. Read by the Python runtime; restored after the
|
|
24
|
+
* TS migration. The web UI + CLI are intentionally unlimited and do not use this.
|
|
25
|
+
* Runtime-agnostic env read (Deno EF safely falls back to the default).
|
|
26
|
+
*/
|
|
27
|
+
export function getMaxResponseBytes(): number {
|
|
28
|
+
const raw = (globalThis as { process?: { env?: Record<string, string | undefined> } })
|
|
29
|
+
.process?.env?.CEREFOX_MAX_RESPONSE_BYTES;
|
|
30
|
+
if (raw === undefined || raw === "") return MAX_RESPONSE_BYTES;
|
|
31
|
+
const n = Number.parseInt(raw, 10);
|
|
32
|
+
return Number.isNaN(n) || n <= 0 ? MAX_RESPONSE_BYTES : n;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Built-in default cosine-similarity floor for hybrid/semantic search. */
|
|
36
|
+
export const DEFAULT_MIN_SEARCH_SCORE = 0.5;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Resolve the minimum cosine-similarity floor for hybrid/semantic search
|
|
40
|
+
* (vector-only matches below this are dropped; FTS matches always pass).
|
|
41
|
+
* Overridable via the `CEREFOX_MIN_SEARCH_SCORE` env var (0.0–1.0). The Python
|
|
42
|
+
* runtime read this; the TS migration dropped it — restored here as the single
|
|
43
|
+
* default used by the CLI, local/remote MCP, and the web API.
|
|
44
|
+
*
|
|
45
|
+
* Runtime-agnostic env read: works in Node/Bun; in the Deno Edge Function
|
|
46
|
+
* `process` may be absent, so it falls back to the built-in default (the cloud
|
|
47
|
+
* EF path doesn't use the host `.env` anyway).
|
|
48
|
+
*/
|
|
49
|
+
export function getMinSearchScore(): number {
|
|
50
|
+
const raw = (globalThis as { process?: { env?: Record<string, string | undefined> } })
|
|
51
|
+
.process?.env?.CEREFOX_MIN_SEARCH_SCORE;
|
|
52
|
+
if (raw === undefined || raw === "") return DEFAULT_MIN_SEARCH_SCORE;
|
|
53
|
+
const n = Number.parseFloat(raw);
|
|
54
|
+
return Number.isNaN(n) || n < 0 || n > 1 ? DEFAULT_MIN_SEARCH_SCORE : n;
|
|
55
|
+
}
|
|
56
|
+
|
|
21
57
|
export function applyByteBudget(
|
|
22
58
|
rows: unknown[],
|
|
23
59
|
maxBytes: number,
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import type { MCPSupabaseClient } from "./types.ts";
|
|
9
9
|
|
|
10
|
-
import { applyByteBudget,
|
|
10
|
+
import { applyByteBudget, getMaxResponseBytes, logUsage } from "./_utils.ts";
|
|
11
11
|
import { lookupProjectId } from "./_projects.ts";
|
|
12
12
|
import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
|
|
13
13
|
|
|
@@ -39,8 +39,9 @@ async function handler(
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
// Enforce byte ceiling for content mode
|
|
42
|
+
const ceiling = getMaxResponseBytes();
|
|
42
43
|
const max_bytes = include_content
|
|
43
|
-
? Math.min(requested_max_bytes ??
|
|
44
|
+
? Math.min(requested_max_bytes ?? ceiling, ceiling)
|
|
44
45
|
: null;
|
|
45
46
|
|
|
46
47
|
const params: Record<string, unknown> = {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
import type { MCPSupabaseClient } from "./types.ts";
|
|
19
19
|
|
|
20
20
|
import { getEmbedding } from "../embeddings/index.ts";
|
|
21
|
-
import { applyByteBudget,
|
|
21
|
+
import { applyByteBudget, getMaxResponseBytes, getMinSearchScore, logUsage } from "./_utils.ts";
|
|
22
22
|
import { lookupProjectId } from "./_projects.ts";
|
|
23
23
|
import { McpInvalidParams, type ToolContext, type ToolDefinition } from "./types.ts";
|
|
24
24
|
|
|
@@ -32,12 +32,13 @@ async function handler(
|
|
|
32
32
|
const match_count = (args.match_count as number | undefined) ?? 5;
|
|
33
33
|
const mode = (args.mode as string | undefined) ?? "docs";
|
|
34
34
|
const alpha = (args.alpha as number | undefined) ?? 0.7;
|
|
35
|
-
const min_score = (args.min_score as number | undefined) ??
|
|
35
|
+
const min_score = (args.min_score as number | undefined) ?? getMinSearchScore();
|
|
36
36
|
const metadata_filter =
|
|
37
37
|
(args.metadata_filter as Record<string, string> | null | undefined) ?? null;
|
|
38
38
|
const requested_max_bytes = args.max_bytes as number | undefined;
|
|
39
39
|
|
|
40
|
-
const
|
|
40
|
+
const ceiling = getMaxResponseBytes();
|
|
41
|
+
const max_bytes = Math.min(requested_max_bytes ?? ceiling, ceiling);
|
|
41
42
|
|
|
42
43
|
if (
|
|
43
44
|
metadata_filter !== null &&
|
package/docs/guides/cli.md
CHANGED
|
@@ -14,7 +14,7 @@ Every command reads configuration from `.env` in the working directory (or envir
|
|
|
14
14
|
|
|
15
15
|
The CLI is the TypeScript `@cerefox/memory` package. Invoke any command as plain `cerefox <subcommand>` (installed via the installer or `npm install -g @cerefox/memory` — see [`quickstart.md`](quickstart.md#1-install)).
|
|
16
16
|
|
|
17
|
-
> **v0.9 verb rename**: commands now follow a `resource verb` shape (e.g. `cerefox document get`, `cerefox project list`). The old flat verbs (`get-doc`, `list-docs`, `ingest`, `list-versions`, `config-get`, `deploy-server`, `docs`, …)
|
|
17
|
+
> **v0.9 verb rename**: commands now follow a `resource verb` shape (e.g. `cerefox document get`, `cerefox project list`). The old flat verbs (`get-doc`, `list-docs`, `ingest`, `list-versions`, `config-get`, `deploy-server`, `docs`, …) survive as hidden husks — they still run but print a pointer to the new form and exit non-zero (removed only at v1.0). Use the new forms below.
|
|
18
18
|
|
|
19
19
|
## Commands
|
|
20
20
|
|
|
@@ -141,7 +141,7 @@ cerefox search [OPTIONS] QUERY
|
|
|
141
141
|
```bash
|
|
142
142
|
cerefox search "OAuth design"
|
|
143
143
|
cerefox search "decisions" --metadata-filter '{"type":"decision-log"}' --match-count 5
|
|
144
|
-
cerefox search "what we tried" --mode
|
|
144
|
+
cerefox search "what we tried" --mode hybrid --requestor "claude-code"
|
|
145
145
|
cerefox search "design docs" --only-metadata
|
|
146
146
|
```
|
|
147
147
|
|
|
@@ -24,6 +24,13 @@ CEREFOX_CONFIG_DIR=~/.cerefox-personal cerefox search "…"
|
|
|
24
24
|
|
|
25
25
|
Full rule documented in [`docs/specs/polish-and-distribution-design.md` §7](../specs/polish-and-distribution-design.md).
|
|
26
26
|
|
|
27
|
+
> **Local / self-hosted (World B).** For the Docker backend (`cerefox-local`), do **not**
|
|
28
|
+
> set the Supabase or `CEREFOX_DATABASE_URL` vars below — the container generates and owns
|
|
29
|
+
> them. Put `OPENAI_API_KEY` plus any of the `CEREFOX_*` **tuning** options on this page
|
|
30
|
+
> (search, chunking, retrieval, versioning, embedding base-url/model, caller identity) in
|
|
31
|
+
> `~/.cerefox/local/.env`; the installer + `cerefox-local` forward them into the container.
|
|
32
|
+
> Apply changes with `cerefox-local init`. See [`setup-local.md`](setup-local.md).
|
|
33
|
+
|
|
27
34
|
---
|
|
28
35
|
|
|
29
36
|
## Supabase / Database
|
|
@@ -45,34 +52,27 @@ Full rule documented in [`docs/specs/polish-and-distribution-design.md` §7](../
|
|
|
45
52
|
|
|
46
53
|
Cerefox uses cloud-based embedding APIs. Local models (mpnet, Ollama) are not supported — they require large downloads, fail on some hardware, and add installation complexity.
|
|
47
54
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
55
|
+
> **TS runtime: OpenAI only (today).** The current TypeScript runtime implements the
|
|
56
|
+
> OpenAI embedder. `CEREFOX_EMBEDDER` and the `CEREFOX_FIREWORKS_*` variables are
|
|
57
|
+
> documented (they worked in the retired Python runtime) but are **not yet wired in TS** —
|
|
58
|
+
> they're currently no-ops, tracked for a future release.
|
|
51
59
|
|
|
52
60
|
### OpenAI (default, recommended)
|
|
53
61
|
|
|
54
62
|
| Variable | Default | Description |
|
|
55
63
|
|----------|---------|-------------|
|
|
56
64
|
| `OPENAI_API_KEY` | `""` | OpenAI API key. Also accepted as `CEREFOX_OPENAI_API_KEY`. Get one at [platform.openai.com/api-keys](https://platform.openai.com/api-keys). |
|
|
57
|
-
| `CEREFOX_OPENAI_BASE_URL` | `https://api.openai.com/v1` | API base URL.
|
|
58
|
-
| `CEREFOX_OPENAI_EMBEDDING_MODEL` | `text-embedding-3-small` | OpenAI embedding model. |
|
|
59
|
-
| `CEREFOX_OPENAI_EMBEDDING_DIMENSIONS` | `768` | Output dimensions. Must match the
|
|
60
|
-
|
|
61
|
-
For cost estimates see `docs/guides/operational-cost.md`.
|
|
62
|
-
|
|
63
|
-
### Fireworks AI (alternative, lower cost)
|
|
65
|
+
| `CEREFOX_OPENAI_BASE_URL` | `https://api.openai.com/v1` | API base URL. Safe to override for proxies or OpenAI-compatible gateways. |
|
|
66
|
+
| `CEREFOX_OPENAI_EMBEDDING_MODEL` | `text-embedding-3-small` | OpenAI embedding model. ⚠ see warning below. |
|
|
67
|
+
| `CEREFOX_OPENAI_EMBEDDING_DIMENSIONS` | `768` | Output dimensions. Must match the DB schema (`VECTOR(768)`). ⚠ see warning below. |
|
|
64
68
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
69
|
+
> **⚠ Changing the model or dimensions is breaking.** Query vectors must match the stored
|
|
70
|
+
> vectors. After changing `CEREFOX_OPENAI_EMBEDDING_MODEL` you MUST re-embed the whole
|
|
71
|
+
> corpus (`cerefox server reindex`); changing `CEREFOX_OPENAI_EMBEDDING_DIMENSIONS` away
|
|
72
|
+
> from 768 also requires a schema change. `CEREFOX_OPENAI_BASE_URL` is the only one safe to
|
|
73
|
+
> flip on an existing knowledge base.
|
|
70
74
|
|
|
71
|
-
|
|
72
|
-
```env
|
|
73
|
-
CEREFOX_EMBEDDER=fireworks
|
|
74
|
-
CEREFOX_FIREWORKS_API_KEY=fw_...
|
|
75
|
-
```
|
|
75
|
+
For cost estimates see `docs/guides/operational-cost.md`.
|
|
76
76
|
|
|
77
77
|
### Edge Functions (for agents)
|
|
78
78
|
|
|
@@ -37,7 +37,7 @@ jobs / CI / make targets that invoke them.
|
|
|
37
37
|
|
|
38
38
|
### TS scripts and `.env` resolution
|
|
39
39
|
|
|
40
|
-
`bun scripts/<name>.ts` reads the same `.env` the
|
|
40
|
+
`bun scripts/<name>.ts` reads the same `.env` the `cerefox` CLI does. Precedence:
|
|
41
41
|
|
|
42
42
|
1. `CEREFOX_CONFIG_DIR` env var (explicit override; supports `~`).
|
|
43
43
|
2. `./.env` in the current working directory (dev mode).
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# Quickstart -- Zero to First Document
|
|
2
2
|
|
|
3
|
-
Get Cerefox running on your machine via the npm install path
|
|
4
|
-
|
|
5
|
-
prerequisite — provisioning a free one takes a few minutes),
|
|
6
|
-
install and setup below
|
|
3
|
+
Get Cerefox running on your machine via the npm install path (the **cloud /
|
|
4
|
+
Supabase** backend). **No source clone required.** Once you have a Supabase
|
|
5
|
+
project (the one prerequisite — provisioning a free one takes a few minutes),
|
|
6
|
+
the Cerefox install and setup below takes ~15 minutes.
|
|
7
7
|
|
|
8
8
|
> **Upgrading from an earlier version?** See [`upgrading.md`](upgrading.md)
|
|
9
9
|
> for migration steps instead.
|
|
@@ -114,8 +114,9 @@ You should see results from the bundled self-docs.
|
|
|
114
114
|
The path above is for **end users** (no clone). If you want to hack on Cerefox,
|
|
115
115
|
clone the repo, run `bun install`, and use the contributor scripts
|
|
116
116
|
(`bun scripts/db_deploy.ts`, `bun scripts/db_migrate.ts`). `uv` is only needed
|
|
117
|
-
for the legacy Python MCP fallback. See [`
|
|
118
|
-
|
|
117
|
+
for the legacy Python MCP fallback. See [`CONTRIBUTING.md`](../../CONTRIBUTING.md).
|
|
118
|
+
(Want a no-cloud install instead? That's the self-hosted Docker backend —
|
|
119
|
+
[`setup-local.md`](setup-local.md).)
|
|
119
120
|
|
|
120
121
|
---
|
|
121
122
|
|
|
@@ -125,7 +126,7 @@ for the legacy Python MCP fallback. See [`setup-local.md`](setup-local.md) and
|
|
|
125
126
|
`cerefox document ingest-dir ./notes/` (recurses into sub-directories automatically)
|
|
126
127
|
- **Search from the CLI**: `cerefox search "your query"`
|
|
127
128
|
- **Discover all commands**: `cerefox --help`
|
|
128
|
-
- **Run the web UI**: `cerefox web` (TypeScript — Hono backend + React SPA); see [`
|
|
129
|
+
- **Run the web UI**: `cerefox web` (TypeScript — Hono backend + React SPA); see [`cli.md`](cli.md)
|
|
129
130
|
- **Connect more AI clients** (Cursor, Codex, ChatGPT GPT Actions, etc.):
|
|
130
131
|
[`connect-agents.md`](connect-agents.md)
|
|
131
132
|
- **Configuration reference**: [`configuration.md`](configuration.md)
|
|
@@ -47,7 +47,7 @@ You need three values from Supabase: a URL, an API key, and a direct Postgres co
|
|
|
47
47
|
|
|
48
48
|
See the **[Supabase API keys (2026)](#supabase-api-keys-2026)** section near the end of this guide for the full picture. The short version:
|
|
49
49
|
|
|
50
|
-
- For `CEREFOX_SUPABASE_KEY` (this guide,
|
|
50
|
+
- For `CEREFOX_SUPABASE_KEY` (this guide, the web UI, and the CLI): use the new **secret key** (`sb_secret_…`) from **Project Settings → API Keys → Secret key**. The legacy `service_role` JWT also still works during the transition.
|
|
51
51
|
- For `CEREFOX_SUPABASE_ANON_KEY` (only if you'll use Edge Functions / MCP / GPT Actions; not needed for this guide's deployment step): you must use the **legacy anon JWT** (`eyJ…`). The new `sb_publishable_…` key fails at the Edge Function gateway. See the reference section for why.
|
|
52
52
|
|
|
53
53
|
Either way: keep this key secret — it bypasses Row Level Security and grants full database access.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cerefox/memory",
|
|
3
|
-
"version": "0.10.
|
|
3
|
+
"version": "0.10.1",
|
|
4
4
|
"description": "Cerefox — user-owned shared memory for AI agents. The local TypeScript runtime: stdio MCP server in v0.4; CLI binary added in v0.5; in-process web server in v0.6; ingestion pipeline in v0.7.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"homepage": "https://github.com/fstamatelopoulos/cerefox",
|