@cerefox/memory 1.15.2 → 1.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cerefox.js +126 -108
- package/dist/server-assets/_shared/ef-meta/index.ts +3 -3
- package/dist/server-assets/_shared/embeddings/onnx-embedder.ts +34 -0
- package/dist/server-assets/_shared/mcp-tools/ingest.ts +7 -6
- package/dist/server-assets/supabase/functions/cerefox-ingest/index.ts +2 -2
- package/docs/api/openapi.json +5790 -0
- package/docs/guides/api.md +297 -0
- package/docs/guides/linking.md +118 -0
- package/docs/guides/securing-local-access.md +253 -0
- package/docs/guides/staging-env.md +334 -0
- package/package.json +1 -1
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* doesn't touch `supabase/functions/` leaves it alone).
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
export const EF_VERSION = "1.15.
|
|
21
|
+
export const EF_VERSION = "1.15.3";
|
|
22
22
|
|
|
23
23
|
/**
|
|
24
24
|
* The Cerefox RELEASE version — what `cerefox --version` reports and what npm
|
|
@@ -36,7 +36,7 @@ export const EF_VERSION = "1.15.2";
|
|
|
36
36
|
* is imported by the Deno Edge Functions, which cannot reach into the npm
|
|
37
37
|
* package.
|
|
38
38
|
*/
|
|
39
|
-
export const CEREFOX_VERSION = "1.15.
|
|
39
|
+
export const CEREFOX_VERSION = "1.15.3";
|
|
40
40
|
|
|
41
41
|
/**
|
|
42
42
|
* The most recent version whose EF-side SOURCE actually changed (#127).
|
|
@@ -46,7 +46,7 @@ export const CEREFOX_VERSION = "1.15.2";
|
|
|
46
46
|
* `cut_release.ts` ONLY when EF source changed since the last tag; doctor
|
|
47
47
|
* uses it to stay silent on label-only drift.
|
|
48
48
|
*/
|
|
49
|
-
export const EF_LAST_CHANGED = "1.15.
|
|
49
|
+
export const EF_LAST_CHANGED = "1.15.3";
|
|
50
50
|
|
|
51
51
|
/**
|
|
52
52
|
* The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp
|
|
@@ -46,6 +46,25 @@ export function buildPrefixedInputs(texts: string[], role: EmbedRole): string[]
|
|
|
46
46
|
return texts.map((t) => p + t);
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
/** What the q8 pipeline reads, relative to the cache dir. Same layout under
|
|
50
|
+
* transformers.js 3.x (the Local image) and 4.x (npm installs). */
|
|
51
|
+
const MODEL_FILES = [
|
|
52
|
+
"config.json",
|
|
53
|
+
"tokenizer.json",
|
|
54
|
+
"tokenizer_config.json",
|
|
55
|
+
`onnx/model_quantized.onnx`,
|
|
56
|
+
].map((f) => join(ONNX_MODEL_ID, f));
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Is every file the pipeline needs already on disk? Decides whether loading is
|
|
60
|
+
* a download worth narrating or a local read that should stay silent: the
|
|
61
|
+
* "loading from HuggingFace… first-run only" banner used to print on EVERY
|
|
62
|
+
* process start, which on the CLI is every command (#301).
|
|
63
|
+
*/
|
|
64
|
+
export function isModelCached(dir: string = getCacheDir()): boolean {
|
|
65
|
+
return MODEL_FILES.every((f) => existsSync(join(dir, f)));
|
|
66
|
+
}
|
|
67
|
+
|
|
49
68
|
function getCacheDir(): string {
|
|
50
69
|
const env = (globalThis as { process?: { env?: Record<string, string | undefined> } })
|
|
51
70
|
.process?.env ?? {};
|
|
@@ -70,6 +89,13 @@ async function loadTransformers(): Promise<typeof transformersModule> {
|
|
|
70
89
|
// This is the embedder that exists so nothing leaves the machine, so opt out
|
|
71
90
|
// before the native library loads. `??=`: an operator who sets it explicitly
|
|
72
91
|
// (to anything) keeps their choice.
|
|
92
|
+
//
|
|
93
|
+
// This works under NODE, which is what an npm install runs (the bin's
|
|
94
|
+
// shebang). Under BUN a process.env assignment does not reach the native
|
|
95
|
+
// library: measured, a Bun process queued events despite this line, while
|
|
96
|
+
// the same variable set before launch queued none. Anything that runs the
|
|
97
|
+
// embedder under Bun must set it in the launching environment; the Local
|
|
98
|
+
// image does so in its Dockerfile.
|
|
73
99
|
process.env.ORT_DISABLE_TELEMETRY ??= "1";
|
|
74
100
|
transformersModule = await import(spec);
|
|
75
101
|
const dir = getCacheDir();
|
|
@@ -111,6 +137,14 @@ async function ensurePipeline(): Promise<FeaturePipeline> {
|
|
|
111
137
|
if (pipelinePromise) return pipelinePromise;
|
|
112
138
|
pipelinePromise = (async () => {
|
|
113
139
|
const transformers = await loadTransformers();
|
|
140
|
+
// A cached model is a local read: no banner, no per-file lines, no
|
|
141
|
+
// "ready" line. The narration is for the one run that downloads.
|
|
142
|
+
if (isModelCached()) {
|
|
143
|
+
const pipe = await transformers.pipeline("feature-extraction", ONNX_MODEL_ID, {
|
|
144
|
+
dtype: ONNX_MODEL_DTYPE,
|
|
145
|
+
});
|
|
146
|
+
return pipe as unknown as FeaturePipeline;
|
|
147
|
+
}
|
|
114
148
|
const mb = ONNX_MODEL_APPROX_MB;
|
|
115
149
|
const fmt = (s: number) => (s < 60 ? `${s}s` : `${Math.round(s / 60)}m`);
|
|
116
150
|
process.stderr.write(
|
|
@@ -109,6 +109,11 @@ async function handler(
|
|
|
109
109
|
);
|
|
110
110
|
}
|
|
111
111
|
const update_if_exists = (args.update_if_exists as boolean | undefined) ?? false;
|
|
112
|
+
// The override note is for a flag the caller actually SET to false. An
|
|
113
|
+
// omitted flag overrode nothing, and appending the note to every update by
|
|
114
|
+
// document_id (the recommended workflow) read as a warning.
|
|
115
|
+
const overrideNote =
|
|
116
|
+
args.update_if_exists === false ? " Note: update_if_exists flag was overridden by document_id." : "";
|
|
112
117
|
const author = callerIdentity(args) ?? DEFAULT_IDENTITY;
|
|
113
118
|
const author_type = "agent"; // MCP path is always agent
|
|
114
119
|
const expected_content_hash = (args.expected_content_hash as string | undefined)?.trim() || null;
|
|
@@ -168,9 +173,7 @@ async function handler(
|
|
|
168
173
|
}
|
|
169
174
|
|
|
170
175
|
if (existingDoc.content_hash === contentHash) {
|
|
171
|
-
const note =
|
|
172
|
-
? ""
|
|
173
|
-
: " Note: update_if_exists flag was overridden by document_id.";
|
|
176
|
+
const note = overrideNote;
|
|
174
177
|
return `Document already up-to-date: "${existingDoc.title}" (id: ${existingDoc.id}). Content hash unchanged (${contentHash}).${note}`;
|
|
175
178
|
}
|
|
176
179
|
|
|
@@ -229,9 +232,7 @@ async function handler(
|
|
|
229
232
|
await ensureDocumentInProject(supabase, existingDoc.id, project_name, { author, authorType: author_type });
|
|
230
233
|
}
|
|
231
234
|
|
|
232
|
-
const note =
|
|
233
|
-
? ""
|
|
234
|
-
: " Note: update_if_exists flag was overridden by document_id.";
|
|
235
|
+
const note = overrideNote;
|
|
235
236
|
return `Document updated: "${title}" (id: ${existingDoc.id}), ${chunks.length} chunk(s), ${totalChars} chars. New content_hash: ${contentHash}.${note}${escapedContentNote(content)}`;
|
|
236
237
|
}
|
|
237
238
|
|
|
@@ -374,7 +374,7 @@ Deno.serve(async (req: Request) => {
|
|
|
374
374
|
|
|
375
375
|
// Content unchanged -- skip re-indexing
|
|
376
376
|
if (existingDoc.content_hash === contentHash) {
|
|
377
|
-
const note = update_if_exists
|
|
377
|
+
const note = body.update_if_exists === false ? "update_if_exists flag was overridden by document_id" : undefined;
|
|
378
378
|
return new Response(
|
|
379
379
|
JSON.stringify({
|
|
380
380
|
document_id: existingDoc.id,
|
|
@@ -462,7 +462,7 @@ Deno.serve(async (req: Request) => {
|
|
|
462
462
|
await ensureDocumentInProject(supabase, existingDoc.id, project_name, { author, authorType: author_type });
|
|
463
463
|
}
|
|
464
464
|
|
|
465
|
-
const note = update_if_exists
|
|
465
|
+
const note = body.update_if_exists === false ? "update_if_exists flag was overridden by document_id" : undefined;
|
|
466
466
|
return new Response(
|
|
467
467
|
JSON.stringify({
|
|
468
468
|
document_id: existingDoc.id,
|