@cerefox/memory 1.17.0 → 1.17.1

package/dist/bin/cerefox.js

@@ -5318,7 +5318,7 @@ var init_cli_core = __esm(() => {
 });
 
 // src/meta.ts
-var PKG_VERSION = "1.17.0";
+var PKG_VERSION = "1.17.1";
 var init_meta = () => {};
 
 // ../../_shared/config/paths.ts
@@ -26230,7 +26230,7 @@ var init_bundled_docs = __esm(() => {
 });
 
 // ../../_shared/ef-meta/index.ts
-var EF_VERSION = "1.17.0", CEREFOX_VERSION = "1.17.0", EF_LAST_CHANGED = "1.17.0";
+var EF_VERSION = "1.17.1", CEREFOX_VERSION = "1.17.1", EF_LAST_CHANGED = "1.17.1";
 var init_ef_meta = () => {};
 
 // ../../_shared/compatibility/index.ts
@@ -26369,20 +26369,22 @@ var init_compatibility = __esm(() => {
 // ../../_shared/embeddings/onnx-embedder.ts
 var exports_onnx_embedder = {};
 __export(exports_onnx_embedder, {
+  EmbedderDownloadError: () => EmbedderDownloadError,
   ONNX_MODEL_APPROX_MB: () => ONNX_MODEL_APPROX_MB,
   ONNX_MODEL_DIM: () => ONNX_MODEL_DIM,
   ONNX_MODEL_DTYPE: () => ONNX_MODEL_DTYPE,
   ONNX_MODEL_ID: () => ONNX_MODEL_ID,
   ONNX_MODEL_NAME: () => ONNX_MODEL_NAME,
   buildPrefixedInputs: () => buildPrefixedInputs,
+  downloadModelFiles: () => downloadModelFiles,
   isModelCached: () => isModelCached,
   nomicPrefix: () => nomicPrefix,
   onnxEmbed: () => onnxEmbed,
   warmup: () => warmup
 });
-import { existsSync as existsSync10, mkdirSync as mkdirSync4 } from "node:fs";
+import { existsSync as existsSync10, mkdirSync as mkdirSync4, renameSync, statSync as statSync2, writeFileSync as writeFileSync5, appendFileSync } from "node:fs";
 import { homedir as homedir6 } from "node:os";
-import { join as join9 } from "node:path";
+import { dirname as dirname4, join as join9 } from "node:path";
 function nomicPrefix(role) {
   return role === "query" ? "search_query: " : "search_document: ";
 }
@@ -26390,6 +26392,86 @@ function buildPrefixedInputs(texts, role) {
   const p = nomicPrefix(role);
   return texts.map((t) => p + t);
 }
+async function fetchRange(url, start, end, timeoutMs) {
+  const res = await fetch(url, { headers: { Range: `bytes=${start}-${end}` }, signal: AbortSignal.timeout(timeoutMs) });
+  if (res.status === 200) {
+    await res.body?.cancel();
+    return "no-range";
+  }
+  const total = Number(/\/(\d+)\s*$/.exec(res.headers.get("content-range") ?? "")?.[1] ?? 0);
+  if (res.status === 416 && total > 0) {
+    await res.body?.cancel();
+    return { bytes: new Uint8Array(0), total };
+  }
+  if (res.status !== 206)
+    throw new Error(`HTTP ${res.status}`);
+  if (total === 0)
+    throw new Error("partial response without a total size (Content-Range)");
+  const bytes = new Uint8Array(await res.arrayBuffer());
+  const want = Math.min(end, total > 0 ? total - 1 : end) - start + 1;
+  if (bytes.byteLength !== want)
+    throw new Error(`segment ${start}-${end}: got ${bytes.byteLength} bytes, expected ${want}`);
+  return { bytes, total };
+}
+async function downloadModelFiles(remoteHost, onProgress, onDone, opts = {}) {
+  const attempts = opts.attempts ?? DOWNLOAD_ATTEMPTS;
+  const backoffMs = opts.backoffMs ?? ((attempt) => Math.min(30, 2 ** attempt) * 1000);
+  const segment = opts.segmentBytes ?? SEGMENT_BYTES;
+  const timeoutMs = opts.timeoutMs ?? REQUEST_TIMEOUT_MS;
+  const dir = getCacheDir();
+  const base = remoteHost.endsWith("/") ? remoteHost : `${remoteHost}/`;
+  for (const rel of MODEL_REPO_FILES) {
+    const dest = join9(dir, ONNX_MODEL_ID, rel);
+    if (existsSync10(dest))
+      continue;
+    mkdirSync4(dirname4(dest), { recursive: true });
+    const part = `${dest}.part`;
+    const url = `${base}${ONNX_MODEL_ID}/resolve/main/${rel}`;
+    let failures = 0;
+    let lastError = "";
+    let total = 0;
+    for (;; ) {
+      const have = existsSync10(part) ? statSync2(part).size : 0;
+      if (total > 0 && have >= total)
+        break;
+      try {
+        const got = await fetchRange(url, have, have + segment - 1, timeoutMs);
+        if (got === "no-range") {
+          const res = await fetch(url, { signal: AbortSignal.timeout(Math.max(timeoutMs, ONNX_MODEL_APPROX_MB * 1e5)) });
+          if (!res.ok)
+            throw new Error(`HTTP ${res.status}`);
+          const declared = Number(res.headers.get("content-length") ?? 0);
+          const bytes = new Uint8Array(await res.arrayBuffer());
+          if (declared > 0 && bytes.byteLength !== declared) {
+            throw new Error(`got ${bytes.byteLength} bytes, expected ${declared}`);
+          }
+          writeFileSync5(part, bytes);
+          total = bytes.byteLength;
+        } else {
+          total = got.total;
+          if (have > total) {
+            writeFileSync5(part, new Uint8Array(0));
+            continue;
+          }
+          appendFileSync(part, got.bytes);
+        }
+        failures = 0;
+        onProgress(rel, statSync2(part).size, total);
+      } catch (err) {
+        lastError = err instanceof Error ? err.message : String(err);
+        failures++;
+        if (failures >= attempts)
+          throw new EmbedderDownloadError(`${rel}: ${lastError}`);
+        const wait = backoffMs(failures);
+        process.stderr.write(`[cerefox-embed] ${rel}: ${lastError}; retrying in ${Math.round(wait / 1000)}s (${failures}/${attempts - 1})
+`);
+        await new Promise((r) => setTimeout(r, wait));
+      }
+    }
+    renameSync(part, dest);
+    onDone(rel, statSync2(dest).size);
+  }
+}
 function isModelCached(dir = getCacheDir()) {
   return MODEL_FILES.every((f) => existsSync10(join9(dir, f)));
 }
@@ -26518,9 +26600,9 @@ async function ensurePipeline() {
           progressState.set(file, { ...prior, done: true });
       }
     };
+    await downloadModelFiles(String(transformers.env.remoteHost ?? "https://huggingface.co/"), (file, loaded, total) => progressCallback({ status: "progress", file, loaded, total }), (file, size) => progressCallback({ status: "done", file, total: size }));
     const pipe = await transformers.pipeline("feature-extraction", ONNX_MODEL_ID, {
-      dtype: ONNX_MODEL_DTYPE,
-      progress_callback: progressCallback
+      dtype: ONNX_MODEL_DTYPE
     });
     process.stderr.write(`[cerefox-embed] embedder ready.
 `);
@@ -26564,14 +26646,23 @@ async function onnxEmbed(texts, role) {
   }
   return vectors;
 }
-var ONNX_MODEL_ID = "nomic-ai/nomic-embed-text-v1.5", ONNX_MODEL_NAME = "nomic-embed-text-v1.5", ONNX_MODEL_DTYPE = "q8", ONNX_MODEL_DIM = 768, ONNX_MODEL_APPROX_MB = 130, MODEL_FILES, transformersModule = null, pipelinePromise = null, DEFAULT_ONNX_BATCH = 4;
+var ONNX_MODEL_ID = "nomic-ai/nomic-embed-text-v1.5", ONNX_MODEL_NAME = "nomic-embed-text-v1.5", ONNX_MODEL_DTYPE = "q8", ONNX_MODEL_DIM = 768, ONNX_MODEL_APPROX_MB = 130, MODEL_REPO_FILES, MODEL_FILES, EmbedderDownloadError, DOWNLOAD_ATTEMPTS = 6, SEGMENT_BYTES, REQUEST_TIMEOUT_MS = 120000, transformersModule = null, pipelinePromise = null, DEFAULT_ONNX_BATCH = 4;
 var init_onnx_embedder = __esm(() => {
-  MODEL_FILES = [
+  MODEL_REPO_FILES = [
     "config.json",
     "tokenizer.json",
     "tokenizer_config.json",
     `onnx/model_quantized.onnx`
-  ].map((f) => join9(ONNX_MODEL_ID, f));
+  ];
+  MODEL_FILES = MODEL_REPO_FILES.map((f) => join9(ONNX_MODEL_ID, f));
+  EmbedderDownloadError = class EmbedderDownloadError extends Error {
+    status = 503;
+    constructor(detail) {
+      super(`The local embedding model (${ONNX_MODEL_NAME}, ~${ONNX_MODEL_APPROX_MB} MB) could not be downloaded: ${detail}. ` + "What was downloaded is kept, and the next request resumes it. Check the network and try again.");
+      this.name = "EmbedderDownloadError";
+    }
+  };
+  SEGMENT_BYTES = 4 * 1024 * 1024;
 });
 
 // ../../_shared/embeddings/index.ts
@@ -71773,7 +71864,7 @@ init_config();
 init_config();
 init_compatibility();
 init_server_assets();
-import { existsSync as existsSync11, readFileSync as readFileSync9, realpathSync, statSync as statSync2 } from "node:fs";
+import { existsSync as existsSync11, readFileSync as readFileSync9, realpathSync, statSync as statSync3 } from "node:fs";
 import { homedir as homedir7 } from "node:os";
 import { join as join10 } from "node:path";
 
@@ -72014,7 +72105,7 @@ function checkConfig() {
   let modeDetail = "";
   if (process.platform !== "win32") {
     try {
-      const mode = statSync2(envPath).mode & 511;
+      const mode = statSync3(envPath).mode & 511;
       if (mode & 63) {
         return {
           name: "config",
@@ -72371,6 +72462,17 @@ async function checkEmbedderMismatch() {
     const recorded = [...new Set(rows.map((r) => r.embedder_primary))];
     const stale = recorded.filter((r) => r !== active);
     if (stale.length === 0) {
+      if (resolveEmbedderKind() === "local") {
+        await Promise.resolve().then(() => init_onnx_embedder());
+        if (!isModelCached()) {
+          return {
+            name: EMBEDDER_CHECK_NAME,
+            status: "warn",
+            detail: `configured "${active}", but the model is not downloaded yet; the first search or write will fetch it (~130 MB).`,
+            hint: "Fetch it now with `cerefox embedder-warmup` (Cerefox Local: `cerefox-local embedder-warmup`); it resumes if the connection drops."
+          };
+        }
+      }
       return {
         name: EMBEDDER_CHECK_NAME,
         status: "ok",
@@ -74030,7 +74132,7 @@ init_dist4();
 init_cli_core();
 init_config();
 var import_cli_progress = __toESM(require_cli_progress(), 1);
-import { readdirSync as readdirSync4, statSync as statSync3 } from "node:fs";
+import { readdirSync as readdirSync4, statSync as statSync4 } from "node:fs";
 import { basename as basename3, extname as extname4, join as join11 } from "node:path";
 function walk(dir, extensions) {
   let entries;
@@ -74044,7 +74146,7 @@ function walk(dir, extensions) {
     const full = join11(dir, name);
     let stat;
     try {
-      stat = statSync3(full);
+      stat = statSync4(full);
     } catch {
       continue;
     }
@@ -74147,10 +74249,10 @@ import {
   existsSync as existsSync12,
   mkdirSync as mkdirSync5,
   readFileSync as readFileSync14,
-  writeFileSync as writeFileSync5
+  writeFileSync as writeFileSync6
 } from "node:fs";
 import { homedir as homedir8 } from "node:os";
-import { dirname as dirname4, join as join12 } from "node:path";
+import { dirname as dirname5, join as join12 } from "node:path";
 async function readConfigFile(path) {
   if (!existsSync12(path)) {
     throw userError(`--config file not found: ${path}`);
@@ -74454,8 +74556,8 @@ async function postWriteLifecycle(envPath, options) {
   println(c.dim(`  Config in effect: ${envPath}`));
 }
 function writeAnswersTo(target, answers) {
-  mkdirSync5(dirname4(target), { recursive: true });
-  writeFileSync5(target, buildEnvFile(answers), "utf8");
+  mkdirSync5(dirname5(target), { recursive: true });
+  writeFileSync6(target, buildEnvFile(answers), "utf8");
   if (process.platform !== "win32") {
     try {
       chmodSync(target, 384);
@@ -74518,7 +74620,7 @@ async function action23(options) {
     const ch = await promptMigrationChoice();
     println("");
     if (ch === "c") {
-      mkdirSync5(dirname4(homeEnv), { recursive: true });
+      mkdirSync5(dirname5(homeEnv), { recursive: true });
       copyFileSync2(cwdEnv, homeEnv);
       if (process.platform !== "win32") {
         try {
@@ -75135,7 +75237,7 @@ function registerMigrateFormat(program) {
 // src/cli/commands/restore.ts
 init_cli_core();
 init_client();
-import { existsSync as existsSync13, readFileSync as readFileSync15, readdirSync as readdirSync5, statSync as statSync4 } from "node:fs";
+import { existsSync as existsSync13, readFileSync as readFileSync15, readdirSync as readdirSync5, statSync as statSync5 } from "node:fs";
 import { homedir as homedir9 } from "node:os";
 import { join as join13, resolve as resolve4 } from "node:path";
 function expandHome2(path) {
@@ -75150,10 +75252,10 @@ function resolveBackupFile(target) {
   if (!existsSync13(path)) {
     throw userError(`Backup path not found: ${target}`);
   }
-  const stat = statSync4(path);
+  const stat = statSync5(path);
   if (stat.isFile())
     return path;
-  const candidates = readdirSync5(path).filter((n) => n.endsWith(".json") && n.startsWith("cerefox-")).map((n) => ({ name: n, mtime: statSync4(join13(path, n)).mtimeMs })).sort((a, b) => b.mtime - a.mtime);
+  const candidates = readdirSync5(path).filter((n) => n.endsWith(".json") && n.startsWith("cerefox-")).map((n) => ({ name: n, mtime: statSync5(join13(path, n)).mtimeMs })).sort((a, b) => b.mtime - a.mtime);
   if (candidates.length === 0) {
     throw userError(`No cerefox-*.json files in ${path}`);
   }
@@ -75638,8 +75740,8 @@ init_config();
 import { randomBytes } from "node:crypto";
 
 // src/cli/util/env-file.ts
-import { copyFileSync as copyFileSync3, existsSync as existsSync14, readFileSync as readFileSync16, writeFileSync as writeFileSync6 } from "node:fs";
-import { dirname as dirname5 } from "node:path";
+import { copyFileSync as copyFileSync3, existsSync as existsSync14, readFileSync as readFileSync16, writeFileSync as writeFileSync7 } from "node:fs";
+import { dirname as dirname6 } from "node:path";
 import { spawnSync as spawnSync6 } from "node:child_process";
 function escapeRegExp(s) {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
@@ -75649,7 +75751,7 @@ function upsertEnvVar(path, key, value, opts = {}) {
   const header = opts.comment ? `# ${opts.comment}
 ` : "";
   if (!existsSync14(path)) {
-    writeFileSync6(path, `${header}${line}
+    writeFileSync7(path, `${header}${line}
 `, { mode: 384 });
     return { path, action: "created" };
   }
@@ -75674,7 +75776,7 @@ ${header}${line}
 `;
     action = "added";
   }
-  writeFileSync6(path, next);
+  writeFileSync7(path, next);
   return { path, action, backupPath };
 }
 function readEnvVar(path, key) {
@@ -75685,7 +75787,7 @@ function readEnvVar(path, key) {
 }
 function envGitignoreWarning(path) {
   try {
-    const dir = dirname5(path);
+    const dir = dirname6(path);
     const inTree = spawnSync6("git", ["-C", dir, "rev-parse", "--is-inside-work-tree"], {
       encoding: "utf8"
     });
@@ -77645,7 +77747,7 @@ var _baseMimes = {
 var baseMimes = _baseMimes;
 
 // ../../node_modules/.bun/@hono+node-server@2.1.1+2ac783cc5e75a70c/node_modules/@hono/node-server/dist/serve-static.mjs
-import { createReadStream, existsSync as existsSync16, statSync as statSync5 } from "node:fs";
+import { createReadStream, existsSync as existsSync16, statSync as statSync6 } from "node:fs";
 import { join as join14 } from "node:path";
 var COMPRESSIBLE_CONTENT_TYPE_REGEX = /^\s*(?:text\/[^;\s]+|application\/(?:javascript|json|xml|xml-dtd|ecmascript|dart|postscript|rtf|tar|toml|vnd\.dart|vnd\.ms-fontobject|vnd\.ms-opentype|wasm|x-httpd-php|x-javascript|x-ns-proxy-autoconfig|x-sh|x-tar|x-virtualbox-hdd|x-virtualbox-ova|x-virtualbox-ovf|x-virtualbox-vbox|x-virtualbox-vdi|x-virtualbox-vhd|x-virtualbox-vmdk|x-www-form-urlencoded)|font\/(?:otf|ttf)|image\/(?:bmp|vnd\.adobe\.photoshop|vnd\.microsoft\.icon|vnd\.ms-dds|x-icon|x-ms-bmp)|message\/rfc822|model\/gltf-binary|x-shader\/x-fragment|x-shader\/x-vertex|[^;\s]+?\+(?:json|text|xml|yaml))(?:[;\s]|$)/i;
 var ENCODINGS = {
@@ -77657,7 +77759,7 @@ var ENCODINGS_ORDERED_KEYS = Object.keys(ENCODINGS);
 var getStats = (path) => {
   let stats;
   try {
-    stats = statSync5(path);
+    stats = statSync6(path);
   } catch {}
   return stats;
 };
@@ -77803,7 +77905,7 @@ var serveStatic = (options = { root: "" }) => {
 
 // src/web/server.ts
 import { existsSync as existsSync20 } from "node:fs";
-import { readFileSync as readFileSync19, statSync as statSync8 } from "node:fs";
+import { readFileSync as readFileSync19, statSync as statSync9 } from "node:fs";
 import { join as join18 } from "node:path";
 
 // ../../node_modules/.bun/hono@4.13.7/node_modules/hono/dist/compose.js
@@ -79993,6 +80095,9 @@ function registerDiscoveryRoutes(app, ctx) {
       if (err instanceof HttpError) {
         return c.json({ detail: err.message }, err.status);
       }
+      if (err instanceof Error && err.name === "EmbedderDownloadError") {
+        return c.json({ detail: err.message }, 503);
+      }
       throw err;
     }
     const who = resolveCallerIdentity(c);
@@ -80792,6 +80897,8 @@ function notReady(error) {
   return { success: false, error, detail: error };
 }
 function ingestFailure(err) {
+  if (err instanceof Error && err.name === "EmbedderDownloadError")
+    return [notReady(err.message), 503];
   if (err instanceof ConcurrencyConflictError) {
     return [
       {
@@ -81058,7 +81165,7 @@ function registerIngestRoutes(app, ctx) {
 init_server_assets();
 init_meta();
 import { execFileSync } from "node:child_process";
-import { dirname as dirname7 } from "node:path";
+import { dirname as dirname8 } from "node:path";
 import { fileURLToPath as fileURLToPath4 } from "node:url";
 
 // src/web/docs.ts
@@ -81066,9 +81173,9 @@ import {
   existsSync as existsSync17,
   readFileSync as readFileSync17,
   readdirSync as readdirSync6,
-  statSync as statSync6
+  statSync as statSync7
 } from "node:fs";
-import { basename as basename5, dirname as dirname6, join as join15, resolve as resolve5 } from "node:path";
+import { basename as basename5, dirname as dirname7, join as join15, resolve as resolve5 } from "node:path";
 import { fileURLToPath as fileURLToPath3 } from "node:url";
 var TOP_LEVEL_DOCS = [
   { filename: "README.md", path: "README.md", category: "readme" },
@@ -81084,7 +81191,7 @@ var TOP_LEVEL_DOCS = [
   }
 ];
 function moduleDir2() {
-  return dirname6(fileURLToPath3(import.meta.url));
+  return dirname7(fileURLToPath3(import.meta.url));
 }
 function resolveDocsRoots() {
   const here = moduleDir2();
@@ -81096,7 +81203,7 @@ function resolveDocsRoots() {
   let pkgTopLevel = null;
   for (const pkg of pkgRootCandidates) {
     const guides = join15(pkg, "docs", "guides");
-    if (existsSync17(guides) && statSync6(guides).isDirectory()) {
+    if (existsSync17(guides) && statSync7(guides).isDirectory()) {
       pkgGuides = guides;
       pkgTopLevel = pkg;
       break;
@@ -81162,7 +81269,7 @@ function readDoc(docPath) {
   if (!candidate.startsWith(rootResolved + "/") && candidate !== rootResolved) {
     return null;
   }
-  if (existsSync17(candidate) && statSync6(candidate).isFile()) {
+  if (existsSync17(candidate) && statSync7(candidate).isFile()) {
     try {
       return readFileSync17(candidate, "utf8");
     } catch {
@@ -81188,7 +81295,7 @@ function readOpenApiDocument() {
 
 // src/web/routes/meta.ts
 init_compatibility();
-function resolveGitCommitShort(dir = dirname7(fileURLToPath4(import.meta.url))) {
+function resolveGitCommitShort(dir = dirname8(fileURLToPath4(import.meta.url))) {
   const env = process.env.CEREFOX_GIT_COMMIT;
   if (env)
     return env.slice(0, 7);
@@ -81320,7 +81427,7 @@ function registerPostgrestProxy(app) {
 
 // src/web/routes/preferences.ts
 init_config();
-import { existsSync as existsSync18, mkdirSync as mkdirSync6, readFileSync as readFileSync18, writeFileSync as writeFileSync7 } from "node:fs";
+import { existsSync as existsSync18, mkdirSync as mkdirSync6, readFileSync as readFileSync18, writeFileSync as writeFileSync8 } from "node:fs";
 import { join as join16 } from "node:path";
 function isTheme(v) {
   return v === "auto" || v === "light" || v === "dark";
@@ -81348,7 +81455,7 @@ function registerPreferencesRoutes(app) {
       const dir = userStateDir();
       if (!existsSync18(dir))
         mkdirSync6(dir, { recursive: true });
-      writeFileSync7(prefsFile(), `${JSON.stringify(next, null, 2)}
+      writeFileSync8(prefsFile(), `${JSON.stringify(next, null, 2)}
 `);
     } catch (err) {
       return c.json({ detail: err instanceof Error ? err.message : String(err) }, 500);
@@ -81483,14 +81590,14 @@ function registerProjectsRoutes(app, ctx) {
 }
 
 // src/web/static.ts
-import { existsSync as existsSync19, statSync as statSync7 } from "node:fs";
-import { dirname as dirname8, join as join17 } from "node:path";
+import { existsSync as existsSync19, statSync as statSync8 } from "node:fs";
+import { dirname as dirname9, join as join17 } from "node:path";
 import { fileURLToPath as fileURLToPath5 } from "node:url";
 function moduleDir3() {
-  return dirname8(fileURLToPath5(import.meta.url));
+  return dirname9(fileURLToPath5(import.meta.url));
 }
 function isUsableSpaDir(dir) {
-  return existsSync19(dir) && statSync7(dir).isDirectory() && existsSync19(join17(dir, "index.html"));
+  return existsSync19(dir) && statSync8(dir).isDirectory() && existsSync19(join17(dir, "index.html"));
 }
 function resolveSpaDist() {
   const here = moduleDir3();
@@ -81512,7 +81619,7 @@ function resolveStaticDir() {
     join17(here, "..", "..", "..", "..", "web", "static")
   ];
   for (const c of candidates) {
-    if (existsSync19(c) && statSync7(c).isDirectory())
+    if (existsSync19(c) && statSync8(c).isDirectory())
       return c;
   }
   return null;
@@ -81613,7 +81720,7 @@ function buildApp(ctx = buildWebContext()) {
       let cached = null;
       const readIndex = () => {
         try {
-          const { mtimeMs, size } = statSync8(indexPath);
+          const { mtimeMs, size } = statSync9(indexPath);
           if (!cached || cached.mtimeMs !== mtimeMs || cached.size !== size) {
             cached = { mtimeMs, size, html: readFileSync19(indexPath, "utf8") };
           }

package/dist/server-assets/_shared/ef-meta/index.ts

@@ -18,7 +18,7 @@
  * doesn't touch `supabase/functions/` leaves it alone).
  */
 
-export const EF_VERSION = "1.17.0";
+export const EF_VERSION = "1.17.1";
 
 /**
  * The Cerefox RELEASE version — what `cerefox --version` reports and what npm
@@ -36,7 +36,7 @@ export const EF_VERSION = "1.17.0";
  * is imported by the Deno Edge Functions, which cannot reach into the npm
  * package.
  */
-export const CEREFOX_VERSION = "1.17.0";
+export const CEREFOX_VERSION = "1.17.1";
 
 /**
  * The most recent version whose EF-side SOURCE actually changed (#127).
@@ -46,7 +46,7 @@ export const CEREFOX_VERSION = "1.17.0";
  * `cut_release.ts` ONLY when EF source changed since the last tag; doctor
  * uses it to stay silent on label-only drift.
  */
-export const EF_LAST_CHANGED = "1.17.0";
+export const EF_LAST_CHANGED = "1.17.1";
 
 /**
  * The 8 peer EFs the cerefox-mcp aggregator probes (excludes cerefox-mcp

package/dist/server-assets/_shared/embeddings/onnx-embedder.ts

@@ -21,9 +21,9 @@
  * volume so models survive container recreate/upgrade); default `~/.cerefox/models`.
  */
 
-import { existsSync, mkdirSync } from "node:fs";
+import { existsSync, mkdirSync, renameSync, statSync, writeFileSync, appendFileSync } from "node:fs";
 import { homedir } from "node:os";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
 
 export const ONNX_MODEL_ID = "nomic-ai/nomic-embed-text-v1.5";
 export const ONNX_MODEL_NAME = "nomic-embed-text-v1.5";
@@ -46,14 +46,149 @@ export function buildPrefixedInputs(texts: string[], role: EmbedRole): string[]
   return texts.map((t) => p + t);
 }
 
-/** What the q8 pipeline reads, relative to the cache dir. Same layout under
+/** What the q8 pipeline reads, relative to the model dir. Same layout under
  *  transformers.js 3.x (the Local image) and 4.x (npm installs). */
-const MODEL_FILES = [
+const MODEL_REPO_FILES = [
   "config.json",
   "tokenizer.json",
   "tokenizer_config.json",
   `onnx/model_quantized.onnx`,
-].map((f) => join(ONNX_MODEL_ID, f));
+];
+const MODEL_FILES = MODEL_REPO_FILES.map((f) => join(ONNX_MODEL_ID, f));
+
+/**
+ * The model could not be fetched (#314). Carries a status so HTTP surfaces
+ * answer 503 (try again) rather than 500, and a message that says what was
+ * happening instead of the transport error underneath.
+ */
+export class EmbedderDownloadError extends Error {
+  readonly status = 503;
+  constructor(detail: string) {
+    super(
+      `The local embedding model (${ONNX_MODEL_NAME}, ~${ONNX_MODEL_APPROX_MB} MB) could not be downloaded: ${detail}. ` +
+        "What was downloaded is kept, and the next request resumes it. Check the network and try again.",
+    );
+    this.name = "EmbedderDownloadError";
+  }
+}
+
+/** Consecutive failed requests per file before giving up. */
+const DOWNLOAD_ATTEMPTS = 6;
+/** Bytes per Range request. Small enough that a failure loses little. */
+const SEGMENT_BYTES = 4 * 1024 * 1024;
+/**
+ * Per-request limit. A connection that dies without a reset (a network that
+ * disappears rather than refuses) otherwise leaves fetch waiting forever. 4 MB
+ * in 2 minutes is ~35 KB/s, far below any link that could finish the model.
+ */
+const REQUEST_TIMEOUT_MS = 120_000;
+
+/**
+ * One exact byte range, or an error. The length check is the point: when a
+ * connection drops, Bun's fetch can silently re-issue the request and splice
+ * the new response onto the old stream (observed: one fetch, two server
+ * requests, the bytes of both). Trusting the stream would write a corrupt model;
+ * a segment of the wrong length is retried instead.
+ */
+async function fetchRange(
+  url: string,
+  start: number,
+  end: number,
+  timeoutMs: number,
+): Promise<{ bytes: Uint8Array; total: number } | "no-range"> {
+  const res = await fetch(url, { headers: { Range: `bytes=${start}-${end}` }, signal: AbortSignal.timeout(timeoutMs) });
+  if (res.status === 200) {
+    await res.body?.cancel();
+    return "no-range";
+  }
+  const total = Number(/\/(\d+)\s*$/.exec(res.headers.get("content-range") ?? "")?.[1] ?? 0);
+  if (res.status === 416 && total > 0) {
+    // Nothing at or past `start`: the part file already holds the whole file.
+    await res.body?.cancel();
+    return { bytes: new Uint8Array(0), total };
+  }
+  if (res.status !== 206) throw new Error(`HTTP ${res.status}`);
+  if (total === 0) throw new Error("partial response without a total size (Content-Range)");
+  const bytes = new Uint8Array(await res.arrayBuffer());
+  const want = Math.min(end, total > 0 ? total - 1 : end) - start + 1;
+  if (bytes.byteLength !== want) throw new Error(`segment ${start}-${end}: got ${bytes.byteLength} bytes, expected ${want}`);
+  return { bytes, total };
+}
+
+/**
+ * Fetch any missing model file into the cache path transformers.js reads,
+ * resumably and with retries (#314). Before this the download was one stream
+ * inside transformers.js: on a slow link the connection dropped minutes in, the
+ * request that triggered it failed with a raw socket error, and the next attempt
+ * refetched the 131 MB from zero.
+ *
+ * Works in verified Range segments appended to `<file>.part`, renamed into
+ * place only when the declared size is reached, so neither a partial nor a
+ * spliced file is ever mistaken for the model. A host that ignores Range gets
+ * one whole-file request, checked against Content-Length.
+ */
+export async function downloadModelFiles(
+  remoteHost: string,
+  onProgress: (file: string, loaded: number, total: number) => void,
+  onDone: (file: string, size: number) => void,
+  opts: { attempts?: number; backoffMs?: (attempt: number) => number; segmentBytes?: number; timeoutMs?: number } = {},
+): Promise<void> {
+  const attempts = opts.attempts ?? DOWNLOAD_ATTEMPTS;
+  const backoffMs = opts.backoffMs ?? ((attempt: number) => Math.min(30, 2 ** attempt) * 1000);
+  const segment = opts.segmentBytes ?? SEGMENT_BYTES;
+  const timeoutMs = opts.timeoutMs ?? REQUEST_TIMEOUT_MS;
+  const dir = getCacheDir();
+  const base = remoteHost.endsWith("/") ? remoteHost : `${remoteHost}/`;
+  for (const rel of MODEL_REPO_FILES) {
+    const dest = join(dir, ONNX_MODEL_ID, rel);
+    if (existsSync(dest)) continue;
+    mkdirSync(dirname(dest), { recursive: true });
+    const part = `${dest}.part`;
+    const url = `${base}${ONNX_MODEL_ID}/resolve/main/${rel}`;
+    let failures = 0;
+    let lastError = "";
+    let total = 0;
+    for (;;) {
+      const have = existsSync(part) ? statSync(part).size : 0;
+      if (total > 0 && have >= total) break;
+      try {
+        const got = await fetchRange(url, have, have + segment - 1, timeoutMs);
+        if (got === "no-range") {
+          // Whole file in one request; only a length-checked body is kept.
+          // No segment to bound it, so a generous whole-file limit (10 KB/s).
+          const res = await fetch(url, { signal: AbortSignal.timeout(Math.max(timeoutMs, ONNX_MODEL_APPROX_MB * 100_000)) });
+          if (!res.ok) throw new Error(`HTTP ${res.status}`);
+          const declared = Number(res.headers.get("content-length") ?? 0);
+          const bytes = new Uint8Array(await res.arrayBuffer());
+          if (declared > 0 && bytes.byteLength !== declared) {
+            throw new Error(`got ${bytes.byteLength} bytes, expected ${declared}`);
+          }
+          writeFileSync(part, bytes);
+          total = bytes.byteLength;
+        } else {
+          total = got.total;
+          if (have > total) {
+            // A part file longer than the file itself is not ours to trust.
+            writeFileSync(part, new Uint8Array(0));
+            continue;
+          }
+          appendFileSync(part, got.bytes);
+        }
+        failures = 0;
+        onProgress(rel, statSync(part).size, total);
+      } catch (err) {
+        lastError = err instanceof Error ? err.message : String(err);
+        failures++;
+        if (failures >= attempts) throw new EmbedderDownloadError(`${rel}: ${lastError}`);
+        const wait = backoffMs(failures);
+        process.stderr.write(`[cerefox-embed] ${rel}: ${lastError}; retrying in ${Math.round(wait / 1000)}s (${failures}/${attempts - 1})\n`);
+        await new Promise((r) => setTimeout(r, wait));
+      }
+    }
+    renameSync(part, dest);
+    onDone(rel, statSync(dest).size);
+  }
+}
 
 /**
  * Is every file the pipeline needs already on disk? Decides whether loading is
@@ -223,9 +358,17 @@ async function ensurePipeline(): Promise<FeaturePipeline> {
       }
     };
 
+    // Our own resumable, retried fetch first (#314); transformers.js then
+    // finds every file in its cache and loads locally.
+    await downloadModelFiles(
+      String(transformers.env.remoteHost ?? "https://huggingface.co/"),
+      (file, loaded, total) => progressCallback({ status: "progress", file, loaded, total }),
+      (file, size) => progressCallback({ status: "done", file, total: size }),
+    );
+    // Every file is on disk now, so this is a local load: no progress callback,
+    // which would only report each file "done" a second time.
     const pipe = await transformers.pipeline("feature-extraction", ONNX_MODEL_ID, {
       dtype: ONNX_MODEL_DTYPE,
-      progress_callback: progressCallback,
     });
     process.stderr.write(`[cerefox-embed] embedder ready.\n`);
     return pipe as unknown as FeaturePipeline;

package/dist/server-assets/db/rpcs.sql

@@ -237,6 +237,15 @@ BEGIN
                 -- scored 1.2 or 0.3 depending on which phrasing found it
                 -- (docs/specs/search-calibration.md).
                 ts_rank_cd(c.fts, query_fts, 32)::FLOAT AS fts_score,
+                -- 0.18.1: the exact cosine of every keyword match, so a chunk
+                -- that matched by keyword is never fused as if its meaning
+                -- scored 0 merely because it fell outside vec_results.
+                CASE
+                    WHEN p_use_upgrade AND c.embedding_upgrade IS NOT NULL
+                        THEN (1.0 - (c.embedding_upgrade <=> p_query_embedding))::FLOAT
+                    ELSE
+                        (1.0 - (c.embedding_primary <=> p_query_embedding))::FLOAT
+                END AS fts_vec_score,
                 -- v1.0.4 coverage gate: in AND mode a match means 100% of the
                 -- query's terms are present, so the unconditional pass is
                 -- earned by construction. In OR-fallback mode, earn it only
@@ -283,21 +292,24 @@ BEGIN
                   ))
               AND (p_metadata_filter IS NULL OR d.metadata @> p_metadata_filter)
               AND (p_review_status IS NULL OR d.review_status = p_review_status)
-            ORDER BY
-                CASE
-                    WHEN p_use_upgrade AND c.embedding_upgrade IS NOT NULL
-                        THEN c.embedding_upgrade <=> p_query_embedding
-                    ELSE c.embedding_primary <=> p_query_embedding
-                END
+            -- 0.18.1: ordered by the computed similarity, NOT by the bare
+            -- `embedding <=> query` distance. The bare form (which the CASE
+            -- folds to once p_use_upgrade is known) lets the planner serve this
+            -- from the HNSW index, which is approximate and returns at most
+            -- hnsw.ef_search (40) rows: on a store with heavy version churn it
+            -- returned ~38 of the 250 candidates asked for, mostly not the
+            -- nearest ones. An exact scan is also faster at knowledge-base
+            -- scale (thousands of chunks).
+            ORDER BY vec_score DESC
             LIMIT candidate_count
         ),
         combined AS (
             SELECT
                 COALESCE(f.id, v.id) AS id,
-                (   v_alpha * COALESCE(v.vec_score, 0.0) +
+                (   v_alpha * COALESCE(v.vec_score, f.fts_vec_score, 0.0) +
                     (1.0 - v_alpha) * COALESCE(f.fts_score, 0.0)
                 ) AS score,
-                COALESCE(v.vec_score, 0.0) AS vec_score,
+                COALESCE(v.vec_score, f.fts_vec_score, 0.0) AS vec_score,
                 -- TRUE when the chunk matched the @@ FTS operator WITH enough
                 -- term coverage to earn the unconditional pass (v1.0.4; always
                 -- true for AND-mode matches). We use this flag rather than
@@ -565,12 +577,10 @@ BEGIN
                   THEN (1.0 - (c.embedding_upgrade <=> p_query_embedding))::FLOAT
               ELSE (1.0 - (c.embedding_primary <=> p_query_embedding))::FLOAT
           END >= p_min_score
-    ORDER BY
-        CASE
-            WHEN p_use_upgrade AND c.embedding_upgrade IS NOT NULL
-                THEN c.embedding_upgrade <=> p_query_embedding
-            ELSE c.embedding_primary <=> p_query_embedding
-        END
+    -- 0.18.1: by the computed similarity, so the approximate HNSW index cannot
+    -- serve (and truncate) it; see vec_results in cerefox_hybrid_search.
+    -- By position: `score` would be ambiguous with the OUT column here.
+    ORDER BY 8 DESC
     LIMIT p_match_count;
 END;
 $$;
@@ -3219,6 +3229,11 @@ SET search_path = public, pg_catalog
 AS $$
     -- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
     -- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
+    -- 0.18.1: exact vector candidates. Hybrid and semantic search ordered by
+    -- the bare distance, which let the planner use the approximate HNSW index
+    -- (at most ef_search = 40 rows, poor recall after heavy version churn);
+    -- they now order by the computed similarity (exact scan). Keyword matches
+    -- carry their exact cosine into fusion. RPC-only.
     -- 0.18.0 (iteration 48): search calibration, RPC-only. Bounded keyword
     -- score in hybrid fusion (ts_rank_cd normalisation 32); min_term_coverage
     -- default 0.67; built-in min_search_score follows the store's embedder
@@ -3264,7 +3279,7 @@ AS $$
     -- 0.11.0 supersedes 0.10.6 (v1.2.1, #191): this branch carries that fix plus
     -- the partial-edit surface, and both migrations (0019, 0020) are in the
     -- sequence, so a store deploying this gets everything from both lines.
-    SELECT '0.18.0'::TEXT;
+    SELECT '0.18.1'::TEXT;
 $$;
 
 -- ── cerefox_find_dead_links ──────────────────────────────────────────────────

package/dist/server-assets/db/schema.sql

@@ -5,7 +5,7 @@
 -- Requires extensions: vector (pgvector), uuid-ossp
 -- These are enabled at the top of db_deploy.py before this file is applied.
 --
--- @version: 0.18.0
+-- @version: 0.18.1
 -- The `@version` marker above is read by the schema-version-mismatch banner
 -- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
 -- changes in a way that requires `cerefox server deploy` to be re-run —

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerefox/memory",
-  "version": "1.17.0",
+  "version": "1.17.1",
   "description": "Cerefox \u2014 user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
   "license": "Apache-2.0",
   "homepage": "https://github.com/fstamatelopoulos/cerefox",