memorix 1.6.0 → 1.6.2

package/dist/vector-backfill-runner.js

@@ -1369,8 +1369,8 @@ async function acquireLock(lockPath) {
       const code = err instanceof Error && "code" in err ? err.code : void 0;
       if (code === "EEXIST" || code === "EPERM") {
         try {
-          const stat4 = await fs2.stat(lockPath);
-          if (Date.now() - stat4.mtimeMs > LOCK_STALE_MS) {
+          const stat5 = await fs2.stat(lockPath);
+          if (Date.now() - stat5.mtimeMs > LOCK_STALE_MS) {
             await fs2.unlink(lockPath).catch(() => {
             });
             continue;
@@ -2481,24 +2481,131 @@ var init_transformers_provider = __esm({
 // src/embedding/api-provider.ts
 var api_provider_exports = {};
 __export(api_provider_exports, {
-  APIEmbeddingProvider: () => APIEmbeddingProvider
+  APIEmbeddingProvider: () => APIEmbeddingProvider,
+  convertArrayCacheToJsonl: () => convertArrayCacheToJsonl,
+  ensureDiskCacheLoaded: () => ensureDiskCacheLoaded,
+  getApiEmbeddingCachePayloadBytesForTests: () => getApiEmbeddingCachePayloadBytesForTests,
+  getApiEmbeddingDiskCacheLoadCountForTests: () => getApiEmbeddingDiskCacheLoadCountForTests,
+  getApiEmbeddingJsonlLineCountCallsForTests: () => getApiEmbeddingJsonlLineCountCallsForTests,
+  getApiEmbeddingMaxEntriesForDimensionsForTests: () => getApiEmbeddingMaxEntriesForDimensionsForTests,
+  migrateJsonArrayCacheToJsonl: () => migrateJsonArrayCacheToJsonl,
+  parseCacheJsonInWorker: () => parseCacheJsonInWorker,
+  resetApiEmbeddingCacheForTests: () => resetApiEmbeddingCacheForTests,
+  saveDiskCacheNow: () => saveDiskCacheNow,
+  seedApiEmbeddingCacheForTests: () => seedApiEmbeddingCacheForTests
 });
 import { createHash as createHash2 } from "crypto";
-import { readFile as readFile2, writeFile as writeFile2, mkdir as mkdir2 } from "fs/promises";
+import { createReadStream, createWriteStream, constants as fsConstants } from "fs";
+import { access, readFile as readFile2, writeFile as writeFile2, mkdir as mkdir2, rename, stat, unlink } from "fs/promises";
 import { join as join3 } from "path";
 import { homedir as homedir7 } from "os";
+import { createInterface } from "readline";
 function cacheDir() {
   return process.env.MEMORIX_DATA_DIR || join3(homedir7(), ".memorix", "data");
 }
 function cacheFile() {
   return join3(cacheDir(), ".embedding-api-cache.json");
 }
+function cacheJsonlFile() {
+  return join3(cacheDir(), ".embedding-api-cache.jsonl");
+}
 function dimsCacheFile() {
   return join3(cacheDir(), ".embedding-dims-cache.json");
 }
 function cacheMetaFile() {
   return join3(cacheDir(), ".embedding-api-cache-meta.json");
 }
+function cacheStatFile() {
+  return join3(cacheDir(), ".embedding-api-cache-stat.json");
+}
+function payloadBytes(vector) {
+  return vector.length * BYTES_PER_DIM;
+}
+function maxCachePayloadBytes() {
+  const raw = process.env.MEMORIX_EMBEDDING_CACHE_MAX_BYTES;
+  if (raw) {
+    const parsed = Number.parseInt(raw, 10);
+    if (Number.isFinite(parsed) && parsed > 0) return parsed;
+  }
+  return MAX_CACHE_PAYLOAD_BYTES;
+}
+function maxEntriesForDimensions(dimensions) {
+  const dim = Math.max(1, Math.floor(dimensions));
+  const byBytes = Math.floor(maxCachePayloadBytes() / (dim * BYTES_PER_DIM));
+  return Math.max(1, Math.min(MAX_CACHE_ENTRIES, byBytes));
+}
+function rememberDimensions(vector) {
+  if (vector.length > 0 && (cacheDimensions === null || vector.length > cacheDimensions)) {
+    cacheDimensions = vector.length;
+  }
+}
+function evictOldest() {
+  const firstKey = cache3.keys().next().value;
+  if (firstKey === void 0) return;
+  const old = cache3.get(firstKey);
+  cache3.delete(firstKey);
+  if (old) cachePayloadBytes -= payloadBytes(old);
+}
+function evictToFit(incomingBytes, incomingDims) {
+  const limit2 = maxEntriesForDimensions(incomingDims);
+  while (cache3.size > 0 && (cache3.size >= limit2 || cachePayloadBytes + incomingBytes > maxCachePayloadBytes())) {
+    evictOldest();
+  }
+}
+function isNumberArray(value) {
+  return Array.isArray(value) && value.every((entry) => typeof entry === "number");
+}
+function parseCacheRow(raw) {
+  if (Array.isArray(raw)) {
+    const hash4 = raw[0];
+    const vector = raw[1];
+    if (typeof hash4 === "string" && isNumberArray(vector)) {
+      return { hash: hash4, vector };
+    }
+    return null;
+  }
+  if (raw && typeof raw === "object") {
+    const row = raw;
+    if (typeof row.h === "string" && isNumberArray(row.v)) {
+      return { hash: row.h, vector: row.v };
+    }
+  }
+  return null;
+}
+function cacheInsert(hash4, value, dirty) {
+  rememberDimensions(value);
+  const incoming = payloadBytes(value);
+  const existing = cache3.get(hash4);
+  if (existing) {
+    cachePayloadBytes -= payloadBytes(existing);
+    cache3.delete(hash4);
+  }
+  evictToFit(incoming, value.length || cacheDimensions || DEFAULT_ASSUMED_DIMENSIONS);
+  cache3.set(hash4, value);
+  cachePayloadBytes += incoming;
+  if (dirty) diskCacheDirty2 = true;
+}
+async function readCacheStat() {
+  try {
+    const data = JSON.parse(await readFile2(cacheStatFile(), "utf-8"));
+    if (typeof data.entryCount === "number" && Number.isFinite(data.entryCount) && data.entryCount >= 0) {
+      return {
+        version: 1,
+        entryCount: Math.floor(data.entryCount),
+        dimensions: isValidDimension(data.dimensions) ? data.dimensions : null
+      };
+    }
+  } catch {
+  }
+  return null;
+}
+async function writeCacheStat(stat5) {
+  try {
+    await mkdir2(cacheDir(), { recursive: true });
+    await writeFile2(cacheStatFile(), JSON.stringify(stat5));
+  } catch {
+  }
+}
 function normalizeText(text2) {
   return text2.replace(/\s+/g, " ").trim().slice(0, MAX_INPUT_CHARS);
 }
@@ -2584,29 +2691,300 @@ function toProviderInput(input, baseUrl) {
   }
   return { type: input.modality, url: input.url };
 }
+function parseCacheJsonInWorker(filePath) {
+  return new Promise((resolve, reject) => {
+    void import("worker_threads").then(({ Worker }) => {
+      const worker = new Worker(
+        `
+        const { parentPort, workerData } = require('node:worker_threads');
+        const { readFileSync } = require('node:fs');
+        try {
+          const raw = readFileSync(workerData.filePath, 'utf8');
+          parentPort.postMessage({ ok: true, entries: JSON.parse(raw) });
+        } catch (err) {
+          parentPort.postMessage({
+            ok: false,
+            error: err instanceof Error ? err.message : String(err),
+          });
+        }
+        `,
+        { eval: true, workerData: { filePath } }
+      );
+      worker.once("message", (msg) => {
+        void worker.terminate();
+        if (msg?.ok && Array.isArray(msg.entries)) {
+          resolve(msg.entries);
+          return;
+        }
+        reject(new Error(msg?.error || "embedding cache worker parse failed"));
+      });
+      worker.once("error", reject);
+    }).catch(reject);
+  });
+}
+function findJsonValueEnd(source) {
+  let i2 = 0;
+  while (i2 < source.length && /\s/.test(source[i2])) i2 += 1;
+  if (i2 >= source.length) return -1;
+  const start = source[i2];
+  if (start === '"') {
+    i2 += 1;
+    while (i2 < source.length) {
+      if (source[i2] === "\\") {
+        i2 += 2;
+        continue;
+      }
+      if (source[i2] === '"') return i2 + 1;
+      i2 += 1;
+    }
+    return -1;
+  }
+  if (start === "{" || start === "[") {
+    const close = start === "{" ? "}" : "]";
+    let depth = 0;
+    let inString = false;
+    for (; i2 < source.length; i2 += 1) {
+      const ch = source[i2];
+      if (inString) {
+        if (ch === "\\") {
+          i2 += 1;
+          continue;
+        }
+        if (ch === '"') inString = false;
+        continue;
+      }
+      if (ch === '"') {
+        inString = true;
+        continue;
+      }
+      if (ch === start) depth += 1;
+      else if (ch === close) {
+        depth -= 1;
+        if (depth === 0) return i2 + 1;
+      }
+    }
+    return -1;
+  }
+  const rest = source.slice(i2);
+  const match2 = rest.match(/^(?:-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?|true|false|null)/);
+  return match2 ? i2 + match2[0].length : -1;
+}
+async function streamJsonArrayToJsonl(filePath, tmpPath) {
+  const input = createReadStream(filePath, { encoding: "utf8" });
+  const out = createWriteStream(tmpPath);
+  let buf = "";
+  let started = false;
+  let count2 = 0;
+  const writeLine = async (line) => {
+    if (!out.write(line)) {
+      await new Promise((resolve, reject) => {
+        out.once("drain", resolve);
+        out.once("error", reject);
+      });
+    }
+  };
+  try {
+    for await (const chunk of input) {
+      buf += chunk;
+      if (!started) {
+        const idx = buf.search(/\S/);
+        if (idx < 0) continue;
+        if (buf[idx] !== "[") throw new Error("embedding cache is not a JSON array");
+        buf = buf.slice(idx + 1);
+        started = true;
+      }
+      while (true) {
+        buf = buf.replace(/^\s*,?\s*/, "");
+        if (!buf) break;
+        if (buf[0] === "]") {
+          buf = "";
+          break;
+        }
+        const end = findJsonValueEnd(buf);
+        if (end < 0) break;
+        const value = JSON.parse(buf.slice(0, end));
+        buf = buf.slice(end);
+        const row = parseCacheRow(value);
+        if (row) {
+          await writeLine(`${JSON.stringify({ h: row.hash, v: row.vector })}
+`);
+          count2 += 1;
+          if (count2 % 50 === 0) {
+            await new Promise((resolve) => setImmediate(resolve));
+          }
+        }
+      }
+      if (buf.length > MAX_MIGRATION_VALUE_BYTES) {
+        throw new Error("embedding cache entry too large to migrate safely");
+      }
+    }
+  } finally {
+    await new Promise((resolve, reject) => {
+      out.end(() => resolve());
+      out.once("error", reject);
+    });
+  }
+  return count2;
+}
+async function migrateJsonArrayCacheToJsonl(filePath, jsonlPath) {
+  const tmpPath = `${jsonlPath}.tmp`;
+  try {
+    const count2 = await streamJsonArrayToJsonl(filePath, tmpPath);
+    await rename(tmpPath, jsonlPath);
+    diskCacheEntryCount = count2;
+    await writeCacheStat({
+      version: 1,
+      entryCount: count2,
+      dimensions: cacheDimensions
+    });
+    return count2;
+  } catch (err) {
+    await unlink(tmpPath).catch(() => void 0);
+    throw err;
+  }
+}
+function convertArrayCacheToJsonl(filePath, jsonlPath) {
+  return migrateJsonArrayCacheToJsonl(filePath, jsonlPath);
+}
+async function loadCacheJsonl(filePath) {
+  const rl = createInterface({
+    input: createReadStream(filePath, { encoding: "utf8" }),
+    crlfDelay: Infinity
+  });
+  let loaded = 0;
+  let onDiskCount = 0;
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const row = parseCacheRow(JSON.parse(trimmed));
+      if (!row) continue;
+      onDiskCount += 1;
+      cacheInsert(row.hash, row.vector, false);
+      loaded = cache3.size;
+    } catch {
+    }
+    if (onDiskCount % 250 === 0) {
+      await new Promise((resolve) => setImmediate(resolve));
+    }
+  }
+  diskCacheEntryCount = onDiskCount;
+  await writeCacheStat({
+    version: 1,
+    entryCount: onDiskCount,
+    dimensions: cacheDimensions
+  });
+  return loaded;
+}
+async function countJsonlLines(filePath) {
+  jsonlLineCountCalls += 1;
+  try {
+    const rl = createInterface({
+      input: createReadStream(filePath, { encoding: "utf8" }),
+      crlfDelay: Infinity
+    });
+    let count2 = 0;
+    for await (const line of rl) {
+      if (line.trim()) count2 += 1;
+    }
+    return count2;
+  } catch {
+    return 0;
+  }
+}
+async function resolveOnDiskCount(jsonlPath) {
+  if (diskCacheEntryCount > 0) return diskCacheEntryCount;
+  const sidecar = await readCacheStat();
+  if (sidecar) {
+    diskCacheEntryCount = sidecar.entryCount;
+    if (sidecar.dimensions && cacheDimensions === null) cacheDimensions = sidecar.dimensions;
+    return sidecar.entryCount;
+  }
+  try {
+    await access(jsonlPath, fsConstants.R_OK);
+  } catch {
+    return 0;
+  }
+  const counted = await countJsonlLines(jsonlPath);
+  diskCacheEntryCount = counted;
+  await writeCacheStat({
+    version: 1,
+    entryCount: counted,
+    dimensions: cacheDimensions
+  });
+  return counted;
+}
 async function loadDiskCache2() {
-  if (diskCacheLoaded) return;
+  if (diskCacheLoaded) return cache3.size;
+  let loaded = 0;
   try {
-    const raw = await readFile2(cacheFile(), "utf-8");
-    const entries = JSON.parse(raw);
-    for (const [k, v] of entries) cache3.set(k, v);
-    console.error(`[memorix] Loaded ${entries.length} cached API embeddings from disk`);
+    const jsonlPath = cacheJsonlFile();
+    let hasJsonl = false;
+    try {
+      await access(jsonlPath, fsConstants.R_OK);
+      hasJsonl = true;
+    } catch {
+      hasJsonl = false;
+    }
+    if (hasJsonl) {
+      loaded = await loadCacheJsonl(jsonlPath);
+    } else {
+      const filePath = cacheFile();
+      await stat(filePath);
+      await migrateJsonArrayCacheToJsonl(filePath, jsonlPath);
+      loaded = await loadCacheJsonl(jsonlPath);
+    }
+    diskCacheLoadCount += 1;
+    console.error(`[memorix] Loaded ${loaded} cached API embeddings from disk`);
   } catch {
   }
   diskCacheLoaded = true;
+  return loaded;
 }
 function startDiskCacheLoad() {
   if (diskCacheLoaded || diskCacheLoadPromise) return;
-  diskCacheLoadPromise = loadDiskCache2().catch(() => {
+  diskCacheLoadPromise = loadDiskCache2().then(() => void 0).catch(() => {
   });
 }
 async function ensureDiskCacheLoaded() {
-  if (diskCacheLoaded) return;
-  if (diskCacheLoadPromise) {
-    await diskCacheLoadPromise;
-    return;
-  }
-  await loadDiskCache2();
+  if (diskCacheLoaded) return cache3.size;
+  if (!diskCacheLoadPromise) {
+    diskCacheLoadPromise = loadDiskCache2().then(() => void 0).catch(() => {
+    });
+  }
+  await diskCacheLoadPromise;
+  return cache3.size;
+}
+function getApiEmbeddingDiskCacheLoadCountForTests() {
+  return diskCacheLoadCount;
+}
+function resetApiEmbeddingCacheForTests() {
+  cache3.clear();
+  cachePayloadBytes = 0;
+  cacheDimensions = null;
+  diskCacheEntryCount = 0;
+  diskCacheLoaded = false;
+  diskCacheLoadPromise = null;
+  diskCacheLoadCount = 0;
+  diskCacheDirty2 = false;
+  jsonlLineCountCalls = 0;
+}
+function seedApiEmbeddingCacheForTests(entries) {
+  cache3.clear();
+  cachePayloadBytes = 0;
+  cacheDimensions = null;
+  for (const [key, value] of entries) cacheInsert(key, value, true);
+  diskCacheLoaded = true;
+  diskCacheDirty2 = true;
+}
+function getApiEmbeddingCachePayloadBytesForTests() {
+  return cachePayloadBytes;
+}
+function getApiEmbeddingJsonlLineCountCallsForTests() {
+  return jsonlLineCountCalls;
+}
+function getApiEmbeddingMaxEntriesForDimensionsForTests(dimensions) {
+  return maxEntriesForDimensions(dimensions);
 }
 function dimsCacheKey(config) {
   return [
@@ -2722,8 +3100,46 @@ async function saveDiskCacheNow() {
   if (!diskCacheDirty2) return;
   try {
     await mkdir2(cacheDir(), { recursive: true });
-    const entries = Array.from(cache3.entries());
-    await writeFile2(cacheFile(), JSON.stringify(entries));
+    const jsonlPath = cacheJsonlFile();
+    const existing = await resolveOnDiskCount(jsonlPath);
+    if (existing > 0 && cache3.size < Math.ceil(existing * 0.5)) {
+      console.error(
+        `[memorix] Refusing to overwrite embedding cache (${existing} on disk vs ${cache3.size} in memory)`
+      );
+      diskCacheDirty2 = false;
+      return;
+    }
+    const tmpPath = `${jsonlPath}.tmp`;
+    await new Promise((resolve, reject) => {
+      const out = createWriteStream(tmpPath);
+      const entries = cache3.entries();
+      const writeBatch = () => {
+        for (let i2 = 0; i2 < 50; i2++) {
+          const next = entries.next();
+          if (next.done) {
+            out.end();
+            return;
+          }
+          const [hash4, vector] = next.value;
+          if (!out.write(`${JSON.stringify({ h: hash4, v: vector })}
+`)) {
+            out.once("drain", writeBatch);
+            return;
+          }
+        }
+        setImmediate(writeBatch);
+      };
+      out.on("finish", resolve);
+      out.on("error", reject);
+      writeBatch();
+    });
+    await rename(tmpPath, jsonlPath);
+    diskCacheEntryCount = cache3.size;
+    await writeCacheStat({
+      version: 1,
+      entryCount: cache3.size,
+      dimensions: cacheDimensions
+    });
     diskCacheDirty2 = false;
   } catch {
   }
@@ -2737,12 +3153,7 @@ function scheduleDiskSave() {
   }, DISK_SAVE_DEBOUNCE_MS);
 }
 function cacheSet(hash4, value) {
-  if (cache3.size >= MAX_CACHE_SIZE3) {
-    const firstKey = cache3.keys().next().value;
-    if (firstKey !== void 0) cache3.delete(firstKey);
-  }
-  cache3.set(hash4, value);
-  diskCacheDirty2 = true;
+  cacheInsert(hash4, value, true);
 }
 function resolveEnvEmbeddingApiKey() {
   return process.env.MEMORIX_EMBEDDING_API_KEY;
@@ -2861,18 +3272,27 @@ async function waitForRetry(waitMs, signal) {
     abortSignal.addEventListener("abort", onAbort, { once: true });
   });
 }
-var cache3, MAX_CACHE_SIZE3, diskCacheDirty2, diskSaveTimer, diskCacheLoaded, diskCacheLoadPromise, MAX_INPUT_CHARS, MAX_CONCURRENCY, DISK_SAVE_DEBOUNCE_MS, DEFAULT_MAX_BATCH_SIZE, DASHSCOPE_MAX_BATCH_SIZE, MAX_RETRIES2, BASE_DELAY_MS, EmbeddingAPIError, APIEmbeddingProvider;
+var cache3, MAX_CACHE_ENTRIES, MAX_CACHE_PAYLOAD_BYTES, BYTES_PER_DIM, DEFAULT_ASSUMED_DIMENSIONS, MAX_MIGRATION_VALUE_BYTES, cachePayloadBytes, cacheDimensions, diskCacheEntryCount, diskCacheDirty2, diskSaveTimer, diskCacheLoaded, diskCacheLoadPromise, diskCacheLoadCount, jsonlLineCountCalls, MAX_INPUT_CHARS, MAX_CONCURRENCY, DISK_SAVE_DEBOUNCE_MS, DEFAULT_MAX_BATCH_SIZE, DASHSCOPE_MAX_BATCH_SIZE, MAX_RETRIES2, BASE_DELAY_MS, EmbeddingAPIError, APIEmbeddingProvider;
 var init_api_provider = __esm({
   "src/embedding/api-provider.ts"() {
     "use strict";
     init_esm_shims();
     init_provider();
     cache3 = /* @__PURE__ */ new Map();
-    MAX_CACHE_SIZE3 = 1e4;
+    MAX_CACHE_ENTRIES = 2e5;
+    MAX_CACHE_PAYLOAD_BYTES = 256 * 1024 * 1024;
+    BYTES_PER_DIM = 8;
+    DEFAULT_ASSUMED_DIMENSIONS = 4096;
+    MAX_MIGRATION_VALUE_BYTES = 16 * 1024 * 1024;
+    cachePayloadBytes = 0;
+    cacheDimensions = null;
+    diskCacheEntryCount = 0;
     diskCacheDirty2 = false;
     diskSaveTimer = null;
     diskCacheLoaded = false;
     diskCacheLoadPromise = null;
+    diskCacheLoadCount = 0;
+    jsonlLineCountCalls = 0;
     MAX_INPUT_CHARS = 32e3;
     MAX_CONCURRENCY = 4;
     DISK_SAVE_DEBOUNCE_MS = 5e3;
@@ -2930,7 +3350,9 @@ var init_api_provider = __esm({
         if (config.requestedDimensions) {
           console.error(`[memorix] Dimension shortening: ${config.requestedDimensions}d requested`);
         }
-        startDiskCacheLoad();
+        if (allowNetworkProbe) {
+          startDiskCacheLoad();
+        }
         return new _APIEmbeddingProvider(config, probeDimensions);
       }
       static resolveConfig() {
@@ -3206,6 +3628,7 @@ __export(provider_exports, {
   MAX_INLINE_EMBEDDING_BYTES: () => MAX_INLINE_EMBEDDING_BYTES,
   UnsupportedEmbeddingModalityError: () => UnsupportedEmbeddingModalityError,
   getEmbeddingProvider: () => getEmbeddingProvider,
+  getEmbeddingRuntimeHealth: () => getEmbeddingRuntimeHealth,
   isEmbeddingExplicitlyDisabled: () => isEmbeddingExplicitlyDisabled,
   isVectorSearchAvailable: () => isVectorSearchAvailable,
   resetProvider: () => resetProvider,
@@ -3267,7 +3690,12 @@ function warnOnce(message) {
   lastEmbeddingWarningAt = now3;
   console.error(message);
 }
+function embeddingModeToken() {
+  return process.env.MEMORIX_EMBEDDING?.toLowerCase()?.trim() ?? "";
+}
 function getEmbeddingMode2() {
+  const token = embeddingModeToken();
+  if (cachedEmbeddingMode && cachedEmbeddingModeToken === token) return cachedEmbeddingMode;
   try {
     let cfg;
     try {
@@ -3276,10 +3704,17 @@ function getEmbeddingMode2() {
       cfg = (init_config(), __toCommonJS(config_exports));
     }
     const { getEmbeddingMode: cfgMode } = cfg;
-    return cfgMode();
+    cachedEmbeddingMode = cfgMode();
+    cachedEmbeddingModeToken = token;
+    return cachedEmbeddingMode;
   } catch {
-    const env2 = process.env.MEMORIX_EMBEDDING?.toLowerCase()?.trim();
-    if (env2 === "fastembed" || env2 === "transformers" || env2 === "api" || env2 === "auto") return env2;
+    if (token === "fastembed" || token === "transformers" || token === "api" || token === "auto") {
+      cachedEmbeddingMode = token;
+      cachedEmbeddingModeToken = token;
+      return token;
+    }
+    cachedEmbeddingMode = "off";
+    cachedEmbeddingModeToken = token;
     return "off";
   }
 }
@@ -3313,13 +3748,36 @@ function isTemporaryEmbeddingFailure(error) {
   if (!(error instanceof Error)) return false;
   return /embedding api error \(401\)/i.test(error.message) || /invalid_api_key/i.test(error.message) || /incorrect api key/i.test(error.message) || /unauthorized/i.test(error.message) || /embedding api timeout/i.test(error.message) || /embedding api error \((402|429|5\d\d)\)/i.test(error.message) || /quota exceeded/i.test(error.message) || /account balance/i.test(error.message) || /fetch failed/i.test(error.message) || /econnreset/i.test(error.message) || /econnrefused/i.test(error.message) || /temporarily unavailable/i.test(error.message);
 }
+function classifyEmbeddingFailure(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  if (/embedding api error \(401\)|invalid_api_key|incorrect api key|unauthorized/i.test(message)) {
+    return "authorization";
+  }
+  if (/embedding api error \(402\)|quota exceeded|account balance/i.test(message)) {
+    return "billing";
+  }
+  if (/embedding api timeout|abort/i.test(message)) {
+    return "timeout";
+  }
+  return "upstream";
+}
 function markTemporaryFailure(reason) {
   provider = null;
   initPromise = null;
   lastInitWasTemporaryFailure = true;
   lastFailureTimestamp = Date.now();
+  lastEmbeddingFailure = {
+    kind: classifyEmbeddingFailure(reason),
+    occurredAt: lastFailureTimestamp
+  };
   const message = reason instanceof Error ? reason.message : String(reason);
-  warnOnce(`[memorix] Embedding provider temporarily unavailable at runtime: ${message}`);
+  warnOnce(`[memorix] Embedding provider unavailable at runtime: ${message}`);
+}
+function markProviderReady(candidate) {
+  provider = wrapProvider(candidate);
+  lastInitWasTemporaryFailure = false;
+  lastEmbeddingFailure = null;
+  return provider;
 }
 async function createFastEmbedProvider() {
   try {
@@ -3435,35 +3893,35 @@ async function getEmbeddingProvider(options = {}) {
     if (mode === "fastembed") {
       const initialized = await createFastEmbedProvider();
       if (!initialized) return null;
-      provider = wrapProvider(initialized);
+      provider = markProviderReady(initialized);
       warnOnce(`[memorix] Embedding provider: ${provider.name} (${provider.dimensions}d)`);
       return provider;
     }
     if (mode === "transformers") {
       const initialized = await createTransformersProvider();
       if (!initialized) return null;
-      provider = wrapProvider(initialized);
+      provider = markProviderReady(initialized);
       warnOnce(`[memorix] Embedding provider: ${provider.name} (${provider.dimensions}d)`);
       return provider;
     }
     if (mode === "api") {
       const initialized = await createAPIProvider(options);
       if (!initialized) return null;
-      provider = wrapProvider(initialized);
+      provider = markProviderReady(initialized);
       warnOnce(`[memorix] Embedding provider: ${provider.name} (${provider.dimensions}d)`);
       return provider;
     }
     if (hasAPIEmbeddingConfig()) {
       const initialized = await createAPIProvider(options);
       if (initialized) {
-        provider = wrapProvider(initialized);
+        provider = markProviderReady(initialized);
         warnOnce(`[memorix] Embedding provider: ${provider.name} (${provider.dimensions}d)`);
         return provider;
       }
     }
     const localFallback = await createLocalFallbackProvider();
     if (localFallback) {
-      provider = wrapProvider(localFallback);
+      provider = markProviderReady(localFallback);
       warnOnce(`[memorix] Embedding provider: ${provider.name} (${provider.dimensions}d)`);
       return provider;
     }
@@ -3485,18 +3943,42 @@ async function isVectorSearchAvailable() {
   const p = await getEmbeddingProvider();
   return p !== null;
 }
+function getEmbeddingRuntimeHealth() {
+  const mode = getEmbeddingMode2();
+  if (mode === "off") return { status: "disabled" };
+  if (lastInitWasTemporaryFailure && lastEmbeddingFailure) {
+    return {
+      status: "degraded",
+      failure: {
+        kind: lastEmbeddingFailure.kind,
+        occurredAt: new Date(lastEmbeddingFailure.occurredAt).toISOString(),
+        retryAfterMs: Math.max(0, RETRY_COOLDOWN_MS - (Date.now() - lastEmbeddingFailure.occurredAt))
+      }
+    };
+  }
+  if (provider) {
+    return {
+      status: "ready",
+      provider: { name: provider.name, dimensions: provider.dimensions }
+    };
+  }
+  return { status: "uninitialized" };
+}
 function isEmbeddingExplicitlyDisabled() {
   return getEmbeddingMode2() === "off";
 }
 function resetProvider() {
   provider = null;
   initPromise = null;
+  cachedEmbeddingMode = null;
+  cachedEmbeddingModeToken = null;
   lastInitWasTemporaryFailure = false;
   lastFailureTimestamp = 0;
+  lastEmbeddingFailure = null;
   lastEmbeddingWarning = "";
   lastEmbeddingWarningAt = 0;
 }
-var EmbeddingInputError, UnsupportedEmbeddingModalityError, MAX_INLINE_EMBEDDING_BYTES, provider, initPromise, lastInitWasTemporaryFailure, lastEmbeddingWarning, lastEmbeddingWarningAt, WARNING_COOLDOWN_MS, RETRY_COOLDOWN_MS, lastFailureTimestamp;
+var EmbeddingInputError, UnsupportedEmbeddingModalityError, MAX_INLINE_EMBEDDING_BYTES, provider, initPromise, cachedEmbeddingMode, cachedEmbeddingModeToken, lastInitWasTemporaryFailure, lastEmbeddingFailure, lastEmbeddingWarning, lastEmbeddingWarningAt, WARNING_COOLDOWN_MS, RETRY_COOLDOWN_MS, lastFailureTimestamp;
 var init_provider = __esm({
   "src/embedding/provider.ts"() {
     "use strict";
@@ -3520,7 +4002,10 @@ var init_provider = __esm({
     MAX_INLINE_EMBEDDING_BYTES = 5 * 1024 * 1024;
     provider = null;
     initPromise = null;
+    cachedEmbeddingMode = null;
+    cachedEmbeddingModeToken = null;
     lastInitWasTemporaryFailure = false;
+    lastEmbeddingFailure = null;
     lastEmbeddingWarning = "";
     lastEmbeddingWarningAt = 0;
     WARNING_COOLDOWN_MS = 3e4;
@@ -5276,6 +5761,9 @@ async function attachCachedVectors(database, candidates) {
       await update(database, candidates[index].id, { ...existing, embedding: vector });
     } catch {
     }
+    if (index % 50 === 0) {
+      await new Promise((resolve) => setImmediate(resolve));
+    }
   }
 }
 async function hydrateIndex(observations2, options = {}) {
@@ -5326,17 +5814,24 @@ async function hydrateIndex(observations2, options = {}) {
       await insert2(database, doc);
       rememberObservationDoc(doc);
       inserted++;
+      if (inserted % 50 === 0) {
+        await new Promise((resolve) => setImmediate(resolve));
+      }
+      if (inserted % 200 === 0) {
+        await new Promise((resolve) => setTimeout(resolve, 0));
+      }
     } catch {
     }
   }
-  deferredCachedVectorHydration = deferCachedVectors ? attachCachedVectors(database, candidates).catch(() => {
+  deferredCachedVectorHydration = options.skipCachedVectors ? null : deferCachedVectors ? attachCachedVectors(database, candidates).catch(() => {
   }) : null;
   return inserted;
 }
-async function hydrateIndexForStartup(observations2) {
+async function hydrateIndexForStartup(observations2, options = {}) {
   return hydrateIndex(observations2, {
     allowNetworkProbe: false,
-    deferCachedVectors: true
+    deferCachedVectors: true,
+    skipCachedVectors: options.skipCachedVectors
   });
 }
 function getDeferredCachedVectorHydration() {
@@ -8124,12 +8619,25 @@ __export(observations_exports, {
   prepareSearchIndex: () => prepareSearchIndex,
   probeSearchIndex: () => probeSearchIndex,
   reindexObservations: () => reindexObservations,
+  resetObservationRuntime: () => resetObservationRuntime,
   resolveObservations: () => resolveObservations,
   storeObservation: () => storeObservation,
   suggestTopicKey: () => suggestTopicKey,
   updateObservationAdmission: () => updateObservationAdmission,
   withFreshObservations: () => withFreshObservations
 });
+function resetObservationRuntime() {
+  observations = [];
+  nextId = 1;
+  projectDir = null;
+  searchIndexPrepared = false;
+  corpusLoaded = false;
+  loadedObservationStore = null;
+  embeddingWriteMode = "background";
+  embeddingWorkerProjectRoot = void 0;
+  vectorMissingIds.clear();
+  vectorSchemaUpgradePromise = null;
+}
 function logEmbeddingFailureOnce(key, message) {
   const now3 = Date.now();
   const last = embeddingFailureLogTimestamps.get(key) ?? 0;
@@ -8238,23 +8746,33 @@ async function upgradeVectorSchemaAfterFirstEmbedding(embedding) {
 async function initObservations(dir, options = {}) {
   embeddingWriteMode = options.embeddingWriteMode ?? "background";
   embeddingWorkerProjectRoot = options.projectRoot;
-  if (projectDir === dir) return;
   await initObservationStore(dir);
   const store = getObservationStore();
-  observations = await store.loadAll();
+  if (projectDir === dir && loadedObservationStore === store) {
+    if (options.skipCorpusLoad || corpusLoaded) return;
+  }
   nextId = await store.loadIdCounter();
   projectDir = dir;
   searchIndexPrepared = false;
   vectorSchemaUpgradePromise = null;
+  if (options.skipCorpusLoad) {
+    observations = [];
+    corpusLoaded = false;
+    loadedObservationStore = store;
+    return;
+  }
+  observations = await store.loadAll();
+  corpusLoaded = true;
+  loadedObservationStore = store;
 }
 function scheduleObservationEmbedding(input) {
   const { observationId: observationId2, projectId, searchableText, doc } = input;
   vectorMissingIds.add(observationId2);
+  if (isEmbeddingExplicitlyDisabled()) {
+    vectorMissingIds.delete(observationId2);
+    return;
+  }
   if (embeddingWriteMode === "deferred") {
-    if (isEmbeddingExplicitlyDisabled()) {
-      vectorMissingIds.delete(observationId2);
-      return;
-    }
     queueVectorBackfill(projectId, {
       detachedWorker: true,
       projectRoot: embeddingWorkerProjectRoot
@@ -8301,7 +8819,7 @@ function scheduleObservationEmbedding(input) {
   });
 }
 async function ensureFreshObservations() {
-  if (!projectDir) return false;
+  if (!projectDir || !corpusLoaded) return false;
   try {
     const store = getObservationStore();
     const wasStale = await store.ensureFresh();
@@ -8342,7 +8860,7 @@ async function storeObservation(input) {
   });
   if (safeAttachments) input = { ...input, attachments: safeAttachments };
   await ensureFreshObservations();
-  if (input.topicKey) {
+  if (input.topicKey && corpusLoaded) {
     const existing = observations.find(
       (o) => o.topicKey === input.topicKey && o.projectId === input.projectId
     );
@@ -8439,13 +8957,17 @@ async function storeObservation(input) {
       });
       observation.writeGeneration = store.getGeneration();
       if (upsertedInsideLock || reloadCacheAfterCommit) {
-        observations = await store.loadAll();
-        nextId = await store.loadIdCounter();
-        if (upsertedInsideLock) {
-          observation = observations.find((candidate) => candidate.id === observation.id) ?? observation;
+        if (corpusLoaded) {
+          observations = await store.loadAll();
+          nextId = await store.loadIdCounter();
+          if (upsertedInsideLock) {
+            observation = observations.find((candidate) => candidate.id === observation.id) ?? observation;
+          }
+        } else {
+          nextId = await store.loadIdCounter();
         }
       } else {
-        observations.push(observation);
+        if (corpusLoaded) observations.push(observation);
         nextId = observation.id + 1;
       }
       if (upsertedInsideLock) return;
@@ -8513,7 +9035,9 @@ async function storeObservation(input) {
       createdByAgentId: input.createdByAgentId ?? "",
       sharedWithAgentIds: JSON.stringify(input.sharedWithAgentIds ?? [])
     };
-    await insertObservation(doc);
+    if (corpusLoaded) {
+      await insertObservation(doc);
+    }
   };
   await assignAndPersist();
   if (upsertedInsideLock) {
@@ -8875,9 +9399,11 @@ async function reindexObservations() {
   searchIndexPrepared = true;
   return count2;
 }
-async function prepareSearchIndex() {
+async function prepareSearchIndex(options = {}) {
   if (searchIndexPrepared) return 0;
-  const count2 = await hydrateIndexForStartup(observations);
+  const count2 = await hydrateIndexForStartup(observations, {
+    skipCachedVectors: options.skipCachedVectors
+  });
   if (count2 === 0) {
     searchIndexPrepared = true;
     return 0;
@@ -9069,7 +9595,7 @@ async function backfillVectorEmbeddings(options = {}) {
     ...lastFailure ? { lastError: lastFailure } : {}
   };
 }
-var observations, nextId, projectDir, searchIndexPrepared, embeddingWriteMode, embeddingWorkerProjectRoot, vectorMissingIds, vectorBackfillRunning, vectorSchemaUpgradePromise, lastVectorBackfill, embeddingFailureLogTimestamps, EMBEDDING_FAILURE_LOG_COOLDOWN_MS;
+var observations, nextId, projectDir, searchIndexPrepared, embeddingWriteMode, embeddingWorkerProjectRoot, corpusLoaded, loadedObservationStore, vectorMissingIds, vectorBackfillRunning, vectorSchemaUpgradePromise, lastVectorBackfill, embeddingFailureLogTimestamps, EMBEDDING_FAILURE_LOG_COOLDOWN_MS;
 var init_observations = __esm({
   "src/memory/observations.ts"() {
     "use strict";
@@ -9088,6 +9614,8 @@ var init_observations = __esm({
     projectDir = null;
     searchIndexPrepared = false;
     embeddingWriteMode = "background";
+    corpusLoaded = false;
+    loadedObservationStore = null;
     vectorMissingIds = /* @__PURE__ */ new Set();
     vectorBackfillRunning = false;
     vectorSchemaUpgradePromise = null;
@@ -10433,7 +10961,7 @@ var init_types2 = __esm({
 
 // src/media/asset-store.ts
 import { createHash as createHash5, randomUUID as randomUUID6 } from "crypto";
-import { copyFile, lstat, mkdir as mkdir3, open, readFile as readFile3, readdir, rename, rm, stat, writeFile as writeFile3 } from "fs/promises";
+import { copyFile, lstat, mkdir as mkdir3, open, readFile as readFile3, readdir, rename as rename2, rm, stat as stat2, writeFile as writeFile3 } from "fs/promises";
 import path9 from "path";
 function mediaRoot(dataDir) {
   return path9.resolve(dataDir, "media");
@@ -10506,7 +11034,7 @@ async function persistBuffer(bytes, destination) {
   const temporary = `${destination}.${randomUUID6()}.tmp`;
   try {
     await writeFile3(temporary, bytes, { flag: "wx" });
-    await rename(temporary, destination);
+    await rename2(temporary, destination);
     return true;
   } catch (error) {
     if (error?.code === "EEXIST" && await destinationExists(destination)) return false;
@@ -61586,9 +62114,9 @@ var require_node_domexception = __commonJS({
 });
 
 // node_modules/fetch-blob/from.js
-import { statSync as statSync2, createReadStream, promises as fs7 } from "fs";
+import { statSync as statSync2, createReadStream as createReadStream2, promises as fs7 } from "fs";
 import { basename } from "path";
-var import_node_domexception, stat2, blobFromSync, blobFrom, fileFrom, fileFromSync, fromBlob, fromFile, BlobDataItem;
+var import_node_domexception, stat3, blobFromSync, blobFrom, fileFrom, fileFromSync, fromBlob, fromFile, BlobDataItem;
 var init_from = __esm({
   "node_modules/fetch-blob/from.js"() {
     "use strict";
@@ -61596,23 +62124,23 @@ var init_from = __esm({
     import_node_domexception = __toESM(require_node_domexception(), 1);
     init_file();
     init_fetch_blob();
-    ({ stat: stat2 } = fs7);
+    ({ stat: stat3 } = fs7);
     blobFromSync = (path20, type) => fromBlob(statSync2(path20), path20, type);
-    blobFrom = (path20, type) => stat2(path20).then((stat4) => fromBlob(stat4, path20, type));
-    fileFrom = (path20, type) => stat2(path20).then((stat4) => fromFile(stat4, path20, type));
+    blobFrom = (path20, type) => stat3(path20).then((stat5) => fromBlob(stat5, path20, type));
+    fileFrom = (path20, type) => stat3(path20).then((stat5) => fromFile(stat5, path20, type));
     fileFromSync = (path20, type) => fromFile(statSync2(path20), path20, type);
-    fromBlob = (stat4, path20, type = "") => new fetch_blob_default([new BlobDataItem({
+    fromBlob = (stat5, path20, type = "") => new fetch_blob_default([new BlobDataItem({
       path: path20,
-      size: stat4.size,
-      lastModified: stat4.mtimeMs,
+      size: stat5.size,
+      lastModified: stat5.mtimeMs,
       start: 0
     })], { type });
-    fromFile = (stat4, path20, type = "") => new file_default([new BlobDataItem({
+    fromFile = (stat5, path20, type = "") => new file_default([new BlobDataItem({
       path: path20,
-      size: stat4.size,
-      lastModified: stat4.mtimeMs,
+      size: stat5.size,
+      lastModified: stat5.mtimeMs,
       start: 0
-    })], basename(path20), { type, lastModified: stat4.mtimeMs });
+    })], basename(path20), { type, lastModified: stat5.mtimeMs });
     BlobDataItem = class _BlobDataItem {
       #path;
       #start;
@@ -61635,11 +62163,11 @@ var init_from = __esm({
         });
       }
       async *stream() {
-        const { mtimeMs } = await stat2(this.#path);
+        const { mtimeMs } = await stat3(this.#path);
         if (mtimeMs > this.lastModified) {
           throw new import_node_domexception.default("The requested file could not be read, typically due to permission problems that have occurred after a reference to a file was acquired.", "NotReadableError");
         }
-        yield* createReadStream(this.#path, {
+        yield* createReadStream2(this.#path, {
           start: this.#start,
           end: this.#start + this.size - 1
         });
@@ -77269,7 +77797,7 @@ var init_wrapper = __esm({
 });
 
 // node_modules/@google/genai/dist/node/index.mjs
-import { createWriteStream } from "fs";
+import { createWriteStream as createWriteStream2 } from "fs";
 import * as fs8 from "fs/promises";
 import { writeFile as writeFile4 } from "fs/promises";
 import { Readable } from "stream";
@@ -94549,7 +95077,7 @@ ${underline}`);
         if (params.downloadPath) {
           const response = await downloadFile(params, apiClient);
           if (response instanceof HttpResponse) {
-            const writer = createWriteStream(params.downloadPath);
+            const writer = createWriteStream2(params.downloadPath);
             const body = Readable.fromWeb(response.responseInternal.body);
             body.pipe(writer);
             await finished(writer);
@@ -144467,7 +144995,7 @@ function explainRetention(doc, referenceTime) {
   };
 }
 function toRetentionDocument(obs, accessMap) {
-  const access = accessMap?.get(obs.id);
+  const access2 = accessMap?.get(obs.id);
   return {
     id: `obs-${obs.id}`,
     observationId: obs.id,
@@ -144481,8 +145009,8 @@ function toRetentionDocument(obs, accessMap) {
     tokens: obs.tokens,
     createdAt: obs.createdAt,
     projectId: obs.projectId,
-    accessCount: access?.accessCount ?? 0,
-    lastAccessedAt: access?.lastAccessedAt ?? "",
+    accessCount: access2?.accessCount ?? 0,
+    lastAccessedAt: access2?.lastAccessedAt ?? "",
     status: obs.status ?? "active",
     source: obs.source ?? "agent",
     sourceDetail: obs.sourceDetail ?? "",
@@ -145761,7 +146289,7 @@ function indexFileLite(projectId, projectRoot, abs, indexedAt, fileStat) {
   const rel = normalizeCodePath(relative(projectRoot, abs));
   try {
     const text2 = readFileSync6(abs, "utf-8");
-    const stat4 = fileStat ?? statSync3(abs);
+    const stat5 = fileStat ?? statSync3(abs);
     const file = {
       id: makeCodeFileId(projectId, rel),
       projectId,
@@ -145770,8 +146298,8 @@ function indexFileLite(projectId, projectRoot, abs, indexedAt, fileStat) {
       contentHash: hashText(text2),
       // Preserve filesystem precision. Rounding here made same-size edits in
       // a short Windows timestamp window indistinguishable to refresh.
-      mtimeMs: Number(stat4.mtimeMs),
-      sizeBytes: Number(stat4.size),
+      mtimeMs: Number(stat5.mtimeMs),
+      sizeBytes: Number(stat5.size),
       indexedAt
     };
     return {
@@ -145798,18 +146326,18 @@ async function indexProjectLite(options) {
   let skippedOversizedFiles = 0;
   let unreadableFiles = 0;
   for (const abs of paths) {
-    let stat4;
+    let stat5;
     try {
-      stat4 = statSync3(abs);
+      stat5 = statSync3(abs);
     } catch {
       unreadableFiles++;
       continue;
     }
-    if (Number(stat4.size) > maxFileBytes) {
+    if (Number(stat5.size) > maxFileBytes) {
       skippedOversizedFiles++;
       continue;
     }
-    const attempt = indexFileLite(options.projectId, options.projectRoot, abs, indexedAt, stat4);
+    const attempt = indexFileLite(options.projectId, options.projectRoot, abs, indexedAt, stat5);
     if (!attempt.delta) {
       if (attempt.unreadable) unreadableFiles++;
       continue;
@@ -145837,16 +146365,16 @@ async function refreshProjectLite(store, options) {
   for (const abs of paths) {
     const rel = normalizeCodePath(relative(options.projectRoot, abs));
     seenPaths.add(rel);
-    let stat4;
+    let stat5;
     try {
-      stat4 = statSync3(abs);
+      stat5 = statSync3(abs);
     } catch {
       unreadableFiles++;
       continue;
     }
     const existing = existingByPath.get(rel);
-    const mtimeMs = Number(stat4.mtimeMs);
-    const sizeBytes = Number(stat4.size);
+    const mtimeMs = Number(stat5.mtimeMs);
+    const sizeBytes = Number(stat5.size);
     if (sizeBytes > maxFileBytes) {
       skippedOversizedFiles++;
       if (existing) oversizedExistingFileIds.add(existing.id);
@@ -145856,7 +146384,7 @@ async function refreshProjectLite(store, options) {
       unchangedFiles++;
       continue;
     }
-    const attempt = indexFileLite(options.projectId, options.projectRoot, abs, indexedAt, stat4);
+    const attempt = indexFileLite(options.projectId, options.projectRoot, abs, indexedAt, stat5);
     if (!attempt.delta) {
       if (attempt.unreadable) unreadableFiles++;
       continue;