memorix 1.1.11 → 1.1.13

package/dist/maintenance-runner.js

@@ -4773,6 +4773,45 @@ function dimsCacheKey(config) {
     config.requestedDimensions ?? "native"
   ].join("|");
 }
+function isValidDimension(value) {
+  return typeof value === "number" && Number.isInteger(value) && value > 0;
+}
+async function loadCachedVectorDimensions(config) {
+  try {
+    const raw = await readFile2(CACHE_META_FILE, "utf-8");
+    const data = JSON.parse(raw);
+    const namespace = cacheNamespace(config);
+    if (!Array.isArray(data?.entries)) return null;
+    const entry = data.entries.find(
+      (candidate) => typeof candidate === "object" && candidate !== null && candidate.namespace === namespace && isValidDimension(candidate.dimensions)
+    );
+    return entry?.dimensions ?? null;
+  } catch {
+    return null;
+  }
+}
+async function saveCachedVectorDimensions(config, dimensions) {
+  if (!isValidDimension(dimensions)) return;
+  try {
+    await mkdir2(CACHE_DIR2, { recursive: true });
+    let entries = [];
+    try {
+      const raw = await readFile2(CACHE_META_FILE, "utf-8");
+      const data = JSON.parse(raw);
+      if (Array.isArray(data?.entries)) {
+        entries = data.entries.filter(
+          (candidate) => typeof candidate === "object" && candidate !== null && typeof candidate.namespace === "string" && isValidDimension(candidate.dimensions) && typeof candidate.ts === "number"
+        );
+      }
+    } catch {
+    }
+    const namespace = cacheNamespace(config);
+    entries = entries.filter((entry) => entry.namespace !== namespace);
+    entries.push({ namespace, dimensions, ts: Date.now() });
+    await writeFile2(CACHE_META_FILE, JSON.stringify({ version: 1, entries }));
+  } catch {
+  }
+}
 async function loadCachedDims(config) {
   try {
     const raw = await readFile2(DIMS_CACHE_FILE, "utf-8");
@@ -4835,6 +4874,7 @@ async function saveCachedDims(config, dimensions) {
     await writeFile2(DIMS_CACHE_FILE, JSON.stringify({ entries }));
   } catch {
   }
+  await saveCachedVectorDimensions(config, dimensions);
 }
 async function saveDiskCacheNow() {
   if (!diskCacheDirty2) return;
@@ -4917,7 +4957,7 @@ async function fetchWithRetry(url, apiKey, body, attempt = 0) {
   const errorText2 = await response.text().catch(() => "unknown error");
   throw new Error(`Embedding API error (${response.status}): ${errorText2}`);
 }
-var CACHE_DIR2, CACHE_FILE2, DIMS_CACHE_FILE, cache3, MAX_CACHE_SIZE3, diskCacheDirty2, diskSaveTimer, diskCacheLoaded, diskCacheLoadPromise, MAX_INPUT_CHARS, MAX_CONCURRENCY, DISK_SAVE_DEBOUNCE_MS, DEFAULT_MAX_BATCH_SIZE, DASHSCOPE_MAX_BATCH_SIZE, MAX_RETRIES, BASE_DELAY_MS, APIEmbeddingProvider;
+var CACHE_DIR2, CACHE_FILE2, DIMS_CACHE_FILE, CACHE_META_FILE, cache3, MAX_CACHE_SIZE3, diskCacheDirty2, diskSaveTimer, diskCacheLoaded, diskCacheLoadPromise, MAX_INPUT_CHARS, MAX_CONCURRENCY, DISK_SAVE_DEBOUNCE_MS, DEFAULT_MAX_BATCH_SIZE, DASHSCOPE_MAX_BATCH_SIZE, MAX_RETRIES, BASE_DELAY_MS, APIEmbeddingProvider;
 var init_api_provider = __esm({
   "src/embedding/api-provider.ts"() {
     "use strict";
@@ -4925,6 +4965,7 @@ var init_api_provider = __esm({
     CACHE_DIR2 = process.env.MEMORIX_DATA_DIR || join4(homedir7(), ".memorix", "data");
     CACHE_FILE2 = join4(CACHE_DIR2, ".embedding-api-cache.json");
     DIMS_CACHE_FILE = join4(CACHE_DIR2, ".embedding-dims-cache.json");
+    CACHE_META_FILE = join4(CACHE_DIR2, ".embedding-api-cache-meta.json");
     cache3 = /* @__PURE__ */ new Map();
     MAX_CACHE_SIZE3 = 1e4;
     diskCacheDirty2 = false;
@@ -4951,13 +4992,22 @@ var init_api_provider = __esm({
         this.dimensions = detectedDimensions;
         this.name = `api-${config.model.replace(/\//g, "-")}`;
       }
-      static async create() {
+      static async create(options = {}) {
         const config = _APIEmbeddingProvider.resolveConfig();
-        startDiskCacheLoad();
+        const allowNetworkProbe = options.allowNetworkProbe !== false;
         let probeDimensions = await loadCachedDims(config);
+        let dimensionSource = "dims-cache";
+        if (probeDimensions === null) {
+          probeDimensions = await loadCachedVectorDimensions(config);
+          if (probeDimensions !== null) dimensionSource = "vector-cache";
+        }
         if (probeDimensions !== null) {
-          console.error(`[memorix] API embedding: ${config.model} @ ${config.baseUrl} (${probeDimensions}d) [cached dims]`);
+          console.error(`[memorix] API embedding: ${config.model} @ ${config.baseUrl} (${probeDimensions}d) [${dimensionSource}]`);
+          if (dimensionSource === "dims-cache") {
+            void saveCachedVectorDimensions(config, probeDimensions);
+          }
         } else {
+          if (!allowNetworkProbe) return null;
           probeDimensions = await _APIEmbeddingProvider.probeAPI(config);
           console.error(`[memorix] API embedding: ${config.model} @ ${config.baseUrl} (${probeDimensions}d)`);
           saveCachedDims(config, probeDimensions).catch(() => {
@@ -4966,6 +5016,7 @@ var init_api_provider = __esm({
         if (config.requestedDimensions) {
           console.error(`[memorix] Dimension shortening: ${config.requestedDimensions}d requested`);
         }
+        startDiskCacheLoad();
         return new _APIEmbeddingProvider(config, probeDimensions);
       }
       static resolveConfig() {
@@ -5144,6 +5195,13 @@ var init_api_provider = __esm({
         scheduleDiskSave();
         return results;
       }
+      async getCachedEmbeddings(texts) {
+        await ensureDiskCacheLoaded();
+        return texts.map((text) => {
+          const hash = textHash2(normalizeText(text), this.cacheKeyNamespace);
+          return cache3.get(hash) ?? null;
+        });
+      }
       getStats() {
         return {
           totalTokens: this.totalTokensUsed,
@@ -5252,10 +5310,10 @@ async function createTransformersProvider() {
     return null;
   }
 }
-async function createAPIProvider() {
+async function createAPIProvider(options) {
   try {
     const { APIEmbeddingProvider: APIEmbeddingProvider2 } = await Promise.resolve().then(() => (init_api_provider(), api_provider_exports));
-    return await APIEmbeddingProvider2.create();
+    return await APIEmbeddingProvider2.create(options ?? {});
   } catch (e) {
     warnOnce(`[memorix] Failed to init API embedding: ${e instanceof Error ? e.message : e}`);
     return null;
@@ -5310,11 +5368,24 @@ function wrapProvider(candidate) {
         }
         throw error;
       }
-    }
+    },
+    ...candidate.getCachedEmbeddings ? {
+      async getCachedEmbeddings(texts) {
+        return candidate.getCachedEmbeddings(texts);
+      }
+    } : {}
   };
 }
-async function getEmbeddingProvider() {
+async function getEmbeddingProvider(options = {}) {
   if (provider) return provider;
+  if (options.allowNetworkProbe === false) {
+    const mode = getEmbeddingMode2();
+    if (mode !== "api" && !(mode === "auto" && hasAPIEmbeddingConfig())) {
+      return null;
+    }
+    const cached = await createAPIProvider({ allowNetworkProbe: false });
+    return cached ? wrapProvider(cached) : null;
+  }
   if (lastInitWasTemporaryFailure) {
     const elapsed = Date.now() - lastFailureTimestamp;
     if (elapsed < RETRY_COOLDOWN_MS) {
@@ -6315,12 +6386,15 @@ __export(orama_store_exports, {
   batchGenerateEmbeddings: () => batchGenerateEmbeddings,
   generateEmbedding: () => generateEmbedding,
   getDb: () => getDb,
+  getDeferredCachedVectorHydration: () => getDeferredCachedVectorHydration,
   getLastSearchMode: () => getLastSearchMode,
   getObservationCount: () => getObservationCount,
   getObservationsByIds: () => getObservationsByIds,
   getTimeline: () => getTimeline,
   getVectorDimensions: () => getVectorDimensions,
+  hasObservationVector: () => hasObservationVector,
   hydrateIndex: () => hydrateIndex,
+  hydrateIndexForStartup: () => hydrateIndexForStartup,
   insertObservation: () => insertObservation,
   isEmbeddingEnabled: () => isEmbeddingEnabled,
   makeOramaObservationId: () => makeOramaObservationId,
@@ -6381,11 +6455,10 @@ function stripVectorSearchParams(params) {
   const { mode, vector, similarity, hybridWeights, ...rest } = params;
   return rest;
 }
-async function getDb() {
-  if (db) return db;
-  const provider2 = await getEmbeddingProvider();
-  embeddingEnabled = provider2 !== null;
-  embeddingDimensions = provider2?.dimensions ?? null;
+async function initializeDb(options, generation) {
+  const provider2 = await getEmbeddingProvider({ allowNetworkProbe: options.allowNetworkProbe });
+  const nextEmbeddingEnabled = provider2 !== null;
+  const nextEmbeddingDimensions = provider2?.dimensions ?? null;
   const baseSchema = {
     id: "string",
     observationId: "number",
@@ -6408,15 +6481,39 @@ async function getDb() {
     documentType: "string",
     knowledgeLayer: "string"
   };
-  const dims = embeddingDimensions ?? 384;
-  const schema = embeddingEnabled ? { ...baseSchema, embedding: `vector[${dims}]` } : baseSchema;
-  db = await create({ schema });
-  return db;
+  const dims = nextEmbeddingDimensions ?? 384;
+  const schema = nextEmbeddingEnabled ? { ...baseSchema, embedding: `vector[${dims}]` } : baseSchema;
+  const nextDb = await create({ schema });
+  if (generation !== dbGeneration) {
+    return getDb(options);
+  }
+  indexEmbeddingProvider = provider2;
+  embeddingEnabled = nextEmbeddingEnabled;
+  embeddingDimensions = nextEmbeddingDimensions;
+  db = nextDb;
+  return nextDb;
+}
+async function getDb(options = {}) {
+  if (db) return db;
+  if (dbInitPromise) return dbInitPromise;
+  const initPromise2 = initializeDb(options, dbGeneration);
+  dbInitPromise = initPromise2;
+  try {
+    return await initPromise2;
+  } finally {
+    if (dbInitPromise === initPromise2) {
+      dbInitPromise = null;
+    }
+  }
 }
 async function resetDb() {
+  dbGeneration++;
   db = null;
+  dbInitPromise = null;
   embeddingEnabled = false;
   embeddingDimensions = null;
+  indexEmbeddingProvider = null;
+  deferredCachedVectorHydration = null;
   lastSearchModeByProject.clear();
   docByObservationKey.clear();
 }
@@ -6444,14 +6541,64 @@ async function batchGenerateEmbeddings(texts) {
     return texts.map(() => null);
   }
 }
-async function hydrateIndex(observations2) {
-  const database = await getDb();
+async function getCachedEmbeddings(texts) {
+  if (!embeddingEnabled || texts.length === 0) return texts.map(() => null);
+  const provider2 = indexEmbeddingProvider;
+  if (!provider2?.getCachedEmbeddings) return texts.map(() => null);
+  try {
+    return await provider2.getCachedEmbeddings(texts);
+  } catch {
+    return texts.map(() => null);
+  }
+}
+function observationEmbeddingText(observation) {
+  return [
+    observation.title ?? "",
+    observation.narrative ?? "",
+    ...Array.isArray(observation.facts) ? observation.facts : []
+  ].join(" ");
+}
+function documentEmbeddingText(document) {
+  return [document.title ?? "", document.narrative ?? "", document.facts ?? ""].join(" ");
+}
+function isCompatibleCachedVector(vector) {
+  return Boolean(
+    vector && embeddingDimensions !== null && vector.length === embeddingDimensions && vector.every(Number.isFinite)
+  );
+}
+async function attachCachedVectors(database, candidates) {
+  const cachedVectors = await getCachedEmbeddings(
+    candidates.map(({ observation }) => observationEmbeddingText(observation))
+  );
+  if (db !== database) return;
+  for (let index = 0; index < candidates.length; index++) {
+    const vector = cachedVectors[index];
+    if (!isCompatibleCachedVector(vector)) continue;
+    const existing = getByID(database, candidates[index].id);
+    if (!existing || documentEmbeddingText(existing) !== observationEmbeddingText(candidates[index].observation)) continue;
+    try {
+      await update(database, candidates[index].id, { ...existing, embedding: vector });
+    } catch {
+    }
+  }
+}
+async function hydrateIndex(observations2, options = {}) {
+  const database = await getDb({ allowNetworkProbe: options.allowNetworkProbe });
+  const candidates = [];
+  for (const observation of observations2) {
+    if (!observation || !observation.id || !observation.projectId) continue;
+    const id = makeOramaObservationId(observation.projectId, observation.id);
+    if (getByID(database, id)) continue;
+    candidates.push({ observation, id });
+  }
+  const deferCachedVectors = options.deferCachedVectors === true && Boolean(indexEmbeddingProvider?.getCachedEmbeddings);
+  const cachedVectors = deferCachedVectors ? candidates.map(() => null) : await getCachedEmbeddings(candidates.map(({ observation }) => observationEmbeddingText(observation)));
   let inserted = 0;
-  for (const obs of observations2) {
-    if (!obs || !obs.id || !obs.projectId) continue;
+  for (let index = 0; index < candidates.length; index++) {
+    const { observation: obs, id } = candidates[index];
     try {
-      const id = makeOramaObservationId(obs.projectId, obs.id);
-      if (getByID(database, id)) continue;
+      const vector = cachedVectors[index];
+      const compatibleVector = isCompatibleCachedVector(vector) ? vector : null;
       const doc = {
         id,
         observationId: obs.id,
@@ -6470,7 +6617,8 @@ async function hydrateIndex(observations2) {
         status: obs.status ?? "active",
         source: obs.source || "agent",
         documentType: "observation",
-        knowledgeLayer: resolveKnowledgeLayer("observation", obs.sourceDetail, obs.source)
+        knowledgeLayer: resolveKnowledgeLayer("observation", obs.sourceDetail, obs.source),
+        ...compatibleVector ? { embedding: compatibleVector } : {}
       };
       await insert2(database, doc);
       rememberObservationDoc(doc);
@@ -6478,8 +6626,24 @@ async function hydrateIndex(observations2) {
     } catch {
     }
   }
+  deferredCachedVectorHydration = deferCachedVectors ? attachCachedVectors(database, candidates).catch(() => {
+  }) : null;
   return inserted;
 }
+async function hydrateIndexForStartup(observations2) {
+  return hydrateIndex(observations2, {
+    allowNetworkProbe: false,
+    deferCachedVectors: true
+  });
+}
+function getDeferredCachedVectorHydration() {
+  return deferredCachedVectorHydration;
+}
+function hasObservationVector(projectId, observationId) {
+  if (!db || !embeddingEnabled || embeddingDimensions === null) return false;
+  const document = getByID(db, makeOramaObservationId(projectId, observationId));
+  return Array.isArray(document?.embedding) && document.embedding.length === embeddingDimensions && document.embedding.every(Number.isFinite);
+}
 async function insertObservation(doc) {
   const database = await getDb();
   await insert2(database, doc);
@@ -7013,7 +7177,7 @@ function formatTime(isoDate) {
     return isoDate;
   }
 }
-var db, embeddingEnabled, embeddingDimensions, docByObservationKey, NON_CJK_HYBRID_SIMILARITY, lastSearchModeByProject, SEARCH_MODE_DEFAULT_KEY, COMMAND_LOG_TITLE, COMMAND_STYLE_TITLE, COMMAND_LIKE_QUERY2, COMMAND_INTENT_QUERY, NATURAL_LANGUAGE_MARKERS;
+var db, dbInitPromise, dbGeneration, embeddingEnabled, embeddingDimensions, indexEmbeddingProvider, deferredCachedVectorHydration, docByObservationKey, NON_CJK_HYBRID_SIMILARITY, lastSearchModeByProject, SEARCH_MODE_DEFAULT_KEY, COMMAND_LOG_TITLE, COMMAND_STYLE_TITLE, COMMAND_LIKE_QUERY2, COMMAND_INTENT_QUERY, NATURAL_LANGUAGE_MARKERS;
 var init_orama_store = __esm({
   "src/store/orama-store.ts"() {
     "use strict";
@@ -7025,8 +7189,12 @@ var init_orama_store = __esm({
     init_intent_detector();
     init_query_expansion();
     db = null;
+    dbInitPromise = null;
+    dbGeneration = 0;
     embeddingEnabled = false;
     embeddingDimensions = null;
+    indexEmbeddingProvider = null;
+    deferredCachedVectorHydration = null;
     docByObservationKey = /* @__PURE__ */ new Map();
     NON_CJK_HYBRID_SIMILARITY = 0.45;
     lastSearchModeByProject = /* @__PURE__ */ new Map();
@@ -7217,7 +7385,18 @@ async function bindObservationCodeRefsBestEffort(observation) {
 function isVectorCompatibleWithCurrentIndex(embedding) {
   if (!embedding) return false;
   const vectorDimensions = getVectorDimensions();
-  return vectorDimensions === null || embedding.length === vectorDimensions;
+  return vectorDimensions !== null && embedding.length === vectorDimensions && embedding.every(Number.isFinite);
+}
+async function upgradeVectorSchemaAfterFirstEmbedding(embedding) {
+  if (isVectorCompatibleWithCurrentIndex(embedding)) return true;
+  if (getVectorDimensions() !== null || !embedding.every(Number.isFinite)) return false;
+  const targetProjectDir = projectDir;
+  if (!vectorSchemaUpgradePromise) {
+    vectorSchemaUpgradePromise = reindexObservations().then(() => projectDir === targetProjectDir && isVectorCompatibleWithCurrentIndex(embedding)).catch(() => false).finally(() => {
+      vectorSchemaUpgradePromise = null;
+    });
+  }
+  return vectorSchemaUpgradePromise;
 }
 async function initObservations(dir) {
   if (projectDir === dir) return;
@@ -7227,6 +7406,7 @@ async function initObservations(dir) {
   nextId = await store.loadIdCounter();
   projectDir = dir;
   searchIndexPrepared = false;
+  vectorSchemaUpgradePromise = null;
 }
 async function ensureFreshObservations() {
   if (!projectDir) return false;
@@ -7406,6 +7586,9 @@ async function storeObservation(input) {
   const searchableText = [input.title, input.narrative, ...input.facts ?? []].join(" ");
   generateEmbedding(searchableText).then(async (embedding) => {
     if (embedding) {
+      if (!isVectorCompatibleWithCurrentIndex(embedding)) {
+        await upgradeVectorSchemaAfterFirstEmbedding(embedding);
+      }
       if (!isVectorCompatibleWithCurrentIndex(embedding)) {
         const vectorDimensions = getVectorDimensions();
         console.error(
@@ -7516,6 +7699,13 @@ async function upsertObservation(existing, input, now) {
   vectorMissingIds.add(obsId);
   generateEmbedding(searchableText).then(async (embedding) => {
     if (embedding) {
+      if (!isVectorCompatibleWithCurrentIndex(embedding)) {
+        await upgradeVectorSchemaAfterFirstEmbedding(embedding);
+      }
+      if (!isVectorCompatibleWithCurrentIndex(embedding)) {
+        queueVectorBackfill(existing.projectId);
+        return;
+      }
       try {
         const { removeObservation: removeObs } = await Promise.resolve().then(() => (init_orama_store(), orama_store_exports));
         await removeObs(makeOramaObservationId(existing.projectId, obsId));
@@ -7660,14 +7850,21 @@ async function reindexObservations() {
   let embeddings = observations.map(() => null);
   const provider2 = await getEmbeddingProvider();
   const canBatchEmbedAtStartup = provider2 !== null && !provider2.name.startsWith("api-");
+  await getDb();
+  const texts = observations.map(
+    (obs) => [obs.title, obs.narrative, ...obs.facts].join(" ")
+  );
+  if (provider2?.getCachedEmbeddings) {
+    try {
+      embeddings = await provider2.getCachedEmbeddings(texts);
+    } catch {
+    }
+  }
   if (provider2 && !canBatchEmbedAtStartup) {
-    console.error("[memorix] Startup reindex: skipping synchronous API embeddings; background backfill will hydrate vectors");
+    console.error("[memorix] Startup reindex: restored cached API embeddings; uncached vectors stay in background recovery");
   }
   if (canBatchEmbedAtStartup) {
     try {
-      const texts = observations.map(
-        (obs) => [obs.title, obs.narrative, ...obs.facts].join(" ")
-      );
       embeddings = await batchGenerateEmbeddings(texts);
     } catch {
     }
@@ -7720,16 +7917,31 @@ async function reindexObservations() {
 }
 async function prepareSearchIndex() {
   if (searchIndexPrepared) return 0;
-  const count2 = await hydrateIndex(observations);
+  const count2 = await hydrateIndexForStartup(observations);
   if (count2 === 0) {
     searchIndexPrepared = true;
     return 0;
   }
-  vectorMissingIds.clear();
-  if (isEmbeddingEnabled()) {
-    for (const obs of observations) {
-      vectorMissingIds.add(obs.id);
+  const hydratedProjectDir = projectDir;
+  const queueMissingVectors = () => {
+    if (projectDir !== hydratedProjectDir) return;
+    vectorMissingIds.clear();
+    if (!isEmbeddingExplicitlyDisabled()) {
+      const projectsWithMissingVectors = /* @__PURE__ */ new Set();
+      for (const obs of observations) {
+        if (!hasObservationVector(obs.projectId, obs.id)) {
+          vectorMissingIds.add(obs.id);
+          projectsWithMissingVectors.add(obs.projectId);
+        }
+      }
+      for (const projectId of projectsWithMissingVectors) queueVectorBackfill(projectId);
     }
+  };
+  const cacheHydration = getDeferredCachedVectorHydration();
+  if (cacheHydration) {
+    void cacheHydration.finally(queueMissingVectors);
+  } else {
+    queueMissingVectors();
   }
   searchIndexPrepared = true;
   return count2;
@@ -7789,6 +8001,9 @@ async function backfillVectorEmbeddings(options = {}) {
       try {
         const embedding = await generateEmbedding(text);
         if (embedding) {
+          if (!isVectorCompatibleWithCurrentIndex(embedding)) {
+            await upgradeVectorSchemaAfterFirstEmbedding(embedding);
+          }
           if (!isVectorCompatibleWithCurrentIndex(embedding)) {
             const vectorDimensions = getVectorDimensions();
             console.error(
@@ -7851,7 +8066,7 @@ async function backfillVectorEmbeddings(options = {}) {
   }
   return { attempted: ids.length, succeeded, failed };
 }
-var observations, nextId, projectDir, searchIndexPrepared, vectorMissingIds, vectorBackfillRunning, lastVectorBackfill, embeddingFailureLogTimestamps, EMBEDDING_FAILURE_LOG_COOLDOWN_MS;
+var observations, nextId, projectDir, searchIndexPrepared, vectorMissingIds, vectorBackfillRunning, vectorSchemaUpgradePromise, lastVectorBackfill, embeddingFailureLogTimestamps, EMBEDDING_FAILURE_LOG_COOLDOWN_MS;
 var init_observations = __esm({
   "src/memory/observations.ts"() {
     "use strict";
@@ -7869,6 +8084,7 @@ var init_observations = __esm({
     searchIndexPrepared = false;
     vectorMissingIds = /* @__PURE__ */ new Set();
     vectorBackfillRunning = false;
+    vectorSchemaUpgradePromise = null;
     lastVectorBackfill = null;
     embeddingFailureLogTimestamps = /* @__PURE__ */ new Map();
     EMBEDDING_FAILURE_LOG_COOLDOWN_MS = 3e4;