@gmickel/gno 1.5.2 → 1.7.0

package/src/embed/fingerprint.ts

@@ -0,0 +1,37 @@
+/**
+ * Embedding freshness fingerprint.
+ *
+ * @module src/embed/fingerprint
+ */
+
+import { getEmbeddingCompatibilityProfile } from "../llm/embedding-compatibility";
+
+export const EMBEDDING_CONTEXTUAL_FORMAT_VERSION = "contextual-embedding-v1";
+export const EMBEDDING_CHUNKING_STRATEGY_VERSION = "markdown-char-semantic-v1";
+
+export interface EmbeddingFingerprintInput {
+  modelUri: string;
+  dimensions?: number;
+}
+
+export function getEmbeddingFingerprint(
+  input: EmbeddingFingerprintInput
+): string {
+  const profile = getEmbeddingCompatibilityProfile(input.modelUri);
+  const payload = {
+    chunking: EMBEDDING_CHUNKING_STRATEGY_VERSION,
+    contextualFormatting: EMBEDDING_CONTEXTUAL_FORMAT_VERSION,
+    dimensions: input.dimensions ?? null,
+    modelUri: input.modelUri,
+    profile: {
+      batchEmbeddingTrusted: profile.batchEmbeddingTrusted,
+      documentFormat: profile.documentFormat,
+      id: profile.id,
+      queryFormat: profile.queryFormat,
+    },
+  };
+
+  return new Bun.CryptoHasher("sha256")
+    .update(JSON.stringify(payload))
+    .digest("hex");
+}

package/src/embed/retry.ts

@@ -0,0 +1,137 @@
+import type { EmbeddingPort } from "../llm/types";
+import type { BacklogItem, VectorIndexPort, VectorRow } from "../store/vector";
+
+import { formatDocForEmbedding } from "../pipeline/contextual";
+import { embedTextsWithRecovery } from "./batch";
+
+export const MAX_EMBED_CHUNK_ATTEMPTS = 2;
+export const MAX_EMBED_FAILURE_SAMPLES = 5;
+
+export interface EmbedStoreBatchResult {
+  embedded: number;
+  errors: number;
+  retryItems: BacklogItem[];
+  errorSamples: string[];
+  suggestion?: string;
+  batchFailed: boolean;
+  batchError?: string;
+}
+
+export function chunkRetryKey(item: Pick<BacklogItem, "mirrorHash" | "seq">) {
+  return `${item.mirrorHash}\0${item.seq}`;
+}
+
+export function addUniqueSamples(target: string[], samples: string[]): void {
+  for (const sample of samples) {
+    if (target.length >= MAX_EMBED_FAILURE_SAMPLES) {
+      break;
+    }
+    if (!target.includes(sample)) {
+      target.push(sample);
+    }
+  }
+}
+
+export function formatLlmFailure(
+  error: { message: string; cause?: unknown } | undefined
+): string {
+  if (!error) {
+    return "Unknown embedding failure";
+  }
+  const cause =
+    error.cause &&
+    typeof error.cause === "object" &&
+    "message" in error.cause &&
+    typeof error.cause.message === "string"
+      ? error.cause.message
+      : typeof error.cause === "string"
+        ? error.cause
+        : "";
+  return cause && cause !== error.message
+    ? `${error.message} - ${cause}`
+    : error.message;
+}
+
+export async function embedAndStoreBatch(params: {
+  embedPort: EmbeddingPort;
+  vectorIndex: VectorIndexPort;
+  items: BacklogItem[];
+  modelUri: string;
+  embedFingerprint: string;
+}): Promise<EmbedStoreBatchResult> {
+  const { embedPort, vectorIndex, items, modelUri, embedFingerprint } = params;
+  const embedResult = await embedTextsWithRecovery(
+    embedPort,
+    items.map((item) =>
+      formatDocForEmbedding(item.text, item.title ?? undefined, modelUri)
+    )
+  );
+
+  if (!embedResult.ok) {
+    const formattedError = formatLlmFailure(embedResult.error);
+    return {
+      embedded: 0,
+      errors: embedResult.error.retryable ? 0 : items.length,
+      retryItems: embedResult.error.retryable ? items : [],
+      errorSamples: [formattedError],
+      suggestion: embedResult.error.retryable
+        ? "Try rerunning the same command. If failures persist, rerun with `gno --verbose embed --batch-size 1` to isolate failing chunks."
+        : embedResult.error.suggestion,
+      batchFailed: true,
+      batchError: formattedError,
+    };
+  }
+
+  const vectors: VectorRow[] = [];
+  const retryItems: BacklogItem[] = [];
+  for (const [idx, item] of items.entries()) {
+    const embedding = embedResult.value.vectors[idx];
+    if (!embedding) {
+      retryItems.push(item);
+      continue;
+    }
+    vectors.push({
+      mirrorHash: item.mirrorHash,
+      seq: item.seq,
+      model: modelUri,
+      embedFingerprint,
+      embedding: new Float32Array(embedding),
+    });
+  }
+
+  if (vectors.length === 0) {
+    return {
+      embedded: 0,
+      errors: 0,
+      retryItems,
+      errorSamples: embedResult.value.failureSamples,
+      suggestion: embedResult.value.retrySuggestion,
+      batchFailed: embedResult.value.batchFailed,
+      batchError: embedResult.value.batchError,
+    };
+  }
+
+  const storeResult = await vectorIndex.upsertVectors(vectors);
+  if (!storeResult.ok) {
+    return {
+      embedded: 0,
+      errors: vectors.length,
+      retryItems,
+      errorSamples: [storeResult.error.message],
+      suggestion:
+        "Store write failed. Rerun `gno embed` once more; if it repeats, run `gno doctor` and `gno vec sync`.",
+      batchFailed: embedResult.value.batchFailed,
+      batchError: embedResult.value.batchError,
+    };
+  }
+
+  return {
+    embedded: vectors.length,
+    errors: 0,
+    retryItems,
+    errorSamples: embedResult.value.failureSamples,
+    suggestion: embedResult.value.retrySuggestion,
+    batchFailed: embedResult.value.batchFailed,
+    batchError: embedResult.value.batchError,
+  };
+}

package/src/llm/nodeLlamaCpp/embedding.ts

@@ -39,6 +39,8 @@ interface TokenizingModel {
   detokenize(tokens: readonly number[]): string;
 }
 
+type EmbeddingInput = Parameters<LlamaEmbeddingContext["getEmbeddingFor"]>[0];
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Constants
 // ─────────────────────────────────────────────────────────────────────────────
@@ -46,12 +48,19 @@ interface TokenizingModel {
 // Aim for a small pool so CPU-only runs can exploit parallel contexts without
 // multiplying RAM usage too aggressively. Additional contexts fall back
 // gracefully if memory is tight.
-const MAX_EMBEDDING_CONTEXTS = 4;
+const MAX_DEFAULT_EMBEDDING_CONTEXTS = 2;
+const MAX_EMBEDDING_CONTEXTS_OVERRIDE = 4;
 const TARGET_CORES_PER_EMBEDDING_CONTEXT = 4;
-const LOW_MEMORY_WINDOWS_THRESHOLD_BYTES = 24 * 1024 * 1024 * 1024;
+const CONSTRAINED_WINDOWS_THRESHOLD_BYTES = 16 * 1024 * 1024 * 1024;
+const MID_MEMORY_WINDOWS_THRESHOLD_BYTES = 24 * 1024 * 1024 * 1024;
 const LOW_MEMORY_WINDOWS_CONTEXTS = 1;
+const MID_MEMORY_WINDOWS_CONTEXTS = 2;
 const DEFAULT_EMBEDDING_CONTEXT_SIZE = 2_048;
 
+function embeddingVectorToArray(vector: readonly number[]): number[] {
+  return Array.isArray(vector) ? (vector as number[]) : Array.from(vector);
+}
+
 function resolveEmbeddingContextPoolOverride(
   env: NodeJS.ProcessEnv = process.env
 ): number | undefined {
@@ -63,7 +72,35 @@ function resolveEmbeddingContextPoolOverride(
   if (!(Number.isFinite(parsed) && parsed > 0)) {
     return undefined;
   }
-  return Math.max(1, Math.min(MAX_EMBEDDING_CONTEXTS, parsed));
+  return Math.max(1, Math.min(MAX_EMBEDDING_CONTEXTS_OVERRIDE, parsed));
+}
+
+function resolveThreadsPerContextOverride(
+  env: NodeJS.ProcessEnv = process.env
+): number | undefined {
+  const raw = env.GNO_EMBED_THREADS;
+  if (!raw) {
+    return undefined;
+  }
+  const parsed = Number.parseInt(raw, 10);
+  if (!(Number.isFinite(parsed) && parsed > 0)) {
+    return undefined;
+  }
+  return Math.max(1, parsed);
+}
+
+function resolveEmbeddingContextSizeOverride(
+  env: NodeJS.ProcessEnv = process.env
+): number | undefined {
+  const raw = env.GNO_EMBED_CONTEXT_SIZE;
+  if (!raw) {
+    return undefined;
+  }
+  const parsed = Number.parseInt(raw, 10);
+  if (!(Number.isFinite(parsed) && parsed > 0)) {
+    return undefined;
+  }
+  return Math.max(128, parsed);
 }
 
 export function resolveEmbeddingContextPoolSize(options: {
@@ -86,19 +123,28 @@ export function resolveEmbeddingContextPoolSize(options: {
   const totalMemoryBytes = options.totalMemoryBytes ?? totalmem();
   if (
     platformName === "win32" &&
-    totalMemoryBytes <= LOW_MEMORY_WINDOWS_THRESHOLD_BYTES
+    totalMemoryBytes < CONSTRAINED_WINDOWS_THRESHOLD_BYTES
   ) {
     return LOW_MEMORY_WINDOWS_CONTEXTS;
   }
 
   const cpuMathCores = Math.max(1, options.cpuMathCores);
-  return Math.max(
+  const adaptivePoolSize = Math.max(
     1,
     Math.min(
-      MAX_EMBEDDING_CONTEXTS,
+      MAX_DEFAULT_EMBEDDING_CONTEXTS,
       Math.ceil(cpuMathCores / TARGET_CORES_PER_EMBEDDING_CONTEXT)
     )
   );
+
+  if (
+    platformName === "win32" &&
+    totalMemoryBytes < MID_MEMORY_WINDOWS_THRESHOLD_BYTES
+  ) {
+    return Math.min(MID_MEMORY_WINDOWS_CONTEXTS, adaptivePoolSize);
+  }
+
+  return adaptivePoolSize;
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -145,9 +191,9 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
         return { ok: false, error: prepared.error };
       }
       const embedding = await this.runOnWorker((worker) =>
-        worker.context.getEmbeddingFor(prepared.value.text)
+        worker.context.getEmbeddingFor(prepared.value.input)
       );
-      const vector = Array.from(embedding.vector) as number[];
+      const vector = embeddingVectorToArray(embedding.vector);
 
       // Cache dimensions on first call
       if (this.dims === null) {
@@ -171,13 +217,13 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
     }
 
     try {
-      const preparedTexts: string[] = [];
+      const preparedInputs: EmbeddingInput[] = [];
       for (const text of texts) {
         const prepared = this.truncateForEmbedding(text, "batch");
         if (!prepared.ok) {
           return { ok: false, error: prepared.error };
         }
-        preparedTexts.push(prepared.value.text);
+        preparedInputs.push(prepared.value.input);
       }
 
       const allResults = Array.from(
@@ -191,16 +237,19 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
           while (true) {
             const index = nextIndex;
             nextIndex += 1;
-            if (index >= preparedTexts.length) {
+            if (index >= preparedInputs.length) {
               return;
             }
 
+            const input = preparedInputs[index];
+            if (input === undefined) {
+              return;
+            }
             const embedding = await this.runOnSpecificWorker(
               worker,
-              (current) =>
-                current.context.getEmbeddingFor(preparedTexts[index] as string)
+              (current) => current.context.getEmbeddingFor(input)
             );
-            allResults[index] = Array.from(embedding.vector) as number[];
+            allResults[index] = embeddingVectorToArray(embedding.vector);
           }
         })
       );
@@ -316,6 +365,11 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
       return 0;
     }
 
+    const override = resolveThreadsPerContextOverride();
+    if (override !== undefined) {
+      return override;
+    }
+
     return Math.max(1, Math.floor(Math.max(1, llama.cpuMathCores) / poolSize));
   }
 
@@ -335,6 +389,8 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
       this.llamaModel = llamaModel as TokenizingModel;
       const llama = await this.manager.getLlama();
       const lifecycleVersion = this.lifecycleVersion;
+      this.embeddingContextSize =
+        resolveEmbeddingContextSizeOverride() ?? DEFAULT_EMBEDDING_CONTEXT_SIZE;
       const targetPoolSize = this.resolveTargetPoolSize(llama);
       const threadsPerContext = this.resolveThreadsPerContext(
         llama,
@@ -400,7 +456,7 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
   private truncateForEmbedding(
     text: string,
     mode: "single" | "batch"
-  ): LlmResult<{ text: string }> {
+  ): LlmResult<{ input: EmbeddingInput }> {
     const model = this.llamaModel;
     const modelLimit =
       typeof model?.trainContextSize === "number" &&
@@ -409,7 +465,7 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
         ? Math.floor(model.trainContextSize)
         : undefined;
     if (!model) {
-      return { ok: true, value: { text } };
+      return { ok: true, value: { input: text } };
     }
 
     const rawLimit =
@@ -420,10 +476,13 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
     try {
       const tokens = model.tokenize(text);
       if (tokens.length <= limit) {
-        return { ok: true, value: { text } };
+        return {
+          ok: true,
+          value: { input: tokens as EmbeddingInput },
+        };
       }
 
-      const truncatedText = model.detokenize(tokens.slice(0, limit));
+      const truncatedTokens = tokens.slice(0, limit);
       const shouldWarn =
         mode === "single"
           ? !this.warnedSingleTruncation
@@ -438,7 +497,10 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
           `[llama] Truncated embedding input from ${tokens.length} to ${limit} tokens`
         );
       }
-      return { ok: true, value: { text: truncatedText } };
+      return {
+        ok: true,
+        value: { input: truncatedTokens as EmbeddingInput },
+      };
     } catch (error) {
       return { ok: false, error: inferenceFailedError(this.modelUri, error) };
     }

package/src/sdk/embed.ts

@@ -19,15 +19,15 @@ import type {
 import type { GnoEmbedOptions, GnoEmbedResult } from "./types";
 
 import { embedBacklog } from "../embed";
-import { embedTextsWithRecovery } from "../embed/batch";
+import { getEmbeddingFingerprint } from "../embed/fingerprint";
+import {
+  chunkRetryKey,
+  embedAndStoreBatch,
+  MAX_EMBED_CHUNK_ATTEMPTS,
+} from "../embed/retry";
 import { resolveModelUri } from "../llm/registry";
-import { formatDocForEmbedding } from "../pipeline/contextual";
 import { err, ok } from "../store/types";
-import {
-  createVectorIndexPort,
-  createVectorStatsPort,
-  type VectorRow,
-} from "../store/vector";
+import { createVectorIndexPort, createVectorStatsPort } from "../store/vector";
 import { sdkError } from "./errors";
 
 interface EmbedRuntimeOptions {
@@ -121,6 +121,68 @@ async function forceEmbedAll(
   let embedded = 0;
   let errors = 0;
   let cursor: { mirrorHash: string; seq: number } | undefined;
+  const retryQueue = new Map<string, { item: BacklogItem; attempts: number }>();
+  const embedFingerprint = getEmbeddingFingerprint({
+    modelUri,
+    dimensions: vectorIndex.dimensions,
+  });
+
+  const enqueueRetryItems = (items: BacklogItem[], attempts: number): void => {
+    for (const item of items) {
+      const key = chunkRetryKey(item);
+      const existing = retryQueue.get(key);
+      retryQueue.set(key, {
+        item,
+        attempts: Math.max(existing?.attempts ?? 0, attempts),
+      });
+    }
+  };
+
+  const drainRetryQueue = async (): Promise<number> => {
+    if (retryQueue.size === 0) {
+      return 0;
+    }
+
+    let retryEmbedded = 0;
+    const entries = [...retryQueue.values()].filter(
+      (entry) => entry.attempts < MAX_EMBED_CHUNK_ATTEMPTS
+    );
+
+    for (let idx = 0; idx < entries.length; idx += batchSize) {
+      const slice = entries.slice(idx, idx + batchSize);
+      for (const entry of slice) {
+        retryQueue.delete(chunkRetryKey(entry.item));
+        entry.attempts += 1;
+      }
+
+      const retryResult = await embedAndStoreBatch({
+        embedPort,
+        vectorIndex,
+        items: slice.map((entry) => entry.item),
+        modelUri,
+        embedFingerprint,
+      });
+      embedded += retryResult.embedded;
+      errors += retryResult.errors;
+      retryEmbedded += retryResult.embedded;
+
+      const retryByKey = new Set(
+        retryResult.retryItems.map((item) => chunkRetryKey(item))
+      );
+      for (const entry of slice) {
+        if (!retryByKey.has(chunkRetryKey(entry.item))) {
+          continue;
+        }
+        if (entry.attempts >= MAX_EMBED_CHUNK_ATTEMPTS) {
+          errors += 1;
+        } else {
+          retryQueue.set(chunkRetryKey(entry.item), entry);
+        }
+      }
+    }
+
+    return retryEmbedded;
+  };
 
   while (true) {
     const batchResult = await getActiveChunks(db, batchSize, cursor);
@@ -140,45 +202,27 @@ async function forceEmbedAll(
       cursor = { mirrorHash: lastItem.mirrorHash, seq: lastItem.seq };
     }
 
-    const embedResult = await embedTextsWithRecovery(
+    const beforeEmbedded = embedded;
+    const embedResult = await embedAndStoreBatch({
       embedPort,
-      batch.map((item) =>
-        formatDocForEmbedding(
-          item.text,
-          item.title ?? undefined,
-          embedPort.modelUri
-        )
-      )
-    );
-
-    if (!embedResult.ok) {
-      errors += batch.length;
-      continue;
-    }
+      vectorIndex,
+      items: batch,
+      modelUri,
+      embedFingerprint,
+    });
+    embedded += embedResult.embedded;
+    errors += embedResult.errors;
+    enqueueRetryItems(embedResult.retryItems, 1);
 
-    const vectors: VectorRow[] = [];
-    for (const [idx, item] of batch.entries()) {
-      const embedding = embedResult.value.vectors[idx];
-      if (!embedding) {
-        errors += 1;
-        continue;
-      }
-      vectors.push({
-        mirrorHash: item.mirrorHash,
-        seq: item.seq,
-        model: modelUri,
-        embedding: new Float32Array(embedding),
-      });
+    if (embedded > beforeEmbedded) {
+      await drainRetryQueue();
     }
+  }
 
-    if (vectors.length > 0) {
-      const storeResult = await vectorIndex.upsertVectors(vectors);
-      if (!storeResult.ok) {
-        errors += vectors.length;
-        continue;
-      }
-      embedded += vectors.length;
-    }
+  await drainRetryQueue();
+  if (retryQueue.size > 0) {
+    errors += retryQueue.size;
+    retryQueue.clear();
   }
 
   if (vectorIndex.vecDirty) {
@@ -217,24 +261,25 @@ export async function runEmbed(
   const db = runtime.store.getRawDb();
   const stats: VectorStatsPort = createVectorStatsPort(db);
 
-  const backlogResult = force
-    ? await getActiveChunkCount(db)
-    : await stats.countBacklog(modelUri, { collection: options.collection });
-  if (!backlogResult.ok) {
-    throw sdkError("STORE", backlogResult.error.message, {
-      cause: backlogResult.error.cause,
-    });
-  }
+  let totalToEmbed = 0;
+  if (force) {
+    const forceCount = await getActiveChunkCount(db);
+    if (!forceCount.ok) {
+      throw sdkError("STORE", forceCount.error.message, {
+        cause: forceCount.error.cause,
+      });
+    }
 
-  const totalToEmbed = backlogResult.value;
-  if (totalToEmbed === 0 || dryRun) {
-    return {
-      embedded: totalToEmbed,
-      errors: 0,
-      duration: 0,
-      model: modelUri,
-      searchAvailable: await checkVecAvailable(db),
-    };
+    totalToEmbed = forceCount.value;
+    if (totalToEmbed === 0 || dryRun) {
+      return {
+        embedded: totalToEmbed,
+        errors: 0,
+        duration: 0,
+        model: modelUri,
+        searchAvailable: await checkVecAvailable(db),
+      };
+    }
   }
 
   const embedResult = await runtime.llm.createEmbeddingPort(modelUri, {
@@ -266,6 +311,36 @@ export async function runEmbed(
     }
 
     const vectorIndex = vectorResult.value;
+    if (!force) {
+      const embedFingerprint = getEmbeddingFingerprint({
+        modelUri,
+        dimensions: vectorIndex.dimensions,
+      });
+      const backlogResult = await stats.countBacklog(
+        modelUri,
+        embedFingerprint,
+        {
+          collection: options.collection,
+        }
+      );
+      if (!backlogResult.ok) {
+        throw sdkError("STORE", backlogResult.error.message, {
+          cause: backlogResult.error.cause,
+        });
+      }
+
+      totalToEmbed = backlogResult.value;
+      if (totalToEmbed === 0 || dryRun) {
+        return {
+          embedded: totalToEmbed,
+          errors: 0,
+          duration: 0,
+          model: modelUri,
+          searchAvailable: vectorIndex.searchAvailable,
+        };
+      }
+    }
+
     const startedAt = Date.now();
     let result: { embedded: number; errors: number };
     if (force) {

package/src/store/migrations/008-vector-fingerprints.ts

@@ -0,0 +1,25 @@
+/**
+ * Migration: vector embedding freshness fingerprints.
+ *
+ * @module src/store/migrations/008-vector-fingerprints
+ */
+
+import type { Database } from "bun:sqlite";
+
+import type { Migration } from "./runner";
+
+export const migration: Migration = {
+  version: 8,
+  name: "vector_fingerprints",
+
+  up(db: Database): void {
+    db.exec(`
+      ALTER TABLE content_vectors ADD COLUMN embed_fingerprint TEXT NOT NULL DEFAULT ''
+    `);
+
+    db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_vectors_freshness
+      ON content_vectors(model, embed_fingerprint, mirror_hash, seq, embedded_at)
+    `);
+  },
+};

package/src/store/migrations/index.ts

@@ -21,6 +21,7 @@ import { migration as m004 } from "./004-doc-links";
 import { migration as m005 } from "./005-graph-indexes";
 import { migration as m006 } from "./006-document-metadata";
 import { migration as m007 } from "./007-document-date-fields";
+import { migration as m008 } from "./008-vector-fingerprints";
 
 /** All migrations in order */
-export const migrations = [m001, m002, m003, m004, m005, m006, m007];
+export const migrations = [m001, m002, m003, m004, m005, m006, m007, m008];