@gmickel/gno 1.6.0 → 1.7.0

package/src/embed/fingerprint.ts

@@ -0,0 +1,37 @@
+/**
+ * Embedding freshness fingerprint.
+ *
+ * @module src/embed/fingerprint
+ */
+
+import { getEmbeddingCompatibilityProfile } from "../llm/embedding-compatibility";
+
+export const EMBEDDING_CONTEXTUAL_FORMAT_VERSION = "contextual-embedding-v1";
+export const EMBEDDING_CHUNKING_STRATEGY_VERSION = "markdown-char-semantic-v1";
+
+export interface EmbeddingFingerprintInput {
+  modelUri: string;
+  dimensions?: number;
+}
+
+export function getEmbeddingFingerprint(
+  input: EmbeddingFingerprintInput
+): string {
+  const profile = getEmbeddingCompatibilityProfile(input.modelUri);
+  const payload = {
+    chunking: EMBEDDING_CHUNKING_STRATEGY_VERSION,
+    contextualFormatting: EMBEDDING_CONTEXTUAL_FORMAT_VERSION,
+    dimensions: input.dimensions ?? null,
+    modelUri: input.modelUri,
+    profile: {
+      batchEmbeddingTrusted: profile.batchEmbeddingTrusted,
+      documentFormat: profile.documentFormat,
+      id: profile.id,
+      queryFormat: profile.queryFormat,
+    },
+  };
+
+  return new Bun.CryptoHasher("sha256")
+    .update(JSON.stringify(payload))
+    .digest("hex");
+}

package/src/embed/retry.ts

@@ -0,0 +1,137 @@
+import type { EmbeddingPort } from "../llm/types";
+import type { BacklogItem, VectorIndexPort, VectorRow } from "../store/vector";
+
+import { formatDocForEmbedding } from "../pipeline/contextual";
+import { embedTextsWithRecovery } from "./batch";
+
+export const MAX_EMBED_CHUNK_ATTEMPTS = 2;
+export const MAX_EMBED_FAILURE_SAMPLES = 5;
+
+export interface EmbedStoreBatchResult {
+  embedded: number;
+  errors: number;
+  retryItems: BacklogItem[];
+  errorSamples: string[];
+  suggestion?: string;
+  batchFailed: boolean;
+  batchError?: string;
+}
+
+export function chunkRetryKey(item: Pick<BacklogItem, "mirrorHash" | "seq">) {
+  return `${item.mirrorHash}\0${item.seq}`;
+}
+
+export function addUniqueSamples(target: string[], samples: string[]): void {
+  for (const sample of samples) {
+    if (target.length >= MAX_EMBED_FAILURE_SAMPLES) {
+      break;
+    }
+    if (!target.includes(sample)) {
+      target.push(sample);
+    }
+  }
+}
+
+export function formatLlmFailure(
+  error: { message: string; cause?: unknown } | undefined
+): string {
+  if (!error) {
+    return "Unknown embedding failure";
+  }
+  const cause =
+    error.cause &&
+    typeof error.cause === "object" &&
+    "message" in error.cause &&
+    typeof error.cause.message === "string"
+      ? error.cause.message
+      : typeof error.cause === "string"
+        ? error.cause
+        : "";
+  return cause && cause !== error.message
+    ? `${error.message} - ${cause}`
+    : error.message;
+}
+
+export async function embedAndStoreBatch(params: {
+  embedPort: EmbeddingPort;
+  vectorIndex: VectorIndexPort;
+  items: BacklogItem[];
+  modelUri: string;
+  embedFingerprint: string;
+}): Promise<EmbedStoreBatchResult> {
+  const { embedPort, vectorIndex, items, modelUri, embedFingerprint } = params;
+  const embedResult = await embedTextsWithRecovery(
+    embedPort,
+    items.map((item) =>
+      formatDocForEmbedding(item.text, item.title ?? undefined, modelUri)
+    )
+  );
+
+  if (!embedResult.ok) {
+    const formattedError = formatLlmFailure(embedResult.error);
+    return {
+      embedded: 0,
+      errors: embedResult.error.retryable ? 0 : items.length,
+      retryItems: embedResult.error.retryable ? items : [],
+      errorSamples: [formattedError],
+      suggestion: embedResult.error.retryable
+        ? "Try rerunning the same command. If failures persist, rerun with `gno --verbose embed --batch-size 1` to isolate failing chunks."
+        : embedResult.error.suggestion,
+      batchFailed: true,
+      batchError: formattedError,
+    };
+  }
+
+  const vectors: VectorRow[] = [];
+  const retryItems: BacklogItem[] = [];
+  for (const [idx, item] of items.entries()) {
+    const embedding = embedResult.value.vectors[idx];
+    if (!embedding) {
+      retryItems.push(item);
+      continue;
+    }
+    vectors.push({
+      mirrorHash: item.mirrorHash,
+      seq: item.seq,
+      model: modelUri,
+      embedFingerprint,
+      embedding: new Float32Array(embedding),
+    });
+  }
+
+  if (vectors.length === 0) {
+    return {
+      embedded: 0,
+      errors: 0,
+      retryItems,
+      errorSamples: embedResult.value.failureSamples,
+      suggestion: embedResult.value.retrySuggestion,
+      batchFailed: embedResult.value.batchFailed,
+      batchError: embedResult.value.batchError,
+    };
+  }
+
+  const storeResult = await vectorIndex.upsertVectors(vectors);
+  if (!storeResult.ok) {
+    return {
+      embedded: 0,
+      errors: vectors.length,
+      retryItems,
+      errorSamples: [storeResult.error.message],
+      suggestion:
+        "Store write failed. Rerun `gno embed` once more; if it repeats, run `gno doctor` and `gno vec sync`.",
+      batchFailed: embedResult.value.batchFailed,
+      batchError: embedResult.value.batchError,
+    };
+  }
+
+  return {
+    embedded: vectors.length,
+    errors: 0,
+    retryItems,
+    errorSamples: embedResult.value.failureSamples,
+    suggestion: embedResult.value.retrySuggestion,
+    batchFailed: embedResult.value.batchFailed,
+    batchError: embedResult.value.batchError,
+  };
+}

package/src/sdk/embed.ts

@@ -19,15 +19,15 @@ import type {
 import type { GnoEmbedOptions, GnoEmbedResult } from "./types";
 
 import { embedBacklog } from "../embed";
-import { embedTextsWithRecovery } from "../embed/batch";
+import { getEmbeddingFingerprint } from "../embed/fingerprint";
+import {
+  chunkRetryKey,
+  embedAndStoreBatch,
+  MAX_EMBED_CHUNK_ATTEMPTS,
+} from "../embed/retry";
 import { resolveModelUri } from "../llm/registry";
-import { formatDocForEmbedding } from "../pipeline/contextual";
 import { err, ok } from "../store/types";
-import {
-  createVectorIndexPort,
-  createVectorStatsPort,
-  type VectorRow,
-} from "../store/vector";
+import { createVectorIndexPort, createVectorStatsPort } from "../store/vector";
 import { sdkError } from "./errors";
 
 interface EmbedRuntimeOptions {
@@ -121,6 +121,68 @@ async function forceEmbedAll(
   let embedded = 0;
   let errors = 0;
   let cursor: { mirrorHash: string; seq: number } | undefined;
+  const retryQueue = new Map<string, { item: BacklogItem; attempts: number }>();
+  const embedFingerprint = getEmbeddingFingerprint({
+    modelUri,
+    dimensions: vectorIndex.dimensions,
+  });
+
+  const enqueueRetryItems = (items: BacklogItem[], attempts: number): void => {
+    for (const item of items) {
+      const key = chunkRetryKey(item);
+      const existing = retryQueue.get(key);
+      retryQueue.set(key, {
+        item,
+        attempts: Math.max(existing?.attempts ?? 0, attempts),
+      });
+    }
+  };
+
+  const drainRetryQueue = async (): Promise<number> => {
+    if (retryQueue.size === 0) {
+      return 0;
+    }
+
+    let retryEmbedded = 0;
+    const entries = [...retryQueue.values()].filter(
+      (entry) => entry.attempts < MAX_EMBED_CHUNK_ATTEMPTS
+    );
+
+    for (let idx = 0; idx < entries.length; idx += batchSize) {
+      const slice = entries.slice(idx, idx + batchSize);
+      for (const entry of slice) {
+        retryQueue.delete(chunkRetryKey(entry.item));
+        entry.attempts += 1;
+      }
+
+      const retryResult = await embedAndStoreBatch({
+        embedPort,
+        vectorIndex,
+        items: slice.map((entry) => entry.item),
+        modelUri,
+        embedFingerprint,
+      });
+      embedded += retryResult.embedded;
+      errors += retryResult.errors;
+      retryEmbedded += retryResult.embedded;
+
+      const retryByKey = new Set(
+        retryResult.retryItems.map((item) => chunkRetryKey(item))
+      );
+      for (const entry of slice) {
+        if (!retryByKey.has(chunkRetryKey(entry.item))) {
+          continue;
+        }
+        if (entry.attempts >= MAX_EMBED_CHUNK_ATTEMPTS) {
+          errors += 1;
+        } else {
+          retryQueue.set(chunkRetryKey(entry.item), entry);
+        }
+      }
+    }
+
+    return retryEmbedded;
+  };
 
   while (true) {
     const batchResult = await getActiveChunks(db, batchSize, cursor);
@@ -140,45 +202,27 @@ async function forceEmbedAll(
       cursor = { mirrorHash: lastItem.mirrorHash, seq: lastItem.seq };
     }
 
-    const embedResult = await embedTextsWithRecovery(
+    const beforeEmbedded = embedded;
+    const embedResult = await embedAndStoreBatch({
       embedPort,
-      batch.map((item) =>
-        formatDocForEmbedding(
-          item.text,
-          item.title ?? undefined,
-          embedPort.modelUri
-        )
-      )
-    );
-
-    if (!embedResult.ok) {
-      errors += batch.length;
-      continue;
-    }
+      vectorIndex,
+      items: batch,
+      modelUri,
+      embedFingerprint,
+    });
+    embedded += embedResult.embedded;
+    errors += embedResult.errors;
+    enqueueRetryItems(embedResult.retryItems, 1);
 
-    const vectors: VectorRow[] = [];
-    for (const [idx, item] of batch.entries()) {
-      const embedding = embedResult.value.vectors[idx];
-      if (!embedding) {
-        errors += 1;
-        continue;
-      }
-      vectors.push({
-        mirrorHash: item.mirrorHash,
-        seq: item.seq,
-        model: modelUri,
-        embedding: new Float32Array(embedding),
-      });
+    if (embedded > beforeEmbedded) {
+      await drainRetryQueue();
     }
+  }
 
-    if (vectors.length > 0) {
-      const storeResult = await vectorIndex.upsertVectors(vectors);
-      if (!storeResult.ok) {
-        errors += vectors.length;
-        continue;
-      }
-      embedded += vectors.length;
-    }
+  await drainRetryQueue();
+  if (retryQueue.size > 0) {
+    errors += retryQueue.size;
+    retryQueue.clear();
   }
 
   if (vectorIndex.vecDirty) {
@@ -217,24 +261,25 @@ export async function runEmbed(
   const db = runtime.store.getRawDb();
   const stats: VectorStatsPort = createVectorStatsPort(db);
 
-  const backlogResult = force
-    ? await getActiveChunkCount(db)
-    : await stats.countBacklog(modelUri, { collection: options.collection });
-  if (!backlogResult.ok) {
-    throw sdkError("STORE", backlogResult.error.message, {
-      cause: backlogResult.error.cause,
-    });
-  }
+  let totalToEmbed = 0;
+  if (force) {
+    const forceCount = await getActiveChunkCount(db);
+    if (!forceCount.ok) {
+      throw sdkError("STORE", forceCount.error.message, {
+        cause: forceCount.error.cause,
+      });
+    }
 
-  const totalToEmbed = backlogResult.value;
-  if (totalToEmbed === 0 || dryRun) {
-    return {
-      embedded: totalToEmbed,
-      errors: 0,
-      duration: 0,
-      model: modelUri,
-      searchAvailable: await checkVecAvailable(db),
-    };
+    totalToEmbed = forceCount.value;
+    if (totalToEmbed === 0 || dryRun) {
+      return {
+        embedded: totalToEmbed,
+        errors: 0,
+        duration: 0,
+        model: modelUri,
+        searchAvailable: await checkVecAvailable(db),
+      };
+    }
   }
 
   const embedResult = await runtime.llm.createEmbeddingPort(modelUri, {
@@ -266,6 +311,36 @@ export async function runEmbed(
     }
 
     const vectorIndex = vectorResult.value;
+    if (!force) {
+      const embedFingerprint = getEmbeddingFingerprint({
+        modelUri,
+        dimensions: vectorIndex.dimensions,
+      });
+      const backlogResult = await stats.countBacklog(
+        modelUri,
+        embedFingerprint,
+        {
+          collection: options.collection,
+        }
+      );
+      if (!backlogResult.ok) {
+        throw sdkError("STORE", backlogResult.error.message, {
+          cause: backlogResult.error.cause,
+        });
+      }
+
+      totalToEmbed = backlogResult.value;
+      if (totalToEmbed === 0 || dryRun) {
+        return {
+          embedded: totalToEmbed,
+          errors: 0,
+          duration: 0,
+          model: modelUri,
+          searchAvailable: vectorIndex.searchAvailable,
+        };
+      }
+    }
+
     const startedAt = Date.now();
     let result: { embedded: number; errors: number };
     if (force) {

package/src/store/migrations/008-vector-fingerprints.ts

@@ -0,0 +1,25 @@
+/**
+ * Migration: vector embedding freshness fingerprints.
+ *
+ * @module src/store/migrations/008-vector-fingerprints
+ */
+
+import type { Database } from "bun:sqlite";
+
+import type { Migration } from "./runner";
+
+export const migration: Migration = {
+  version: 8,
+  name: "vector_fingerprints",
+
+  up(db: Database): void {
+    db.exec(`
+      ALTER TABLE content_vectors ADD COLUMN embed_fingerprint TEXT NOT NULL DEFAULT ''
+    `);
+
+    db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_vectors_freshness
+      ON content_vectors(model, embed_fingerprint, mirror_hash, seq, embedded_at)
+    `);
+  },
+};

package/src/store/migrations/index.ts

@@ -21,6 +21,7 @@ import { migration as m004 } from "./004-doc-links";
 import { migration as m005 } from "./005-graph-indexes";
 import { migration as m006 } from "./006-document-metadata";
 import { migration as m007 } from "./007-document-date-fields";
+import { migration as m008 } from "./008-vector-fingerprints";
 
 /** All migrations in order */
-export const migrations = [m001, m002, m003, m004, m005, m006, m007];
+export const migrations = [m001, m002, m003, m004, m005, m006, m007, m008];

package/src/store/sqlite/adapter.ts

@@ -53,6 +53,7 @@ import { analyzeGraphCommunities } from "../../core/graph-analysis";
 import { normalizeWikiName, stripWikiMdExt } from "../../core/links";
 import { migrations, runMigrations } from "../migrations";
 import { err, ok } from "../types";
+import { getStoredEmbeddingFingerprint } from "../vector/freshness";
 import { modelTableName } from "../vector/sqlite-vec";
 import { loadFts5Snowball } from "./fts5-snowball";
 
@@ -3065,10 +3066,14 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
 
   async getStatus(options?: {
     embedModel?: string;
+    embedFingerprint?: string;
   }): Promise<StoreResult<IndexStatus>> {
     try {
       const db = this.ensureOpen();
       const embedModel = options?.embedModel ?? null;
+      const embedFingerprint =
+        options?.embedFingerprint ??
+        (embedModel ? getStoredEmbeddingFingerprint(db, embedModel) : null);
 
       // Get version
       const versionRow = db
@@ -3097,7 +3102,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       }
 
       const collectionStats = db
-        .query<CollectionStat, [string | null, string | null]>(
+        .query<CollectionStat, [string | null, string | null, string | null]>(
           `
           SELECT
             c.name,
@@ -3120,7 +3125,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
                SELECT 1 FROM content_vectors cv
                WHERE cv.mirror_hash = cc.mirror_hash
                  AND cv.seq = cc.seq
-                 AND (? IS NULL OR cv.model = ?)
+                 AND (? IS NULL OR (
+                   cv.model = ?
+                   AND cv.embed_fingerprint = ?
+                 ))
                  AND cv.embedded_at >= cc.created_at
              )) as embedded_count
           FROM collections c
@@ -3128,7 +3136,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           GROUP BY c.name, c.path
         `
         )
-        .all(embedModel, embedModel);
+        .all(embedModel, embedModel, embedFingerprint);
 
       // Get totals
       const totalsRow = db
@@ -3152,7 +3160,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       // Embedding backlog: chunks from active docs without vectors
       // Uses EXISTS to avoid duplicates when multiple docs share mirror_hash
       const backlogRow = db
-        .query<{ count: number }, [string | null, string | null]>(
+        .query<
+          { count: number },
+          [string | null, string | null, string | null]
+        >(
           `
           SELECT COUNT(*) as count FROM content_chunks c
           WHERE EXISTS (
@@ -3163,12 +3174,15 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
             SELECT 1 FROM content_vectors v
             WHERE v.mirror_hash = c.mirror_hash
               AND v.seq = c.seq
-              AND (? IS NULL OR v.model = ?)
+              AND (? IS NULL OR (
+                v.model = ?
+                AND v.embed_fingerprint = ?
+              ))
               AND v.embedded_at >= c.created_at
           )
         `
         )
-        .get(embedModel, embedModel);
+        .get(embedModel, embedModel, embedFingerprint);
 
       // Recent errors (last 24h)
       const recentErrorsRow = db

package/src/store/types.ts

@@ -984,6 +984,7 @@ export interface StorePort {
    */
   getStatus(options?: {
     embedModel?: string;
+    embedFingerprint?: string;
   }): Promise<StoreResult<IndexStatus>>;
 
   // ─────────────────────────────────────────────────────────────────────────

package/src/store/vector/freshness.ts

@@ -0,0 +1,34 @@
+/**
+ * Vector freshness helpers.
+ *
+ * @module src/store/vector/freshness
+ */
+
+import type { Database } from "bun:sqlite";
+
+import { getEmbeddingFingerprint } from "../../embed/fingerprint";
+
+export function getStoredEmbeddingDimensions(
+  db: Database,
+  model: string
+): number | undefined {
+  const row = db
+    .prepare("SELECT embedding FROM content_vectors WHERE model = ? LIMIT 1")
+    .get(model) as { embedding: Uint8Array } | undefined;
+
+  if (!row?.embedding) {
+    return undefined;
+  }
+
+  return row.embedding.byteLength / Float32Array.BYTES_PER_ELEMENT;
+}
+
+export function getStoredEmbeddingFingerprint(
+  db: Database,
+  modelUri: string
+): string {
+  return getEmbeddingFingerprint({
+    modelUri,
+    dimensions: getStoredEmbeddingDimensions(db, modelUri),
+  });
+}

package/src/store/vector/sqlite-vec.ts

@@ -116,8 +116,10 @@ export async function createVectorIndexPort(
 
   // Prepared statements for content_vectors table
   const upsertVectorStmt = db.prepare(`
-    INSERT OR REPLACE INTO content_vectors (mirror_hash, seq, model, embedding, embedded_at)
-    VALUES (?, ?, ?, ?, datetime('now'))
+    INSERT OR REPLACE INTO content_vectors (
+      mirror_hash, seq, model, embed_fingerprint, embedding, embedded_at
+    )
+    VALUES (?, ?, ?, ?, ?, datetime('now'))
   `);
 
   const deleteVectorStmt = db.prepare(`
@@ -172,6 +174,7 @@ export async function createVectorIndexPort(
               row.mirrorHash,
               row.seq,
               row.model,
+              row.embedFingerprint,
               encodeEmbedding(row.embedding)
             );
           }

package/src/store/vector/stats.ts

@@ -65,6 +65,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
 
     countBacklog(
       model: string,
+      embedFingerprint: string,
       options?: { collection?: string }
     ): Promise<StoreResult<number>> {
       try {
@@ -80,10 +81,13 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
             WHERE v.mirror_hash = c.mirror_hash
               AND v.seq = c.seq
               AND v.model = ?
+              AND v.embed_fingerprint = ?
               AND v.embedded_at >= c.created_at
           )
         `;
-        const result = db.prepare(sql).get(...activeDoc.params, model) as {
+        const result = db
+          .prepare(sql)
+          .get(...activeDoc.params, model, embedFingerprint) as {
           count: number;
         };
         return Promise.resolve(ok(result.count));
@@ -99,6 +103,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
 
     getBacklog(
       model: string,
+      embedFingerprint: string,
       options?: {
         limit?: number;
         after?: { mirrorHash: string; seq: number };
@@ -123,6 +128,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
                 WHERE v.mirror_hash = c.mirror_hash
                   AND v.seq = c.seq
                   AND v.model = ?
+                  AND v.embed_fingerprint = ?
               ) THEN 'new'
               ELSE 'changed'
             END as reason
@@ -133,6 +139,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
             WHERE v.mirror_hash = c.mirror_hash
               AND v.seq = c.seq
               AND v.model = ?
+              AND v.embed_fingerprint = ?
               AND v.embedded_at >= c.created_at
           )
           AND (c.mirror_hash > ? OR (c.mirror_hash = ? AND c.seq > ?))
@@ -148,6 +155,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
                 WHERE v.mirror_hash = c.mirror_hash
                   AND v.seq = c.seq
                   AND v.model = ?
+                  AND v.embed_fingerprint = ?
               ) THEN 'new'
               ELSE 'changed'
             END as reason
@@ -158,6 +166,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
             WHERE v.mirror_hash = c.mirror_hash
               AND v.seq = c.seq
               AND v.model = ?
+              AND v.embed_fingerprint = ?
               AND v.embedded_at >= c.created_at
           )
           ORDER BY c.mirror_hash, c.seq
@@ -167,14 +176,23 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
         const params = after
           ? [
               model,
+              embedFingerprint,
               ...activeDoc.params,
               model,
+              embedFingerprint,
               after.mirrorHash,
               after.mirrorHash,
               after.seq,
               limit,
             ]
-          : [model, ...activeDoc.params, model, limit];
+          : [
+              model,
+              embedFingerprint,
+              ...activeDoc.params,
+              model,
+              embedFingerprint,
+              limit,
+            ];
 
         const results = db.prepare(sql).all(...params) as BacklogItem[];
         return Promise.resolve(ok(results));