@gmickel/gno 2.12.1 → 2.13.0

package/src/store/sqlite/adapter.ts

@@ -164,7 +164,7 @@ import {
 import { getSchemaVersion, migrations, runMigrations } from "../migrations";
 import { err, ok } from "../types";
 import { getStoredEmbeddingFingerprint } from "../vector/freshness";
-import { modelTableName } from "../vector/sqlite-vec";
+import { bindReadConnection, modelTableName } from "../vector/sqlite-vec";
 import {
   getVariantStatus,
   listVectorPartitions,
@@ -622,7 +622,12 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
   private activeTransaction?: {
     token: { revoked: boolean };
     release: () => void;
+    db: Database;
   };
+  /** Set while a read-only caller runs; its reads use `readDb`. */
+  private readonly readScope = new AsyncLocalStorage<true>();
+  /** Second connection for read-only callers, opened on first use. */
+  private readDb: Database | null = null;
   private shutdownFenced = false;
   private shutdownDeadline?: number;
   private contextGeneration = 0;
@@ -741,6 +746,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
 
   async close(): Promise<void> {
     this.fenceForShutdown();
+    this.readDb?.close(false);
+    this.readDb = null;
     if (this.db) {
       try {
         // Current Bun finalizes all statements and releases Windows handles now.
@@ -791,7 +798,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     // JS cannot interleave this synchronous rollback with a running callback.
     // A callback suspended at await must never commit or use the store again.
     transaction.token.revoked = true;
-    this.db?.exec("ROLLBACK");
+    transaction.db.exec("ROLLBACK");
     transaction.release();
     this.activeTransaction = undefined;
   }
@@ -800,6 +807,47 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     return this.db !== null;
   }
 
+  /**
+   * Run a read-only caller on a second connection to the same index. All
+   * writes stay on the shared connection, so a write transaction there (a
+   * sync batch held open across file conversions) is never visible to these
+   * reads: WAL gives this connection the last committed state. An in-memory
+   * index has no second connection and reads on the shared one.
+   */
+  withReadConnection<T>(fn: () => Promise<T>): Promise<T> {
+    if (this.dbPath === "" || this.dbPath === ":memory:") return fn();
+    return this.readScope.run(true, fn);
+  }
+
+  private openReadDb(main: Database): Database {
+    if (this.readDb) return this.readDb;
+    const db = new Database(this.dbPath);
+    try {
+      // A write here would wait out the busy timeout while the shared
+      // connection's batch, on the same event loop, cannot commit: fail fast.
+      db.exec("PRAGMA query_only = ON");
+      db.exec("PRAGMA foreign_keys = ON");
+      const busyTimeout =
+        main.query<{ timeout: number }, []>("PRAGMA busy_timeout").get()
+          ?.timeout ?? DEFAULT_BUSY_TIMEOUT_MS;
+      db.exec(`PRAGMA busy_timeout = ${busyTimeout}`);
+      if (this.ftsTokenizer.startsWith("snowball")) {
+        const snowball = loadFts5Snowball(db);
+        if (!snowball.loaded)
+          throw new Error(`Failed to load fts5-snowball: ${snowball.error}`);
+      }
+    } catch (cause) {
+      db.close(false);
+      throw cause;
+    }
+    bindReadConnection(db, main, async () => {
+      const release = await this.acquireTransactionWriter();
+      release();
+    });
+    this.readDb = db;
+    return db;
+  }
+
   /**
    * Run an async function within a single SQLite transaction.
    * Uses SAVEPOINT for nesting safety.
@@ -808,6 +856,9 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
    * explicit BEGIN/COMMIT to support async callbacks.
    */
   async withTransaction<T>(fn: () => Promise<T>): Promise<StoreResult<T>> {
+    // Writes always run on the shared connection, even from a read scope.
+    if (this.readScope.getStore())
+      return this.readScope.exit(() => this.withTransaction(fn));
     const parent = this.txContext.getStore();
     const connection = this.ensureOpen();
     const isOuter = parent === undefined;
@@ -826,7 +877,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       if (isOuter) {
         // IMMEDIATE reduces lock churn for bulk writes
         db.exec("BEGIN IMMEDIATE");
-        this.activeTransaction = { token, release: releaseWriter! };
+        this.activeTransaction = { token, release: releaseWriter!, db };
       } else {
         db.exec(`SAVEPOINT ${savepoint}`);
       }
@@ -872,6 +923,11 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     }
   }
 
+  /** Write outside a transaction, on the shared connection even from a read scope. */
+  private async admitWrite<T>(write: (db: Database) => T): Promise<T> {
+    return this.readScope.exit(() => write(this.ensureOpen()));
+  }
+
   private async acquireTransactionWriter(): Promise<() => void> {
     const previous = this.txTail;
     let release!: () => void;
@@ -913,8 +969,12 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     if (!this.db) {
       throw new Error("Database not open");
     }
-    this.capShutdownBusyWait(this.db);
-    return this.db;
+    const db =
+      this.readScope.getStore() && !this.txContext.getStore()
+        ? this.openReadDb(this.db)
+        : this.db;
+    this.capShutdownBusyWait(db);
+    return db;
   }
 
   // ─────────────────────────────────────────────────────────────────────────
@@ -1507,14 +1567,16 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
   async createRetrievalTrace(
     input: RetrievalTraceInput
   ): Promise<StoreResult<RetrievalTraceAppendResult>> {
-    return createStoredTrace(this.ensureOpen(), input);
+    return this.admitWrite((db) => createStoredTrace(db, input));
   }
 
   async mergeRetrievalTraceEgressLineage(
     traceId: string,
     lineage: EgressLineage
   ): Promise<StoreResult<RetrievalTraceAppendResult>> {
-    return mergeStoredTraceEgressLineage(this.ensureOpen(), traceId, lineage);
+    return this.admitWrite((db) =>
+      mergeStoredTraceEgressLineage(db, traceId, lineage)
+    );
   }
 
   async getRetrievalTrace(
@@ -1542,37 +1604,39 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     status: RetrievalTraceTerminalStatus,
     updatedAtMs: number
   ): Promise<StoreResult<RetrievalTraceAppendResult>> {
-    return finalizeStoredTrace(this.ensureOpen(), traceId, status, updatedAtMs);
+    return this.admitWrite((db) =>
+      finalizeStoredTrace(db, traceId, status, updatedAtMs)
+    );
   }
 
   async appendRetrievalTraceRun(
     input: RetrievalTraceRunInput
   ): Promise<StoreResult<RetrievalTraceAppendResult>> {
-    return appendStoredTraceRun(this.ensureOpen(), input);
+    return this.admitWrite((db) => appendStoredTraceRun(db, input));
   }
 
   async appendRetrievalTraceEvent(
     input: RetrievalTraceEventInput
   ): Promise<StoreResult<RetrievalTraceAppendResult>> {
-    return appendStoredTraceEvent(this.ensureOpen(), input);
+    return this.admitWrite((db) => appendStoredTraceEvent(db, input));
   }
 
   async appendRetrievalTraceJudgment(
     input: RetrievalTraceJudgmentInput
   ): Promise<StoreResult<RetrievalTraceAppendResult>> {
-    return appendStoredTraceJudgment(this.ensureOpen(), input);
+    return this.admitWrite((db) => appendStoredTraceJudgment(db, input));
   }
 
   async appendRetrievalTraceExport(
     input: RetrievalTraceExportInput
   ): Promise<StoreResult<RetrievalTraceAppendResult>> {
-    return appendStoredTraceExport(this.ensureOpen(), input);
+    return this.admitWrite((db) => appendStoredTraceExport(db, input));
   }
 
   async appendRetrievalTraceExportManifest(
     input: RetrievalTraceExportManifestInput
   ): Promise<StoreResult<RetrievalTraceAppendResult>> {
-    return appendStoredTraceExportManifest(this.ensureOpen(), input);
+    return this.admitWrite((db) => appendStoredTraceExportManifest(db, input));
   }
 
   async getRetrievalTraceExportManifest(
@@ -1590,32 +1654,34 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
   async getOrCreateRetrievalTraceRedactionSecret(): Promise<
     StoreResult<string>
   > {
-    return getOrCreateStoredTraceRedactionSecret(this.ensureOpen());
+    return this.admitWrite((db) => getOrCreateStoredTraceRedactionSecret(db));
   }
 
   async deleteRetrievalTrace(
     traceId: string
   ): Promise<StoreResult<RetrievalTraceDeleteCounts>> {
-    return deleteStoredTrace(this.ensureOpen(), traceId);
+    return this.admitWrite((db) => deleteStoredTrace(db, traceId));
   }
 
   async purgeRetrievalTraces(): Promise<
     StoreResult<RetrievalTracePurgeResult>
   > {
-    return purgeStoredTraces(this.ensureOpen());
+    return this.admitWrite((db) => purgeStoredTraces(db));
   }
 
   async enforceRetrievalTraceRetention(
     policy: RetrievalTraceRetentionPolicy,
     nowMs: number
   ): Promise<StoreResult<RetrievalTraceRetentionResult>> {
-    return enforceStoredTraceRetention(this.ensureOpen(), policy, nowMs);
+    return this.admitWrite((db) =>
+      enforceStoredTraceRetention(db, policy, nowMs)
+    );
   }
 
   async appendEgressAuditReceipt(
     receipt: EgressAuditReceiptInput
   ): Promise<StoreResult<RetrievalTraceAppendResult>> {
-    return appendStoredEgressAuditReceipt(this.ensureOpen(), receipt);
+    return this.admitWrite((db) => appendStoredEgressAuditReceipt(db, receipt));
   }
 
   async appendEgressAuditReceiptWithRetention(
@@ -1623,11 +1689,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     policy: EgressAuditRetentionPolicy,
     nowMs: number
   ): Promise<StoreResult<RetrievalTraceAppendResult>> {
-    return appendStoredEgressAuditReceiptWithRetention(
-      this.ensureOpen(),
-      receipt,
-      policy,
-      nowMs
+    return this.admitWrite((db) =>
+      appendStoredEgressAuditReceiptWithRetention(db, receipt, policy, nowMs)
     );
   }
 
@@ -1647,7 +1710,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
   async deleteEgressAuditReceipt(
     auditId: string
   ): Promise<StoreResult<EgressAuditDeleteResult>> {
-    return deleteStoredEgressAuditReceipt(this.ensureOpen(), auditId);
+    return this.admitWrite((db) => deleteStoredEgressAuditReceipt(db, auditId));
   }
 
   async getEgressAuditStatus(): Promise<StoreResult<EgressAuditStatusResult>> {
@@ -1658,13 +1721,15 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     policy: EgressAuditRetentionPolicy,
     nowMs: number
   ): Promise<StoreResult<EgressAuditRetentionResult>> {
-    return enforceStoredEgressAuditRetention(this.ensureOpen(), policy, nowMs);
+    return this.admitWrite((db) =>
+      enforceStoredEgressAuditRetention(db, policy, nowMs)
+    );
   }
 
   async purgeEgressAuditReceipts(): Promise<
     StoreResult<EgressAuditPurgeResult>
   > {
-    return purgeStoredEgressAuditReceipts(this.ensureOpen());
+    return this.admitWrite((db) => purgeStoredEgressAuditReceipts(db));
   }
 
   getContextGeneration(): number {
@@ -2330,6 +2395,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     mirrorHashes: string[],
     options: {
       collection?: string;
+      collections?: string[];
       activeOnly?: boolean;
     } = {}
   ): Promise<StoreResult<DocumentRow[]>> {
@@ -2350,7 +2416,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
 
       // SQLite SQLITE_LIMIT_VARIABLE_NUMBER defaults to 999.
       // Reserve headroom for optional non-IN parameters.
-      const SQL_PARAM_LIMIT = options.collection ? 899 : 900;
+      const SQL_PARAM_LIMIT = 898;
 
       for (let i = 0; i < uniqueHashes.length; i += SQL_PARAM_LIMIT) {
         const batch = uniqueHashes.slice(i, i + SQL_PARAM_LIMIT);
@@ -2365,6 +2431,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           clauses.push("collection = ?");
           params.push(options.collection);
         }
+        if (options.collections !== undefined) {
+          clauses.push("collection IN (SELECT value FROM json_each(?))");
+          params.push(JSON.stringify(options.collections));
+        }
 
         const sql = `SELECT * FROM documents INDEXED BY idx_documents_mirror_hash WHERE ${clauses.join(" AND ")} ORDER BY id`;
         rows.push(...db.query<DbDocumentRow, string[]>(sql).all(...params));

package/src/store/sqlite/eligibility.ts

@@ -90,6 +90,10 @@ export function buildEligibleDocumentQuery(
     conditions.push("d.collection = ?");
     params.push(options.collection);
   }
+  if (options.collections !== undefined) {
+    conditions.push("d.collection IN (SELECT value FROM json_each(?))");
+    params.push(JSON.stringify(options.collections));
+  }
   if (options.relPathPrefix !== undefined) {
     conditions.push(
       "(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) = ? OR substr(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path), 1, length(?) + 1) = ? || '/')"

package/src/store/types.ts

@@ -718,6 +718,11 @@ export interface DocumentEligibilityOptions {
   exclude?: string[];
   /** Filter by collection */
   collection?: string;
+  /**
+   * Filter to documents in ANY of these collections, before ranking and LIMIT.
+   * Undefined is unrestricted; an empty array denies all.
+   */
+  collections?: string[];
   /** Internal exact relative-path boundary applied before ranking and LIMIT. */
   relPathPrefix?: string;
   /** Filter to docs with ANY of these tags */
@@ -1945,6 +1950,8 @@ export interface StorePort {
     mirrorHashes: string[],
     options?: {
       collection?: string;
+      /** Keep only documents in ANY of these collections; empty denies all. */
+      collections?: string[];
       activeOnly?: boolean;
     }
   ): Promise<StoreResult<DocumentRow[]>>;

package/src/store/vector/sqlite-vec.ts

@@ -68,6 +68,37 @@ export function modelTableName(modelUri: string): string {
 // ─────────────────────────────────────────────────────────────────────────────
 
 const indexDatabases = new WeakMap<VectorIndexPort, Database>();
+/** Query-only read connections, mapped to the shared connection that writes. */
+const writerDatabases = new WeakMap<
+  Database,
+  { writer: Database; idle: () => Promise<void> }
+>();
+
+/**
+ * Register `reader` as a query-only connection whose writes go to `writer`;
+ * `idle` resolves once no write transaction is open on `writer`.
+ */
+export function bindReadConnection(
+  reader: Database,
+  writer: Database,
+  idle: () => Promise<void>
+): void {
+  writerDatabases.set(reader, { writer, idle });
+}
+
+/**
+ * The connection to write on for `db`: itself, or for a query-only read
+ * connection, the shared connection behind it. Partition resolution records
+ * runtime callers and migrates legacy partitions, so it runs there.
+ */
+export function writerConnection(db: Database): Database {
+  return writerDatabases.get(db)?.writer ?? db;
+}
+
+/** Wait until the shared connection behind `db` has committed its open batch. */
+export function writerIdle(db: Database): Promise<void> {
+  return writerDatabases.get(db)?.idle() ?? Promise.resolve();
+}
 
 /** Partition resolution needs the database behind a vector index port. */
 export function getVectorIndexDatabase(
@@ -131,9 +162,22 @@ export async function createVectorIndexPort(
     loadError = e instanceof Error ? e.message : String(e);
   }
 
+  // A query-only (read) connection cannot create the table. A missing table
+  // there means no vectors were ever stored for this model: search is empty.
+  const tableExists =
+    db
+      .query<{ found: number }, [string]>(
+        "SELECT 1 AS found FROM sqlite_master WHERE name = ?"
+      )
+      .get(tableName) !== null;
+  const queryOnly =
+    db.query<{ query_only: number }, []>("PRAGMA query_only").get()
+      ?.query_only === 1;
+  const noVectorsYet = searchAvailable && !tableExists && queryOnly;
+
   // Create per-model vec0 table if extension available
   // Graceful degradation: if table creation fails, storage still works
-  if (searchAvailable) {
+  if (searchAvailable && !tableExists && !queryOnly) {
     try {
       db.exec(`
         CREATE VIRTUAL TABLE IF NOT EXISTS ${tableName} USING vec0(
@@ -161,26 +205,32 @@ export async function createVectorIndexPort(
   `);
 
   // Prepared statements for vec0 table (if available)
-  const deleteVecChunkStmt = searchAvailable
-    ? db.prepare(`DELETE FROM ${tableName} WHERE chunk_id = ?`)
-    : null;
-
-  const insertVecStmt = searchAvailable
-    ? db.prepare(`INSERT INTO ${tableName} (chunk_id, embedding) VALUES (?, ?)`)
-    : null;
+  const deleteVecChunkStmt =
+    searchAvailable && !noVectorsYet
+      ? db.prepare(`DELETE FROM ${tableName} WHERE chunk_id = ?`)
+      : null;
+
+  const insertVecStmt =
+    searchAvailable && !noVectorsYet
+      ? db.prepare(
+          `INSERT INTO ${tableName} (chunk_id, embedding) VALUES (?, ?)`
+        )
+      : null;
 
-  const searchStmt = searchAvailable
-    ? db.prepare(`
+  const searchStmt =
+    searchAvailable && !noVectorsYet
+      ? db.prepare(`
         SELECT chunk_id, distance
         FROM ${tableName}
         WHERE embedding MATCH ?
           AND k = ?
       `)
-    : null;
+      : null;
 
-  const deleteVecStmt = searchAvailable
-    ? db.prepare(`DELETE FROM ${tableName} WHERE chunk_id LIKE ? || ':%'`)
-    : null;
+  const deleteVecStmt =
+    searchAvailable && !noVectorsYet
+      ? db.prepare(`DELETE FROM ${tableName} WHERE chunk_id LIKE ? || ':%'`)
+      : null;
 
   // Track if vec0 writes failed this session (needs sync)
   let vecDirty = false;
@@ -291,7 +341,7 @@ export async function createVectorIndexPort(
       k: number,
       searchOptions?: VectorSearchOptions
     ): Promise<StoreResult<VectorSearchResult[]>> {
-      if (!(searchAvailable && searchStmt)) {
+      if (!searchAvailable || !(searchStmt || noVectorsYet)) {
         return Promise.resolve(
           err("VEC_SEARCH_UNAVAILABLE", formatUnavailableMessage(loadError))
         );
@@ -397,6 +447,8 @@ export async function createVectorIndexPort(
             )
           );
         }
+        // No per-model table on a read connection: nothing was ever stored.
+        if (!searchStmt) return Promise.resolve(ok([]));
         const results = searchStmt.all(encodeEmbedding(embedding), k) as {
           chunk_id: string;
           distance: number;

package/src/store/vector/variant-search.ts

@@ -13,10 +13,16 @@ import { buildEligibleDocumentQuery } from "../sqlite/eligibility";
 import {
   embeddingPartitionIdentity,
   identityPartitionId,
+  isPartitionActivated,
   resolveRuntimePartition,
   retrievalUse,
 } from "./runtime-compat";
-import { encodeEmbedding, getVectorIndexDatabase } from "./sqlite-vec";
+import {
+  encodeEmbedding,
+  getVectorIndexDatabase,
+  writerConnection,
+  writerIdle,
+} from "./sqlite-vec";
 import { embeddingInputHash } from "./variants";
 
 /** Fallback/warning code: this runtime may not read the activated vectors. */
@@ -35,13 +41,22 @@ export type SearchPartition =
  * an activated partition gets a reason instead of an identity; vector spaces
  * are never mixed.
  */
-export async function resolveVectorSearchIdentity(
+export function resolveVectorSearchIdentity(
   port: EmbeddingPort,
   vectorIndex: VectorIndexPort
+): Promise<SearchPartition> {
+  return resolveSearchPartition(port, vectorIndex, false);
+}
+
+async function resolveSearchPartition(
+  port: EmbeddingPort,
+  vectorIndex: VectorIndexPort,
+  waited: boolean
 ): Promise<SearchPartition> {
   const primary = embeddingPartitionIdentity(port);
-  const db = getVectorIndexDatabase(vectorIndex);
-  if (!primary || !db) return { identity: primary };
+  const indexDb = getVectorIndexDatabase(vectorIndex);
+  if (!primary || !indexDb) return { identity: primary };
+  const db = writerConnection(indexDb);
   let resolved;
   try {
     resolved = await resolveRuntimePartition(db, port, primary);
@@ -61,6 +76,28 @@ export async function resolveVectorSearchIdentity(
   );
   if (use.kind === "unavailable")
     return { unavailable: { reason: use.reason, building: use.building } };
+  // A first query on an upgraded index can migrate its legacy partition inside
+  // a sync batch still open on the shared connection; the read connection sees
+  // it only once that batch commits. Wait for that one batch, then resolve
+  // again (the migration commits with it, or reruns if the batch rolled back).
+  const partitionId = identityPartitionId(resolved.identity);
+  if (
+    indexDb !== db &&
+    isPartitionActivated(db, partitionId) &&
+    !isPartitionActivated(indexDb, partitionId)
+  ) {
+    if (!waited) {
+      await writerIdle(indexDb);
+      return resolveSearchPartition(port, vectorIndex, true);
+    }
+    return {
+      unavailable: {
+        reason:
+          "its vector partition is being migrated while the index syncs; retry shortly",
+        building: false,
+      },
+    };
+  }
   return { identity: resolved.identity };
 }
 

package/browser-extension/artifacts/gno-browser-clipper-v2.12.1.zip.sha256

@@ -1 +0,0 @@
-0b2bb11d4bf3e827e847426464808224e803bc7a84a4e065541a7d057f5cb651  gno-browser-clipper-v2.12.1.zip