@gmickel/gno 2.9.0 → 2.9.2

package/src/serve/watch-service.ts

@@ -15,6 +15,7 @@ import type { CollectionSyncResult, SyncOptions } from "../ingestion";
 import type { SqliteAdapter } from "../store/sqlite/adapter";
 import type { DocumentEvent, DocumentEventBus } from "./doc-events";
 import type { EmbedScheduler } from "./embed-scheduler";
+import type { WatcherRetryKind } from "./watch-service-events";
 import type {
   WatcherSnapshot,
   WatcherSnapshotBuildResult,
@@ -48,7 +49,11 @@ import {
 } from "./watch-service-lifecycle";
 import { runOwnedCollectionFlush } from "./watch-service-run-flush";
 import { beginSnapshotInit } from "./watch-service-snapshot";
-import { pendingHasWork, type CollectionPending } from "./watch-service-state";
+import {
+  emptyPending,
+  pendingHasWork,
+  type CollectionPending,
+} from "./watch-service-state";
 
 export interface CollectionWatchState {
   expectedCollections: string[];
@@ -110,6 +115,12 @@ interface CollectionWatchServiceOptions {
   snapshotEntryCeiling?: number;
   /** Shared writer lease taken (no wait) around each flush's writes. */
   acquireWriteLease?: () => Promise<(() => Promise<void>) | null>;
+  /**
+   * Queue a full reconcile of every watched collection at start, so changes
+   * made while no resident ran reach the index. Off for the daemon, whose
+   * initial sync (unless --no-sync-on-start) covers the same ground.
+   */
+  reconcileOnStart?: boolean;
 }
 
 export class CollectionWatchService {
@@ -126,7 +137,9 @@ export class CollectionWatchService {
   readonly #pendingByCollection = new Map<string, CollectionPending>();
   readonly #timers = new Map<string, ReturnType<typeof setTimeout>>();
   readonly #flushDeadlineAt = new Map<string, number>();
-  readonly #retryScheduled = new Set<string>();
+  readonly #retryScheduled = new Map<string, WatcherRetryKind>();
+  readonly #retryFailures = new Map<string, number>();
+  readonly #freshEvents = new Set<string>();
   readonly #syncing = new Set<string>();
   readonly #inFlightSyncs = new Set<Promise<void>>();
   readonly #suppressedPaths = new Map<string, number>();
@@ -149,6 +162,7 @@ export class CollectionWatchService {
   readonly #snapshotFs: WatcherSnapshotFs | undefined;
   readonly #snapshotEntryCeiling: number | undefined;
   readonly #acquireWriteLease: CollectionWatchServiceOptions["acquireWriteLease"];
+  readonly #reconcileOnStart: boolean;
   #nextCollectionGeneration = 0;
   #disposed = false;
   #lastEventAt: string | null = null;
@@ -173,11 +187,24 @@ export class CollectionWatchService {
     this.#snapshotFs = options.snapshotFs;
     this.#snapshotEntryCeiling = options.snapshotEntryCeiling;
     this.#acquireWriteLease = options.acquireWriteLease;
+    this.#reconcileOnStart = options.reconcileOnStart ?? false;
   }
 
   start(): void {
-    if (!this.#disposed) {
-      this.updateCollections(this.#collections);
+    if (this.#disposed) {
+      return;
+    }
+    this.updateCollections(this.#collections);
+    if (!this.#reconcileOnStart) {
+      return;
+    }
+    // Generation work waits for the snapshot baseline, then runs a full
+    // syncCollection through the normal flush path (write lease, failure
+    // backoff, source availability). Unwatchable collections queue nothing.
+    for (const name of this.#watchers.keys()) {
+      const pending = this.#pendingByCollection.get(name) ?? emptyPending();
+      pending.generationReconcile = true;
+      this.#pendingByCollection.set(name, pending);
     }
   }
 
@@ -216,6 +243,8 @@ export class CollectionWatchService {
     this.#timers.clear();
     this.#flushDeadlineAt.clear();
     this.#retryScheduled.clear();
+    this.#retryFailures.clear();
+    this.#freshEvents.clear();
     this.#watchers.clear();
     this.#watchRoots.clear();
     this.#collectionGenerations.clear();
@@ -303,6 +332,8 @@ export class CollectionWatchService {
       flushDeadlineAt: this.#flushDeadlineAt,
       timers: this.#timers,
       retryScheduled: this.#retryScheduled,
+      retryFailures: this.#retryFailures,
+      freshEvents: this.#freshEvents,
       inFlightSyncs: this.#inFlightSyncs,
       runFlush: (name) => this.#flushCollection(name),
     };
@@ -316,6 +347,8 @@ export class CollectionWatchService {
     this.#snapshotInit.delete(collectionName);
     this.#flushDeadlineAt.delete(collectionName);
     this.#retryScheduled.delete(collectionName);
+    this.#retryFailures.delete(collectionName);
+    this.#freshEvents.delete(collectionName);
     const timer = this.#timers.get(collectionName);
     if (timer) {
       clearTimeout(timer);

package/src/sessions/import-child.ts

@@ -9,8 +9,11 @@
  * store connection: the archive import lock, receipts and error codes are
  * unchanged.
  *
- * The child reads one request from stdin and writes one JSON result line to
- * stdout. The request carries the caller's loaded config: the child opens
+ * The child reads one request line from stdin and writes one JSON result line
+ * to stdout. The parent keeps stdin open for the child's lifetime: EOF means
+ * the parent died, and the child exits instead of importing on as an orphan.
+ * A resident kills its import children when it shuts down, and every import
+ * has a timeout. The request carries the caller's loaded config: the child opens
  * the index without syncing any config into it and imports with exactly the
  * config the server holds. From source the child runs this module; a
  * compiled executable re-runs itself with `IMPORT_CHILD_ENV` set, which the
@@ -87,12 +90,24 @@ async function importInThisProcess(
   }
 }
 
+/** Longest a single import may run before its child is killed. */
+export const SESSION_IMPORT_TIMEOUT_MS = 30 * 60_000;
+
+/** Import children still running in this process. */
+const liveChildren = new Set<Bun.Subprocess>();
+
+/** Kill every running import child (resident shutdown). */
+export function killImportChildren(): void {
+  for (const child of liveChildren) child.kill("SIGKILL");
+}
+
 /**
  * Import a registered source without blocking the caller's event loop.
  * Throws a `SessionsError` for typed failures and a plain `Error` otherwise.
  */
 export async function importInChildProcess(
-  request: ImportChildRequest
+  request: ImportChildRequest,
+  options: { timeoutMs?: number } = {}
 ): Promise<SessionImportReceipt> {
   // A compiled executable cannot run an external TS entry: it re-runs itself.
   // Shared check: covers the POSIX `/$bunfs/` and Windows `B:/~BUN/` roots.
@@ -100,15 +115,38 @@ export async function importInChildProcess(
   const child = Bun.spawn({
     cmd: compiled ? [process.execPath] : [process.execPath, import.meta.path],
     env: compiled ? { ...process.env, [IMPORT_CHILD_ENV]: "1" } : process.env,
-    stdin: new Blob([JSON.stringify(request)]),
+    // The request line, then held open as the child's lifeline.
+    stdin: "pipe",
     stdout: "pipe",
     stderr: "pipe",
   });
-  const [stdout, stderr, exitCode] = await Promise.all([
-    new Response(child.stdout).text(),
-    new Response(child.stderr).text(),
-    child.exited,
-  ]);
+  liveChildren.add(child);
+  const timeoutMs = options.timeoutMs ?? SESSION_IMPORT_TIMEOUT_MS;
+  let timedOut = false;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    child.kill("SIGKILL");
+  }, timeoutMs);
+  let output: [string, string, number];
+  try {
+    void child.stdin.write(`${JSON.stringify(request)}\n`);
+    void child.stdin.flush();
+    output = await Promise.all([
+      new Response(child.stdout).text(),
+      new Response(child.stderr).text(),
+      child.exited,
+    ]);
+  } finally {
+    clearTimeout(timer);
+    liveChildren.delete(child);
+    void Promise.resolve(child.stdin.end()).catch(() => undefined);
+  }
+  const [stdout, stderr, exitCode] = output;
+  if (timedOut) {
+    throw new Error(
+      `session import timed out after ${Math.round(timeoutMs / 1000)} s`
+    );
+  }
   const line = stdout.trim().split("\n").at(-1) ?? "";
   let result: ImportChildResult | null = null;
   try {
@@ -126,15 +164,43 @@ export async function importInChildProcess(
   throw new Error(result.message);
 }
 
-/** Child entry: one request on stdin, one JSON result line on stdout. */
+/** Read the request line; afterwards stdin is only the lifeline. */
+async function readRequestLine(
+  reader: ReadableStreamDefaultReader<Uint8Array>
+): Promise<string> {
+  const decoder = new TextDecoder();
+  let text = "";
+  while (!text.includes("\n")) {
+    const chunk = await reader.read();
+    if (chunk.done) break;
+    text += decoder.decode(chunk.value, { stream: true });
+  }
+  return text.split("\n")[0] ?? "";
+}
+
+/** Child entry: one request line on stdin, one JSON result line on stdout. */
 export async function runImportChild(): Promise<void> {
   // Nothing this child starts may re-enter child mode.
   delete process.env[IMPORT_CHILD_ENV];
+  const reader = Bun.stdin.stream().getReader();
+  let finished = false;
   let result: ImportChildResult;
   try {
     const request = JSON.parse(
-      await new Response(Bun.stdin.stream()).text()
+      await readRequestLine(reader)
     ) as ImportChildRequest;
+    // The parent holds stdin open until it has our result; EOF before that
+    // means it died. Noticed between import steps.
+    void (async () => {
+      try {
+        while (!(await reader.read()).done) {
+          // nothing is written after the request
+        }
+      } catch {
+        // a broken pipe is the same as EOF
+      }
+      if (!finished) process.exit(1);
+    })();
     result = { ok: true, receipt: await importInThisProcess(request) };
   } catch (error) {
     result =
@@ -146,7 +212,9 @@ export async function runImportChild(): Promise<void> {
             message: error instanceof Error ? error.message : String(error),
           };
   }
+  finished = true;
   process.stdout.write(`${JSON.stringify(result)}\n`);
+  void reader.cancel().catch(() => undefined);
 }
 
 if (import.meta.main) await runImportChild();

package/src/store/migrations/035-vector-length-index.ts

@@ -0,0 +1,30 @@
+/**
+ * Migration: index stored vector byte lengths per model.
+ *
+ * Status checks the stored embedding dimensions for the active model by
+ * comparing the smallest and largest vector in its partition. Without an
+ * index that is a scan of every vector on each status build (about 0.6 ms
+ * per thousand vectors), run on the resident's main thread. With this
+ * covering index each bound is a single index seek.
+ *
+ * @module src/store/migrations/035-vector-length-index
+ */
+
+import type { Database } from "bun:sqlite";
+
+import type { Migration } from "./runner";
+
+export const migration: Migration = {
+  version: 35,
+  name: "vector_length_index",
+
+  up(db: Database): void {
+    db.exec(
+      "CREATE INDEX IF NOT EXISTS idx_vectors_model_bytes ON content_vectors(model, length(embedding))"
+    );
+  },
+
+  down(db: Database): void {
+    db.exec("DROP INDEX IF EXISTS idx_vectors_model_bytes");
+  },
+};

package/src/store/migrations/index.ts

@@ -48,6 +48,7 @@ import { migration as m031 } from "./031-runtime-independent-vectors";
 import { migration as m032 } from "./032-vector-runtime-callers";
 import { migration as m033 } from "./033-drop-documents-active-index";
 import { migration as m034 } from "./034-collection-link-workspace";
+import { migration as m035 } from "./035-vector-length-index";
 
 /** All migrations in order */
 export const migrations = [
@@ -85,4 +86,5 @@ export const migrations = [
   m032,
   m033,
   m034,
+  m035,
 ];

package/src/store/sqlite/adapter.ts

@@ -979,7 +979,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         }
       });
 
-      transaction();
+      // Read-then-write: take the write lock up front. A deferred transaction
+      // whose snapshot goes stale (another process commits meanwhile) fails at
+      // once with "database is locked" instead of waiting out the busy timeout.
+      transaction.immediate();
       return ok(undefined);
     } catch (cause) {
       return err(

package/src/store/vector/freshness.ts

@@ -35,14 +35,21 @@ export function getStoredEmbeddingDimensions(
     }
   }
   // Validate the entire model partition: never trust an arbitrary first blob.
-  const row = db
-    .prepare(`SELECT MIN(length(embedding)) AS smallest,
-    MAX(length(embedding)) AS largest FROM content_vectors WHERE model = ?`)
-    .get(model) as { smallest: number | null; largest: number | null };
-  const bytes = row.smallest;
+  // One aggregate per query so each bound is a seek on idx_vectors_model_bytes
+  // (MIN and MAX together would scan every vector), read in one transaction
+  // so both bounds see the same snapshot while another process writes.
+  const bound = (aggregate: "MIN" | "MAX"): number | null =>
+    (
+      db
+        .prepare(
+          `SELECT ${aggregate}(length(embedding)) AS bytes FROM content_vectors WHERE model = ?`
+        )
+        .get(model) as { bytes: number | null }
+    ).bytes;
+  const [bytes, largest] = db.transaction(() => [bound("MIN"), bound("MAX")])();
   if (
     !bytes ||
-    bytes !== row.largest ||
+    bytes !== largest ||
     bytes % Float32Array.BYTES_PER_ELEMENT !== 0
   ) {
     return undefined;

package/browser-extension/artifacts/gno-browser-clipper-v2.9.0.zip.sha256

@@ -1 +0,0 @@
-2d8fa6721fe305e2fd70450405e255900d028b187c884bbf74f29a3a33d27d3c  gno-browser-clipper-v2.9.0.zip