@gmickel/gno 1.34.4 → 1.34.6

package/src/serve/watch-service-lifecycle.ts

@@ -0,0 +1,219 @@
+/**
+ * Collection watcher lifecycle: fingerprinting and updateCollections.
+ *
+ * @module src/serve/watch-service-lifecycle
+ */
+
+import type { FSWatcher } from "node:fs";
+
+// node:path — Bun has no path utilities
+import { normalize } from "node:path";
+
+import type { Collection } from "../config/types";
+import type { SyncOptions } from "../ingestion";
+import type { WatcherSnapshot } from "./watch-snapshot";
+
+import { emptyPending, type CollectionPending } from "./watch-service-state";
+
+export interface WatchLifecycleHost {
+  disposed: () => boolean;
+  getCollections: () => Collection[];
+  setCollections: (collections: Collection[]) => void;
+  getSyncOptions: () => SyncOptions;
+  setSyncOptions: (syncOptions: SyncOptions) => void;
+  watchers: Map<string, FSWatcher>;
+  watchRoots: Map<string, string>;
+  collectionFingerprints: Map<string, string>;
+  collectionGenerations: Map<string, number>;
+  nextGeneration: () => number;
+  failedCollections: Map<string, string>;
+  snapshots: Map<string, WatcherSnapshot>;
+  snapshotReady: Map<string, boolean>;
+  snapshotInit: Map<string, Promise<void>>;
+  /** Collections currently mid-flush (ABA ownership). */
+  syncing: Set<string>;
+  /**
+   * Pending work by collection. Config/generation invalidation marks dirty work
+   * forceFallback + durable generation reconcile so a new baseline cannot absorb it.
+   */
+  pendingByCollection: Map<string, CollectionPending>;
+  clearCollectionRuntimeState: (collectionName: string) => void;
+  beginSnapshotInit: (collection: Collection) => void;
+  watchFactory: (
+    path: string,
+    options: { recursive: boolean },
+    listener: (
+      eventType: string,
+      filename: string | Buffer | null | undefined
+    ) => void
+  ) => FSWatcher;
+  onWatchEvent: (
+    collectionName: string,
+    watchedRoot: string,
+    filename: string | Buffer | null | undefined
+  ) => void;
+}
+
+export function watcherCollectionFingerprint(
+  collection: Collection,
+  syncOptions: SyncOptions
+): string {
+  return JSON.stringify({
+    path: normalize(collection.path),
+    pattern: collection.pattern,
+    include: collection.include,
+    exclude: collection.exclude,
+    languageHint: collection.languageHint ?? null,
+    recordAdapters: collection.recordAdapters ?? null,
+    limits: syncOptions.limits ?? null,
+    concurrency: syncOptions.concurrency ?? null,
+    contentTypeRules: syncOptions.contentTypeRules ?? null,
+    contentTypeRulesFingerprint:
+      syncOptions.contentTypeRulesFingerprint ?? null,
+    projectTypedEdges: syncOptions.projectTypedEdges ?? null,
+  });
+}
+
+/**
+ * Drop generation/failed tombstones when the name is no longer configured and
+ * no in-flight flush still needs the ownership token.
+ */
+export function clearLifecycleTombstones(
+  host: WatchLifecycleHost,
+  collectionName: string
+): void {
+  if (host.syncing.has(collectionName)) {
+    return;
+  }
+  if (host.getCollections().some((entry) => entry.name === collectionName)) {
+    return;
+  }
+  host.collectionGenerations.delete(collectionName);
+  host.failedCollections.delete(collectionName);
+  host.collectionFingerprints.delete(collectionName);
+}
+
+/**
+ * Reconcile active watchers with the desired collection set.
+ * Closes removed/moved roots, bumps generations on config change, and starts
+ * new recursive watchers (capturing events before snapshot baseline init).
+ */
+export function applyCollectionUpdate(
+  host: WatchLifecycleHost,
+  collections: Collection[],
+  syncOptions?: SyncOptions
+): void {
+  if (host.disposed()) {
+    return;
+  }
+  if (syncOptions) {
+    host.setSyncOptions(syncOptions);
+  }
+  const nextByName = new Map(
+    collections.map((collection) => [collection.name, collection])
+  );
+
+  for (const [collectionName, watcher] of host.watchers) {
+    const nextCollection = nextByName.get(collectionName);
+    const nextRoot = nextCollection
+      ? normalize(nextCollection.path)
+      : undefined;
+    if (
+      nextRoot === undefined ||
+      nextRoot !== host.watchRoots.get(collectionName)
+    ) {
+      watcher.close();
+      host.watchers.delete(collectionName);
+      host.clearCollectionRuntimeState(collectionName);
+      if (nextRoot === undefined) {
+        // Removed: bump generation for in-flight ABA, then clear if idle.
+        host.collectionGenerations.set(collectionName, host.nextGeneration());
+        host.failedCollections.delete(collectionName);
+        clearLifecycleTombstones(host, collectionName);
+      } else {
+        host.failedCollections.delete(collectionName);
+      }
+    }
+  }
+
+  // Fingerprints / generations / failed for names no longer configured.
+  for (const collectionName of [
+    ...host.collectionFingerprints.keys(),
+    ...host.collectionGenerations.keys(),
+    ...host.failedCollections.keys(),
+  ]) {
+    if (nextByName.has(collectionName)) {
+      continue;
+    }
+    host.collectionFingerprints.delete(collectionName);
+    if (!host.collectionGenerations.has(collectionName)) {
+      host.collectionGenerations.set(collectionName, host.nextGeneration());
+    } else if (!host.syncing.has(collectionName)) {
+      host.collectionGenerations.set(collectionName, host.nextGeneration());
+    }
+    host.clearCollectionRuntimeState(collectionName);
+    clearLifecycleTombstones(host, collectionName);
+  }
+
+  host.setCollections(collections);
+  for (const collection of collections) {
+    const fingerprint = watcherCollectionFingerprint(
+      collection,
+      host.getSyncOptions()
+    );
+    const previousFingerprint = host.collectionFingerprints.get(
+      collection.name
+    );
+    if (previousFingerprint === fingerprint) {
+      continue;
+    }
+    host.collectionFingerprints.set(collection.name, fingerprint);
+    host.collectionGenerations.set(collection.name, host.nextGeneration());
+    host.snapshots.delete(collection.name);
+    host.snapshotReady.set(collection.name, false);
+    host.snapshotInit.delete(collection.name);
+    // Initial start (no prior fingerprint): snapshot only — no full sync.
+    // Every later material change (idle same-root, root replacement, exact-only
+    // pending, dirty work) durably enqueues generation reconciliation.
+    if (previousFingerprint !== undefined) {
+      const pending =
+        host.pendingByCollection.get(collection.name) ?? emptyPending();
+      pending.generationReconcile = true;
+      // Queued ambiguous work must not be absorbed by the new baseline.
+      if (pending.dirty.size > 0 || pending.overflow || pending.forceFallback) {
+        pending.forceFallback = true;
+      }
+      host.pendingByCollection.set(collection.name, pending);
+    }
+    if (host.watchers.has(collection.name)) {
+      host.beginSnapshotInit(collection);
+    }
+  }
+
+  for (const collection of host.getCollections()) {
+    if (host.watchers.has(collection.name)) {
+      continue;
+    }
+    try {
+      const watchedRoot = normalize(collection.path);
+      // Capture events BEFORE async snapshot baseline construction.
+      const watcher = host.watchFactory(
+        collection.path,
+        { recursive: true },
+        (_eventType, filename) => {
+          host.onWatchEvent(collection.name, watchedRoot, filename);
+        }
+      );
+      host.watchers.set(collection.name, watcher);
+      host.watchRoots.set(collection.name, watchedRoot);
+      host.failedCollections.delete(collection.name);
+      host.snapshotReady.set(collection.name, false);
+      host.beginSnapshotInit(collection);
+    } catch (error) {
+      host.failedCollections.set(
+        collection.name,
+        error instanceof Error ? error.message : "watch unavailable"
+      );
+    }
+  }
+}

package/src/serve/watch-service-run-flush.ts

@@ -0,0 +1,236 @@
+/**
+ * Owned collection flush orchestration for CollectionWatchService.
+ *
+ * @module src/serve/watch-service-run-flush
+ */
+
+// node:path — Bun has no path utilities
+import { normalize } from "node:path";
+
+import type { Collection } from "../config/types";
+import type { CollectionSyncResult, SyncOptions } from "../ingestion";
+import type { SqliteAdapter } from "../store/sqlite/adapter";
+import type { WatchQueueHost } from "./watch-service-events";
+import type { CollectionPending } from "./watch-service-state";
+import type { WatcherSnapshot, WatcherSnapshotFs } from "./watch-snapshot";
+
+import {
+  requeueAfterFailure,
+  requeueGenerationReconcile,
+  startFlush,
+} from "./watch-service-events";
+import { flushCollectionOnce } from "./watch-service-flush";
+import {
+  emptyPending,
+  pendingHasWork,
+  takePending,
+} from "./watch-service-state";
+
+export interface RunFlushContext {
+  collectionName: string;
+  disposed: () => boolean;
+  collections: () => Collection[];
+  store: SqliteAdapter;
+  syncOptions: () => SyncOptions;
+  pendingByCollection: Map<string, CollectionPending>;
+  flushDeadlineAt: Map<string, number>;
+  syncing: Set<string>;
+  retryScheduled: Set<string>;
+  collectionGenerations: Map<string, number>;
+  snapshots: Map<string, WatcherSnapshot>;
+  snapshotReady: Map<string, boolean>;
+  snapshotInit: Map<string, Promise<void>>;
+  suppressedPaths: Map<string, number>;
+  clock: () => number;
+  queueHost: WatchQueueHost;
+  callbacks: {
+    onSyncStart?: (event: { collection: string; relPaths: string[] }) => void;
+    onSyncComplete?: (event: {
+      collection: string;
+      relPaths: string[];
+      result: CollectionSyncResult;
+    }) => void;
+    onSyncError?: (event: {
+      collection: string;
+      relPaths: string[];
+      error: unknown;
+    }) => void;
+  } | null;
+  onAfterSync: (collection: Collection, relPaths: string[]) => void;
+  beginSnapshotInit: (collection: Collection) => void;
+  clearLifecycleTombstones: (collectionName: string) => void;
+  pruneSuppression: () => void;
+  notifySettledIfIdle: () => void;
+  /** Optional injectable FS for unsupported-handle / test seams. */
+  snapshotFs?: WatcherSnapshotFs;
+  /** Test seam: lower snapshot entry ceiling for overflow→full proofs. */
+  snapshotEntryCeiling?: number;
+}
+
+/**
+ * Drain one collection's pending work under a generation+root ownership token.
+ */
+export async function runOwnedCollectionFlush(
+  ctx: RunFlushContext
+): Promise<void> {
+  const { collectionName } = ctx;
+  if (ctx.disposed()) {
+    return;
+  }
+  const pending = ctx.pendingByCollection.get(collectionName);
+  if (!pendingHasWork(pending)) {
+    ctx.flushDeadlineAt.delete(collectionName);
+    return;
+  }
+  if (ctx.syncing.has(collectionName) || !pending) {
+    return;
+  }
+
+  const collection = ctx
+    .collections()
+    .find((entry) => entry.name === collectionName);
+  if (!collection) {
+    ctx.pendingByCollection.delete(collectionName);
+    ctx.flushDeadlineAt.delete(collectionName);
+    return;
+  }
+
+  const taken = takePending(pending);
+  ctx.pendingByCollection.set(collectionName, emptyPending());
+  ctx.flushDeadlineAt.delete(collectionName);
+  ctx.syncing.add(collectionName);
+
+  const ownerGeneration = ctx.collectionGenerations.get(collectionName) ?? 0;
+  const ownerRoot = normalize(collection.path);
+  const stillOwner = (): boolean => {
+    if (ctx.disposed()) {
+      return false;
+    }
+    const current = ctx
+      .collections()
+      .find((entry) => entry.name === collectionName);
+    return (
+      current !== undefined &&
+      (ctx.collectionGenerations.get(collectionName) ?? 0) ===
+        ownerGeneration &&
+      normalize(current.path) === ownerRoot
+    );
+  };
+
+  try {
+    const outcome = await flushCollectionOnce({
+      collection,
+      collectionName,
+      store: ctx.store,
+      syncOptions: ctx.syncOptions(),
+      exactTaken: taken.exact,
+      dirtyTaken: taken.dirty,
+      forceFallback: taken.forceFallback,
+      overflow: taken.overflow,
+      generationReconcile: taken.generationReconcile,
+      previousSnapshot: ctx.snapshots.get(collectionName) ?? null,
+      ownerGeneration,
+      ownerRoot,
+      disposed: ctx.disposed,
+      getCurrentCollection: () =>
+        ctx.collections().find((entry) => entry.name === collectionName),
+      getCurrentGeneration: () =>
+        ctx.collectionGenerations.get(collectionName) ?? 0,
+      getCurrentSyncOptions: ctx.syncOptions,
+      clock: ctx.clock,
+      suppressedPaths: ctx.suppressedPaths,
+      snapshotFs: ctx.snapshotFs,
+      snapshotEntryCeiling: ctx.snapshotEntryCeiling,
+      onSyncStart: (relPaths) => {
+        if (!stillOwner()) {
+          return;
+        }
+        ctx.callbacks?.onSyncStart?.({
+          collection: collection.name,
+          relPaths,
+        });
+      },
+      onSyncComplete: (relPaths, result) => {
+        ctx.callbacks?.onSyncComplete?.({
+          collection: collectionName,
+          relPaths,
+          result,
+        });
+      },
+      onSyncError: (relPaths, error) => {
+        ctx.callbacks?.onSyncError?.({
+          collection: collectionName,
+          relPaths,
+          error,
+        });
+      },
+      onAfterSync: ctx.onAfterSync,
+      commitSnapshot: (snapshot) => {
+        if (!stillOwner()) {
+          return;
+        }
+        ctx.snapshots.set(collectionName, snapshot);
+      },
+      invalidateSnapshot: (current) => {
+        const live = ctx
+          .collections()
+          .find((entry) => entry.name === collectionName);
+        if (!live || normalize(live.path) !== normalize(current.path)) {
+          return;
+        }
+        ctx.snapshots.delete(collectionName);
+        ctx.snapshotReady.set(collectionName, false);
+        ctx.snapshotInit.delete(collectionName);
+        ctx.beginSnapshotInit(current);
+      },
+      requeue: (exact, dirty, forceFlags) => {
+        if (!stillOwner()) {
+          return;
+        }
+        requeueAfterFailure(
+          ctx.queueHost,
+          collectionName,
+          exact,
+          dirty,
+          forceFlags
+        );
+      },
+      requeueGeneration: () => {
+        const live = ctx
+          .collections()
+          .find((entry) => entry.name === collectionName);
+        if (!live) {
+          return;
+        }
+        requeueGenerationReconcile(ctx.queueHost, collectionName);
+      },
+    });
+    if (
+      outcome.status === "failed" &&
+      outcome.error &&
+      !(
+        outcome.error instanceof Error &&
+        (outcome.error.message ===
+          "One or more paths failed during watcher sync" ||
+          outcome.error.message ===
+            "One or more paths failed during watcher generation reconcile")
+      )
+    ) {
+      throw outcome.error;
+    }
+  } finally {
+    ctx.syncing.delete(collectionName);
+    ctx.clearLifecycleTombstones(collectionName);
+    ctx.pruneSuppression();
+    if (!ctx.disposed()) {
+      if (
+        pendingHasWork(ctx.pendingByCollection.get(collectionName)) &&
+        !ctx.retryScheduled.has(collectionName)
+      ) {
+        startFlush(ctx.queueHost, collectionName);
+      } else if (!pendingHasWork(ctx.pendingByCollection.get(collectionName))) {
+        ctx.notifySettledIfIdle();
+      }
+    }
+  }
+}

package/src/serve/watch-service-snapshot.ts

@@ -0,0 +1,101 @@
+/**
+ * Per-collection snapshot baseline initialization for the resident watcher.
+ *
+ * @module src/serve/watch-service-snapshot
+ */
+
+// node:path — Bun has no path utilities
+import { normalize } from "node:path";
+
+import type { Collection } from "../config/types";
+import type {
+  WatcherSnapshot,
+  WatcherSnapshotBuildResult,
+  WatcherSnapshotOptions,
+} from "./watch-snapshot";
+
+import { buildWatcherSnapshot } from "./watch-snapshot";
+
+export interface SnapshotInitHost {
+  disposed: () => boolean;
+  getGeneration: (collectionName: string) => number;
+  getRoot: (collectionName: string) => string | undefined;
+  setSnapshot: (collectionName: string, snapshot: WatcherSnapshot) => void;
+  clearSnapshot: (collectionName: string) => void;
+  setReady: (collectionName: string, ready: boolean) => void;
+  getInit: (collectionName: string) => Promise<void> | undefined;
+  setInit: (collectionName: string, init: Promise<void> | undefined) => void;
+  onReadyWithPending: (collectionName: string) => void;
+  /** Optional injectable builder for hung/slow-init tests. */
+  buildSnapshot?: (
+    rootAbs: string,
+    options?: WatcherSnapshotOptions
+  ) => Promise<WatcherSnapshotBuildResult>;
+}
+
+/** Start (or supersede) baseline construction; events may already be buffering. */
+export function beginSnapshotInit(
+  host: SnapshotInitHost,
+  collection: Collection
+): void {
+  if (host.disposed()) {
+    return;
+  }
+  const generation = host.getGeneration(collection.name);
+  const root = normalize(collection.path);
+  let init!: Promise<void>;
+  init = runSnapshotInit(host, collection.name, root, generation, () => init);
+  host.setInit(collection.name, init);
+  void init.catch(() => undefined);
+}
+
+async function runSnapshotInit(
+  host: SnapshotInitHost,
+  collectionName: string,
+  root: string,
+  generation: number,
+  getInit: () => Promise<void>
+): Promise<void> {
+  try {
+    const builder = host.buildSnapshot ?? buildWatcherSnapshot;
+    const built = await builder(root);
+    if (host.disposed()) {
+      return;
+    }
+    if (
+      host.getGeneration(collectionName) !== generation ||
+      host.getRoot(collectionName) !== root
+    ) {
+      return;
+    }
+    if (built.status === "ok") {
+      host.setSnapshot(collectionName, built.snapshot);
+    } else {
+      host.clearSnapshot(collectionName);
+    }
+  } catch {
+    if (
+      !host.disposed() &&
+      host.getGeneration(collectionName) === generation &&
+      host.getRoot(collectionName) === root
+    ) {
+      host.clearSnapshot(collectionName);
+    }
+  }
+
+  if (host.getInit(collectionName) !== getInit()) {
+    return;
+  }
+  host.setInit(collectionName, undefined);
+  if (host.disposed()) {
+    return;
+  }
+  // Readiness flips even on init failure so forceFallback classification can run.
+  if (
+    host.getGeneration(collectionName) === generation &&
+    host.getRoot(collectionName) === root
+  ) {
+    host.setReady(collectionName, true);
+    host.onReadyWithPending(collectionName);
+  }
+}

package/src/serve/watch-service-state.ts

@@ -0,0 +1,146 @@
+/**
+ * Per-collection pending queues, caps, and flush scheduling helpers.
+ *
+ * @module src/serve/watch-service-state
+ */
+
+import {
+  WATCHER_MAX_DIRTY_HINTS,
+  WATCHER_MAX_EXACT_PATHS,
+  WATCHER_MAX_FLUSH_DELAY_MS,
+  WATCHER_FLUSH_DEBOUNCE_MS,
+  addToCappedSet,
+} from "./watch-reconciliation";
+
+export interface CollectionPending {
+  exact: Set<string>;
+  dirty: Set<string>;
+  /** When true, dirty overflow forced a root-wide fallback hint. */
+  overflow: boolean;
+  /**
+   * Ambiguous work arrived before snapshot readiness for this generation.
+   * Classification must use store/disk fallback so baseline absorption cannot
+   * suppress content-hash of present eligible finals.
+   */
+  forceFallback: boolean;
+  /**
+   * Config-generation full reconcile failed or was interrupted; next flush must
+   * retry even if exact/dirty event hints are exhausted.
+   */
+  generationReconcile: boolean;
+}
+
+export function emptyPending(): CollectionPending {
+  return {
+    exact: new Set(),
+    dirty: new Set(),
+    overflow: false,
+    forceFallback: false,
+    generationReconcile: false,
+  };
+}
+
+export function pendingHasWork(
+  pending: CollectionPending | undefined
+): boolean {
+  return Boolean(
+    pending &&
+    (pending.exact.size > 0 ||
+      pending.dirty.size > 0 ||
+      pending.generationReconcile)
+  );
+}
+
+export function queueExactPath(
+  pending: CollectionPending,
+  relPath: string,
+  maxExact: number = WATCHER_MAX_EXACT_PATHS
+): CollectionPending {
+  const result = addToCappedSet(pending.exact, relPath, maxExact);
+  if (result === "overflow") {
+    pending.overflow = true;
+    pending.dirty.add("");
+  }
+  return pending;
+}
+
+export function queueDirtyHint(
+  pending: CollectionPending,
+  hint: string,
+  maxDirty: number = WATCHER_MAX_DIRTY_HINTS
+): CollectionPending {
+  const result = addToCappedSet(pending.dirty, hint, maxDirty);
+  if (result === "overflow") {
+    pending.overflow = true;
+    pending.dirty.clear();
+    pending.dirty.add("");
+  }
+  return pending;
+}
+
+/** Flags that must survive take → failed flush → requeue. */
+export interface PendingForceFlags {
+  forceFallback: boolean;
+  overflow: boolean;
+}
+
+/** Drain pending work for one flush attempt. */
+export function takePending(pending: CollectionPending): {
+  exact: string[];
+  dirty: string[];
+  forceFallback: boolean;
+  overflow: boolean;
+  generationReconcile: boolean;
+} {
+  return {
+    exact: [...pending.exact],
+    dirty: pending.overflow ? ["", ...pending.dirty] : [...pending.dirty],
+    forceFallback: pending.forceFallback || pending.overflow,
+    overflow: pending.overflow,
+    generationReconcile: pending.generationReconcile,
+  };
+}
+
+/** Restore durable force flags after a failed classification/sync attempt. */
+export function applyPendingForceFlags(
+  pending: CollectionPending,
+  flags: PendingForceFlags | undefined
+): CollectionPending {
+  if (!flags) {
+    return pending;
+  }
+  if (flags.forceFallback) {
+    pending.forceFallback = true;
+  }
+  if (flags.overflow) {
+    pending.overflow = true;
+    pending.dirty.add("");
+  }
+  return pending;
+}
+
+export interface FlushSchedule {
+  delayMs: number;
+  deadlineAt: number;
+}
+
+/**
+ * Compute debounce delay respecting the hard maximum flush deadline.
+ * `deadlineAt` is created on the first event of a window and retained until drain.
+ */
+export function computeFlushDelay(options: {
+  nowMs: number;
+  existingDeadlineAt: number | undefined;
+  debounceMs?: number;
+  maxFlushDelayMs?: number;
+}): FlushSchedule {
+  const debounceMs = options.debounceMs ?? WATCHER_FLUSH_DEBOUNCE_MS;
+  const maxFlushDelayMs = options.maxFlushDelayMs ?? WATCHER_MAX_FLUSH_DELAY_MS;
+  const deadlineAt =
+    options.existingDeadlineAt ?? options.nowMs + maxFlushDelayMs;
+  const untilDeadline = Math.max(0, deadlineAt - options.nowMs);
+  return {
+    delayMs: Math.min(debounceMs, untilDeadline),
+    deadlineAt,
+  };
+}