@gmickel/gno 1.34.5 → 1.35.0

package/src/serve/watch-service-lifecycle.ts

@@ -0,0 +1,221 @@
+/**
+ * Collection watcher lifecycle: fingerprinting and updateCollections.
+ *
+ * @module src/serve/watch-service-lifecycle
+ */
+
+import type { FSWatcher } from "node:fs";
+
+// node:path — Bun has no path utilities
+import { normalize } from "node:path";
+
+import type { Collection } from "../config/types";
+import type { SyncOptions } from "../ingestion";
+import type { WatcherSnapshot } from "./watch-snapshot";
+
+import { resolveSourceAvailability } from "../ingestion/source-availability";
+import { emptyPending, type CollectionPending } from "./watch-service-state";
+
+export interface WatchLifecycleHost {
+  disposed: () => boolean;
+  getCollections: () => Collection[];
+  setCollections: (collections: Collection[]) => void;
+  getSyncOptions: () => SyncOptions;
+  setSyncOptions: (syncOptions: SyncOptions) => void;
+  watchers: Map<string, FSWatcher>;
+  watchRoots: Map<string, string>;
+  collectionFingerprints: Map<string, string>;
+  collectionGenerations: Map<string, number>;
+  nextGeneration: () => number;
+  failedCollections: Map<string, string>;
+  snapshots: Map<string, WatcherSnapshot>;
+  snapshotReady: Map<string, boolean>;
+  snapshotInit: Map<string, Promise<void>>;
+  /** Collections currently mid-flush (ABA ownership). */
+  syncing: Set<string>;
+  /**
+   * Pending work by collection. Config/generation invalidation marks dirty work
+   * forceFallback + durable generation reconcile so a new baseline cannot absorb it.
+   */
+  pendingByCollection: Map<string, CollectionPending>;
+  clearCollectionRuntimeState: (collectionName: string) => void;
+  beginSnapshotInit: (collection: Collection) => void;
+  watchFactory: (
+    path: string,
+    options: { recursive: boolean },
+    listener: (
+      eventType: string,
+      filename: string | Buffer | null | undefined
+    ) => void
+  ) => FSWatcher;
+  onWatchEvent: (
+    collectionName: string,
+    watchedRoot: string,
+    filename: string | Buffer | null | undefined
+  ) => void;
+}
+
+export function watcherCollectionFingerprint(
+  collection: Collection,
+  syncOptions: SyncOptions
+): string {
+  return JSON.stringify({
+    path: normalize(collection.path),
+    pattern: collection.pattern,
+    include: collection.include,
+    exclude: collection.exclude,
+    languageHint: collection.languageHint ?? null,
+    recordAdapters: collection.recordAdapters ?? null,
+    sourceAvailability: resolveSourceAvailability(collection, syncOptions),
+    limits: syncOptions.limits ?? null,
+    concurrency: syncOptions.concurrency ?? null,
+    contentTypeRules: syncOptions.contentTypeRules ?? null,
+    contentTypeRulesFingerprint:
+      syncOptions.contentTypeRulesFingerprint ?? null,
+    projectTypedEdges: syncOptions.projectTypedEdges ?? null,
+  });
+}
+
+/**
+ * Drop generation/failed tombstones when the name is no longer configured and
+ * no in-flight flush still needs the ownership token.
+ */
+export function clearLifecycleTombstones(
+  host: WatchLifecycleHost,
+  collectionName: string
+): void {
+  if (host.syncing.has(collectionName)) {
+    return;
+  }
+  if (host.getCollections().some((entry) => entry.name === collectionName)) {
+    return;
+  }
+  host.collectionGenerations.delete(collectionName);
+  host.failedCollections.delete(collectionName);
+  host.collectionFingerprints.delete(collectionName);
+}
+
+/**
+ * Reconcile active watchers with the desired collection set.
+ * Closes removed/moved roots, bumps generations on config change, and starts
+ * new recursive watchers (capturing events before snapshot baseline init).
+ */
+export function applyCollectionUpdate(
+  host: WatchLifecycleHost,
+  collections: Collection[],
+  syncOptions?: SyncOptions
+): void {
+  if (host.disposed()) {
+    return;
+  }
+  if (syncOptions) {
+    host.setSyncOptions(syncOptions);
+  }
+  const nextByName = new Map(
+    collections.map((collection) => [collection.name, collection])
+  );
+
+  for (const [collectionName, watcher] of host.watchers) {
+    const nextCollection = nextByName.get(collectionName);
+    const nextRoot = nextCollection
+      ? normalize(nextCollection.path)
+      : undefined;
+    if (
+      nextRoot === undefined ||
+      nextRoot !== host.watchRoots.get(collectionName)
+    ) {
+      watcher.close();
+      host.watchers.delete(collectionName);
+      host.clearCollectionRuntimeState(collectionName);
+      if (nextRoot === undefined) {
+        // Removed: bump generation for in-flight ABA, then clear if idle.
+        host.collectionGenerations.set(collectionName, host.nextGeneration());
+        host.failedCollections.delete(collectionName);
+        clearLifecycleTombstones(host, collectionName);
+      } else {
+        host.failedCollections.delete(collectionName);
+      }
+    }
+  }
+
+  // Fingerprints / generations / failed for names no longer configured.
+  for (const collectionName of [
+    ...host.collectionFingerprints.keys(),
+    ...host.collectionGenerations.keys(),
+    ...host.failedCollections.keys(),
+  ]) {
+    if (nextByName.has(collectionName)) {
+      continue;
+    }
+    host.collectionFingerprints.delete(collectionName);
+    if (!host.collectionGenerations.has(collectionName)) {
+      host.collectionGenerations.set(collectionName, host.nextGeneration());
+    } else if (!host.syncing.has(collectionName)) {
+      host.collectionGenerations.set(collectionName, host.nextGeneration());
+    }
+    host.clearCollectionRuntimeState(collectionName);
+    clearLifecycleTombstones(host, collectionName);
+  }
+
+  host.setCollections(collections);
+  for (const collection of collections) {
+    const fingerprint = watcherCollectionFingerprint(
+      collection,
+      host.getSyncOptions()
+    );
+    const previousFingerprint = host.collectionFingerprints.get(
+      collection.name
+    );
+    if (previousFingerprint === fingerprint) {
+      continue;
+    }
+    host.collectionFingerprints.set(collection.name, fingerprint);
+    host.collectionGenerations.set(collection.name, host.nextGeneration());
+    host.snapshots.delete(collection.name);
+    host.snapshotReady.set(collection.name, false);
+    host.snapshotInit.delete(collection.name);
+    // Initial start (no prior fingerprint): snapshot only — no full sync.
+    // Every later material change (idle same-root, root replacement, exact-only
+    // pending, dirty work) durably enqueues generation reconciliation.
+    if (previousFingerprint !== undefined) {
+      const pending =
+        host.pendingByCollection.get(collection.name) ?? emptyPending();
+      pending.generationReconcile = true;
+      // Queued ambiguous work must not be absorbed by the new baseline.
+      if (pending.dirty.size > 0 || pending.overflow || pending.forceFallback) {
+        pending.forceFallback = true;
+      }
+      host.pendingByCollection.set(collection.name, pending);
+    }
+    if (host.watchers.has(collection.name)) {
+      host.beginSnapshotInit(collection);
+    }
+  }
+
+  for (const collection of host.getCollections()) {
+    if (host.watchers.has(collection.name)) {
+      continue;
+    }
+    try {
+      const watchedRoot = normalize(collection.path);
+      // Capture events BEFORE async snapshot baseline construction.
+      const watcher = host.watchFactory(
+        collection.path,
+        { recursive: true },
+        (_eventType, filename) => {
+          host.onWatchEvent(collection.name, watchedRoot, filename);
+        }
+      );
+      host.watchers.set(collection.name, watcher);
+      host.watchRoots.set(collection.name, watchedRoot);
+      host.failedCollections.delete(collection.name);
+      host.snapshotReady.set(collection.name, false);
+      host.beginSnapshotInit(collection);
+    } catch (error) {
+      host.failedCollections.set(
+        collection.name,
+        error instanceof Error ? error.message : "watch unavailable"
+      );
+    }
+  }
+}

package/src/serve/watch-service-run-flush.ts

@@ -0,0 +1,236 @@
+/**
+ * Owned collection flush orchestration for CollectionWatchService.
+ *
+ * @module src/serve/watch-service-run-flush
+ */
+
+// node:path — Bun has no path utilities
+import { normalize } from "node:path";
+
+import type { Collection } from "../config/types";
+import type { CollectionSyncResult, SyncOptions } from "../ingestion";
+import type { SqliteAdapter } from "../store/sqlite/adapter";
+import type { WatchQueueHost } from "./watch-service-events";
+import type { CollectionPending } from "./watch-service-state";
+import type { WatcherSnapshot, WatcherSnapshotFs } from "./watch-snapshot";
+
+import {
+  requeueAfterFailure,
+  requeueGenerationReconcile,
+  startFlush,
+} from "./watch-service-events";
+import { flushCollectionOnce } from "./watch-service-flush";
+import {
+  emptyPending,
+  pendingHasWork,
+  takePending,
+} from "./watch-service-state";
+
+export interface RunFlushContext {
+  collectionName: string;
+  disposed: () => boolean;
+  collections: () => Collection[];
+  store: SqliteAdapter;
+  syncOptions: () => SyncOptions;
+  pendingByCollection: Map<string, CollectionPending>;
+  flushDeadlineAt: Map<string, number>;
+  syncing: Set<string>;
+  retryScheduled: Set<string>;
+  collectionGenerations: Map<string, number>;
+  snapshots: Map<string, WatcherSnapshot>;
+  snapshotReady: Map<string, boolean>;
+  snapshotInit: Map<string, Promise<void>>;
+  suppressedPaths: Map<string, number>;
+  clock: () => number;
+  queueHost: WatchQueueHost;
+  callbacks: {
+    onSyncStart?: (event: { collection: string; relPaths: string[] }) => void;
+    onSyncComplete?: (event: {
+      collection: string;
+      relPaths: string[];
+      result: CollectionSyncResult;
+    }) => void;
+    onSyncError?: (event: {
+      collection: string;
+      relPaths: string[];
+      error: unknown;
+    }) => void;
+  } | null;
+  onAfterSync: (collection: Collection, relPaths: string[]) => void;
+  beginSnapshotInit: (collection: Collection) => void;
+  clearLifecycleTombstones: (collectionName: string) => void;
+  pruneSuppression: () => void;
+  notifySettledIfIdle: () => void;
+  /** Optional injectable FS for unsupported-handle / test seams. */
+  snapshotFs?: WatcherSnapshotFs;
+  /** Test seam: lower snapshot entry ceiling for overflow→full proofs. */
+  snapshotEntryCeiling?: number;
+}
+
+/**
+ * Drain one collection's pending work under a generation+root ownership token.
+ */
+export async function runOwnedCollectionFlush(
+  ctx: RunFlushContext
+): Promise<void> {
+  const { collectionName } = ctx;
+  if (ctx.disposed()) {
+    return;
+  }
+  const pending = ctx.pendingByCollection.get(collectionName);
+  if (!pendingHasWork(pending)) {
+    ctx.flushDeadlineAt.delete(collectionName);
+    return;
+  }
+  if (ctx.syncing.has(collectionName) || !pending) {
+    return;
+  }
+
+  const collection = ctx
+    .collections()
+    .find((entry) => entry.name === collectionName);
+  if (!collection) {
+    ctx.pendingByCollection.delete(collectionName);
+    ctx.flushDeadlineAt.delete(collectionName);
+    return;
+  }
+
+  const taken = takePending(pending);
+  ctx.pendingByCollection.set(collectionName, emptyPending());
+  ctx.flushDeadlineAt.delete(collectionName);
+  ctx.syncing.add(collectionName);
+
+  const ownerGeneration = ctx.collectionGenerations.get(collectionName) ?? 0;
+  const ownerRoot = normalize(collection.path);
+  const stillOwner = (): boolean => {
+    if (ctx.disposed()) {
+      return false;
+    }
+    const current = ctx
+      .collections()
+      .find((entry) => entry.name === collectionName);
+    return (
+      current !== undefined &&
+      (ctx.collectionGenerations.get(collectionName) ?? 0) ===
+        ownerGeneration &&
+      normalize(current.path) === ownerRoot
+    );
+  };
+
+  try {
+    const outcome = await flushCollectionOnce({
+      collection,
+      collectionName,
+      store: ctx.store,
+      syncOptions: ctx.syncOptions(),
+      exactTaken: taken.exact,
+      dirtyTaken: taken.dirty,
+      forceFallback: taken.forceFallback,
+      overflow: taken.overflow,
+      generationReconcile: taken.generationReconcile,
+      previousSnapshot: ctx.snapshots.get(collectionName) ?? null,
+      ownerGeneration,
+      ownerRoot,
+      disposed: ctx.disposed,
+      getCurrentCollection: () =>
+        ctx.collections().find((entry) => entry.name === collectionName),
+      getCurrentGeneration: () =>
+        ctx.collectionGenerations.get(collectionName) ?? 0,
+      getCurrentSyncOptions: ctx.syncOptions,
+      clock: ctx.clock,
+      suppressedPaths: ctx.suppressedPaths,
+      snapshotFs: ctx.snapshotFs,
+      snapshotEntryCeiling: ctx.snapshotEntryCeiling,
+      onSyncStart: (relPaths) => {
+        if (!stillOwner()) {
+          return;
+        }
+        ctx.callbacks?.onSyncStart?.({
+          collection: collection.name,
+          relPaths,
+        });
+      },
+      onSyncComplete: (relPaths, result) => {
+        ctx.callbacks?.onSyncComplete?.({
+          collection: collectionName,
+          relPaths,
+          result,
+        });
+      },
+      onSyncError: (relPaths, error) => {
+        ctx.callbacks?.onSyncError?.({
+          collection: collectionName,
+          relPaths,
+          error,
+        });
+      },
+      onAfterSync: ctx.onAfterSync,
+      commitSnapshot: (snapshot) => {
+        if (!stillOwner()) {
+          return;
+        }
+        ctx.snapshots.set(collectionName, snapshot);
+      },
+      invalidateSnapshot: (current) => {
+        const live = ctx
+          .collections()
+          .find((entry) => entry.name === collectionName);
+        if (!live || normalize(live.path) !== normalize(current.path)) {
+          return;
+        }
+        ctx.snapshots.delete(collectionName);
+        ctx.snapshotReady.set(collectionName, false);
+        ctx.snapshotInit.delete(collectionName);
+        ctx.beginSnapshotInit(current);
+      },
+      requeue: (exact, dirty, forceFlags) => {
+        if (!stillOwner()) {
+          return;
+        }
+        requeueAfterFailure(
+          ctx.queueHost,
+          collectionName,
+          exact,
+          dirty,
+          forceFlags
+        );
+      },
+      requeueGeneration: () => {
+        const live = ctx
+          .collections()
+          .find((entry) => entry.name === collectionName);
+        if (!live) {
+          return;
+        }
+        requeueGenerationReconcile(ctx.queueHost, collectionName);
+      },
+    });
+    if (
+      outcome.status === "failed" &&
+      outcome.error &&
+      !(
+        outcome.error instanceof Error &&
+        (outcome.error.message ===
+          "One or more paths failed during watcher sync" ||
+          outcome.error.message ===
+            "One or more paths failed during watcher generation reconcile")
+      )
+    ) {
+      throw outcome.error;
+    }
+  } finally {
+    ctx.syncing.delete(collectionName);
+    ctx.clearLifecycleTombstones(collectionName);
+    ctx.pruneSuppression();
+    if (!ctx.disposed()) {
+      if (
+        pendingHasWork(ctx.pendingByCollection.get(collectionName)) &&
+        !ctx.retryScheduled.has(collectionName)
+      ) {
+        startFlush(ctx.queueHost, collectionName);
+      } else if (!pendingHasWork(ctx.pendingByCollection.get(collectionName))) {
+        ctx.notifySettledIfIdle();
+      }
+    }
+  }
+}

package/src/serve/watch-service-snapshot.ts

@@ -0,0 +1,125 @@
+/**
+ * Per-collection snapshot baseline initialization for the resident watcher.
+ *
+ * @module src/serve/watch-service-snapshot
+ */
+
+// node:path — Bun has no path utilities
+import { normalize } from "node:path";
+
+import type { Collection } from "../config/types";
+import type { SyncOptions } from "../ingestion";
+import type {
+  WatcherSnapshot,
+  WatcherSnapshotBuildResult,
+  WatcherSnapshotOptions,
+} from "./watch-snapshot";
+
+import {
+  createDirectoryAvailability,
+  memoizeDirectoryAvailability,
+  resolveSourceAvailability,
+} from "../ingestion";
+import { buildWatcherSnapshot } from "./watch-snapshot";
+
+export interface SnapshotInitHost {
+  disposed: () => boolean;
+  getGeneration: (collectionName: string) => number;
+  getRoot: (collectionName: string) => string | undefined;
+  setSnapshot: (collectionName: string, snapshot: WatcherSnapshot) => void;
+  clearSnapshot: (collectionName: string) => void;
+  setReady: (collectionName: string, ready: boolean) => void;
+  getInit: (collectionName: string) => Promise<void> | undefined;
+  setInit: (collectionName: string, init: Promise<void> | undefined) => void;
+  onReadyWithPending: (collectionName: string) => void;
+  /** Current run-level overrides; omitted by narrow unit-test hosts. */
+  getSyncOptions?: () => SyncOptions;
+  /** Optional injectable builder for hung/slow-init tests. */
+  buildSnapshot?: (
+    rootAbs: string,
+    options?: WatcherSnapshotOptions
+  ) => Promise<WatcherSnapshotBuildResult>;
+}
+
+/** Start (or supersede) baseline construction; events may already be buffering. */
+export function beginSnapshotInit(
+  host: SnapshotInitHost,
+  collection: Collection
+): void {
+  if (host.disposed()) {
+    return;
+  }
+  const generation = host.getGeneration(collection.name);
+  const root = normalize(collection.path);
+  let init!: Promise<void>;
+  init = runSnapshotInit(
+    host,
+    collection.name,
+    root,
+    generation,
+    () => init,
+    collection
+  );
+  host.setInit(collection.name, init);
+  void init.catch(() => undefined);
+}
+
+async function runSnapshotInit(
+  host: SnapshotInitHost,
+  collectionName: string,
+  root: string,
+  generation: number,
+  getInit: () => Promise<void>,
+  collection: Collection
+): Promise<void> {
+  try {
+    const builder = host.buildSnapshot ?? buildWatcherSnapshot;
+    const availabilityMode = resolveSourceAvailability(
+      collection,
+      host.getSyncOptions?.()
+    );
+    const built = await builder(root, {
+      directoryAvailability: memoizeDirectoryAvailability(
+        createDirectoryAvailability(availabilityMode)
+      ),
+    });
+    if (host.disposed()) {
+      return;
+    }
+    if (
+      host.getGeneration(collectionName) !== generation ||
+      host.getRoot(collectionName) !== root
+    ) {
+      return;
+    }
+    if (built.status === "ok") {
+      host.setSnapshot(collectionName, built.snapshot);
+    } else {
+      host.clearSnapshot(collectionName);
+    }
+  } catch {
+    if (
+      !host.disposed() &&
+      host.getGeneration(collectionName) === generation &&
+      host.getRoot(collectionName) === root
+    ) {
+      host.clearSnapshot(collectionName);
+    }
+  }
+
+  if (host.getInit(collectionName) !== getInit()) {
+    return;
+  }
+  host.setInit(collectionName, undefined);
+  if (host.disposed()) {
+    return;
+  }
+  // Readiness flips even on init failure so forceFallback classification can run.
+  if (
+    host.getGeneration(collectionName) === generation &&
+    host.getRoot(collectionName) === root
+  ) {
+    host.setReady(collectionName, true);
+    host.onReadyWithPending(collectionName);
+  }
+}