@gmickel/gno 1.34.4 → 1.34.6

package/src/serve/watch-service-events.ts

@@ -0,0 +1,261 @@
+/**
+ * Watcher event intake, pending queues, and flush timer scheduling.
+ *
+ * @module src/serve/watch-service-events
+ */
+
+// node:path — Bun has no path utilities
+import { join, normalize } from "node:path";
+
+import type { Collection } from "../config/types";
+
+import {
+  WATCHER_FLUSH_DEBOUNCE_MS,
+  WATCHER_MAX_FLUSH_DELAY_MS,
+  WATCHER_RETRY_BACKOFF_MS,
+  classifyWatcherFilename,
+} from "./watch-reconciliation";
+import {
+  applyPendingForceFlags,
+  computeFlushDelay,
+  emptyPending,
+  queueDirtyHint,
+  queueExactPath,
+  type CollectionPending,
+  type PendingForceFlags,
+} from "./watch-service-state";
+
+export interface WatchEventHost {
+  disposed: () => boolean;
+  findCollection: (collectionName: string) => Collection | undefined;
+  clock: () => number;
+  suppressedPaths: Map<string, number>;
+  setLastEventAt: (iso: string) => void;
+  enqueueExact: (collectionName: string, relPath: string) => void;
+  enqueueDirty: (collectionName: string, hint: string) => void;
+}
+
+export interface WatchQueueHost {
+  disposed: () => boolean;
+  clock: () => number;
+  flushDebounceMs: number;
+  maxFlushDelayMs: number;
+  maxExactPaths: number;
+  maxDirtyHints: number;
+  pendingByCollection: Map<string, CollectionPending>;
+  flushDeadlineAt: Map<string, number>;
+  timers: Map<string, ReturnType<typeof setTimeout>>;
+  /** Collections with an explicit retry timer; finally must not bypass. */
+  retryScheduled: Set<string>;
+  snapshotReady: Map<string, boolean>;
+  inFlightSyncs: Set<Promise<void>>;
+  runFlush: (collectionName: string) => Promise<void>;
+}
+
+/** Classify a filesystem watch callback and enqueue exact or dirty work. */
+export function handleWatchEvent(
+  host: WatchEventHost,
+  collectionName: string,
+  watchedRoot: string,
+  filename: string | Buffer | null | undefined
+): void {
+  if (host.disposed()) {
+    return;
+  }
+  const currentCollection = host.findCollection(collectionName);
+  if (!currentCollection || normalize(currentCollection.path) !== watchedRoot) {
+    return;
+  }
+
+  const classified = classifyWatcherFilename(filename, currentCollection);
+  if (classified.kind === "reject" || classified.kind === "ignore") {
+    return;
+  }
+
+  if (classified.kind === "exact") {
+    const fullPath = normalize(join(watchedRoot, classified.relPath));
+    const suppressedUntil = host.suppressedPaths.get(fullPath);
+    if (suppressedUntil && suppressedUntil > host.clock()) {
+      return;
+    }
+    host.setLastEventAt(new Date(host.clock()).toISOString());
+    host.enqueueExact(collectionName, classified.relPath);
+    return;
+  }
+
+  if (classified.hint !== "") {
+    const fullPath = normalize(join(watchedRoot, classified.hint));
+    const suppressedUntil = host.suppressedPaths.get(fullPath);
+    if (suppressedUntil && suppressedUntil > host.clock()) {
+      return;
+    }
+  }
+  host.setLastEventAt(new Date(host.clock()).toISOString());
+  host.enqueueDirty(collectionName, classified.hint);
+}
+
+export function enqueueExactPath(
+  host: WatchQueueHost,
+  collectionName: string,
+  relPath: string
+): void {
+  const pending =
+    host.pendingByCollection.get(collectionName) ?? emptyPending();
+  host.pendingByCollection.set(
+    collectionName,
+    queueExactPath(pending, relPath, host.maxExactPaths)
+  );
+  scheduleFlush(host, collectionName);
+}
+
+export function enqueueDirtyHint(
+  host: WatchQueueHost,
+  collectionName: string,
+  hint: string
+): void {
+  const pending =
+    host.pendingByCollection.get(collectionName) ?? emptyPending();
+  // Ambiguous events before baseline readiness must force store/disk reconcile.
+  if (!host.snapshotReady.get(collectionName)) {
+    pending.forceFallback = true;
+  }
+  host.pendingByCollection.set(
+    collectionName,
+    queueDirtyHint(pending, hint, host.maxDirtyHints)
+  );
+  scheduleFlush(host, collectionName);
+}
+
+/** Queue work without arming timers (used by failure requeue). */
+function queueWithoutSchedule(
+  host: WatchQueueHost,
+  collectionName: string,
+  exact: string[],
+  dirty: string[],
+  forceFlags?: PendingForceFlags
+): void {
+  let pending = host.pendingByCollection.get(collectionName) ?? emptyPending();
+  for (const path of exact) {
+    pending = queueExactPath(pending, path, host.maxExactPaths);
+  }
+  for (const hint of dirty) {
+    pending = queueDirtyHint(pending, hint, host.maxDirtyHints);
+  }
+  pending = applyPendingForceFlags(pending, forceFlags);
+  host.pendingByCollection.set(collectionName, pending);
+}
+
+export function scheduleFlush(
+  host: WatchQueueHost,
+  collectionName: string
+): void {
+  if (host.disposed()) {
+    return;
+  }
+  // Explicit retry owns the single timer; do not replace it with debounce.
+  if (host.retryScheduled.has(collectionName)) {
+    return;
+  }
+  const schedule = computeFlushDelay({
+    nowMs: host.clock(),
+    existingDeadlineAt: host.flushDeadlineAt.get(collectionName),
+    debounceMs: host.flushDebounceMs,
+    maxFlushDelayMs: host.maxFlushDelayMs,
+  });
+  host.flushDeadlineAt.set(collectionName, schedule.deadlineAt);
+
+  const existingTimer = host.timers.get(collectionName);
+  if (existingTimer) {
+    clearTimeout(existingTimer);
+  }
+  host.timers.set(
+    collectionName,
+    setTimeout(() => {
+      host.timers.delete(collectionName);
+      startFlush(host, collectionName);
+    }, schedule.delayMs)
+  );
+}
+
+/**
+ * Begin a flush when the debounce fires. Dirty/overflow work waits for snapshot
+ * readiness; exact-only paths content-hash without a baseline.
+ */
+export function startFlush(host: WatchQueueHost, collectionName: string): void {
+  if (host.disposed()) {
+    return;
+  }
+  const pending = host.pendingByCollection.get(collectionName);
+  const exactOnly =
+    pending !== undefined &&
+    pending.exact.size > 0 &&
+    pending.dirty.size === 0 &&
+    !pending.overflow &&
+    !pending.forceFallback &&
+    !pending.generationReconcile;
+  // Exact eligible paths content-hash without a snapshot baseline. Dirty /
+  // overflow / init-fallback / generation work waits so init-time ambiguous
+  // events reconcile against a newer generation rather than empty unproven state.
+  // Leave work queued with no timer; onReadyWithPending schedules exactly once.
+  if (!host.snapshotReady.get(collectionName) && !exactOnly) {
+    return;
+  }
+  const sync = host.runFlush(collectionName);
+  host.inFlightSyncs.add(sync);
+  void sync
+    .finally(() => {
+      host.inFlightSyncs.delete(sync);
+    })
+    .catch(() => undefined);
+}
+
+/**
+ * Re-arm pending work after a file-level failure with retry backoff.
+ * At most one retry timer per collection; finally must not startFlush while set.
+ * forceFallback/overflow survive until forced classification + sync succeed.
+ */
+export function requeueAfterFailure(
+  host: WatchQueueHost,
+  collectionName: string,
+  exact: string[],
+  dirty: string[],
+  forceFlags?: PendingForceFlags
+): void {
+  queueWithoutSchedule(host, collectionName, exact, dirty, forceFlags);
+  if (host.disposed()) {
+    return;
+  }
+  if (host.retryScheduled.has(collectionName)) {
+    // Pending already merged; existing retry timer remains the sole attempt.
+    return;
+  }
+  host.retryScheduled.add(collectionName);
+  const existingTimer = host.timers.get(collectionName);
+  if (existingTimer) {
+    clearTimeout(existingTimer);
+  }
+  host.flushDeadlineAt.set(collectionName, host.clock() + host.maxFlushDelayMs);
+  host.timers.set(
+    collectionName,
+    setTimeout(() => {
+      host.timers.delete(collectionName);
+      host.retryScheduled.delete(collectionName);
+      startFlush(host, collectionName);
+    }, WATCHER_RETRY_BACKOFF_MS)
+  );
+}
+
+/** Mark durable generation-reconcile work and schedule a single retry. */
+export function requeueGenerationReconcile(
+  host: WatchQueueHost,
+  collectionName: string
+): void {
+  const pending =
+    host.pendingByCollection.get(collectionName) ?? emptyPending();
+  pending.generationReconcile = true;
+  host.pendingByCollection.set(collectionName, pending);
+  requeueAfterFailure(host, collectionName, [], []);
+}
+
+// Re-export defaults used by the service constructor for a single import site.
+export { WATCHER_FLUSH_DEBOUNCE_MS, WATCHER_MAX_FLUSH_DELAY_MS };

package/src/serve/watch-service-flush-generation.ts

@@ -0,0 +1,140 @@
+/**
+ * Config-generation / overflow / unsupported-FS full collection reconcile.
+ *
+ * @module src/serve/watch-service-flush-generation
+ */
+
+// node:path — Bun has no path utilities
+import { normalize } from "node:path";
+
+import type {
+  FlushCollectionInput,
+  FlushCollectionOutcome,
+} from "./watch-service-flush";
+
+import { defaultSyncService } from "../ingestion";
+import { hasFileLevelSyncError } from "./watch-reconciliation";
+import {
+  contentChangedPaths,
+  notifyCompletedSync,
+} from "./watch-service-flush-helpers";
+
+/**
+ * When collection generation advanced during/before flush, run full
+ * syncCollection. Failures leave durable generation work and never advance
+ * snapshot ownership. Options are read fresh each iteration.
+ *
+ * If generation/root/options change again after a completed syncCollection,
+ * continue with the latest collection/options rather than returning stale with
+ * empty pending.
+ */
+export async function runGenerationReconcile(
+  input: FlushCollectionInput
+): Promise<FlushCollectionOutcome | null> {
+  let completedGeneration: number | null = null;
+  let needsWork =
+    input.generationReconcile ||
+    input.getCurrentGeneration() !== input.ownerGeneration;
+
+  while (needsWork) {
+    const currentCollection = input.getCurrentCollection();
+    if (!currentCollection) {
+      return { status: "stale" };
+    }
+    const currentGeneration = input.getCurrentGeneration();
+    const currentRoot = normalize(currentCollection.path);
+
+    // Already reconciled this generation and nothing newer is pending.
+    if (
+      completedGeneration !== null &&
+      completedGeneration === currentGeneration
+    ) {
+      return null;
+    }
+
+    try {
+      // Always use live options — content rules may change mid-generation.
+      const result = await defaultSyncService.syncCollection(
+        currentCollection,
+        input.store,
+        {
+          ...input.getCurrentSyncOptions(),
+          runUpdateCmd: false,
+        }
+      );
+      if (input.disposed()) {
+        return { status: "disposed" };
+      }
+
+      const stillCurrent = input.getCurrentCollection();
+      if (!stillCurrent) {
+        return { status: "stale" };
+      }
+      const latestGen = input.getCurrentGeneration();
+      const latestRoot = normalize(stillCurrent.path);
+
+      // Root replaced mid-reconcile: durable requeue for the new owner.
+      if (latestRoot !== currentRoot) {
+        input.requeueGeneration();
+        return { status: "stale" };
+      }
+
+      // Generation advanced during syncCollection: continue with latest, no
+      // intermediate snapshot/callback commit for the superseded generation.
+      if (latestGen !== currentGeneration) {
+        needsWork = true;
+        continue;
+      }
+
+      const ownership = {
+        generation: currentGeneration,
+        root: currentRoot,
+      };
+      // Operation scope: file receipts when present, else empty (full coll).
+      const operationPaths =
+        result.files?.map((file) => file.relPath) ??
+        contentChangedPaths(result);
+
+      if (hasFileLevelSyncError(result)) {
+        // Notify completed sync once; keep durable generation work.
+        notifyCompletedSync(input, operationPaths, result, ownership);
+        const error = new Error(
+          "One or more paths failed during watcher generation reconcile"
+        );
+        input.onSyncError([], error);
+        input.requeueGeneration();
+        return { status: "failed", error };
+      }
+
+      // Only after successful full reconcile: rebuild snapshot ownership.
+      input.invalidateSnapshot(stillCurrent);
+      notifyCompletedSync(input, operationPaths, result, ownership);
+      completedGeneration = currentGeneration;
+      needsWork = input.getCurrentGeneration() !== completedGeneration;
+    } catch (error) {
+      if (input.disposed()) {
+        return { status: "disposed" };
+      }
+      const stillCurrent = input.getCurrentCollection();
+      if (!stillCurrent) {
+        return { status: "stale" };
+      }
+      const latestGen = input.getCurrentGeneration();
+      const latestRoot = normalize(stillCurrent.path);
+      if (latestRoot !== currentRoot) {
+        input.requeueGeneration();
+        return { status: "stale" };
+      }
+      // Gen advanced under a thrown reconcile: continue toward latest rather
+      // than return stale with empty pending.
+      if (latestGen !== currentGeneration) {
+        needsWork = true;
+        continue;
+      }
+      input.onSyncError([], error);
+      input.requeueGeneration();
+      return { status: "failed", error };
+    }
+  }
+  return null;
+}

package/src/serve/watch-service-flush-helpers.ts

@@ -0,0 +1,147 @@
+/**
+ * Settlement helpers for one collection watcher flush.
+ *
+ * @module src/serve/watch-service-flush-helpers
+ */
+
+// node:path — Bun has no path utilities
+import { normalize } from "node:path";
+
+import type { Collection } from "../config/types";
+import type { CollectionSyncResult } from "../ingestion";
+
+import { collectionToWalkConfig, matchesWalkPath } from "../ingestion";
+import {
+  failedSyncPaths,
+  successfulChangedPaths,
+} from "./watch-reconciliation";
+
+export interface FlushNotifyInput {
+  disposed: () => boolean;
+  getCurrentCollection: () => Collection | undefined;
+  getCurrentGeneration: () => number;
+  ownerGeneration: number;
+  ownerRoot: string;
+  onSyncComplete: (relPaths: string[], result: CollectionSyncResult) => void;
+  onAfterSync: (collection: Collection, relPaths: string[]) => void;
+}
+
+/**
+ * Paths that content-changed (added/updated). Used for scheduler/events.
+ * Does not include unchanged or pure inactive-only summary counts.
+ */
+export function contentChangedPaths(
+  result: CollectionSyncResult,
+  fallbackPaths: string[] = []
+): string[] {
+  const fromFiles = successfulChangedPaths(result);
+  if (result.files) {
+    return fromFiles;
+  }
+  return result.filesAdded + result.filesUpdated > 0 ? fallbackPaths : [];
+}
+
+/**
+ * onSyncComplete fires exactly once per completed sync result with the
+ * operation path scope. onAfterSync/scheduler remain ownership-checked and
+ * limited to successful added/updated paths.
+ */
+export function notifyCompletedSync(
+  input: FlushNotifyInput,
+  operationPaths: string[],
+  result: CollectionSyncResult,
+  ownership?: { generation: number; root: string }
+): string[] {
+  input.onSyncComplete(operationPaths, result);
+  const changed = contentChangedPaths(result, operationPaths);
+  if (changed.length === 0 || input.disposed()) {
+    return changed;
+  }
+  const currentCollection = input.getCurrentCollection();
+  if (!currentCollection) {
+    return changed;
+  }
+  const expectedGen = ownership?.generation ?? input.ownerGeneration;
+  const expectedRoot = ownership?.root ?? input.ownerRoot;
+  if (
+    input.getCurrentGeneration() !== expectedGen ||
+    normalize(currentCollection.path) !== expectedRoot
+  ) {
+    return changed;
+  }
+  const filtered = changed.filter((relPath) =>
+    matchesWalkPath(relPath, collectionToWalkConfig(currentCollection, 0))
+  );
+  if (filtered.length > 0) {
+    input.onAfterSync(currentCollection, filtered);
+  }
+  return changed;
+}
+
+/**
+ * Retry authority after a targeted sync. Top-level result.errors retain the
+ * original exact operation even when every file receipt succeeded, unless
+ * durable dirty or generation authority already covers the work.
+ *
+ * Dirty-derived candidate/removal failures retain the original dirty hints and
+ * force flags until a full classified generation succeeds and nextSnapshot
+ * commits — never replace dirty authority solely with the failed exact path.
+ */
+export function computeTargetedRetry(options: {
+  result: CollectionSyncResult;
+  submittedPaths: readonly string[];
+  liveExact: readonly string[];
+  settled: ReadonlySet<string>;
+  dirtyHints: readonly string[];
+  dirtyFailed: boolean;
+  /**
+   * True when this submission included dirty-classified candidates or
+   * proven removals (not merely live exact paths).
+   */
+  dirtyDerivedSubmission: boolean;
+  generationAuthority: boolean;
+}): { retryExact: string[]; retainDirty: boolean } {
+  const {
+    result,
+    submittedPaths,
+    liveExact,
+    settled,
+    dirtyHints,
+    dirtyFailed,
+    dirtyDerivedSubmission,
+    generationAuthority,
+  } = options;
+  const failed = failedSyncPaths(result, submittedPaths).filter(
+    (path) => !settled.has(path)
+  );
+  const topLevelErrors = result.errors.length > 0;
+  const dirtyDerivedFailure =
+    dirtyDerivedSubmission &&
+    (failed.length > 0 || topLevelErrors || result.filesErrored > 0);
+  // Retain original dirty hints until classified work fully succeeds.
+  const retainDirty =
+    dirtyHints.length > 0 &&
+    (dirtyFailed || dirtyDerivedFailure || topLevelErrors);
+
+  if (generationAuthority) {
+    // Full-collection reconcile covers exact + dirty authority.
+    return { retryExact: failed, retainDirty: false };
+  }
+
+  if (topLevelErrors) {
+    // Projection/top-level failure: requeue original exact paths unless
+    // dirty-only operation (no exact) already retains authority via dirty.
+    if (liveExact.length === 0) {
+      return { retryExact: failed, retainDirty: true };
+    }
+    // Exact (and mixed exact+dirty): requeue all original exact for projection.
+    // Successful content changes settle via afterSync only once; requeue still
+    // re-runs authority so top-level errors can clear.
+    return {
+      retryExact: [...new Set([...failed, ...liveExact])],
+      retainDirty,
+    };
+  }
+
+  return { retryExact: failed, retainDirty };
+}