@ultimat3/cache 27.6.1 → 27.8.0

package/CLAUDE.md

@@ -21,6 +21,10 @@ Tier 1. Tagged caching + THE invalidation graph.
 - **The fan-out clears FARTHEST tier first, and reports in read order** (core's `CACHE_TIERS`, what `/_x`
   renders). Near-to-far lets a racing read promote the stale far value straight back up.
   `CacheStack.drop` reverses for the same reason. `invalidation-race.test.ts`.
+- **The EDGE is purged last**: read tiers → ISR revalidators → broadcast → `cdn`. Purged first, a
+  request before the origin's own delete was a public hit the purged edge cached again for a whole
+  `s-maxage`. A refused broadcast purges it again when the deferred one is published
+  (`purgeEdgeAgain`, called by cli); a flush purges it last too. `invalidate-order.test.ts`.
 - **A fill is fenced: sample before `load()`, ask before the write** (`fence.ts`). `sampleFence({ key,
   tags })` → `fence.isValid()`; `markInvalidated` is the write half (`fanOut`, `CacheStack.write`,
   `CacheStack.drop`). Exported so a store outside this package reuses it rather than growing a
@@ -42,6 +46,9 @@ Tier 1. Tagged caching + THE invalidation graph.
   `TIER_ORDER` alias (`X_HELPER_COPY` refuses a value of core's under a second name). Adding a
   rung is an edit to `packages/core/src/cache-vocabulary.ts` plus a factory here; `scripts/render-modes.ts` refuses a second declaration of the set. **`isr` is
   not a tier** — the `'isr'` in `invalidate.ts` is an ISR route (`DependentKind = 'isr-route'`).
+- **`flushProcessTiers(source)` is for a process that cannot name what it missed** (its bus was
+  away): `CacheTier.clear()` on in-process tiers only, tagged ISR pages stale, fills fenced out
+  (`markAllInvalidated`). Never re-emits. A shared tier must NOT implement `clear`.
 - **`bestEffort()` is the only sanctioned way to swallow a cache refusal.** Its label is `TierLabel`
   (`TierName` plus `'query-read'`), deliberately not a widening of `TierName`.
 - **A refusal is rendered with `renderThrowable()`, never `error.message`** — at all five absorbing

package/README.md

@@ -283,6 +283,22 @@ one naming the span that triggered it. That is the log the `/_x` cache panel ren
 actually clear?" is answerable without a log dive because the one fan-out path retained the
 answer, not because a second recorder was wired next to it.
 
+### The order of one bust
+
+Read tiers, farthest first (`redis` → `lru` → `request-memo`) → the ISR revalidators → the
+broadcast to peers → the `cdn` tier **last**. The edge holds responses, not values a render reads:
+purged before the origin dropped its own page, it was handed that page again by the next request
+and kept it for a whole `s-maxage`. `report.tiers` is still the ladder in read order.
+
+### The ISR holder is asked by tag
+
+`registerRevalidator(byPath, byTags?)` takes both halves in one call (`@ultimat3/render`'s
+controller is the caller). `byPath` is told each `isr-route` dependent the graph holds; `byTags`
+(`TagRevalidator`) is handed the busted tags and answers the paths it revalidated — the pages its
+STORE holds under them, which the graph cannot know when two controllers share a store or an entry
+outlived the process that rendered it. Both lists are one `report.isr`; a rejection is a
+`report.errors` row (`tier: 'isr'`), never a failed bust. Omitting `byTags` clears the previous one.
+
 ### Across instances
 
 `invalidateTags` clears the tiers of the process that called it. On a fleet that is one pod: a user
@@ -304,8 +320,17 @@ bus.subscribe('cache.invalidate', (wireTags) => receiveInvalidationBroadcast(wir
 The inbound half **cannot** re-emit, and that is structural rather than a flag: `emit` lives on the
 private fan-out options and `receiveInvalidationBroadcast` is the only caller that passes `false`.
 A receiver that re-broadcast would be a storm bounded by nothing. A failed send lands in
-`report.errors` under `tier: "broadcast"` — the other pods then clear on TTL, and the write that
-triggered the bust still succeeds. An inbound tag this process has not declared is dropped and
+`report.errors` under `tier: "broadcast"` and the write that triggered the bust still succeeds.
+What happens to that bust is the sender's (`@ultimat3/cli` keeps the refused tags, bounded, and
+publishes them when the bus returns); with no such sender the other pods clear on TTL.
+
+**`flushProcessTiers(source)` is the repair for a process that cannot know what it missed** — its
+bus connection was down for a window, or a peer says it had more refused busts than it could keep.
+It calls `CacheTier.clear()` on every tier that has one (only a tier whose store is this process's
+heap implements it: `lruTier` does, `redisTier` and `cdnTier` do not), marks every ISR page that
+depends on a tag stale through the registered revalidator, and fences out every fill in flight
+(`markAllInvalidated`). It never re-emits and never throws; its report and its `/_x` event carry
+`tags: ['*']` (`FLUSH_ALL_TAG`). An inbound tag this process has not declared is dropped and
 reported rather than thrown, because mid-deploy the new pods know an entity the old ones do not and
 a throw would kill the subscriber loop that delivered it.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/cache",
-  "version": "27.6.1",
+  "version": "27.8.0",
   "description": "Tagged caching: request memo, LRU, Redis, CDN — one invalidation graph",
   "license": "MIT",
   "type": "module",
@@ -32,6 +32,6 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "27.6.1"
+    "@ultimat3/core": "27.8.0"
   }
 }

package/src/fence.ts

@@ -70,6 +70,17 @@ export function markInvalidated(scope: FenceScope): void {
   }
 }
 
+/**
+ * Everything, invalidated: every fence sampled before this call answers `false`. For a process
+ * that cannot name what it missed (`flushProcessTiers`) — the same "older than the ring remembers:
+ * unprovable, so refused" rule, reached on purpose. One refetch per in-flight fill.
+ */
+export function markAllInvalidated(): void {
+  generation += 1;
+  forgottenThrough = generation;
+  marks.length = 0;
+}
+
 function hits(mark: Mark, scope: FenceScope): boolean {
   if (mark.key !== undefined) return scope.key !== undefined && mark.key === scope.key;
   const owned = mark.tag;

package/src/index.ts

@@ -39,11 +39,16 @@ export type {
   InvalidationEvent,
   InvalidationReport,
   Revalidator,
+  TagRevalidator,
 } from './invalidate';
 export {
+  EVERY_TAG,
+  FLUSH_ALL_TAG,
+  flushProcessTiers,
   invalidateTags,
   invalidateWireTags,
   isolateTiers,
+  purgeEdgeAgain,
   receiveInvalidationBroadcast,
   recentInvalidations,
   registeredTiers,

package/src/invalidate.ts

@@ -12,8 +12,9 @@ import {
   systemClock,
   withSpan,
 } from '@ultimat3/core';
-import { markInvalidated } from './fence';
-import { dependentsOfKind } from './graph';
+import { markAllInvalidated, markInvalidated } from './fence';
+import { dependentsOfKind, graphSnapshot } from './graph';
+import { logPartial, resetPartialLog } from './partial-log';
 import type { CacheTag } from './tags';
 import { assertKnownTags, knownTags, parseTag, serializeTags } from './tags';
 import { isolateTierFailures, resetTierFailures } from './tier-failures';
@@ -23,6 +24,18 @@ import { sortTiers } from './tiers';
 /** Revalidates one ISR route path. Provided by `@ultimat3/render`; absent on a worker. */
 export type Revalidator = (path: string) => Promise<void> | void;
 
+/**
+ * Revalidates every ISR page HELD under these tags, and answers their paths. The graph only knows
+ * the pages this process rendered; a store that outlives the process, or that two controllers
+ * share, holds pages with no edge here — so the holder is asked by tag as well as told by path.
+ */
+export type TagRevalidator = (
+  tags: readonly CacheTag[] | typeof EVERY_TAG,
+) => Promise<readonly string[]> | readonly string[];
+
+/** What a flush hands the `TagRevalidator` in place of tags: every page it holds under ANY tag. */
+export const EVERY_TAG = 'every-tag';
+
 /**
  * Carries wire tags to every OTHER process. The seam, never the transport: `cache` is tier 1 and
  * may not reach `realtime` (tier 3) or NATS, so `@ultimat3/cli` registers the sender at boot the
@@ -94,6 +107,7 @@ export function recentInvalidations(): readonly InvalidationEvent[] {
 
 const registry: CacheTier[] = [];
 let revalidator: Revalidator | undefined;
+let tagRevalidator: TagRevalidator | undefined;
 let broadcast: InvalidationBroadcast | undefined;
 
 /** Tiers register at boot from `app.config.ts`; order is normalised, not trusted. */
@@ -114,9 +128,11 @@ export function registeredTiers(): readonly CacheTier[] {
 export function resetTiers(): void {
   registry.length = 0;
   revalidator = undefined;
+  tagRevalidator = undefined;
   broadcast = undefined;
   invalidationLog.length = 0;
   resetTierFailures();
+  resetPartialLog();
 }
 
 /**
@@ -134,6 +150,7 @@ export function resetTiers(): void {
 export function isolateTiers(): () => void {
   const capturedTiers = [...registry];
   const capturedRevalidator = revalidator;
+  const capturedTagRevalidator = tagRevalidator;
   const capturedBroadcast = broadcast;
   const capturedLog = [...invalidationLog];
   const restoreFailures = isolateTierFailures();
@@ -142,14 +159,20 @@ export function isolateTiers(): () => void {
     resetTiers();
     registry.push(...capturedTiers);
     revalidator = capturedRevalidator;
+    tagRevalidator = capturedTagRevalidator;
     broadcast = capturedBroadcast;
     invalidationLog.push(...capturedLog);
     restoreFailures();
   };
 }
 
-export function registerRevalidator(next: Revalidator): void {
+/**
+ * One registration for both halves, so they cannot belong to two controllers: `byTags` is replaced
+ * with `next` — omitted, it is cleared, never left pointing at the previous owner's store.
+ */
+export function registerRevalidator(next: Revalidator, byTags?: TagRevalidator): void {
   revalidator = next;
+  tagRevalidator = byTags;
 }
 
 /**
@@ -221,13 +244,18 @@ function fanOut(tags: readonly CacheTag[], options: FanOutOptions): Promise<Inva
 
     const tiers: TierInvalidation[] = [];
     const errors = options.errors;
+    /** Every tier that was asked and answered: what ends that tier's run of refusals. */
+    const cleared: string[] = [];
 
     // FARTHEST tier first. Near-to-far leaves the far tier holding the old value after the near
     // ones are clear, and a read racing the bust promotes it straight back up into them — the
     // report says every tier cleared, and the LRU is stale again before the call returns.
-    for (const tier of [...sortTiers(registry)].reverse()) {
+    // The EDGE is not one of them here: it is purged last, below (`purgeEdge`).
+    const farthestFirst = [...sortTiers(registry)].reverse();
+    for (const tier of farthestFirst.filter((one) => !isEdgeTier(one))) {
       try {
         tiers.push(await tier.invalidateTags(tags));
+        cleared.push(tier.name);
       } catch (error) {
         // `renderThrowable`, never `error.message`: a tier is app-supplied, so the value it
         // rejects with is too, and both `instanceof` and `String()` RUN app code on it. A render
@@ -236,21 +264,32 @@ function fanOut(tags: readonly CacheTag[], options: FanOutOptions): Promise<Inva
         errors.push({ tier: tier.name, message: renderThrowable(error) });
       }
     }
-    // The report is read order, not clear order: it is what the `/_x` panel renders, and a ladder
-    // printed upside down is a second thing for a reader to learn.
-    tiers.sort((a, b) => CACHE_TIERS.indexOf(a.tier) - CACHE_TIERS.indexOf(b.tier));
-
-    const isr = dependentsOfKind(tags, 'isr-route');
+    const isrPaths = new Set(dependentsOfKind(tags, 'isr-route'));
     const cdn = dependentsOfKind(tags, 'cdn-path');
     const liveQueries = dependentsOfKind(tags, 'live-query');
 
-    for (const path of isr) {
+    let isrFailed = false;
+    for (const path of isrPaths) {
       try {
         await revalidator?.(path);
       } catch (error) {
+        isrFailed = true;
         errors.push({ tier: 'isr', message: renderThrowable(error) });
       }
     }
+    // After the graph's own paths, and reported with them: what the holder found under the tags
+    // that this process never registered.
+    try {
+      for (const path of (await tagRevalidator?.(tags)) ?? []) isrPaths.add(path);
+    } catch (error) {
+      isrFailed = true;
+      errors.push({ tier: 'isr', message: renderThrowable(error) });
+    }
+    const isr = [...isrPaths];
+    // Asked and answered: a path revalidated, or the holder read by tag — never a bust that
+    // reached no ISR holder at all, which has said nothing about an earlier refusal.
+    const isrAsked = (isr.length > 0 && revalidator !== undefined) || tagRevalidator !== undefined;
+    if (isrAsked && !isrFailed) cleared.push('isr');
 
     const wire = serializeTags(tags);
 
@@ -259,11 +298,20 @@ function fanOut(tags: readonly CacheTag[], options: FanOutOptions): Promise<Inva
     if (emit && wire.length > 0 && broadcast !== undefined) {
       try {
         await broadcast(wire);
+        cleared.push('broadcast');
       } catch (error) {
         errors.push({ tier: 'broadcast', message: renderThrowable(error) });
       }
     }
 
+    // The edge LAST, after this origin dropped its own pages and told its peers to. Purged first,
+    // a request between the purge and the origin's delete was answered the old page as a public
+    // hit, and the edge — already purged — kept it for a whole `s-maxage` with no purge to come.
+    await purgeEdge(farthestFirst, tags, tiers, errors, cleared);
+    // The report is read order, not clear order: it is what the `/_x` panel renders, and a ladder
+    // printed upside down is a second thing for a reader to learn.
+    tiers.sort((a, b) => CACHE_TIERS.indexOf(a.tier) - CACHE_TIERS.indexOf(b.tier));
+
     const report: InvalidationReport = {
       tags: wire,
       tiers,
@@ -287,7 +335,139 @@ function fanOut(tags: readonly CacheTag[], options: FanOutOptions): Promise<Inva
       errors: report.errors,
     });
 
-    if (errors.length > 0) logger.warn('cache.invalidate.partial', { ...report });
+    // Thinned per failing tier (`partial-log.ts`): an outage is not one line per bust.
+    logPartial(report, cleared);
+    return report;
+  });
+}
+
+/** The one rung that holds RESPONSES rather than values a render reads: core's last `CACHE_TIERS` name. */
+const isEdgeTier = (tier: CacheTier): boolean => tier.name === CACHE_TIERS[CACHE_TIERS.length - 1];
+
+/** The edge's own half of the fan-out, with a read tier's isolation: a refusal is a report row. */
+async function purgeEdge(
+  ordered: readonly CacheTier[],
+  tags: readonly CacheTag[],
+  tiers: TierInvalidation[],
+  errors: { tier: string; message: string }[],
+  /** Told each edge tier that was asked and answered — what ends its run of refusals in the log. */
+  cleared?: string[],
+): Promise<void> {
+  for (const tier of ordered.filter(isEdgeTier)) {
+    try {
+      tiers.push(await tier.invalidateTags(tags));
+      cleared?.push(tier.name);
+    } catch (error) {
+      errors.push({ tier: tier.name, message: renderThrowable(error) });
+    }
+  }
+}
+
+/**
+ * The edge, purged AGAIN for tags this process already busted — for whoever finally got a refused
+ * broadcast through (`@ultimat3/cli`'s deferred publish). The first purge ran with the bust, while
+ * the peers still held their copies: any request they answered since handed the edge the old page
+ * back. Only now, told, do they drop it, so only now does a purge of the edge hold. Never throws,
+ * and touches no other tier — the bust itself cleared those.
+ */
+export async function purgeEdgeAgain(
+  wire: readonly string[],
+): Promise<readonly TierInvalidation[]> {
+  const tiers: TierInvalidation[] = [];
+  const errors: { tier: string; message: string }[] = [];
+  const cleared: string[] = [];
+  await purgeEdge(sortTiers(registry), wire.map(parseTag), tiers, errors, cleared);
+  // The same thinned line a bust writes, on the same per-tier count: an edge that refuses for a
+  // whole outage is one condition, whichever of the two purges met it.
+  logPartial({ tags: wire, tiers, errors }, cleared);
+  return tiers;
+}
+
+/** What a flush reports and records as its tags: not a wire tag, and never parsed as one. */
+export const FLUSH_ALL_TAG = '*';
+
+/**
+ * Drop everything this PROCESS holds, because it cannot know what it missed.
+ *
+ * A peer's bust reaches this process by the broadcast and by nothing else: its in-process tier and
+ * its tag-revalidated ISR pages are cleared by `receiveInvalidationBroadcast`, or they wait for
+ * their TTL — and a `revalidate: { tags }` page has none. So a process whose bus connection was
+ * down for a window, or a sender that had more refused busts than it could keep, has one correct
+ * answer left. Every in-process tier is cleared (`CacheTier.clear`), every ISR page that depends
+ * on a tag is revalidated as its route declared — marked stale, or DELETED under
+ * `onInvalidate: 'purge'`, the store's unregistered pages included — and every fill in flight is
+ * fenced out. Shared read tiers are untouched — the bust's sender cleared those. The edge is
+ * purged, last, for the tags of the ISR pages this process held: what it answered while deaf may
+ * be what the edge holds now. Nothing is re-emitted: this is local repair.
+ * Never throws; the cost is one cold cache.
+ */
+export function flushProcessTiers(source: string): Promise<InvalidationReport> {
+  return withSpan('cache.flush', async (): Promise<InvalidationReport> => {
+    const startedAt = performance.now();
+    markAllInvalidated();
+    const tiers: TierInvalidation[] = [];
+    const errors: { tier: string; message: string }[] = [];
+    for (const tier of sortTiers(registry)) {
+      if (tier.clear === undefined) continue;
+      try {
+        await tier.clear();
+        tiers.push({ tier: tier.name, keys: [] });
+      } catch (error) {
+        errors.push({ tier: tier.name, message: renderThrowable(error) });
+      }
+    }
+    // Read BEFORE the pages are revalidated: a purged page leaves the graph, and its tags are
+    // what the edge is purged by below.
+    const tagged = graphSnapshot().filter((entry) =>
+      entry.dependents.some((dep) => dep.kind === 'isr-route'),
+    );
+    const isrPaths = new Set(
+      tagged.flatMap((entry) =>
+        entry.dependents.filter((dep) => dep.kind === 'isr-route').map((dep) => dep.id),
+      ),
+    );
+    for (const path of isrPaths) {
+      try {
+        await revalidator?.(path);
+      } catch (error) {
+        errors.push({ tier: 'isr', message: renderThrowable(error) });
+      }
+    }
+    // And every page the holder has that the graph never named — as a bust asks it, by tag. A
+    // flush is at least a bust of everything: under `onInvalidate: 'purge'` those pages are deleted.
+    try {
+      for (const path of (await tagRevalidator?.(EVERY_TAG)) ?? []) isrPaths.add(path);
+    } catch (error) {
+      errors.push({ tier: 'isr', message: renderThrowable(error) });
+    }
+    const isr = [...isrPaths];
+    // The edge LAST, as in a bust, and for the pages THIS process could have handed it while it
+    // was deaf: a hit it answered from a copy a peer had already purged went out public, and the
+    // peer's own edge purge had run by then. Shared read tiers stay untouched — the sender's.
+    await purgeEdge(
+      sortTiers(registry),
+      tagged.map((entry) => parseTag(entry.tag)),
+      tiers,
+      errors,
+    );
+    const report: InvalidationReport = {
+      tags: [FLUSH_ALL_TAG],
+      tiers,
+      isr,
+      cdn: [],
+      liveQueries: [],
+      durationMs: Math.round((performance.now() - startedAt) * 100) / 100,
+      errors,
+    };
+    recordInvalidation({
+      at: systemClock.now().toISOString(),
+      tags: report.tags,
+      busted: [...isr],
+      source,
+      durationMs: report.durationMs,
+      errors,
+    });
+    if (errors.length > 0) logger.warn('cache.flush.partial', { ...report });
     return report;
   });
 }

package/src/lru.ts

@@ -305,5 +305,8 @@ export function lruTier(options: LruOptions = {}): CacheTier & { readonly cache:
     invalidateTags(tags: readonly CacheTag[]): Promise<TierInvalidation> {
       return Promise.resolve({ tier: 'lru', keys: cache.invalidateTags(tags) });
     },
+    clear(): void {
+      cache.clear();
+    },
   };
 }

package/src/partial-log.ts

@@ -0,0 +1,54 @@
+// Single responsibility: how often `cache.invalidate.partial` is said. A tier that keeps refusing
+// — the broadcast, while the bus is away — is ONE condition on a process that is still serving:
+// a warning on its 1st, 2nd, 4th, 8th, … refusal and one line when it clears again, never one
+// line per bust per publisher for as long as an outage lasts. The report itself is unchanged:
+// every caller still gets every error, and the `/_x` panel still records every bust.
+
+import { isOutageMilestone, logger } from '@ultimat3/core';
+
+/** The line a partial bust writes; `<event> recovered` when a failing tier clears again. */
+export const INVALIDATE_PARTIAL = 'cache.invalidate.partial';
+
+/**
+ * Tier names tracked at once. A name is a registered tier's, `isr` or `broadcast` — a handful —
+ * so this is never reached by a real app; past it a refusal is simply said every time.
+ */
+const MAX_TRACKED_TIERS = 64;
+
+/** Consecutive refusals per tier name, since that tier last cleared. */
+const refusals = new Map<string, number>();
+
+/**
+ * One finished bust. `errors` is what it could not clear; `cleared` names every tier that was
+ * ASKED and answered — a tier this bust never reached (a bust with nothing to broadcast) has
+ * said nothing about its own outage, so it ends none.
+ */
+export function logPartial<R extends { readonly errors: readonly { readonly tier: string }[] }>(
+  report: R,
+  cleared: readonly string[],
+): void {
+  for (const tier of cleared) {
+    const after = refusals.get(tier);
+    if (after === undefined) continue;
+    refusals.delete(tier);
+    logger.info(`${INVALIDATE_PARTIAL} recovered`, { tier, after });
+  }
+  let said = false;
+  let failures = 0;
+  for (const tier of new Set(report.errors.map((error) => error.tier))) {
+    if (!refusals.has(tier) && refusals.size >= MAX_TRACKED_TIERS) {
+      said = true;
+      continue;
+    }
+    const count = (refusals.get(tier) ?? 0) + 1;
+    refusals.set(tier, count);
+    failures = Math.max(failures, count);
+    if (isOutageMilestone(count)) said = true;
+  }
+  if (said) logger.warn(INVALIDATE_PARTIAL, { ...report, failures });
+}
+
+/** Test seam, and `resetTiers()`'s: every tier starts unrefused. */
+export function resetPartialLog(): void {
+  refusals.clear();
+}

package/src/tiers.ts

@@ -195,6 +195,12 @@ export interface CacheTier {
    * may never be told about — still stops the fill (`tier-fence.ts`). Omitted by in-process tiers.
    */
   fence?(scope: FenceScope): Promise<TierFence>;
+  /**
+   * Drop every entry — implemented ONLY by a tier whose store is this process's own heap (`lru`).
+   * `flushProcessTiers()` calls it when the process cannot know which busts it missed. A shared
+   * tier omits it: whoever ran a bust cleared that store for everyone.
+   */
+  clear?(): Promise<void> | void;
 }
 
 export interface CacheStack {