caracms 0.2.0 → 0.3.1

package/src/live/status-cache.ts

@@ -0,0 +1,219 @@
+/**
+ * The status poll's write-through cache (Redis diet Decisions 1–5).
+ *
+ * Every Redis write a site makes goes through its one server, so what the status route
+ * reads can be kept here and updated (or dropped) by the writes themselves, instead of
+ * being asked of Redis on every poll.
+ *
+ * - **What it holds is Redis as last read or written here, by key**: strings, hashes,
+ *   sets, and scans (every key under a prefix, `caracms:autosave:`, with its value).
+ *   Records, never views made from them: whether a lock holds, or which notices are a
+ *   person's, is worked out per poll (Watch-outs).
+ * - **Only the status route reads it** (Decision 3). Writes, create, delete, move, the
+ *   sweep, Publish and sign-in read Redis directly.
+ * - **An entry is re-read once it's `statusCacheMs` old, counted from its last Redis
+ *   read** (Decision 1): a write-through keeps it current but doesn't make it younger, so
+ *   what bypasses this server (the other server during a deploy, a key expiring, a hand
+ *   edit) shows within that time (Decision 4).
+ * - **The writes are the modules'** (Decision 2): autosaves.ts, locks.ts, git.ts and
+ *   notices.ts tell it each write as it lands. A write it can apply exactly (a value just
+ *   set, a field just written) updates the entry; anything else (a compare-and-swap that
+ *   lost, so the value is unknown) drops it. A write to an entry not held is only noted.
+ * - **Refreshes are single-flight** (Decision 5): a poll needing an entry another poll is
+ *   already reading waits for that read. The ones it's missing go in one batch (SCANs
+ *   with the rest, then their MGETs). A refresh that fails caches nothing, and nothing
+ *   refreshes on a timer: with no poll there's no Redis read, so Redis can sleep.
+ * - **A write that lands while an entry is being read** may be missing from what the
+ *   read brings back: that entry is kept for the polls waiting on it, but as already
+ *   old, so the next poll reads it again.
+ */
+import type { BatchCommand, RedisClient } from './redis.ts';
+
+type Entry =
+	| { kind: 'string'; value: string | null; at: number }
+	| { kind: 'hash'; value: Map<string, string>; at: number }
+	| { kind: 'set'; value: Set<string>; at: number }
+	| { kind: 'scan'; value: Map<string, string>; at: number };
+
+type Kind = Entry['kind'];
+
+/** What a poll reads: keys by kind, and scans by key prefix (`caracms:lock:`). */
+export interface StatusReads {
+	scans?: readonly string[];
+	hashes?: readonly string[];
+	sets?: readonly string[];
+	strings?: readonly string[];
+}
+
+/** One poll's view of the cache. Its entries are the cache's own, so the poll's later write-throughs show in it. */
+export interface StatusSnapshot {
+	/** Every key under the prefix, with its value. */
+	scan(prefix: string): ReadonlyMap<string, string>;
+	hash(key: string): ReadonlyMap<string, string>;
+	set(key: string): ReadonlySet<string>;
+	string(key: string): string | null;
+}
+
+const name = (kind: Kind, key: string) => `${kind} ${key}`;
+
+export class StatusCache {
+	private readonly redis: RedisClient;
+	private readonly maxAgeMs: number;
+	private readonly now: () => number;
+	private readonly entries = new Map<string, Entry>();
+	private readonly reading = new Map<string, Promise<Map<string, Entry>>>();
+	/** Write-throughs so far, and the count at each entry's latest (for a read that a write overtook). */
+	private writes = 0;
+	private readonly written = new Map<string, number>();
+
+	constructor(options: { redis: RedisClient; maxAgeMs: number; now: () => number }) {
+		this.redis = options.redis;
+		this.maxAgeMs = options.maxAgeMs;
+		this.now = options.now;
+	}
+
+	/** What a poll reads, from memory where it's fresh enough, else from Redis (one batch, shared with any poll waiting on the same). */
+	async read(reads: StatusReads): Promise<StatusSnapshot> {
+		const wanted: [Kind, string][] = [
+			...(reads.scans ?? []).map((k) => ['scan', k] as [Kind, string]),
+			...(reads.hashes ?? []).map((k) => ['hash', k] as [Kind, string]),
+			...(reads.sets ?? []).map((k) => ['set', k] as [Kind, string]),
+			...(reads.strings ?? []).map((k) => ['string', k] as [Kind, string]),
+		];
+		const now = this.now();
+		const waits = new Set<Promise<Map<string, Entry>>>();
+		const missing: [Kind, string][] = [];
+		for (const [kind, key] of wanted) {
+			const n = name(kind, key);
+			const inFlight = this.reading.get(n);
+			if (inFlight) waits.add(inFlight);
+			else {
+				const e = this.entries.get(n);
+				if (!e || now - e.at >= this.maxAgeMs) missing.push([kind, key]);
+			}
+		}
+		if (missing.length) {
+			const read = this.refresh(missing);
+			for (const [kind, key] of missing) this.reading.set(name(kind, key), read);
+			const done = () => {
+				for (const [kind, key] of missing) if (this.reading.get(name(kind, key)) === read) this.reading.delete(name(kind, key));
+				if (this.reading.size === 0) this.written.clear();
+			};
+			read.then(done, done);
+			waits.add(read);
+		}
+		const fetched = new Map<string, Entry>();
+		for (const got of await Promise.all(waits)) for (const [n, e] of got) fetched.set(n, e);
+		// The cache's own entry where it still has one (later write-throughs land in it), else what the read brought back,
+		// pinned now: a drop while the poll is still in its writes leaves it the records it read, never none.
+		const pinned = new Map<string, Entry | undefined>(wanted.map(([kind, key]) => [name(kind, key), this.entries.get(name(kind, key)) ?? fetched.get(name(kind, key))]));
+		const pick = (kind: Kind, key: string) => pinned.get(name(kind, key));
+		return {
+			scan: (prefix) => (pick('scan', prefix)?.value as Map<string, string> | undefined) ?? new Map(),
+			hash: (key) => (pick('hash', key)?.value as Map<string, string> | undefined) ?? new Map(),
+			set: (key) => (pick('set', key)?.value as Set<string> | undefined) ?? new Set(),
+			string: (key) => (pick('string', key)?.value as string | null | undefined) ?? null,
+		};
+	}
+
+	/** Read these entries from Redis: one batch, then one more for the scans' MGETs. */
+	private async refresh(entries: [Kind, string][]): Promise<Map<string, Entry>> {
+		const started = this.writes;
+		const at = this.now();
+		const commands = entries.map(([kind, key]): BatchCommand => {
+			if (kind === 'scan') return ['scan', `${key}*`];
+			if (kind === 'hash') return ['hgetall', key];
+			if (kind === 'set') return ['smembers', key];
+			return ['get', key];
+		});
+		const replies = (await this.redis.batch(commands)) as unknown[];
+		const scanned = entries.flatMap(([kind], i) => (kind === 'scan' && (replies[i] as string[]).length ? [i] : []));
+		const values = scanned.length ? ((await this.redis.batch(scanned.map((i): BatchCommand => ['mget', replies[i] as string[]]))) as (string | null)[][]) : [];
+		const out = new Map<string, Entry>();
+		entries.forEach(([kind, key], i) => {
+			const n = name(kind, key);
+			// Written since this read began: kept for now, but already old.
+			const age = (this.written.get(n) ?? 0) > started ? -Infinity : at;
+			let entry: Entry;
+			if (kind === 'scan') {
+				const keys = replies[i] as string[];
+				const got = values[scanned.indexOf(i)] ?? [];
+				const map = new Map<string, string>();
+				keys.forEach((k, j) => {
+					const v = got[j];
+					if (v !== null && v !== undefined) map.set(k, v);
+				});
+				entry = { kind, value: map, at: age };
+			} else if (kind === 'hash') entry = { kind, value: new Map(Object.entries(replies[i] as Record<string, string>)), at: age };
+			else if (kind === 'set') entry = { kind, value: new Set(replies[i] as string[]), at: age };
+			else entry = { kind, value: replies[i] as string | null, at: age };
+			this.entries.set(n, entry);
+			out.set(n, entry);
+		});
+		return out;
+	}
+
+	// ── Write-throughs (the modules call these once a write has landed) ─────────────
+
+	/** Note a write to these entry names; return the ones held. */
+	private touch(names: string[]): Entry[] {
+		this.writes++;
+		const held: Entry[] = [];
+		for (const n of names) {
+			// Only a read in flight needs to know (`refresh`); with none, nothing is remembered.
+			if (this.reading.size) this.written.set(n, this.writes);
+			const e = this.entries.get(n);
+			if (e) held.push(e);
+		}
+		return held;
+	}
+
+	/** The scan entries (held or not) whose prefix this key is under. */
+	private scansOf(key: string): string[] {
+		const out: string[] = [];
+		for (const n of [...this.entries.keys(), ...this.reading.keys()]) if (n.startsWith('scan ') && key.startsWith(n.slice(5))) out.push(n);
+		return [...new Set(out)];
+	}
+
+	/** A string key now holds `value` (null: deleted). */
+	string(key: string, value: string | null): void {
+		for (const e of this.touch([name('string', key), ...this.scansOf(key)])) {
+			if (e.kind === 'string') e.value = value;
+			else if (e.kind === 'scan') {
+				if (value === null) e.value.delete(key);
+				else e.value.set(key, value);
+			}
+		}
+	}
+
+	/** A hash's fields were set, or deleted. */
+	hash(key: string, set: Record<string, string>, del: readonly string[] = []): void {
+		for (const e of this.touch([name('hash', key)])) {
+			if (e.kind !== 'hash') continue;
+			for (const [f, v] of Object.entries(set)) e.value.set(f, v);
+			for (const f of del) e.value.delete(f);
+		}
+	}
+
+	/** Members were added to a set. */
+	add(key: string, ...members: string[]): void {
+		for (const e of this.touch([name('set', key)])) if (e.kind === 'set') for (const m of members) e.value.add(m);
+	}
+
+	/** The key changed in a way not known here (a compare-and-swap that lost): forget it, and any scan it's under. */
+	drop(key: string): void {
+		const names = [...(['string', 'hash', 'set'] as const).map((k) => name(k, key)), ...this.scansOf(key)];
+		this.touch(names);
+		for (const n of names) this.entries.delete(n);
+	}
+}
+
+/** Run a write to `key`; one that fails (lost in flight, so it may or may not have landed) drops the key from the cache. */
+export async function landed<T>(cache: StatusCache | null, key: string, write: () => Promise<T>): Promise<T> {
+	try {
+		return await write();
+	} catch (error) {
+		cache?.drop(key);
+		throw error;
+	}
+}

package/src/live/status.ts

@@ -62,7 +62,11 @@ export interface PublishRecord {
 	tryAgainAt: string | null;
 }
 
-export type PublishState = 'publishing' | 'live' | 'failed';
+/**
+ * `slow` (runs-anywhere Decision 10): still not live well after it was made, or after its
+ * last Try again, with no failure known: "taking longer than usual. It may have failed."
+ */
+export type PublishState = 'publishing' | 'live' | 'failed' | 'slow';
 
 /** One GitHub Deployment and its newest status (null: no status yet). */
 export interface DeploymentFact {
@@ -76,7 +80,7 @@ const FINAL = new Set(['success', 'failure', 'error', 'inactive']);
 
 export interface PublishFacts {
 	record: PublishRecord;
-	/** The running server's commit (RAILWAY_GIT_COMMIT_SHA / CARACMS_RUNNING_COMMIT), or null with no deploy pipeline. */
+	/** The running server's commit (mode.ts → `runningCommitFrom`), or null with none known. */
 	running: string | null;
 	/** Is `running` the publish commit or a descendant of it? null when not known (not asked). */
 	runningIncludesPublish: boolean | null;
@@ -84,6 +88,9 @@ export interface PublishFacts {
 	laterOnMain: ReadonlySet<string>;
 	/** The repo's deployments, any order. */
 	deployments: DeploymentFact[];
+	/** Now (ms), and how long a publish may take before it reads as `slow` (`LiveTimings.publishSlowMs`). Either unset: never slow. */
+	now?: number;
+	slowAfterMs?: number;
 }
 
 /**
@@ -94,6 +101,10 @@ export interface PublishFacts {
  * - **failed**: the newest deployment that has *answered*, for the publish commit or
  *   a later commit on `main` (since the last Try again), says `failure` or `error`.
  *   A deployment stuck `in_progress` is never read as finished or failed.
+ * - **slow** (runs-anywhere Decision 10): not live and not failed, `slowAfterMs` after the
+ *   publish was made, or after its last Try again (`tryAgainAt`). Without deployments a
+ *   failure can never be known, and on Railway a skipped deploy stays `in_progress` for
+ *   good; a running commit that isn't in the repo never includes the publish either.
  * - **publishing**: otherwise.
  */
 export function publishState(f: PublishFacts): { state: PublishState; assumed: boolean } {
@@ -105,5 +116,6 @@ export function publishState(f: PublishFacts): { state: PublishState; assumed: b
 		.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
 	const newest = answered[0];
 	if (newest && (newest.state === 'failure' || newest.state === 'error')) return { state: 'failed', assumed: false };
+	if (f.now !== undefined && f.slowAfterMs !== undefined && f.now - Date.parse(f.record.tryAgainAt ?? f.record.at) >= f.slowAfterMs) return { state: 'slow', assumed: false };
 	return { state: 'publishing', assumed: false };
 }

package/src/live/timings.ts

@@ -29,6 +29,22 @@ export interface LiveTimings {
 	autosaveTtlMs: number;
 	/** Decision 13: each tab polls the status this often (the editor's POLL_MS; /admin passes it to the page). */
 	pollMs: number;
+	/** Runs-anywhere Decision 10: a publish not live this long after it was made (or its last Try again) reads as taking longer than usual. */
+	publishSlowMs: number;
+	/** Runs-anywhere (Watch-outs): Publish's request answers by this long, well inside Cloudflare's 100 s; the Publish carries on past it. */
+	publishAnswerMs: number;
+	/** Redis diet Decision 1: the status poll's cached reads are re-read from Redis once this old (counted from the last Redis read). */
+	statusCacheMs: number;
+	/** Redis diet Decision 6: a tab's heartbeat goes only once its lock's last beat is this old (or to take the lock back). */
+	beatEveryMs: number;
+	/** Redis diet Decision 8: a visible tab with no input this long polls at `slowPollMs` (/admin passes it to the page). */
+	idleSlowAfterMs: number;
+	/** Redis diet Decision 8: an idle tab's poll interval (/admin passes it to the page). */
+	slowPollMs: number;
+	/** Redis diet Decision 10: once the tab holds the document, a keep goes this long after the last change (/admin passes it to the page). */
+	keepDelayMs: number;
+	/** Redis diet Decision 10: …and never later than this after the first change not yet sent (/admin passes it to the page). */
+	keepCapMs: number;
 }
 
 export const DEFAULT_TIMINGS: LiveTimings = {
@@ -44,4 +60,12 @@ export const DEFAULT_TIMINGS: LiveTimings = {
 	presenceMs: 15_000,
 	autosaveTtlMs: 7 * 24 * 60 * 60_000,
 	pollMs: 5000,
+	publishSlowMs: 15 * 60_000,
+	publishAnswerMs: 60_000,
+	statusCacheMs: 30_000,
+	beatEveryMs: 30_000,
+	idleSlowAfterMs: 2 * 60_000,
+	slowPollMs: 30_000,
+	keepDelayMs: 3000,
+	keepCapMs: 15_000,
 };

package/src/media.ts

@@ -25,7 +25,7 @@
  */
 import { createHmac } from 'node:crypto';
 import type { ImageFormat } from './fields.ts';
-import type { RenderImage } from './media-value.ts';
+import { renderMedia, type RenderImage } from './media-value.ts';
 
 /** The srcset width ladder every fluid image draws from (Decision 2). 1920 at the top: imgproxy doesn't upscale. */
 export const IMAGE_WIDTHS: readonly number[] = [160, 320, 480, 640, 960, 1280, 1920];
@@ -253,3 +253,28 @@ export function brandImageUrl(key: string, cssHeight: number, config: MediaConfi
 	if (config.kind !== 'ready') return null;
 	return imageUrl(config.settings, key, { height: cssHeight * 2 });
 }
+
+/**
+ * Urban Return port, Decision 20: one URL for a Site Settings image, for the page's frame
+ * (a header, a footer, a layout), which has no block's `cms` for `<Media>`. Exported from
+ * `caracms/content`:
+ *
+ *   const settings = await siteSettings(site, Astro.locals);
+ *   const url = settingsImageUrl(settings.megaPhoto, { width: 640, height: 400 });
+ *
+ * `value` is the field as `siteSettings` gives it (resolved, M9), or the stored value. The
+ * rules are `<Media>`'s: the signed imgproxy URL, `fit` to one side, `fill` to both with the
+ * focal point's gravity, a GIF or an SVG passed through unresized, WebP; under
+ * `CARACMS_MEDIA=memory` the in-memory media's URL. No size: `fit` at the ladder's top rung.
+ * Null for an empty value (or a clip, or a file's key) and when the media isn't set up.
+ */
+export function settingsImageUrl(value: unknown, options: { width?: number; height?: number } = {}): string | null {
+	const image = renderMedia(value, { only: 'image' });
+	if (!image || image.kind !== 'image') return null;
+	const config = readMediaConfig();
+	if (config.kind !== 'ready') return null;
+	const width = options.width ?? 0;
+	const height = options.height ?? 0;
+	const sized = width > 0 || height > 0 ? { width, height } : { width: IMAGE_WIDTHS[IMAGE_WIDTHS.length - 1]! };
+	return imageUrl(config.settings, image.key, { ...sized, focal: image.focal });
+}