@growth-labs/cms 0.5.29 → 0.5.31

package/src/engine/fronts-publish.ts

@@ -0,0 +1,417 @@
+// Fronts publish path — the additive, flag-gated, out-of-process publish flow
+// (increment: cms/Masthead side).
+//
+// TWO seams, both driven by the external Fronts publish worker (foundryd):
+//
+//   1. listFrontsPublishSchedule — the poll surface. Returns `scheduled` items
+//      whose `publish_at` has arrived or is arriving within the caller's window
+//      (plus any overdue), so foundryd knows what to publish. Mirrors the
+//      `runScheduledPublish` (engine/cron.ts) due-predicate — `content_items`
+//      scheduling columns are INTEGER unix seconds, so the window is compared in
+//      seconds (NOT the D1 `datetime('now')` TEXT format; there is no
+//      space-vs-T comparison trap on this column).
+//
+//   2. applyFrontsPublishCallback — the receiver. Records foundryd's outbound
+//      callback (idempotent by `event_id`), reflects the lifecycle status onto
+//      `content_items.fronts_publish_status`, and — ONLY on a confirmed `live`
+//      with a receipt — drives the item to published THROUGH the existing
+//      `content_publication_attempts` 2PC ledger via `publishOne`. A
+//      `queued`/`publishing`/`late`/`failed` callback (or a `live` with no
+//      receipt) never flips the item to published.
+//
+// The legacy D1 cron publish path (engine/cron.ts `runScheduledPublish` + the
+// site's cron) is UNTOUCHED and remains the rollback lever; this path is purely
+// additive and gated by FRONTS_PUBLISH_ENABLED (default ON — a rollback lever,
+// not a dark-staging gate).
+
+import type { ContentType, FrontsPublishStatus } from '../schema/types.js'
+import type { D1Database } from './d1.js'
+import { type PublicationReconciler, publishOne } from './publication.js'
+
+/** The five lifecycle states foundryd reports for a Fronts publish intent. */
+export const FRONTS_PUBLISH_STATUSES: readonly FrontsPublishStatus[] = [
+	'queued',
+	'publishing',
+	'live',
+	'late',
+	'failed',
+] as const
+
+export function isFrontsPublishStatus(value: unknown): value is FrontsPublishStatus {
+	return typeof value === 'string' && (FRONTS_PUBLISH_STATUSES as string[]).includes(value)
+}
+
+/**
+ * Feature flag — a ROLLBACK LEVER, not a graduation gate. Default ENABLED; the
+ * additive Fronts publish endpoints only go dark when the host explicitly sets
+ * FRONTS_PUBLISH_ENABLED to a falsy string ('0' | 'false' | 'off' | 'no') or the
+ * boolean false. When off, foundryd's polls/callbacks are refused loudly (503)
+ * and the untouched D1 cron path owns publishing.
+ */
+export function isFrontsPublishEnabled(env: Record<string, unknown> | undefined): boolean {
+	const raw = env?.FRONTS_PUBLISH_ENABLED
+	if (raw === undefined || raw === null) return true
+	if (typeof raw === 'boolean') return raw
+	if (typeof raw === 'number') return raw !== 0
+	if (typeof raw === 'string') {
+		return !['0', 'false', 'off', 'no'].includes(raw.trim().toLowerCase())
+	}
+	return true
+}
+
+// --- 1. Schedule poll ------------------------------------------------------
+
+export interface FrontsScheduleItem {
+	id: string
+	type: ContentType
+	slug: string
+	title: string
+	channel: string | null
+	/** Scheduled publish time, unix epoch seconds. */
+	publishAt: number
+	/** Current lifecycle marker (null until foundryd first acknowledges it). */
+	frontsPublishStatus: FrontsPublishStatus | null
+	/** publish_at is at or before `now` — foundryd is late on this one. */
+	overdue: boolean
+}
+
+export interface ListFrontsPublishScheduleOptions {
+	/** Window upper bound, unix epoch seconds. Items due at/before this are returned. */
+	toSeconds: number
+	/** Reference time for the `overdue` flag, unix epoch seconds. Defaults to Date.now(). */
+	nowSeconds?: number
+	/** Hard cap on rows returned. Default 200, max 1000. */
+	limit?: number
+}
+
+/**
+ * List `scheduled` items due at or before the window end. Includes overdue
+ * items (publish_at < window start) — the window's lower bound is intentionally
+ * not applied, matching foundryd's "next window + any overdue" contract. Ordered
+ * by publish_at ascending so the most-overdue item is published first.
+ */
+export async function listFrontsPublishSchedule(
+	db: D1Database,
+	options: ListFrontsPublishScheduleOptions,
+): Promise<FrontsScheduleItem[]> {
+	const now = options.nowSeconds ?? Math.floor(Date.now() / 1_000)
+	const limit = Math.min(Math.max(1, Math.floor(options.limit ?? 200)), 1000)
+	const result = await db
+		.prepare(
+			`SELECT id, type, slug, title, channel, publish_at, fronts_publish_status
+			 FROM content_items
+			 WHERE status = 'scheduled'
+			   AND publish_at IS NOT NULL
+			   AND publish_at <= ?
+			 ORDER BY publish_at ASC
+			 LIMIT ?`,
+		)
+		.bind(options.toSeconds, limit)
+		.all<{
+			id: string
+			type: ContentType
+			slug: string
+			title: string
+			channel: string | null
+			publish_at: number
+			fronts_publish_status: FrontsPublishStatus | null
+		}>()
+	return (result.results ?? []).map((row) => ({
+		id: row.id,
+		type: row.type,
+		slug: row.slug,
+		title: row.title,
+		channel: row.channel,
+		publishAt: row.publish_at,
+		frontsPublishStatus: row.fronts_publish_status,
+		overdue: row.publish_at <= now,
+	}))
+}
+
+// --- 2. Publish callback receiver -----------------------------------------
+
+export interface FrontsPublishReceipt {
+	published_urls?: string[] | null
+	platform_post_ids?: string[] | null
+	sha256?: string | null
+}
+
+export interface FrontsPublishCallbackInput {
+	eventId: string
+	intentId?: string | null
+	contentId: string
+	version?: number | null
+	correlationId?: string | null
+	status: FrontsPublishStatus
+	/** ISO-8601 string or unix epoch seconds; used as the published_at on `live`. */
+	occurredAt?: string | number | null
+	receipt?: FrontsPublishReceipt | null
+	error?: string | null
+	/** Reference time (unix seconds), for tests. Defaults to Date.now(). */
+	now?: number
+}
+
+export interface FrontsPublishCallbackResult {
+	ok: boolean
+	/** True when this event_id was already processed — no state change this call. */
+	deduped: boolean
+	status: FrontsPublishStatus
+	/** True when the item is published (flipped by this call, or already was). */
+	published: boolean
+	/** The content_publication_attempts ledger row id, when this call published. */
+	attemptId: string | null
+	contentFound: boolean
+	/** Set when a `live` callback could not be committed through the ledger. */
+	ledgerError: string | null
+}
+
+function hasReceipt(receipt: FrontsPublishReceipt | null | undefined): boolean {
+	if (!receipt) return false
+	const urls = Array.isArray(receipt.published_urls) && receipt.published_urls.length > 0
+	const ids = Array.isArray(receipt.platform_post_ids) && receipt.platform_post_ids.length > 0
+	const sha = typeof receipt.sha256 === 'string' && receipt.sha256.trim().length > 0
+	return urls || ids || sha
+}
+
+/** Resolve occurred_at to a bounded, non-future unix-seconds value (or null). */
+function resolveOccurredAtSeconds(
+	occurredAt: string | number | null | undefined,
+	now: number,
+): number | null {
+	let seconds: number | null = null
+	if (typeof occurredAt === 'number' && Number.isFinite(occurredAt)) {
+		seconds = Math.floor(occurredAt)
+	} else if (typeof occurredAt === 'string' && occurredAt.trim()) {
+		const ms = Date.parse(occurredAt)
+		if (!Number.isNaN(ms)) seconds = Math.floor(ms / 1_000)
+	}
+	if (seconds === null || !Number.isSafeInteger(seconds) || seconds < 0) return null
+	// publishOne rejects a published_at later than `now`; clamp for clock skew.
+	return Math.min(seconds, now)
+}
+
+/**
+ * Apply one foundryd publish-callback. Idempotent by `event_id`: a repeat
+ * delivery returns the stored outcome without re-applying. See the module
+ * header for the state-transition contract.
+ */
+export async function applyFrontsPublishCallback(
+	db: D1Database,
+	input: FrontsPublishCallbackInput,
+): Promise<FrontsPublishCallbackResult> {
+	const now = input.now ?? Math.floor(Date.now() / 1_000)
+
+	// 1. Idempotency — replay the stored outcome for a known event_id.
+	const existing = await readStoredOutcome(db, input.eventId)
+	if (existing) return existing
+
+	// 2. Resolve the target item. A missing item is NOT recorded, so a retry can
+	//    still land once the item exists.
+	const item = await db
+		.prepare('SELECT id, status, fronts_publish_status FROM content_items WHERE id = ? LIMIT 1')
+		.bind(input.contentId)
+		.first<{ id: string; status: string; fronts_publish_status: FrontsPublishStatus | null }>()
+	if (!item) {
+		return {
+			ok: false,
+			deduped: false,
+			status: input.status,
+			published: false,
+			attemptId: null,
+			contentFound: false,
+			ledgerError: null,
+		}
+	}
+
+	// 3. On a confirmed `live` WITH a receipt, drive the publish through the 2PC
+	//    ledger — record attempt + commit — flipping the item to published.
+	//    A `live` on an already-published item just confirms it (no re-publish,
+	//    so we never race the cron path into a duplicate revision).
+	let published = item.status === 'published'
+	let attemptId: string | null = null
+	let ledgerError: string | null = null
+	const shouldPublish = input.status === 'live' && hasReceipt(input.receipt)
+	if (shouldPublish && item.status !== 'published') {
+		try {
+			const receiptSummary = summarizeReceipt(input.receipt)
+			const surface: PublicationReconciler = {
+				name: 'fronts',
+				async prepare() {
+					return { name: 'fronts', status: 'ok', phase: 'prepare' }
+				},
+				async commit() {
+					return { name: 'fronts', status: 'ok', phase: 'commit', message: receiptSummary }
+				},
+			}
+			const publishedAt = resolveOccurredAtSeconds(input.occurredAt, now)
+			const receipt = await publishOne(db, {
+				contentId: input.contentId,
+				now,
+				...(publishedAt !== null ? { publishedAt } : {}),
+				createdBy: 'foundry.fronts-publish',
+				surfaces: [surface],
+			})
+			attemptId = receipt.attemptId
+			published = true
+		} catch (error) {
+			// publishOne can throw when a concurrent/superseding publish already
+			// moved the canonical pointer (the item ends up published anyway) — treat
+			// that as success. Any other failure is recorded and surfaced loudly.
+			const reread = await db
+				.prepare('SELECT status FROM content_items WHERE id = ? LIMIT 1')
+				.bind(input.contentId)
+				.first<{ status: string }>()
+			if (reread?.status === 'published') {
+				published = true
+			} else {
+				ledgerError = error instanceof Error ? error.message : String(error)
+			}
+		}
+	}
+
+	const incomingOccurredAt = resolveOccurredAtSeconds(input.occurredAt, now)
+
+	// 4. Reflect the lifecycle marker onto the item — but NEVER regress it. A
+	//    retried/out-of-order `publishing`/`failed` arriving after `live` must not
+	//    make a published item read as publishing/failed, so gate on occurred_at
+	//    ordering and the terminal `live`/published state.
+	const latestSeenRow = await db
+		.prepare(
+			'SELECT MAX(occurred_at) AS latest FROM fronts_publish_events WHERE content_id = ? AND occurred_at IS NOT NULL',
+		)
+		.bind(input.contentId)
+		.first<{ latest: number | null }>()
+	if (
+		shouldApplyFrontsMarker(
+			item.status,
+			item.fronts_publish_status,
+			input.status,
+			incomingOccurredAt,
+			latestSeenRow?.latest ?? null,
+		)
+	) {
+		await db
+			.prepare(
+				'UPDATE content_items SET fronts_publish_status = ?, updated_at = unixepoch() WHERE id = ?',
+			)
+			.bind(input.status, input.contentId)
+			.run()
+	}
+
+	// 5. Record the durable, idempotent event row (the "processed" marker). The
+	//    dedupe SELECT (step 1) and this INSERT are not one transaction, so two
+	//    concurrent same-event_id deliveries can both reach here — ON CONFLICT DO
+	//    NOTHING makes the loser a no-op (never a spurious PK-violation 500), and
+	//    we then replay the winner's stored outcome. publishOne's own CAS already
+	//    prevents a double publish.
+	const insertResult = await db
+		.prepare(
+			`INSERT INTO fronts_publish_events
+			 (event_id, intent_id, content_id, version, correlation_id, status, occurred_at,
+			  receipt_json, error, published, attempt_id, received_at)
+			 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+			 ON CONFLICT(event_id) DO NOTHING`,
+		)
+		.bind(
+			input.eventId,
+			input.intentId ?? null,
+			input.contentId,
+			typeof input.version === 'number' && Number.isFinite(input.version)
+				? Math.floor(input.version)
+				: null,
+			input.correlationId ?? null,
+			input.status,
+			incomingOccurredAt,
+			input.receipt ? JSON.stringify(input.receipt) : null,
+			ledgerError,
+			published ? 1 : 0,
+			attemptId,
+			now,
+		)
+		.run()
+
+	const changes = insertResult.meta?.changes
+	if (typeof changes === 'number' && changes === 0) {
+		// A concurrent callback with the same event_id won the PK race.
+		const winner = await readStoredOutcome(db, input.eventId)
+		if (winner) return winner
+	}
+
+	return {
+		ok: ledgerError === null,
+		deduped: false,
+		status: input.status,
+		published,
+		attemptId,
+		contentFound: true,
+		ledgerError,
+	}
+}
+
+/** Replay a stored callback outcome by event_id, or null when not yet recorded. */
+async function readStoredOutcome(
+	db: D1Database,
+	eventId: string,
+): Promise<FrontsPublishCallbackResult | null> {
+	const row = await db
+		.prepare(
+			'SELECT status, published, attempt_id, error FROM fronts_publish_events WHERE event_id = ? LIMIT 1',
+		)
+		.bind(eventId)
+		.first<{
+			status: FrontsPublishStatus
+			published: number
+			attempt_id: string | null
+			error: string | null
+		}>()
+	if (!row) return null
+	return {
+		ok: !row.error,
+		deduped: true,
+		status: row.status,
+		published: row.published === 1,
+		attemptId: row.attempt_id,
+		contentFound: true,
+		ledgerError: row.error,
+	}
+}
+
+/**
+ * Whether an incoming callback may overwrite `content_items.fronts_publish_status`.
+ * `live` is the authoritative terminal-success marker (always reflected, never
+ * downgraded); older-by-occurred_at callbacks may not regress a newer marker.
+ */
+function shouldApplyFrontsMarker(
+	itemStatus: string,
+	currentMarker: FrontsPublishStatus | null,
+	incomingStatus: FrontsPublishStatus,
+	incomingOccurredAt: number | null,
+	latestSeenOccurredAt: number | null,
+): boolean {
+	if (incomingStatus === 'live') return true
+	if (itemStatus === 'published' || currentMarker === 'live') return false
+	if (
+		incomingOccurredAt !== null &&
+		latestSeenOccurredAt !== null &&
+		incomingOccurredAt < latestSeenOccurredAt
+	) {
+		return false
+	}
+	return true
+}
+
+/** Bounded, human-readable receipt digest recorded on the ledger surface receipt. */
+function summarizeReceipt(receipt: FrontsPublishReceipt | null | undefined): string {
+	if (!receipt) return 'fronts:live'
+	const parts: string[] = []
+	if (Array.isArray(receipt.published_urls) && receipt.published_urls.length > 0) {
+		parts.push(`urls=${receipt.published_urls.length}`)
+	}
+	if (Array.isArray(receipt.platform_post_ids) && receipt.platform_post_ids.length > 0) {
+		parts.push(`posts=${receipt.platform_post_ids.length}`)
+	}
+	if (typeof receipt.sha256 === 'string' && receipt.sha256.trim()) {
+		parts.push(`sha256=${receipt.sha256.trim().slice(0, 64)}`)
+	}
+	return `fronts:live ${parts.join(' ')}`.trim().slice(0, 500)
+}

package/src/engine/index.ts

@@ -83,6 +83,33 @@ export {
 	type ProcessingState,
 	verifyHmac,
 } from './foundry-dispatch.js'
+// Fronts publish path — schedule poll + publish-callback receiver (additive, flag-gated).
+export {
+	applyFrontsPublishCallback,
+	FRONTS_PUBLISH_STATUSES,
+	type FrontsPublishCallbackInput,
+	type FrontsPublishCallbackResult,
+	type FrontsPublishReceipt,
+	type FrontsScheduleItem,
+	isFrontsPublishEnabled,
+	isFrontsPublishStatus,
+	type ListFrontsPublishScheduleOptions,
+	listFrontsPublishSchedule,
+} from './fronts-publish.js'
+// Canonical Fronts publish INTENT contract — the CMS-owned canonical values a
+// pull consumer needs to build a complete publish intent (packages#500).
+export {
+	armFrontsPublishTriggerToken,
+	type FrontsPublishContentRefs,
+	type FrontsPublishIntent,
+	FrontsPublishIntentError,
+	type FrontsPublishIntentErrorCode,
+	type FrontsPublishIntentFailure,
+	frontsPublishTriggerTokenHash,
+	type HydrateFrontsPublishIntentsResult,
+	hydrateFrontsPublishIntents,
+	setFrontsChannelTargets,
+} from './fronts-publish-intent.js'
 // Import parsers — pure string→struct transforms for Markdown/Ghost/Substack.
 export {
 	htmlToMarkdown,
@@ -168,6 +195,35 @@ export {
 	rollbackPublication,
 	type SkippedArchiveItem,
 } from './publication.js'
+export {
+	acceptsProbeStatus,
+	type ContinuityArchiveItem,
+	type ContinuityDrift,
+	type ContinuityDriftField,
+	type ContinuityPublishedRow,
+	isRetiredItem,
+	payloadDrift,
+} from './publication-continuity.js'
+export {
+	CanonicalJsonError,
+	canonicalJson,
+	canonicalJsonBytes,
+	canonicalJsonSha256,
+	type JsonValue,
+} from './publication-receipt/canonical-json.js'
+export {
+	canonicalizeReceiptBody,
+	PublicationReceiptError,
+	SITE_PUBLICATION_RECEIPT_AUDIENCE,
+	SITE_PUBLICATION_RECEIPT_SCHEMA,
+	type SitePublicationReceiptBody,
+	sitePublicationReceiptBodySchema,
+} from './publication-receipt/receipt.js'
+export {
+	type SignedPublicationReceipt,
+	signPublicationReceipt,
+	verifyPublicationReceipt,
+} from './publication-receipt/signature.js'
 // Pre-publish body validation guard.
 export {
 	evaluateArticleBody,
@@ -312,6 +368,10 @@ export {
 	restoreContent,
 	softDeleteContent,
 } from './soft-delete.js'
+export {
+	normalizeSourceSections,
+	type SourceSectionNormalization,
+} from './source-sections.js'
 // Tag CRUD engine — listTags / searchTags / renameTag / deleteTag.
 export {
 	createTag,

package/src/engine/publication-continuity.ts

@@ -0,0 +1,140 @@
+// Publication continuity decisions (#342).
+//
+// Extracted from the Warfronts publish bridge so every site shares one rule set
+// instead of each re-deriving it. These are the decisions a continuity run makes
+// before it touches the database: is this item retired, did the published
+// revision drift from the archive, and does a live probe response mean success.
+//
+// All pure. The bridge's DB, network and filesystem work stays site-owned; only
+// the judgement moves here.
+
+/** The archive-side fields a continuity run compares. */
+export interface ContinuityArchiveItem {
+	title?: string | null
+	description?: string | null
+	featured?: boolean | null
+	heroImageId?: string | null
+	visibility?: string | null
+	/** The archive's name for what the engine stores as `channel`. */
+	primaryCategory?: string | null
+	bodyMarkdown?: string | null
+	metadata?: { originalStatus?: string | null } | null
+}
+
+/** The joined `content_items` + published-revision row, as the engine stores it. */
+export interface ContinuityPublishedRow {
+	rev_title?: string | null
+	rev_description?: string | null
+	rev_featured?: number | null
+	rev_hero_image_id?: string | null
+	rev_visibility?: string | null
+	rev_channel?: string | null
+	rev_body_markdown?: string | null
+}
+
+export type ContinuityDriftField =
+	| 'title'
+	| 'description'
+	| 'featured'
+	| 'heroImage'
+	| 'visibility'
+	| 'channel'
+	| 'body'
+
+export interface ContinuityDrift {
+	drifted: boolean
+	fields: ContinuityDriftField[]
+	title: string
+	description: string | null
+	/** `content_items.featured` is INTEGER; the archive item carries a boolean. */
+	featured: 0 | 1
+	heroImageId: string | null
+	visibility: 'free' | 'premium'
+	channel: string | null
+	body: string
+	bodyDrifted: boolean
+}
+
+/** An item the archive has taken down rather than published. */
+export function isRetiredItem(item: ContinuityArchiveItem | null | undefined): boolean {
+	return item?.metadata?.originalStatus === 'archived'
+}
+
+/**
+ * Compare the fields a reader actually sees against the published revision.
+ *
+ * The site renders from the published revision payload, so an edit to any field
+ * this does not examine is classified "already in sync", skipped, and never
+ * reaches a reader — while the run reports success. That has stranded real work
+ * twice: fourteen restored article bodies, and a `featured` flag flip, both
+ * merged, both reporting `failed=0`, neither ever visible.
+ *
+ * Comparison is exact rather than a length or prefix heuristic. The entire class
+ * of bug being prevented is a cheap test that silently misses a real edit: a
+ * false "stale" costs one idempotent republish, a false "in sync" loses the edit.
+ *
+ * NOT COMPARED: tags. The revision stores them canonicalized and sorted while
+ * the archive carries raw frontmatter strings, and the engine exports no
+ * canonicalizer to reuse. Detecting a field this cannot also REPAIR would be
+ * worse than not detecting it — tag writes go through the engine, so a tag-only
+ * drift would republish unchanged and stay stale on every run, forever.
+ */
+export function payloadDrift(
+	item: ContinuityArchiveItem,
+	row: ContinuityPublishedRow,
+): ContinuityDrift {
+	// The engine stores an absent body as a single space, so normalize the same
+	// way before comparing or the first resync of a body-less article would
+	// churn forever.
+	const body = item.bodyMarkdown ?? ' '
+	const title = String(item.title ?? '')
+	const description = item.description ?? null
+	const featured: 0 | 1 = item.featured ? 1 : 0
+	const heroImageId = item.heroImageId ?? null
+	const visibility: 'free' | 'premium' = item.visibility === 'premium' ? 'premium' : 'free'
+	const channel = item.primaryCategory ?? null
+
+	const fields: ContinuityDriftField[] = []
+	if (row.rev_title !== title) fields.push('title')
+	if ((row.rev_description ?? null) !== description) fields.push('description')
+	if (Number(row.rev_featured ?? 0) !== featured) fields.push('featured')
+	if ((row.rev_hero_image_id ?? null) !== heroImageId) fields.push('heroImage')
+	if ((row.rev_visibility ?? 'free') !== visibility) fields.push('visibility')
+	if ((row.rev_channel ?? null) !== channel) fields.push('channel')
+	// A revision predating this check can lack the key entirely; treat an absent
+	// body as drift rather than silently matching.
+	if ((row.rev_body_markdown ?? null) !== body) fields.push('body')
+
+	return {
+		drifted: fields.length > 0,
+		fields,
+		title,
+		description,
+		featured,
+		heroImageId,
+		visibility,
+		channel,
+		body,
+		bodyDrifted: fields.includes('body'),
+	}
+}
+
+/**
+ * Whether a live probe response means the publication succeeded.
+ *
+ * LIVE → 200 only.
+ * RETIRED → 301/308 (redirected to a surviving twin) or 404 (gone, with no
+ *   honest target to redirect to).
+ *
+ * The invariant: **a retired article must never answer 200.** That is the
+ * stale-row symptom a teardown exists to remove, and it is the only status this
+ * rejects for a retirement. Demanding 200 for retirements instead would fail the
+ * run forever on exactly the articles the archive deliberately took down.
+ *
+ * 301 and 308 are both permanent; a server may answer GET with one and other
+ * methods with the other, so both are accepted.
+ */
+export function acceptsProbeStatus(status: number, retired: boolean): boolean {
+	if (!retired) return status === 200
+	return status === 301 || status === 308 || status === 404
+}