@growth-labs/cms 0.8.17 → 0.8.19

package/src/engine/content-insights.ts

@@ -6,6 +6,7 @@ import type {
 	InsightEvidence,
 } from '../schema/types.js'
 import type { D1Database } from './d1.js'
+import { canonicalJsonSha256, type JsonValue } from './publication-receipt/canonical-json.js'
 
 /** True when an Insights read/write depends on a table missing from an older schema. */
 export function isMissingInsightsTable(error: unknown): boolean {
@@ -99,69 +100,100 @@ export async function undismissInsight(
 }
 
 export type ReplaceOutcome = 'applied' | 'duplicate' | 'stale'
+
+/** Identity read from the committed run row, never echoed from the incoming request. */
+export interface InsightPublicationIdentity {
+	eventId: string
+	siteId: string
+	runDate: string
+	runAttempt: number
+	runId: string | null
+	contentHash: string
+	/** RFC 8785 SHA-256 of the full validated {meta, rows}, including actual row content. */
+	payloadSha256: string | null
+	/** Legacy rows have no digest/run identity and cannot confirm a duplicate. */
+	verified: boolean
+}
+
 export interface ReplaceResult {
 	outcome: ReplaceOutcome
 	landedRunAttempt: number
+	publication: InsightPublicationIdentity
 }
 
+export class InsightPublicationError extends Error {
+	constructor(readonly code: 'publication_identity_conflict' | 'publication_identity_unverified') {
+		super(code)
+		this.name = 'InsightPublicationError'
+	}
+}
+
+/** Only ingest references these columns; do not mask unrelated missing-column failures. */
+export function isMissingInsightPublicationSchema(error: unknown): boolean {
+	return (
+		isMissingInsightsTable(error) ||
+		(error instanceof Error &&
+			/(?:no such table: content_insight_runs\b|no such column:\s*(?:publication_run_id|payload_sha256)\b|table\s+content_insight_runs\s+has no column named\s+(?:publication_run_id|payload_sha256)\b)/i.test(
+				error.message,
+			))
+	)
+}
+
+interface PublicationRow {
+	event_id: string
+	site_id: string
+	run_date: string
+	run_attempt: number
+	publication_run_id: string | null
+	content_hash: string
+	payload_sha256: string | null
+}
+
+const publicationColumns = `event_id, site_id, run_date, run_attempt,
+	publication_run_id, content_hash, payload_sha256`
+
 /**
- * Replay-safe replace of the board for (meta.site_id, meta.run_date).
- *  - 'duplicate': meta.event_id already present in content_insight_runs.event_id → no-op.
- *  - 'stale':     an equal-or-higher run_attempt already landed for (site_id, run_date) → no-op
- *                 (HMAC authenticates the sender; run_attempt is the monotonic replay guard).
- *  - 'applied':   DELETE the existing board scoped by (site_id, run_date), bulk-insert rows
- *                 (mint id, run_id = meta.run_id, beat from payload, created_at = unixepoch()),
- *                 upsert the run row.
- * Dismissals are untouched — they live in their own table keyed (site_id, intent_key).
+ * Every eligibility read, guarded write and identity read executes in ONE D1 batch.
+ * The run table stays unchanged until the final mutation, so every DELETE/INSERT
+ * evaluates the same predicate against the same transaction state. D1 rolls back
+ * the entire batch on any failure. Dismissals are never touched.
  *
- * The INSERT carries 16 columns (id, site_id, run_id, run_date, desk, suggestion_type,
- * signal_type, is_best_bet, best_bet_rank, desk_rank, headline, evidence, brief,
- * intent_key, beat, created_at) matching migration 0016 column order.
+ * Current event IDs are immutable: exact replay is a no-op; divergent or legacy
+ * unverified reuse fails without writes. Once superseded, an old event has no
+ * historical dedupe promise; its original attempt remains stale.
  */
 export async function replaceInsightsBoard(
 	db: D1Database,
 	payload: InsightsIngestPayload,
 ): Promise<ReplaceResult> {
 	const { meta, rows } = payload
-
-	// 1) event_id dedupe (mirrors foundry_callback_events).
-	const dupEvent = await db
-		.prepare('SELECT id FROM content_insight_runs WHERE event_id = ? LIMIT 1')
-		.bind(meta.event_id)
-		.first<{ id: string }>()
-	if (dupEvent) return { outcome: 'duplicate', landedRunAttempt: meta.run_attempt }
-
-	// 2) stale-replay guard: reject equal-or-lower run_attempt for (site_id, run_date).
-	const existing = await db
-		.prepare(
-			'SELECT id, run_attempt FROM content_insight_runs WHERE site_id = ? AND run_date = ? LIMIT 1',
-		)
-		.bind(meta.site_id, meta.run_date)
-		.first<{ id: string; run_attempt: number }>()
-	if (existing && meta.run_attempt <= existing.run_attempt) {
-		return { outcome: 'stale', landedRunAttempt: existing.run_attempt }
-	}
-
+	const payloadSha256 = await canonicalJsonSha256(payload as unknown as JsonValue)
 	const now = Math.floor(Date.now() / 1000)
 
-	// 3) Scoped replace + bulk insert + run upsert in one D1 batch.
+	const eligible = `NOT EXISTS (SELECT 1 FROM content_insight_runs WHERE event_id = ?)
+		AND NOT EXISTS (SELECT 1 FROM content_insight_runs
+			WHERE site_id = ? AND run_date = ? AND run_attempt >= ?)`
+	const eligibilityBindings = [meta.event_id, meta.site_id, meta.run_date, meta.run_attempt]
+	const currentPublication = db
+		.prepare(`SELECT ${publicationColumns} FROM content_insight_runs
+		WHERE site_id = ? AND run_date = ?`)
+		.bind(meta.site_id, meta.run_date)
 	const statements = [
 		db
-			.prepare('DELETE FROM content_insights WHERE site_id = ? AND run_date = ?')
-			.bind(meta.site_id, meta.run_date),
+			.prepare(`SELECT ${publicationColumns} FROM content_insight_runs WHERE event_id = ? LIMIT 2`)
+			.bind(meta.event_id),
+		currentPublication,
+		db
+			.prepare(`DELETE FROM content_insights WHERE site_id = ? AND run_date = ? AND ${eligible}`)
+			.bind(meta.site_id, meta.run_date, ...eligibilityBindings),
 	]
 	for (const row of rows) {
-		// 16 columns: id, site_id, run_id, run_date, desk, suggestion_type, signal_type,
-		// is_best_bet, best_bet_rank, desk_rank, headline, evidence, brief,
-		// intent_key, beat, created_at
 		statements.push(
 			db
-				.prepare(
-					`INSERT INTO content_insights
-					 (id, site_id, run_id, run_date, desk, suggestion_type, signal_type, is_best_bet,
-					  best_bet_rank, desk_rank, headline, evidence, brief, intent_key, beat, created_at)
-					 VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
-				)
+				.prepare(`INSERT INTO content_insights
+				(id, site_id, run_id, run_date, desk, suggestion_type, signal_type, is_best_bet,
+				 best_bet_rank, desk_rank, headline, evidence, brief, intent_key, beat, created_at)
+				SELECT ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,? WHERE ${eligible}`)
 				.bind(
 					crypto.randomUUID(),
 					row.site_id,
@@ -179,30 +211,32 @@ export async function replaceInsightsBoard(
 					row.intent_key,
 					row.beat,
 					now,
+					...eligibilityBindings,
 				),
 		)
 	}
-	// Upsert the run row keyed on UNIQUE (site_id, run_date).
+	// This MUST be the final mutation: preceding row writes share its eligibility.
 	statements.push(
 		db
-			.prepare(
-				`INSERT INTO content_insight_runs
-				 (id, site_id, run_date, status, event_id, run_attempt, generated_at, content_hash,
-				  demand_count, opportunity_count, started_at, completed_at, error)
-				 VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)
-				 ON CONFLICT(site_id, run_date) DO UPDATE SET
-				   status = excluded.status,
-				   event_id = excluded.event_id,
-				   run_attempt = excluded.run_attempt,
-				   generated_at = excluded.generated_at,
-				   content_hash = excluded.content_hash,
-				   demand_count = excluded.demand_count,
-				   opportunity_count = excluded.opportunity_count,
-				   completed_at = excluded.completed_at,
-				   error = excluded.error`,
-			)
+			.prepare(`INSERT INTO content_insight_runs
+			(id, site_id, run_date, status, event_id, run_attempt, generated_at, content_hash,
+			 demand_count, opportunity_count, started_at, completed_at, error,
+			 publication_run_id, payload_sha256)
+			SELECT ?,?,?,?,?,?,?,?,?,?,?,?,?,?,? WHERE ${eligible}
+			ON CONFLICT(site_id, run_date) DO UPDATE SET
+				status = excluded.status,
+				event_id = excluded.event_id,
+				run_attempt = excluded.run_attempt,
+				generated_at = excluded.generated_at,
+				content_hash = excluded.content_hash,
+				demand_count = excluded.demand_count,
+				opportunity_count = excluded.opportunity_count,
+				completed_at = excluded.completed_at,
+				error = excluded.error,
+				publication_run_id = excluded.publication_run_id,
+				payload_sha256 = excluded.payload_sha256`)
 			.bind(
-				existing?.id ?? crypto.randomUUID(),
+				crypto.randomUUID(),
 				meta.site_id,
 				meta.run_date,
 				meta.status,
@@ -215,9 +249,54 @@ export async function replaceInsightsBoard(
 				now,
 				meta.status === 'completed' || meta.status === 'partial' ? now : null,
 				meta.error ?? null,
+				meta.run_id,
+				payloadSha256,
+				...eligibilityBindings,
 			),
+		currentPublication,
 	)
-
-	await db.batch(statements)
-	return { outcome: 'applied', landedRunAttempt: meta.run_attempt }
+	const results = await db.batch(statements)
+	const events = (results[0].results ?? []) as unknown as PublicationRow[]
+	const previous = results[1].results?.[0] as unknown as PublicationRow | undefined
+	if (events.length > 0) {
+		const event = events[0]
+		if (
+			events.length !== 1 ||
+			event.site_id !== meta.site_id ||
+			event.run_date !== meta.run_date ||
+			event.run_attempt !== meta.run_attempt ||
+			event.content_hash !== meta.content_hash ||
+			(event.publication_run_id !== null && event.publication_run_id !== meta.run_id)
+		) {
+			throw new InsightPublicationError('publication_identity_conflict')
+		}
+		if (event.payload_sha256 === null || event.publication_run_id === null) {
+			throw new InsightPublicationError('publication_identity_unverified')
+		}
+		if (event.payload_sha256 !== payloadSha256) {
+			throw new InsightPublicationError('publication_identity_conflict')
+		}
+	}
+	const committed = results[results.length - 1].results?.[0] as unknown as
+		| PublicationRow
+		| undefined
+	if (!committed) throw new Error('Missing committed insight publication')
+	return {
+		outcome: events.length
+			? 'duplicate'
+			: previous && previous.run_attempt >= meta.run_attempt
+				? 'stale'
+				: 'applied',
+		landedRunAttempt: committed.run_attempt,
+		publication: {
+			eventId: committed.event_id,
+			siteId: committed.site_id,
+			runDate: committed.run_date,
+			runAttempt: committed.run_attempt,
+			runId: committed.publication_run_id,
+			contentHash: committed.content_hash,
+			payloadSha256: committed.payload_sha256,
+			verified: committed.payload_sha256 !== null && committed.publication_run_id !== null,
+		},
+	}
 }

package/src/routes/content-insights.ts

@@ -1,7 +1,9 @@
 import { z } from 'zod'
 import {
 	dismissInsight,
+	InsightPublicationError,
 	isMissingInsightDismissalSchema,
+	isMissingInsightPublicationSchema,
 	isMissingInsightsTable,
 	listInsightBoard,
 	replaceInsightsBoard,
@@ -203,9 +205,12 @@ export function createContentInsightsRoutes(
 					ok: true,
 					outcome: result.outcome,
 					landedRunAttempt: result.landedRunAttempt,
+					publication: result.publication,
 				})
 			} catch (error) {
-				if (isMissingInsightsTable(error)) return json({ error: 'schema_not_ready' }, 409)
+				if (isMissingInsightPublicationSchema(error))
+					return json({ error: 'schema_not_ready' }, 409)
+				if (error instanceof InsightPublicationError) return json({ error: error.code }, 409)
 				throw error
 			}
 		},

package/src/routes/content.ts

@@ -399,14 +399,26 @@ const ContentMetadataSchema = z
 // deploy until 0.8.8, because the masthead only rendered the top-level
 // 'Invalid request' and never the field message telling them what to change.
 //
-// Only the onboarded share host is claimed, and only the share-link shape:
-// https, no credentials, default port, /s/<token> with an optional /download.
-// Nextcloud's folder-listing "Copy link" appends ?dir=/ to a single-file share;
-// it carries nothing the download route needs, so it is dropped. Anything else
-// -- another host, another path, a non-URL -- is returned untouched for the
-// normal URL validation to accept or reject on its own terms.
+// The direct form is `https://<host>/public.php/dav/files/<token>`. From 0.8.8
+// to 0.8.17 this produced `/s/<token>/download` instead; on 2026-09-19 that URL
+// began answering HEAD with a 303 to the dav form, which the Go ingest probe does
+// not follow, so every Nextcloud-sourced video blocked at `media-probe-source`
+// (KB incident-uk-video-prep-2026-09-19). Storing the dav form directly means the
+// probe never meets a redirect. A URL already in the dav form is returned as it
+// is, so the function is idempotent, and rows saved earlier with the old
+// `/download` form are not rewritten by a save that does not touch the source.
+//
+// Only the onboarded share host is claimed, and only two shapes: https, no
+// credentials, default port, and either /s/<token> (with an optional /download
+// and trailing slash) or the dav form itself. Nextcloud's folder-listing "Copy
+// link" appends ?dir=/ to a single-file share; it carries nothing the download
+// route needs, so it is dropped. Any other query is a shape this does not know,
+// so the value is returned untouched. Anything else -- another host, another
+// path, a non-URL -- is likewise returned untouched for the normal URL
+// validation to accept or reject on its own terms.
 const NEXTCLOUD_SHARE_HOSTS = ['drive.fulcrum-portal.com']
-const NEXTCLOUD_SHARE_PATH_RE = /^\/s\/[A-Za-z0-9_-]+(?:\/download)?\/?$/
+const NEXTCLOUD_SHARE_PATH_RE = /^\/s\/([A-Za-z0-9_-]+)(?:\/download)?\/?$/
+const NEXTCLOUD_DIRECT_PATH_RE = /^\/public\.php\/dav\/files\/[A-Za-z0-9_-]+$/
 
 export function normaliseNextcloudShareUrl(value: string): string {
 	let url: URL
@@ -415,23 +427,25 @@ export function normaliseNextcloudShareUrl(value: string): string {
 	} catch {
 		return value
 	}
+	const host = url.hostname.toLowerCase().replace(/\.$/, '')
 	if (
 		url.protocol !== 'https:' ||
 		url.username ||
 		url.password ||
 		(url.port && url.port !== '443') ||
-		!NEXTCLOUD_SHARE_HOSTS.includes(url.hostname.toLowerCase().replace(/\.$/, '')) ||
-		!NEXTCLOUD_SHARE_PATH_RE.test(url.pathname)
+		!NEXTCLOUD_SHARE_HOSTS.includes(host) ||
+		url.hash
 	) {
 		return value
 	}
-	if (url.searchParams.get('dir') !== null && url.searchParams.size === 1) {
-		url.search = ''
-	}
-	if (!/\/download\/?$/.test(url.pathname)) {
-		url.pathname = `${url.pathname.replace(/\/$/, '')}/download`
+	const onlyDirQuery =
+		url.search === '' || (url.searchParams.get('dir') !== null && url.searchParams.size === 1)
+	if (NEXTCLOUD_DIRECT_PATH_RE.test(url.pathname)) {
+		return url.search === '' ? value : onlyDirQuery ? `https://${host}${url.pathname}` : value
 	}
-	return url.toString()
+	const share = NEXTCLOUD_SHARE_PATH_RE.exec(url.pathname)
+	if (!share || !onlyDirQuery) return value
+	return `https://${host}/public.php/dav/files/${share[1]}`
 }
 
 // A Google Drive link is not run through the Nextcloud rewrite; it is kept as

package/src/schema/migrations.ts

@@ -914,6 +914,13 @@ ALTER TABLE podcast_content ADD COLUMN processing_state TEXT CHECK (processing_s
 		id: '0031_podcast_processing_error',
 		sql: `ALTER TABLE podcast_content ADD COLUMN processing_error TEXT;`,
 	},
+	{
+		id: '0032_content_insight_publication_identity',
+		sql: `
+ALTER TABLE content_insight_runs ADD COLUMN publication_run_id TEXT;
+ALTER TABLE content_insight_runs ADD COLUMN payload_sha256 TEXT;
+	`,
+	},
 ]
 
 /**

package/src/schema/types.ts

@@ -677,7 +677,7 @@ export interface ContentInsightRunRow {
 	site_id: string
 	run_date: string
 	status: InsightRunStatus
-	/** Dedupe key (mirrors foundry_callback_events.event_id). */
+	/** Immutable event identity while this publication remains current. */
 	event_id: string
 	/** Monotonic; the stale-replay guard rejects equal-or-lower. */
 	run_attempt: number
@@ -685,6 +685,9 @@ export interface ContentInsightRunRow {
 	generated_at: number
 	/** Content hash of the board for change detection. */
 	content_hash: string
+	/** Sender run identity and receiver-computed RFC 8785 payload digest; null on legacy rows. */
+	publication_run_id: string | null
+	payload_sha256: string | null
 	demand_count: number
 	opportunity_count: number
 	started_at: number | null