@growth-labs/cms 0.1.0

package/src/engine/validator/index.ts

@@ -0,0 +1,385 @@
+/**
+ * article-body-validator (modular layout)
+ *
+ * The validator is the single source of truth for ingest-time formatting
+ * QA. Each `checks/<code>.ts` file implements one independent detector;
+ * this index module composes them into the public API used by the
+ * publish guard and the publishing engine.
+ *
+ * Ported verbatim from fulcrum-labs/fronts
+ * src/lib/article-body-validator/index.ts (inventory §1: extract). The
+ * validator runs on PRE-strip source; renderer-side cleanup happens AFTER
+ * the validator has emitted its report so ingest bugs stay visible.
+ */
+import * as bareUrl from './checks/bare-url-not-autolinked.js'
+import * as brokenFootnoteLabel from './checks/broken-footnote-label.js'
+import * as doubleEncoded from './checks/double-encoded-entities.js'
+import * as emptyAlt from './checks/empty-alt-text.js'
+import * as headingHierarchy from './checks/heading-hierarchy-skip.js'
+import * as htmlCommentLeak from './checks/html-comment-leak.js'
+import * as iframeMissingDims from './checks/iframe-missing-dims-and-wrapper.js'
+import * as invisibleControl from './checks/invisible-control-chars.js'
+import * as paywallMarker from './checks/paywall-marker-leak.js'
+import * as rawBlockHtml from './checks/raw-block-html.js'
+import * as unresolvedAnchor from './checks/unresolved-footnote-anchor.js'
+import * as wordGdocs from './checks/word-gdocs-paste-artifacts.js'
+
+import type {
+	ArticleBodyInput,
+	Severity,
+	ValidationError,
+	ValidationErrorCode,
+	ValidationResult,
+	ValidatorOptions,
+} from './types.js'
+import { DEFAULT_OPTIONS, makeError, SEVERITY } from './types.js'
+
+/* -------------------------------------------------------------------------- */
+/*                     Phase 0 / Phase 1 — preserved checks                   */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Astro 500 error template fingerprints. The build-time guard looks for
+ * the same set, so the static-output check and the pre-publish check
+ * agree on what a baked server-error page looks like.
+ */
+export const ASTRO_SERVER_ERROR_PATTERNS: readonly RegExp[] = [
+	/<title>\s*Server Error(?:\s*\|\s*Fronts)?\s*<\/title>/i,
+	/Fronts encountered an unexpected error\./i,
+	/The request reached Fronts but the response failed before rendering completed\./i,
+	/Something broke upstream\./i,
+]
+
+const LEGIT_ESCAPE_CHARS = '*_[]{}|`#+!>\\-'
+
+function isBlank(value: string | null): boolean {
+	return !value || value.trim().length === 0
+}
+
+export function checkDataUriArtifact(input: ArticleBodyInput): ValidationError[] {
+	const errors: ValidationError[] = []
+	const pattern = /data:image\/[a-z0-9+.-]+;base64,/i
+	const checkField = (value: string | null, fieldName: string) => {
+		if (!value) return
+		const match = pattern.exec(value)
+		if (!match) return
+		errors.push(
+			makeError(
+				'data-uri-artifact',
+				`Article body contains an inline data:image base64 payload in ${fieldName}; bodies must reference R2 URLs only.`,
+				{ start: match.index, end: match.index + match[0].length },
+			),
+		)
+	}
+	checkField(input.body_markdown, 'body_markdown')
+	checkField(input.body_html, 'body_html')
+	return errors
+}
+
+export function checkBakedAstroErrorTemplate(input: ArticleBodyInput): ValidationError[] {
+	const errors: ValidationError[] = []
+	const checkField = (value: string | null, fieldName: string) => {
+		if (!value) return
+		for (const pattern of ASTRO_SERVER_ERROR_PATTERNS) {
+			const match = pattern.exec(value)
+			if (!match) continue
+			errors.push(
+				makeError(
+					'baked-astro-error-template',
+					`Article body in ${fieldName} matches a baked Astro server-error template fingerprint.`,
+					{ start: match.index, end: match.index + match[0].length },
+				),
+			)
+			return
+		}
+	}
+	checkField(input.body_markdown, 'body_markdown')
+	checkField(input.body_html, 'body_html')
+	return errors
+}
+
+export function checkEscapedRawHtml(input: ArticleBodyInput): ValidationError[] {
+	const errors: ValidationError[] = []
+	if (!input.body_markdown) return errors
+	const patterns: Array<{ tag: string; re: RegExp }> = [
+		{ tag: 'iframe', re: /&lt;iframe[\s>]/i },
+		{ tag: 'img', re: /&lt;img[\s>]/i },
+	]
+	for (const { tag, re } of patterns) {
+		const match = re.exec(input.body_markdown)
+		if (!match) continue
+		errors.push(
+			makeError(
+				'escaped-raw-html',
+				`body_markdown contains escaped raw <${tag}> markup — this will render as literal text. Decode the entities before saving.`,
+				{ start: match.index, end: match.index + match[0].length },
+			),
+		)
+	}
+	return errors
+}
+
+export function checkMalformedDataImageMarkdown(input: ArticleBodyInput): ValidationError[] {
+	if (!input.body_markdown) return []
+	const pattern = /\[!\[[^\]]*\]\(data:image\/[a-z0-9+.-]+;base64,/i
+	const match = pattern.exec(input.body_markdown)
+	if (!match) return []
+	return [
+		makeError(
+			'malformed-data-image-markdown',
+			'body_markdown contains the [![](data:image/...)] artifact from a broken import. Replace the inline payload with an R2 URL.',
+			{ start: match.index, end: match.index + match[0].length },
+		),
+	]
+}
+
+export function checkUnbalancedMarkdownEmphasis(input: ArticleBodyInput): ValidationError[] {
+	if (!input.body_markdown) return []
+	const errors: ValidationError[] = []
+	const lines = input.body_markdown.split('\n')
+	let lineStart = 0
+	for (const rawLine of lines) {
+		const len = rawLine.length + 1
+		const lineStartHere = lineStart
+		lineStart += len
+		const stripped = rawLine.replace(/`[^`]*`/g, '')
+		const noStrong = stripped.replace(/\*\*/g, '').replace(/__/g, '')
+		const noEscaped = noStrong.replace(/\\[*_]/g, '')
+		const stars = (noEscaped.match(/\*/g) || []).length
+		const unders = (noEscaped.match(/(?<=\w)_(?=\w)|(?<!\w)_|_(?!\w)/g) || []).length
+		if (stars % 2 === 1) {
+			const idx = noEscaped.indexOf('*')
+			errors.push(
+				makeError('unbalanced-markdown-emphasis', "Line has an unbalanced '*' emphasis marker.", {
+					start: lineStartHere + (idx >= 0 ? idx : 0),
+					end: lineStartHere + rawLine.length,
+				}),
+			)
+			break
+		}
+		if (unders % 2 === 1) {
+			const idx = noEscaped.indexOf('_')
+			errors.push(
+				makeError('unbalanced-markdown-emphasis', "Line has an unbalanced '_' emphasis marker.", {
+					start: lineStartHere + (idx >= 0 ? idx : 0),
+					end: lineStartHere + rawLine.length,
+				}),
+			)
+			break
+		}
+	}
+	return errors
+}
+
+export function checkSingleSentenceParagraphsPattern(
+	input: ArticleBodyInput,
+	options: ValidatorOptions = DEFAULT_OPTIONS,
+): ValidationError[] {
+	if (!input.body_markdown) return []
+	const paragraphs = input.body_markdown
+		.split(/\n{2,}/)
+		.map((p) => p.trim())
+		.filter((p) => p.length > 0)
+	if (paragraphs.length < options.singleSentenceParagraphRun) return []
+	let run = 0
+	let runStartIndex = -1
+	for (let i = 0; i < paragraphs.length; i += 1) {
+		const para = paragraphs[i]
+		if (isLikelyProseParagraph(para) && countSentences(para) <= 1) {
+			if (run === 0) runStartIndex = i
+			run += 1
+			if (run >= options.singleSentenceParagraphRun) {
+				return [
+					makeError(
+						'single-sentence-paragraphs-pattern',
+						`Found ${run}+ consecutive single-sentence paragraphs starting at paragraph ${runStartIndex + 1}; this usually indicates an over-split import.`,
+					),
+				]
+			}
+		} else {
+			run = 0
+			runStartIndex = -1
+		}
+	}
+	return []
+}
+
+function isLikelyProseParagraph(paragraph: string): boolean {
+	const trimmed = paragraph.trim()
+	if (trimmed.startsWith('#')) return false
+	if (trimmed.startsWith('>')) return false
+	if (trimmed.startsWith('- ') || /^\d+\.\s/.test(trimmed)) return false
+	if (trimmed.startsWith('```')) return false
+	if (trimmed.startsWith('<') && trimmed.endsWith('>')) return false
+	if (trimmed.startsWith('![') || trimmed.startsWith('[')) return false
+	if (trimmed.length < 20) return false
+	return true
+}
+
+function countSentences(paragraph: string): number {
+	const stripped = paragraph
+		.replace(/`[^`]*`/g, '')
+		.replace(/\*\*([^*]+)\*\*/g, '$1')
+		.replace(/\*([^*]+)\*/g, '$1')
+	const matches = stripped.match(/[.!?](?:\s+|$)/g)
+	return matches ? matches.length : 0
+}
+
+export function checkBackslashOverEscape(input: ArticleBodyInput): ValidationError[] {
+	if (!input.body_markdown) return []
+	const pattern = /\\([^\s\\])/g
+	for (const match of input.body_markdown.matchAll(pattern)) {
+		const ch = match[1] ?? ''
+		if (LEGIT_ESCAPE_CHARS.includes(ch)) continue
+		const start = match.index ?? 0
+		return [
+			makeError(
+				'backslash-over-escape',
+				`body_markdown contains backslash-over-escape (e.g. '\\${ch}') from a broken import. Strip the legacy escapes before publishing.`,
+				{ start, end: start + match[0].length },
+			),
+		]
+	}
+	return []
+}
+
+const HEADING_ENTITY_PATTERN = /&(?:quot|apos|amp|lt|gt|#\d+);/i
+
+export function checkHtmlEntityInHeading(input: ArticleBodyInput): ValidationError[] {
+	if (!input.body_markdown) return []
+	const lines = input.body_markdown.split('\n')
+	let cursor = 0
+	for (const rawLine of lines) {
+		const lineLen = rawLine.length + 1
+		const trimmed = rawLine.trimStart()
+		if (/^#{1,6}\s+/.test(trimmed)) {
+			const headingText = trimmed.replace(/^#{1,6}\s+/, '')
+			const match = HEADING_ENTITY_PATTERN.exec(headingText)
+			if (match) {
+				const entityStart =
+					cursor +
+					(rawLine.length - trimmed.length) +
+					(trimmed.length - headingText.length) +
+					(match.index ?? 0)
+				return [
+					makeError(
+						'html-entity-in-heading',
+						`Heading contains an HTML entity ('${match[0]}'); headings must be raw characters or the TOC will display the entity code.`,
+						{ start: entityStart, end: entityStart + match[0].length },
+					),
+				]
+			}
+		}
+		cursor += lineLen
+	}
+	return []
+}
+
+export function checkEmptyBody(input: ArticleBodyInput): ValidationError[] {
+	if (isBlank(input.body_markdown) && isBlank(input.body_html)) {
+		return [
+			makeError(
+				'empty-body',
+				'Both body_markdown and body_html are empty — refusing to publish an article with no content.',
+			),
+		]
+	}
+	return []
+}
+
+/* -------------------------------------------------------------------------- */
+/*                       Phase 2 — re-exported check fns                      */
+/* -------------------------------------------------------------------------- */
+
+export const checkBrokenFootnoteLabel = brokenFootnoteLabel.check
+export const checkUnresolvedFootnoteAnchor = unresolvedAnchor.check
+export const checkWordGdocsPasteArtifacts = wordGdocs.check
+export const checkHtmlCommentLeak = htmlCommentLeak.check
+export const checkRawBlockHtml = rawBlockHtml.check
+export const checkDoubleEncodedEntities = doubleEncoded.check
+export const checkInvisibleControlChars = invisibleControl.check
+export const checkBareUrlNotAutolinked = bareUrl.check
+export const checkEmptyAltText = emptyAlt.check
+export const checkIframeMissingDimsAndWrapper = iframeMissingDims.check
+export const checkHeadingHierarchySkip = headingHierarchy.check
+export const checkPaywallMarkerLeak = paywallMarker.check
+
+/* -------------------------------------------------------------------------- */
+/*                         Aggregate + public surface                         */
+/* -------------------------------------------------------------------------- */
+
+export interface RunOptions extends ValidatorOptions {
+	requireBody: boolean
+}
+
+export const DEFAULT_RUN_OPTIONS: RunOptions = {
+	...DEFAULT_OPTIONS,
+	requireBody: true,
+}
+
+/**
+ * Run every check in a deterministic order and aggregate the errors.
+ * Structural problems (empty body, baked-error template) come first;
+ * the body-string checks follow; the noisiest heuristic checks
+ * (paragraph-split pattern) come last.
+ */
+export function validateArticleBody(
+	input: ArticleBodyInput,
+	options: Partial<RunOptions> = {},
+): ValidationResult {
+	const opts: RunOptions = { ...DEFAULT_RUN_OPTIONS, ...options }
+	const errors: ValidationError[] = []
+
+	if (opts.requireBody) errors.push(...checkEmptyBody(input))
+	errors.push(...checkBakedAstroErrorTemplate(input))
+	errors.push(...checkMalformedDataImageMarkdown(input))
+	errors.push(...checkDataUriArtifact(input))
+	errors.push(...checkEscapedRawHtml(input))
+	errors.push(...checkHtmlEntityInHeading(input))
+	errors.push(...checkBackslashOverEscape(input))
+	// Phase 2.
+	errors.push(...checkBrokenFootnoteLabel(input))
+	errors.push(...checkUnresolvedFootnoteAnchor(input))
+	errors.push(...checkWordGdocsPasteArtifacts(input))
+	errors.push(...checkHtmlCommentLeak(input))
+	errors.push(...checkRawBlockHtml(input))
+	errors.push(...checkDoubleEncodedEntities(input))
+	errors.push(...checkInvisibleControlChars(input))
+	errors.push(...checkBareUrlNotAutolinked(input))
+	errors.push(...checkEmptyAltText(input))
+	errors.push(...checkIframeMissingDimsAndWrapper(input))
+	errors.push(...checkHeadingHierarchySkip(input))
+	errors.push(...checkPaywallMarkerLeak(input))
+	// Heuristic.
+	errors.push(...checkUnbalancedMarkdownEmphasis(input))
+	errors.push(...checkSingleSentenceParagraphsPattern(input, opts))
+
+	return errors.length === 0 ? { ok: true } : { ok: false, errors }
+}
+
+/**
+ * Filter a list of errors down to those with `block` severity. Callers
+ * use this to decide between 422 (block) and 200 + warnings (warn-only).
+ */
+export function blockErrors(errors: ValidationError[]): ValidationError[] {
+	return errors.filter((e) => SEVERITY[e.code] === 'block')
+}
+
+export function warnErrors(errors: ValidationError[]): ValidationError[] {
+	return errors.filter((e) => SEVERITY[e.code] === 'warn')
+}
+
+export function formatValidationErrors(errors: ValidationError[]): string {
+	if (errors.length === 0) return 'ok'
+	return errors.map((e) => `[${e.code}] ${e.message}`).join('\n')
+}
+
+export type {
+	ArticleBodyInput,
+	Severity,
+	ValidationError,
+	ValidationErrorCode,
+	ValidationResult,
+	ValidatorOptions,
+}
+export { DEFAULT_OPTIONS, SEVERITY }

package/src/engine/validator/scan.ts

@@ -0,0 +1,103 @@
+/**
+ * Shared markdown-aware scanning helpers used by checks that need to
+ * skip fenced code, inline code, HTML attributes, and existing links.
+ *
+ * Ported verbatim from fulcrum-labs/fronts
+ * src/lib/article-body-validator/scan.ts (inventory §1: extract).
+ *
+ * The validator runs on PRE-strip source, so these helpers must match
+ * the textual reality of stored markdown — not the cleaned form that
+ * ships to the renderer. Each helper is pure and operates over the raw
+ * `body_markdown`. We intentionally avoid pulling in a full CommonMark
+ * AST library: the checks are conservative regex-based scans that
+ * classify each character span as "inside a protected region"
+ * (code/link/attr) vs "scannable prose".
+ */
+
+/**
+ * Mark which character offsets in the source are inside fenced code
+ * blocks (` ```...``` `), inline code spans (`` `…` ``), markdown link
+ * targets `[…](url)`, autolinks `<…>` , and HTML attribute values
+ * (`src="…"`). Each offset is `true` if the character is protected.
+ */
+export function buildProtectedMask(source: string): boolean[] {
+	const len = source.length
+	const mask = new Array<boolean>(len).fill(false)
+
+	// 1) Fenced code blocks. Match either ``` or ~~~ fences; cover both
+	//    the fence lines and the body between them.
+	const fenceRe = /(^|\n)([ \t]*)(`{3,}|~{3,})[^\n]*\n([\s\S]*?)\n\2\3/g
+	for (const match of source.matchAll(fenceRe)) {
+		const start = match.index ?? 0
+		const end = start + match[0].length
+		fill(mask, start, end)
+	}
+
+	// 2) Inline code spans. We deliberately accept single-backtick spans
+	//    only — multi-backtick spans are rare and the cost of missing
+	//    them is one extra warning on an unusual page, not a render bug.
+	const inlineCodeRe = /`[^`\n]+`/g
+	for (const match of source.matchAll(inlineCodeRe)) {
+		const start = match.index ?? 0
+		const end = start + match[0].length
+		// Skip if we're already inside a protected (fenced) region.
+		if (mask[start]) continue
+		fill(mask, start, end)
+	}
+
+	// 3) Markdown links — the URL portion of `[label](url)` and the
+	//    label of reference links `[label][id]` plus the ref target
+	//    `[id]: url`. We protect the URL only so e.g. label text still
+	//    scans for normal lint codes.
+	const linkUrlRe = /\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g
+	for (const match of source.matchAll(linkUrlRe)) {
+		const start = (match.index ?? 0) + 2 // skip past `](`
+		const end = start + (match[1]?.length ?? 0)
+		fill(mask, start, end)
+	}
+
+	const refDefRe = /^\s*\[[^\]]+\]:\s*\S+/gm
+	for (const match of source.matchAll(refDefRe)) {
+		const start = match.index ?? 0
+		const end = start + match[0].length
+		fill(mask, start, end)
+	}
+
+	// 4) Autolinks <https://…>.
+	const autolinkRe = /<https?:\/\/[^>\s]+>/g
+	for (const match of source.matchAll(autolinkRe)) {
+		const start = match.index ?? 0
+		const end = start + match[0].length
+		fill(mask, start, end)
+	}
+
+	// 5) HTML attribute values (single or double quoted). This catches
+	//    src="https://…", href="…", style attributes, etc. We do this
+	//    after fenced/inline code so we don't reprotect those regions.
+	const attrRe = /\s[a-zA-Z][\w-]*\s*=\s*("[^"]*"|'[^']*')/g
+	for (const match of source.matchAll(attrRe)) {
+		const valueStart = (match.index ?? 0) + match[0].indexOf(match[1] ?? '')
+		if (valueStart < 0) continue
+		const end = valueStart + (match[1]?.length ?? 0)
+		fill(mask, valueStart, end)
+	}
+
+	return mask
+}
+
+function fill(mask: boolean[], start: number, end: number): void {
+	for (let i = start; i < end && i < mask.length; i += 1) {
+		mask[i] = true
+	}
+}
+
+/**
+ * Returns true if any character in the [start, end) range is inside a
+ * protected region.
+ */
+export function isProtectedSpan(mask: boolean[], start: number, end: number): boolean {
+	for (let i = start; i < end && i < mask.length; i += 1) {
+		if (mask[i]) return true
+	}
+	return false
+}

package/src/engine/validator/types.ts

@@ -0,0 +1,114 @@
+/**
+ * Shared types for the article-body validator. The body validator is
+ * the single source of truth for ingest-time formatting QA. Each
+ * `checks/<code>.ts` module imports from here and returns
+ * `ValidationError[]` for the input it's given.
+ *
+ * Ported verbatim from fulcrum-labs/fronts
+ * src/lib/article-body-validator/types.ts (inventory §1: extract).
+ * Severity defaults to `warn` for new checks; a code is promoted to
+ * `block` only when it's a proven render-breaker.
+ */
+
+export interface ArticleBodyInput {
+	body_markdown: string | null
+	body_html: string | null
+}
+
+export interface ValidationSpan {
+	start: number
+	end: number
+}
+
+export type Severity = 'block' | 'warn'
+
+export interface ValidationError {
+	code: ValidationErrorCode
+	message: string
+	severity?: Severity
+	span?: ValidationSpan
+}
+
+export type ValidationErrorCode =
+	// Phase 0 / Phase 1 codes (preserved verbatim for back-compat with
+	// CMS / Foundry callers that switch on the string).
+	| 'data-uri-artifact'
+	| 'baked-astro-error-template'
+	| 'escaped-raw-html'
+	| 'unbalanced-markdown-emphasis'
+	| 'malformed-data-image-markdown'
+	| 'single-sentence-paragraphs-pattern'
+	| 'backslash-over-escape'
+	| 'html-entity-in-heading'
+	| 'empty-body'
+	// Phase 2 additions.
+	| 'broken-footnote-label'
+	| 'unresolved-footnote-anchor'
+	| 'word-gdocs-paste-artifacts'
+	| 'html-comment-leak'
+	| 'raw-block-html'
+	| 'double-encoded-entities'
+	| 'invisible-control-chars'
+	| 'bare-url-not-autolinked'
+	| 'empty-alt-text'
+	| 'iframe-missing-dims-and-wrapper'
+	| 'heading-hierarchy-skip'
+	| 'paywall-marker-leak'
+	| 'stale-body-html'
+
+export type ValidationResult = { ok: true } | { ok: false; errors: ValidationError[] }
+
+/**
+ * Per-code severity defaults. Used by callers that surface block vs
+ * warn separately (publish-guard returns 422 for any block-severity
+ * code, 200 + warnings for warn-only codes).
+ */
+export const SEVERITY: Readonly<Record<ValidationErrorCode, Severity>> = {
+	// Existing Phase 0 / Phase 1 — keep block for the proven render-breakers.
+	'data-uri-artifact': 'block',
+	'baked-astro-error-template': 'block',
+	'escaped-raw-html': 'block',
+	'unbalanced-markdown-emphasis': 'warn',
+	'malformed-data-image-markdown': 'block',
+	'single-sentence-paragraphs-pattern': 'warn',
+	'backslash-over-escape': 'block',
+	'html-entity-in-heading': 'block',
+	'empty-body': 'block',
+	// Phase 2.
+	'broken-footnote-label': 'block',
+	'unresolved-footnote-anchor': 'warn',
+	'word-gdocs-paste-artifacts': 'block',
+	'html-comment-leak': 'block',
+	'raw-block-html': 'warn',
+	'double-encoded-entities': 'block',
+	'invisible-control-chars': 'block',
+	'bare-url-not-autolinked': 'warn',
+	'empty-alt-text': 'warn',
+	'iframe-missing-dims-and-wrapper': 'warn',
+	'heading-hierarchy-skip': 'warn',
+	'paywall-marker-leak': 'block',
+	'stale-body-html': 'block',
+}
+
+export interface ValidatorOptions {
+	/**
+	 * Threshold for {@link checkSingleSentenceParagraphsPattern}. Kept
+	 * tunable so the heuristic can be relaxed for pull-quote-heavy
+	 * formats (e.g. video transcripts).
+	 */
+	singleSentenceParagraphRun: number
+}
+
+export const DEFAULT_OPTIONS: ValidatorOptions = {
+	singleSentenceParagraphRun: 10,
+}
+
+export function makeError(
+	code: ValidationErrorCode,
+	message: string,
+	span?: ValidationSpan,
+): ValidationError {
+	return span
+		? { code, message, span, severity: SEVERITY[code] }
+		: { code, message, severity: SEVERITY[code] }
+}