@growth-labs/seo 0.4.0 → 0.4.2

package/src/utils/aeo.ts

@@ -0,0 +1,223 @@
+import type { ContentItem } from '../types.js'
+
+// ─── Public API ───
+
+export interface GenerateAeoMarkdownOptions {
+	/** Publisher organization name for the frontmatter `publisher` field. */
+	publisherName: string
+	/** Schema.org type (e.g. 'Article', 'NewsArticle'). */
+	schemaType: string
+	/** Rendered article body in clean markdown (no HTML chrome). */
+	content: string
+	/** If true, wraps semantic sections in `<!-- aeo:section start/end -->` comments. */
+	ragChunkMarkers?: boolean
+	/** Canonical HTML URL to reference from the twin frontmatter. Defaults to item.url. */
+	canonical?: string
+	/** The primary twin URL (what this file will be served at). Stored in frontmatter as `url`. */
+	twinUrl?: string
+	/** Optional summary-twin URL to cross-link from the full twin frontmatter. */
+	summaryUrl?: string
+	/** Pre-computed SHA-256 content hash for staleness validation. */
+	contentHash?: string
+}
+
+/**
+ * Cheap token count estimate: 1 token ≈ 4 characters. Used for the `x-markdown-tokens`
+ * response header and the `tokens` frontmatter field.
+ */
+export function estimateTokenCount(text: string): number {
+	return Math.ceil(text.length / 4)
+}
+
+/**
+ * Generate an AEO markdown twin for a ContentItem.
+ *
+ * Produces: frontmatter (YAML) + body. When `ragChunkMarkers: true`, inserts
+ * `<!-- aeo:section start="<slug>" -->` / `<!-- aeo:section end="<slug>" -->` pairs
+ * around semantic sections (identified by `## ` headings) to guide RAG retrievers
+ * toward boundaries we control.
+ *
+ * Sanitization:
+ *   - Slugs are forced to `[a-z0-9-]{1,64}`; empty results fall back to `section-N`.
+ *   - Any author-supplied `<!--` / `-->` in the body is escaped (`&lt;!--` / `--&gt;`)
+ *     before wrapping, so forged markers can't close real ones.
+ */
+export function generateAeoMarkdown(
+	item: ContentItem,
+	options: GenerateAeoMarkdownOptions,
+): string {
+	const frontmatter = buildFrontmatter(item, options)
+	const safeBody = escapeHtmlComments(options.content)
+	const body = options.ragChunkMarkers ? wrapSectionMarkers(safeBody) : safeBody
+	return `---\n${frontmatter}\n---\n\n${body}`
+}
+
+// ─── Frontmatter ───
+
+function buildFrontmatter(item: ContentItem, options: GenerateAeoMarkdownOptions): string {
+	const canonical = options.canonical ?? item.url
+	const twinUrl = options.twinUrl ?? item.url
+	const firstImage = Array.isArray(item.image) ? item.image[0] : item.image
+
+	const authorEntries: Record<string, unknown>[] = (item.authors ?? []).map((a) => {
+		const o: Record<string, unknown> = { name: a.name }
+		if (a.url) o.url = a.url
+		if (a.jobTitle) o.jobTitle = a.jobTitle
+		if (a.knowsAbout?.length) o.knowsAbout = a.knowsAbout
+		if (a.sameAs?.length) o.sameAs = a.sameAs
+		return o
+	})
+
+	const alternateLanguages = (item.alternateLocales ?? []).map((l) => ({
+		lang: l.lang,
+		url: l.url,
+	}))
+
+	const lines: string[] = []
+	lines.push(yamlScalar('title', item.title))
+	if (item.description) lines.push(yamlScalar('description', item.description))
+	lines.push(yamlScalar('url', twinUrl))
+	lines.push(yamlScalar('canonical', canonical))
+	if (item.datePublished) lines.push(yamlScalar('datePublished', item.datePublished))
+	if (item.dateModified) lines.push(yamlScalar('dateModified', item.dateModified))
+	if (authorEntries.length) {
+		lines.push('author:')
+		for (const a of authorEntries) {
+			lines.push(`  - name: ${yamlValue(a.name as string)}`)
+			if (a.url) lines.push(`    url: ${yamlValue(a.url as string)}`)
+			if (a.jobTitle) lines.push(`    jobTitle: ${yamlValue(a.jobTitle as string)}`)
+			if (a.knowsAbout) {
+				lines.push('    knowsAbout:')
+				for (const k of a.knowsAbout as string[]) lines.push(`      - ${yamlValue(k)}`)
+			}
+			if (a.sameAs) {
+				lines.push('    sameAs:')
+				for (const s of a.sameAs as string[]) lines.push(`      - ${yamlValue(s)}`)
+			}
+		}
+	}
+	lines.push(yamlScalar('publisher', options.publisherName))
+	if (firstImage) lines.push(yamlScalar('image', firstImage))
+	lines.push(yamlScalar('type', options.schemaType))
+	if (item.locale) lines.push(yamlScalar('language', item.locale))
+	if (item.audio) lines.push(yamlScalar('audio', item.audio.url))
+	if (alternateLanguages.length) {
+		lines.push('alternateLanguages:')
+		for (const l of alternateLanguages) {
+			lines.push(`  - lang: ${yamlValue(l.lang)}`)
+			lines.push(`    url: ${yamlValue(l.url)}`)
+		}
+	}
+	if (options.contentHash) lines.push(yamlScalar('contentHash', options.contentHash))
+	const tokens = estimateTokenCount(options.content)
+	lines.push(`tokens: ${tokens}`)
+	if (options.summaryUrl) lines.push(yamlScalar('summaryUrl', options.summaryUrl))
+	return lines.join('\n')
+}
+
+function yamlScalar(key: string, value: string): string {
+	return `${key}: ${yamlValue(value)}`
+}
+
+/**
+ * Emit a value safely for YAML. Unquoted when possible; quoted (with escapes) when
+ * the value would otherwise parse ambiguously.
+ */
+function yamlValue(v: string): string {
+	// Whitelisted forms that can be emitted unquoted even though they contain `:`.
+	const isUrl = /^https?:\/\//.test(v)
+	const isDate = /^\d{4}-\d{2}-\d{2}/.test(v)
+	// Always quote if contains chars with YAML special meaning (excluding `:` which
+	// is handled separately below), or if starts with a character that could be misread.
+	const needsQuotes =
+		/^[\s-]/.test(v) || /[#{}[\],&*?|>!%@`'"\n\r]/.test(v) || (v.includes(':') && !isUrl && !isDate)
+	if (!needsQuotes) return v
+	return `"${v.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n')}"`
+}
+
+// ─── RAG chunk markers ───
+
+/**
+ * Wrap each `##` section in `<!-- aeo:section start="..." -->` / `<!-- end -->`.
+ * Content before the first `##` is wrapped as `lede`; content after the last `##`
+ * terminates with an end marker for that section.
+ */
+function wrapSectionMarkers(body: string): string {
+	const lines = body.split('\n')
+	const out: string[] = []
+	let currentSlug: string | null = null
+	const usedSlugs = new Set<string>()
+	let ordinal = 0
+	let sawContent = false
+
+	// Open with `lede` if we hit body content before any heading.
+	const openSection = (slug: string) => {
+		out.push(`<!-- aeo:section start="${slug}" -->`)
+		currentSlug = slug
+	}
+	const closeSection = () => {
+		if (currentSlug) {
+			out.push(`<!-- aeo:section end="${currentSlug}" -->`)
+			currentSlug = null
+		}
+	}
+
+	for (const line of lines) {
+		const headingMatch = /^##\s+(.+)$/.exec(line)
+		if (headingMatch) {
+			closeSection()
+			ordinal++
+			const slug = sanitizeSlug(headingMatch[1]!) || `section-${ordinal}`
+			const unique = uniqueSlug(slug, usedSlugs)
+			usedSlugs.add(unique)
+			openSection(unique)
+			out.push(line)
+			sawContent = true
+			continue
+		}
+		if (!currentSlug && line.trim() !== '' && !sawContent) {
+			openSection('lede')
+			usedSlugs.add('lede')
+			sawContent = true
+		}
+		out.push(line)
+	}
+	closeSection()
+	return out.join('\n')
+}
+
+/** Strict slug sanitization — [a-z0-9-]{1,64}, no leading/trailing/repeated dashes. */
+function sanitizeSlug(text: string): string {
+	const lowered = text.toLowerCase()
+	const ascii = lowered.normalize('NFKD').replace(/[\u0300-\u036f]/g, '')
+	const cleaned = ascii
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-+|-+$/g, '')
+		.slice(0, 64)
+	return cleaned
+}
+
+function uniqueSlug(base: string, used: Set<string>): string {
+	if (!used.has(base)) return base
+	let n = 2
+	while (used.has(`${base}-${n}`)) n++
+	return `${base}-${n}`
+}
+
+/**
+ * Escape any author-supplied `<!--` / `-->` in the body so forged markers can't
+ * close real ones. Applied BEFORE wrapping (spec "Marker sanitization").
+ */
+function escapeHtmlComments(body: string): string {
+	return body.replace(/<!--/g, '&lt;!--').replace(/-->/g, '--&gt;')
+}
+
+// ─── Exports for testing ───
+
+export const _internals = {
+	sanitizeSlug,
+	uniqueSlug,
+	escapeHtmlComments,
+	wrapSectionMarkers,
+	yamlValue,
+}

package/src/utils/apple-news-anf.ts

@@ -0,0 +1,163 @@
+import type { ContentItem } from '../types.js'
+
+// Apple News Format (ANF) JSON document generator.
+//
+// EXPERIMENTAL in 0.2.0. Ships a conservative minimal-viable document that
+// passes Apple News Publisher's static validation. Full round-trip validation
+// against News Publisher requires an Apple developer account — test your
+// output in your News Publisher dashboard before committing to this format.
+//
+// ANF spec: https://developer.apple.com/documentation/apple_news/apple_news_format
+//
+// Scope of this package: produce a valid ANF document. Submission to the News
+// Publisher API (authentication, HMAC signing, multipart POST) is the
+// consumer's concern — the ANF spec explicitly keeps credentials out of scope.
+
+const ANF_VERSION = '1.7'
+
+export interface GenerateAnfOptions {
+	channelId: string
+	byline?: string
+	language?: string // ISO 639-1 (default: 'en')
+	// Extended customization is intentionally absent in v1. Consumers needing rich
+	// layout control can post-process the returned object.
+}
+
+export interface AnfComponent {
+	role: string
+	[key: string]: unknown
+}
+
+// Layouts and styles are keyed by user-chosen names and referenced by components;
+// they don't have a `role` themselves. Kept loosely-typed so consumers can customize.
+export type AnfLayoutMap = Record<string, Record<string, unknown>>
+export type AnfStyleMap = Record<string, Record<string, unknown>>
+
+export interface AnfDocument {
+	version: string
+	identifier: string
+	title: string
+	subtitle?: string
+	language: string
+	layout: Record<string, unknown>
+	documentStyle?: Record<string, unknown>
+	metadata?: Record<string, unknown>
+	components: AnfComponent[]
+	componentLayouts?: AnfLayoutMap
+	componentStyles?: AnfStyleMap
+}
+
+/**
+ * Generate an Apple News Format document for a ContentItem. The returned value
+ * is a JSON-serializable object; consumers submit it as the `article.json` part
+ * of a News Publisher multipart POST.
+ *
+ * The generated document uses a conservative default layout (7-column grid),
+ * standard component styles, and a body composed of: title, byline, hero image,
+ * then one body component per paragraph-separated block of the article's
+ * description. Consumers producing real articles should replace the body
+ * components with their rendered content before submitting.
+ */
+export function generateAppleNewsAnf(item: ContentItem, options: GenerateAnfOptions): AnfDocument {
+	const identifier = item.appleNewsId ?? deriveIdentifier(item.url)
+	const language = options.language ?? item.locale ?? 'en'
+
+	const components: AnfComponent[] = []
+
+	// Title
+	components.push({
+		role: 'title',
+		layout: 'titleLayout',
+		text: item.title,
+		textStyle: 'titleStyle',
+	})
+
+	// Byline
+	const byline =
+		options.byline ??
+		(item.authors?.length ? `By ${item.authors.map((a) => a.name).join(', ')}` : undefined)
+	if (byline) {
+		components.push({
+			role: 'byline',
+			layout: 'bylineLayout',
+			text: byline,
+			textStyle: 'bylineStyle',
+		})
+	}
+
+	// Hero image (first, or single).
+	const heroImage = Array.isArray(item.image) ? item.image[0] : item.image
+	if (heroImage) {
+		components.push({
+			role: 'photo',
+			layout: 'heroLayout',
+			URL: heroImage,
+			caption: item.description,
+		})
+	}
+
+	// Body (from description, split on double-newline).
+	if (item.description) {
+		const paragraphs = item.description.split(/\n{2,}/).filter((p) => p.trim().length > 0)
+		for (const p of paragraphs) {
+			components.push({
+				role: 'body',
+				layout: 'bodyLayout',
+				text: p.trim(),
+				textStyle: 'bodyStyle',
+			})
+		}
+	}
+
+	return {
+		version: ANF_VERSION,
+		identifier,
+		title: item.title,
+		language,
+		layout: {
+			columns: 7,
+			width: 1024,
+			margin: 75,
+			gutter: 20,
+		},
+		documentStyle: {
+			backgroundColor: '#FFFFFF',
+		},
+		metadata: {
+			canonicalURL: item.url,
+			datePublished: item.datePublished,
+			dateModified: item.dateModified,
+			thumbnailURL: heroImage,
+			excerpt: item.description,
+			authors: item.authors?.map((a) => a.name),
+		},
+		components,
+		componentLayouts: {
+			titleLayout: { columnStart: 0, columnSpan: 7, margin: { bottom: 18 } },
+			bylineLayout: { columnStart: 0, columnSpan: 7, margin: { bottom: 30 } },
+			heroLayout: { ignoreDocumentMargin: true, minimumHeight: 500 },
+			bodyLayout: { columnStart: 0, columnSpan: 7, margin: { top: 15, bottom: 15 } },
+		},
+		componentStyles: {
+			titleStyle: { textAlignment: 'left', fontName: 'HelveticaNeue-Bold', fontSize: 36 },
+			bylineStyle: { textAlignment: 'left', fontName: 'HelveticaNeue-Medium', fontSize: 13 },
+			bodyStyle: { textAlignment: 'left', fontName: 'Georgia', fontSize: 18, lineHeight: 26 },
+		},
+	}
+}
+
+/**
+ * Derive a stable identifier from the article URL when the consumer hasn't
+ * assigned one explicitly. Must be stable across republishes so Apple News
+ * de-dupes correctly.
+ */
+function deriveIdentifier(url: string): string {
+	// Strip scheme + query/hash; replace non-alphanum with '-' to stay within
+	// Apple's identifier constraints.
+	return url
+		.replace(/^https?:\/\//, '')
+		.replace(/[#?].*$/, '')
+		.replace(/[^a-zA-Z0-9-_]/g, '-')
+		.replace(/-+/g, '-')
+		.replace(/^-|-$/g, '')
+}

package/src/utils/apple-news-rss.ts

@@ -0,0 +1,136 @@
+import type { SeoOptionsWithResolvedSite } from '../options.js'
+import type { ContentItem } from '../types.js'
+import { forAppleNews } from './content-filter.js'
+
+function escapeXml(str: string): string {
+	return str
+		.replace(/&/g, '&')
+		.replace(/</g, '&lt;')
+		.replace(/>/g, '&gt;')
+		.replace(/"/g, '&quot;')
+		.replace(/'/g, '&apos;')
+}
+
+function toRfc822(dateStr: string): string {
+	return new Date(dateStr).toUTCString()
+}
+
+/**
+ * Map from a ContentItem to a self-contained HTML fragment suitable for
+ * Apple News `content:encoded`. Apple requires absolute URLs and no page chrome
+ * (headers, sidebars, footers). Consumers typically produce this from their
+ * rendered article's body element.
+ */
+export type ContentHtmlResolver = (item: ContentItem) => string | undefined
+
+export interface GenerateAppleNewsRssOptions {
+	items: ContentItem[]
+	options: SeoOptionsWithResolvedSite
+	// Resolver that produces the full HTML body for an item. When `options.appleNews.fullContent`
+	// is true, items without a resolved body are skipped (silent). When false, only titles/descriptions
+	// are emitted.
+	contentHtml?: ContentHtmlResolver
+}
+
+const MAX_ITEMS = 100
+
+/**
+ * Generate Apple News Publisher RSS (`/apple-news.xml`) per Apple's ingestion
+ * requirements. Apple's ingester is stricter than a generic reader — missing
+ * required fields cause silent rejection.
+ *
+ * Hard requirements implemented:
+ *   - access: 'members' excluded unconditionally (via forAppleNews).
+ *   - appleNewsPublishable: 'no' excluded (resolved against channel default).
+ *   - content:encoded CDATA body when fullContent: true.
+ *   - Hero image via <media:content> (required for Apple News).
+ *   - dc:creator populated from authors[0].
+ *   - Category from item.appleNewsSection or channel defaultSection.
+ *   - guid in permalink form, stable across republishes.
+ *   - Feed cadence: newest-first, max 100 items, no pagination.
+ */
+export function generateAppleNewsRss({
+	items,
+	options,
+	contentHtml,
+}: GenerateAppleNewsRssOptions): string {
+	const appleNews = options.appleNews
+	if (!appleNews?.enabled) {
+		throw new Error('generateAppleNewsRss called without appleNews.enabled')
+	}
+
+	const filtered = forAppleNews(items, { publishable: appleNews.defaultPublishable })
+	// Newest-first ordering.
+	const sorted = filtered
+		.slice()
+		.sort((a, b) => {
+			const at = a.datePublished ? new Date(a.datePublished).getTime() : 0
+			const bt = b.datePublished ? new Date(b.datePublished).getTime() : 0
+			return bt - at
+		})
+		.slice(0, MAX_ITEMS)
+
+	const selfLink = `${options.site.replace(/\/$/, '')}${appleNews.feedPath}`
+	const channelImage = options.organization.logo
+	const language = options.defaults.locale.replace('_', '-')
+	const lastBuildDate = new Date().toUTCString()
+
+	const itemsXml = sorted
+		.map((item) => {
+			const pubDate = item.datePublished ? toRfc822(item.datePublished) : ''
+			const pubDateTag = pubDate ? `\n      <pubDate>${escapeXml(pubDate)}</pubDate>` : ''
+			const creator = item.authors?.[0]?.name
+			const creatorTag = creator ? `\n      <dc:creator>${escapeXml(creator)}</dc:creator>` : ''
+			const section = item.appleNewsSection ?? appleNews.defaultSection
+			const categoryTag = section ? `\n      <category>${escapeXml(section)}</category>` : ''
+			const descTag = item.description
+				? `\n      <description>${escapeXml(item.description)}</description>`
+				: ''
+
+			let contentTag = ''
+			if (appleNews.fullContent) {
+				const html = contentHtml?.(item)
+				if (html) {
+					// CDATA-wrap; guard against CDATA injection by splitting ']]>' sequences.
+					const safe = html.replace(/]]>/g, ']]]]><![CDATA[>')
+					contentTag = `\n      <content:encoded><![CDATA[${safe}]]></content:encoded>`
+				}
+			}
+
+			// Hero image is required by Apple. We emit media:content on the first image URL;
+			// consumers should ensure this is >= 1024x768 per Apple's minimums.
+			const heroImage = Array.isArray(item.image) ? item.image[0] : item.image
+			const mediaContentTag = heroImage
+				? `\n      <media:content url="${escapeXml(heroImage)}" medium="image"/>`
+				: ''
+
+			return `    <item>
+      <title>${escapeXml(item.title)}</title>
+      <link>${escapeXml(item.url)}</link>
+      <guid isPermaLink="true">${escapeXml(item.url)}</guid>${pubDateTag}${creatorTag}${categoryTag}${descTag}${contentTag}${mediaContentTag}
+    </item>`
+		})
+		.join('\n')
+
+	return `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0"
+  xmlns:atom="http://www.w3.org/2005/Atom"
+  xmlns:dc="http://purl.org/dc/elements/1.1/"
+  xmlns:content="http://purl.org/rss/1.0/modules/content/"
+  xmlns:media="http://search.yahoo.com/mrss/">
+  <channel>
+    <title>${escapeXml(appleNews.channelName)}</title>
+    <link>${escapeXml(options.site)}</link>
+    <description>${escapeXml(options.organization.name)}</description>
+    <language>${escapeXml(language)}</language>
+    <atom:link href="${escapeXml(selfLink)}" rel="self" type="application/rss+xml"/>
+    <image>
+      <url>${escapeXml(channelImage)}</url>
+      <title>${escapeXml(appleNews.channelName)}</title>
+      <link>${escapeXml(options.site)}</link>
+    </image>
+    <lastBuildDate>${lastBuildDate}</lastBuildDate>
+${itemsXml}
+  </channel>
+</rss>`
+}

package/src/utils/content-filter.ts

@@ -0,0 +1,87 @@
+import type { ContentItem } from '../types.js'
+
+// Centralizes the access-rule invariants so we don't scatter "members excluded
+// from X" across eight different routes. If the rule ever changes, it changes
+// here, once.
+
+/**
+ * Items eligible for the primary sitemap (sitemap-articles, sitemap-pages, etc.).
+ * Includes members by default — paywalled URLs should be discoverable so search
+ * engines can route users to the paywall landing. Consumers can opt out per item
+ * with `includeInSitemap: false`.
+ */
+export function forSitemap<T extends ContentItem>(items: T[]): T[] {
+	return items.filter((item) => item.includeInSitemap !== false)
+}
+
+/**
+ * Items eligible for the markdown-specific sitemap (/sitemap-markdown.xml).
+ * Excludes members UNCONDITIONALLY — the markdown sitemap advertises AEO twin
+ * URLs that don't exist for gated content. `includeInSitemap: false` on a public
+ * item still hides it from here.
+ */
+export function forMarkdownSitemap<T extends ContentItem>(items: T[]): T[] {
+	return items.filter((item) => item.access !== 'members' && item.includeInSitemap !== false)
+}
+
+/**
+ * Items eligible for the RSS feed (/feed.xml). Excludes members UNCONDITIONALLY;
+ * `includeInFeed: true` on a members item is a no-op. Public items can opt out
+ * with `includeInFeed: false`.
+ */
+export function forRss<T extends ContentItem>(items: T[]): T[] {
+	return items.filter((item) => item.access !== 'members' && item.includeInFeed !== false)
+}
+
+/**
+ * Items eligible for Apple News Publisher RSS (/apple-news.xml). Excludes members
+ * UNCONDITIONALLY. Additionally filters by `appleNewsPublishable` — if set to 'no'
+ * at the item level, excluded regardless of channel default.
+ */
+export function forAppleNews<T extends ContentItem>(
+	items: T[],
+	defaults: { publishable: 'yes' | 'no' },
+): T[] {
+	return items.filter((item) => {
+		if (item.access === 'members') return false
+		if (item.includeInFeed === false) return false
+		const effective = item.appleNewsPublishable ?? defaults.publishable
+		return effective === 'yes'
+	})
+}
+
+/**
+ * Items eligible for the narrated-articles podcast feed (/listen.xml).
+ * Requires `audio` property, excludes members UNCONDITIONALLY.
+ */
+export function forListen<T extends ContentItem>(items: T[]): T[] {
+	return items.filter(
+		(item) => item.audio !== undefined && item.access !== 'members' && item.includeInFeed !== false,
+	)
+}
+
+/**
+ * Items eligible for llms.txt (the link index). Excludes members UNCONDITIONALLY.
+ * Public items with `includeInFeed: false` still appear — llms.txt is a discovery
+ * file, not a feed, so the opt-out only applies to RSS.
+ */
+export function forLlms<T extends ContentItem>(items: T[]): T[] {
+	return items.filter((item) => item.access !== 'members')
+}
+
+/**
+ * Items eligible for the bulk corpus dump (/llms-full.txt). Same rule as llms.txt:
+ * public items only, members unconditionally excluded.
+ */
+export function forLlmsFull<T extends ContentItem>(items: T[]): T[] {
+	return items.filter((item) => item.access !== 'members')
+}
+
+/**
+ * Items eligible for AEO markdown-twin emission (static mode) or middleware
+ * serving (middleware mode). Excludes members UNCONDITIONALLY — gated content
+ * is never exposed as markdown at rest or via middleware.
+ */
+export function forMarkdownTwin<T extends ContentItem>(items: T[]): T[] {
+	return items.filter((item) => item.access !== 'members')
+}