@growth-labs/seo 0.4.0 → 0.4.2

package/src/site-url-core.ts

@@ -0,0 +1,71 @@
+import type { ResolvedSeoOptions, SeoOptionsWithResolvedSite } from './options.js'
+
+export interface SiteUrlEnvVarRef {
+	envVar: string
+}
+
+export type SiteUrlSource = string | SiteUrlEnvVarRef
+
+export function isSiteUrlEnvVarRef(value: unknown): value is SiteUrlEnvVarRef {
+	return (
+		typeof value === 'object' &&
+		value !== null &&
+		'envVar' in value &&
+		typeof value.envVar === 'string'
+	)
+}
+
+export function validateSiteUrl(value: string, label = 'site'): string {
+	try {
+		new URL(value)
+		return value
+	} catch {
+		throw new Error(`@growth-labs/seo: ${label} must resolve to a valid URL.`)
+	}
+}
+
+export function resolveSiteUrl(site: SiteUrlSource, bindingsEnv?: Record<string, unknown>): string {
+	if (typeof site === 'string') {
+		return validateSiteUrl(site)
+	}
+
+	if (!isSiteUrlEnvVarRef(site)) {
+		throw new Error('@growth-labs/seo: site must be a URL string or { envVar }.')
+	}
+
+	const fromCloudflare = bindingsEnv?.[site.envVar]
+	if (typeof fromCloudflare === 'string') {
+		return validateSiteUrl(fromCloudflare, `site env binding ${site.envVar}`)
+	}
+	if (fromCloudflare !== undefined) {
+		throw new Error(`@growth-labs/seo: site env binding ${site.envVar} must be a string URL.`)
+	}
+
+	const fromNode = getNodeEnv(site.envVar)
+	if (typeof fromNode === 'string') {
+		return validateSiteUrl(fromNode, `process.env.${site.envVar}`)
+	}
+
+	throw new Error(
+		`@growth-labs/seo: site env binding ${site.envVar} is not set. ` +
+			'Provide it as a Cloudflare Worker env binding, or use process.env only in Node tooling/tests.',
+	)
+}
+
+export function resolveSeoConfig(
+	config: ResolvedSeoOptions,
+	bindingsEnv?: Record<string, unknown>,
+): SeoOptionsWithResolvedSite {
+	return {
+		...config,
+		site: resolveSiteUrl(config.site, bindingsEnv),
+	}
+}
+
+function getNodeEnv(name: string): string | undefined {
+	return (
+		globalThis as {
+			process?: { env?: Record<string, string | undefined> }
+		}
+	).process?.env?.[name]
+}

package/src/site-url.ts

@@ -0,0 +1,21 @@
+import { env as cloudflareEnv } from 'cloudflare:workers'
+import type { ResolvedSeoOptions, SeoOptionsWithResolvedSite } from './options.js'
+import {
+	resolveSeoConfig as resolveSeoConfigCore,
+	resolveSiteUrl as resolveSiteUrlCore,
+	type SiteUrlSource,
+} from './site-url-core.js'
+
+export function resolveSiteUrl(
+	site: SiteUrlSource,
+	bindingsEnv: Record<string, unknown> = cloudflareEnv as Record<string, unknown>,
+): string {
+	return resolveSiteUrlCore(site, bindingsEnv)
+}
+
+export function resolveSeoConfig(
+	config: ResolvedSeoOptions,
+	bindingsEnv?: Record<string, unknown>,
+): SeoOptionsWithResolvedSite {
+	return resolveSeoConfigCore(config, bindingsEnv ?? (cloudflareEnv as Record<string, unknown>))
+}

package/src/types.ts

@@ -0,0 +1,166 @@
+import type { APIContext } from 'astro'
+
+// ─── Content data interfaces ───
+
+export interface ContentAuthor {
+	name: string
+	url?: string
+	jobTitle?: string
+	knowsAbout?: string[]
+	sameAs?: string[]
+}
+
+export interface ContentAudio {
+	url: string
+	duration: string // ISO 8601, e.g. 'PT8M30S'
+	narrator?: string
+}
+
+export interface ContentLocaleAlternate {
+	lang: string
+	url: string
+}
+
+export interface ProductVariant {
+	name: string
+	sku?: string
+	price: number
+	currency?: string
+	availability: 'InStock' | 'OutOfStock' | 'PreOrder' | 'Discontinued'
+	image?: string
+}
+
+export interface ContentProduct {
+	name: string
+	description: string
+	price: number
+	currency: string
+	availability: 'InStock' | 'OutOfStock' | 'PreOrder' | 'Discontinued'
+	brand?: string
+	sku?: string
+	gtin?: string
+	mpn?: string
+	images: string[]
+	rating?: {
+		value: number
+		count: number
+		bestRating?: number
+	}
+	reviews?: Array<{
+		author: string
+		rating: number
+		body?: string
+		datePublished?: string
+	}>
+	variants?: ProductVariant[]
+	condition?: 'NewCondition' | 'UsedCondition' | 'RefurbishedCondition'
+}
+
+export interface PodcastEpisodeMeta {
+	episodeNumber?: number
+	seasonNumber?: number
+	episodeType?: 'full' | 'trailer' | 'bonus'
+}
+
+// 'public'  — freely available to crawlers, LLMs, AEO twins emitted.
+// 'members' — gated content; no .md twin emitted. Unconditionally excluded from llms.txt,
+//             llms-full.txt, /feed.xml, /apple-news.xml, /listen.xml, and sitemap-markdown.xml.
+//             Still eligible for the primary sitemap so the paywalled URL remains indexable.
+//             `includeInFeed` is a no-op for members.
+export type ContentAccess = 'public' | 'members'
+
+export interface ContentItem {
+	// ─── Core ───
+	url: string
+	title: string
+	description?: string
+	image?: string
+	datePublished?: string
+	dateModified?: string
+	authors?: ContentAuthor[]
+
+	// ─── Audio narration ───
+	audio?: ContentAudio
+	podcastEpisode?: PodcastEpisodeMeta
+
+	// ─── Multilingual ───
+	locale?: string
+	alternateLocales?: ContentLocaleAlternate[]
+
+	// ─── Apple News ───
+	appleNewsId?: string
+	appleNewsPublishable?: 'yes' | 'no'
+	appleNewsSection?: string
+	newsKeywords?: string[]
+
+	// ─── Access / paywall ───
+	access?: ContentAccess
+	includeInSitemap?: boolean // default true
+	includeInFeed?: boolean // public-only; no-op for members
+	summary?: string // consumer-provided summary for the summary-twin generator
+	isAccessibleForFree?: boolean // JSON-LD; derived from `access` if omitted
+	paywallCssSelector?: string // e.g. '.premium-content' — used in JSON-LD hasPart
+
+	// ─── Commerce ───
+	product?: ContentProduct
+
+	// ─── FAQ / HowTo / Video ───
+	faq?: Array<{ question: string; answer: string }>
+	howToSteps?: Array<{ name: string; text: string; image?: string }>
+	video?: {
+		thumbnailUrl: string
+		duration: string
+		contentUrl?: string
+		embedUrl?: string
+	}
+}
+
+// ─── Content provider ───
+
+export type ContentType = 'articles' | 'pages' | 'videos' | 'products' | 'authors'
+
+export interface ContentProviderParams {
+	type: ContentType
+	slugs?: string[]
+}
+
+export type ContentProvider = (
+	params: ContentProviderParams,
+	context: APIContext,
+) => Promise<ContentItem[]>
+
+// ─── Crawler-class dispatch ───
+
+// Set at request time by the SEO middleware on Astro.locals. Drives body-variant selection,
+// cache-key segmentation, and JSON-LD redaction. See spec "Crawler-class policy".
+export type CrawlerClass =
+	| 'verifiedSearchCrawler'
+	| 'llmTrainingCrawler'
+	| 'userDirectedLlmAgent'
+	| 'anonymous'
+
+// Derived segment that incorporates the crawler-class override of raw consumer auth.
+// Consumers use this (never raw authSegment) in cache keys to avoid leaking member bodies
+// to user-directed LLM agents bearing member cookies.
+export type EffectiveAuthSegment = 'anon' | 'member' | 'search-full'
+
+// ─── Utility return types ───
+
+export interface MetaTag {
+	[key: string]: string
+}
+
+export interface CanonicalLink {
+	rel: 'canonical'
+	href: string
+}
+
+export interface HreflangLink {
+	rel: 'alternate'
+	hreflang: string
+	href: string
+}
+
+// ─── JSON-LD base ───
+
+export type JsonLdObject = Record<string, unknown>

package/src/utils/aeo-summary.ts

@@ -0,0 +1,176 @@
+import type { ContentItem } from '../types.js'
+import { estimateTokenCount } from './aeo.js'
+
+// ─── Public API ───
+
+export interface GenerateSummaryTwinOptions {
+	publisherName: string
+	schemaType: string
+	/** Full article body in markdown (used by tiers 2 and 3 when item.summary is absent). */
+	content?: string
+	/** URL of the full twin for the `fullUrl` frontmatter cross-link. */
+	fullUrl: string
+	/** Target cap in tokens. Default 400; hard ceiling. */
+	maxTokens?: number
+	/** Emits a build-time warning callback when tier-4 fallback fires (spec 1483). */
+	onMinimalFallback?: (item: ContentItem) => void
+}
+
+export interface GenerateSummaryTwinResult {
+	markdown: string
+	tier: 1 | 2 | 3 | 4
+}
+
+const DEFAULT_MAX_TOKENS = 400
+
+/**
+ * Generate a summary twin for a ContentItem via the 4-tier fallback chain
+ * (spec "Summary twin"):
+ *
+ *   1. item.summary (consumer-provided) — used verbatim.
+ *   2. Bullet-list extraction from the article body — 3-5 claims pulled from
+ *      top-level `- ` / `* ` list items.
+ *   3. Narrative fallback — description + first sentence of each `## section`
+ *      (max 5) + final sentence of article as conclusion.
+ *   4. Minimal — description only. NO frontmatter flag (LLMs would propagate it
+ *      into user-visible citations per v5 review). Build-time telemetry only,
+ *      via the onMinimalFallback callback.
+ *
+ * The returned markdown includes frontmatter: `type: summary`, `title`, `url`,
+ * `fullUrl`, `datePublished`/`dateModified` if set. First-match wins.
+ */
+export function generateSummaryTwin(
+	item: ContentItem,
+	options: GenerateSummaryTwinOptions,
+): GenerateSummaryTwinResult {
+	const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS
+
+	const { body, tier } = pickBody(item, options.content, options.onMinimalFallback)
+	const capped = capToTokens(body, maxTokens)
+
+	const fm = buildFrontmatter(item, options)
+	const markdown = `---\n${fm}\n---\n\n${capped}`
+
+	return { markdown, tier }
+}
+
+// ─── Internals ───
+
+function pickBody(
+	item: ContentItem,
+	content: string | undefined,
+	onMinimalFallback: ((item: ContentItem) => void) | undefined,
+): { body: string; tier: 1 | 2 | 3 | 4 } {
+	// Tier 1: explicit summary
+	if (item.summary && item.summary.trim().length > 0) {
+		return { body: item.summary.trim(), tier: 1 }
+	}
+
+	// Tier 2: bullet-list extraction
+	if (content) {
+		const bullets = extractTopLevelBullets(content)
+		if (bullets.length >= 3) {
+			const selected = bullets.slice(0, 5)
+			const lede = item.description?.trim() ?? item.title
+			return {
+				body: `${lede}\n\n${selected.map((b) => `- ${b}`).join('\n')}`,
+				tier: 2,
+			}
+		}
+	}
+
+	// Tier 3: narrative fallback
+	if (content) {
+		const sectionFirsts = extractFirstSentencePerSection(content).slice(0, 5)
+		const lastSentence = extractLastSentence(content)
+		const lede = item.description?.trim()
+		const narrativeParts: string[] = []
+		if (lede) narrativeParts.push(lede)
+		if (sectionFirsts.length > 0) {
+			narrativeParts.push(sectionFirsts.map((s) => `- ${s}`).join('\n'))
+		}
+		if (lastSentence) narrativeParts.push(lastSentence)
+		const narrative = narrativeParts.join('\n\n')
+		if (narrative.length >= 100) {
+			return { body: narrative, tier: 3 }
+		}
+	}
+
+	// Tier 4: minimal fallback
+	onMinimalFallback?.(item)
+	return { body: item.description ?? item.title, tier: 4 }
+}
+
+function extractTopLevelBullets(md: string): string[] {
+	const out: string[] = []
+	// Match top-level (no leading whitespace) `- ` or `* ` bullets.
+	const re = /^[ \t]*(?:[-*])\s+(.+)$/gm
+	let m: RegExpExecArray | null = re.exec(md)
+	while (m !== null) {
+		const line = m[1]?.trim()
+		if (line && line.length > 0) out.push(line)
+		m = re.exec(md)
+	}
+	return out
+}
+
+function extractFirstSentencePerSection(md: string): string[] {
+	const out: string[] = []
+	// Split on `## ` headings; skip the heading line itself and find first sentence of body.
+	const sections = md.split(/^## .+$/gm).slice(1)
+	for (const section of sections) {
+		const firstPara = section
+			.split(/\n{2,}/)
+			.map((p) => p.trim())
+			.find((p) => p.length > 0 && !p.startsWith('#'))
+		if (firstPara) {
+			const sentence = firstSentence(firstPara)
+			if (sentence) out.push(sentence)
+		}
+	}
+	return out
+}
+
+function extractLastSentence(md: string): string | undefined {
+	// Take the last non-empty paragraph, pick its last sentence.
+	const paras = md
+		.split(/\n{2,}/)
+		.map((p) => p.trim())
+		.filter((p) => p.length > 0 && !p.startsWith('#'))
+	const lastPara = paras[paras.length - 1]
+	if (!lastPara) return undefined
+	const sentences = lastPara.split(/(?<=[.!?])\s+/)
+	return sentences[sentences.length - 1]?.trim()
+}
+
+function firstSentence(text: string): string | undefined {
+	const match = /^(.+?[.!?])(?:\s|$)/.exec(text)
+	return match?.[1]?.trim() ?? text.split('\n')[0]?.trim()
+}
+
+function capToTokens(body: string, maxTokens: number): string {
+	if (estimateTokenCount(body) <= maxTokens) return body
+	const maxChars = maxTokens * 4
+	return `${body.slice(0, maxChars).trimEnd()}…`
+}
+
+function buildFrontmatter(item: ContentItem, options: GenerateSummaryTwinOptions): string {
+	const lines: string[] = []
+	lines.push(`type: summary`)
+	lines.push(`title: ${yv(item.title)}`)
+	lines.push(`url: ${yv(`${options.fullUrl}.summary.md`)}`)
+	lines.push(`fullUrl: ${yv(options.fullUrl)}`)
+	if (item.datePublished) lines.push(`datePublished: ${yv(item.datePublished)}`)
+	if (item.dateModified) lines.push(`dateModified: ${yv(item.dateModified)}`)
+	lines.push(`publisher: ${yv(options.publisherName)}`)
+	lines.push(`schemaType: ${yv(options.schemaType)}`)
+	return lines.join('\n')
+}
+
+function yv(v: string): string {
+	const isUrl = /^https?:\/\//.test(v)
+	const isDate = /^\d{4}-\d{2}-\d{2}/.test(v)
+	const needsQuotes = /[#{}[\],&*?|>!%@`'"\n]/.test(v) || (v.includes(':') && !isUrl && !isDate)
+	if (!needsQuotes) return v
+	return `"${v.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n')}"`
+}

package/src/utils/aeo-twin-emitter.ts

@@ -0,0 +1,173 @@
+import type { ContentItem } from '../types.js'
+import { type GenerateAeoMarkdownOptions, generateAeoMarkdown } from './aeo.js'
+import { generateSummaryTwin } from './aeo-summary.js'
+import { forMarkdownTwin } from './content-filter.js'
+import { computeContentHash } from './staleness.js'
+
+// Public API ─────────────────────────────────────────────────────────────
+
+export type RenderBody = (item: ContentItem) => string | Promise<string>
+
+export interface EmitAeoTwinsOptions {
+	items: ContentItem[]
+	publisherName: string
+	schemaType: string
+	/** Resolver that returns the article body in markdown. Required; the emitter
+	 *  can't synthesize content. Consumers typically produce this from their CMS. */
+	renderBody: RenderBody
+	/** Map item.url → primary twin URL. Default: append '.md'. */
+	twinUrl?: (articleUrl: string) => string
+	/** Predicate applied after the default members filter. Default: always true. */
+	include?: (item: ContentItem) => boolean
+	/** Emit a summary twin alongside the primary. Default true. */
+	summaryTwin?: boolean
+	/** Wrap semantic sections in <!-- aeo:section --> markers. Default true. */
+	ragChunkMarkers?: boolean
+	/** Stale-hash metadata mode. */
+	stalenessCheck?: 'content-hash' | 'dateModified' | 'none'
+	/** Called when the summary generator falls back to tier 4. Used for build telemetry. */
+	onSummaryMinimalFallback?: (item: ContentItem) => void
+}
+
+export interface EmittedTwin {
+	/** URL path this file will be served at (used by the writer to derive the filesystem path). */
+	urlPath: string
+	/** Full URL (for frontmatter). */
+	url: string
+	/** File body. */
+	content: string
+	/** Content-type for HTTP response headers when this file is served. */
+	contentType: string
+	/** If this emission is the primary twin for an item, the item is populated. */
+	item?: ContentItem
+	/** If this emission is a summary twin, the corresponding primary URL. */
+	primaryUrl?: string
+	/** Primary twins carry a stable content hash for staleness validation; absent on summaries. */
+	contentHash?: string
+}
+
+export interface EmitAeoTwinsResult {
+	twins: EmittedTwin[]
+	/** Map of item.url → contentHash. Persist to disk for staleness validation. */
+	contentHashes: Map<string, string>
+	/** Number of items filtered out (either by access rule or consumer predicate). */
+	skipped: number
+}
+
+/**
+ * Compute all twin files for a given contentProvider output, without touching
+ * the filesystem. Callers (index.ts at astro:build:done time) write the returned
+ * twins to disk and record content hashes for later staleness validation.
+ *
+ * Filtering:
+ *   - Members items excluded unconditionally (via forMarkdownTwin).
+ *   - Consumer-supplied `include` predicate applied after the access filter.
+ *
+ * For each surviving item we emit:
+ *   - Primary twin at twinUrl(item.url).
+ *   - Summary twin at <primary>.summary.md (when summaryTwin: true).
+ *
+ * Aliases (spec twinAliases) are NOT emitted as static files in v7 — they're
+ * middleware-only redirects. Static-mode twins live at the primary URL only.
+ */
+export async function emitAeoTwins(options: EmitAeoTwinsOptions): Promise<EmitAeoTwinsResult> {
+	const {
+		items,
+		publisherName,
+		schemaType,
+		renderBody,
+		twinUrl = defaultTwinUrl,
+		include = () => true,
+		summaryTwin = true,
+		ragChunkMarkers = true,
+		stalenessCheck = 'content-hash',
+		onSummaryMinimalFallback,
+	} = options
+
+	const filtered = forMarkdownTwin(items).filter(include)
+	const skipped = items.length - filtered.length
+
+	const twins: EmittedTwin[] = []
+	const contentHashes = new Map<string, string>()
+
+	for (const item of filtered) {
+		const primaryUrl = twinUrl(item.url)
+		const primaryUrlPath = urlPath(primaryUrl)
+		const body = await renderBody(item)
+
+		const contentHash =
+			stalenessCheck === 'content-hash' ? await computeContentHash(item, body) : undefined
+		if (contentHash) contentHashes.set(item.url, contentHash)
+
+		const summaryUrl = summaryTwin ? `${primaryUrl}.summary.md` : undefined
+
+		const aeoOpts: GenerateAeoMarkdownOptions = {
+			publisherName,
+			schemaType,
+			content: body,
+			ragChunkMarkers,
+			canonical: item.url,
+			twinUrl: primaryUrl,
+			summaryUrl,
+			contentHash,
+		}
+		const primaryContent = generateAeoMarkdown(item, aeoOpts)
+
+		twins.push({
+			urlPath: primaryUrlPath,
+			url: primaryUrl,
+			content: primaryContent,
+			contentType: 'text/markdown; charset=utf-8',
+			item,
+			contentHash,
+		})
+
+		if (summaryTwin && summaryUrl) {
+			const summary = generateSummaryTwin(item, {
+				publisherName,
+				schemaType,
+				content: body,
+				fullUrl: primaryUrl,
+				onMinimalFallback: onSummaryMinimalFallback,
+			})
+			twins.push({
+				urlPath: urlPath(summaryUrl),
+				url: summaryUrl,
+				content: summary.markdown,
+				contentType: 'text/markdown; charset=utf-8',
+				primaryUrl,
+			})
+		}
+	}
+
+	return { twins, contentHashes, skipped }
+}
+
+// ─── Defaults ───
+
+/**
+ * Default twinUrl: strip trailing slashes and append '.md'.
+ * `/article/midway/` -> `/article/midway.md`
+ * `/article/midway`  -> `/article/midway.md`
+ */
+function defaultTwinUrl(articleUrl: string): string {
+	return `${articleUrl.replace(/\/+$/, '')}.md`
+}
+
+/**
+ * Extract the URL-path portion of an absolute or relative URL.
+ * Used by the caller to derive the filesystem write path under dist/client/.
+ */
+function urlPath(url: string): string {
+	try {
+		return new URL(url).pathname
+	} catch {
+		// Relative URL — assume it's already a path.
+		return url.startsWith('/') ? url : `/${url}`
+	}
+}
+
+export const _internals = {
+	defaultTwinUrl,
+	urlPath,
+}