@growth-labs/seo 0.4.0 → 0.4.2

package/src/middleware/seo.ts

@@ -0,0 +1,350 @@
+import 'virtual:growth-labs/seo/config'
+import { getConfig, getContentProvider } from '../_internal/state.js'
+import type { SeoEnv } from '../bindings.js'
+import { type ResolvedSeoOptions, resolveAeoTwins } from '../options.js'
+import { getRuntimeEnv, getWaitUntil } from '../runtime.js'
+import type { ContentItem, ContentProvider, CrawlerClass, EffectiveAuthSegment } from '../types.js'
+import { estimateTokenCount, generateAeoMarkdown } from '../utils/aeo.js'
+import { classifyRequest } from '../utils/crawler-class.js'
+import { computeEffectiveAuthSegment, type RawAuthSegment } from '../utils/effective-auth.js'
+import { createDohResolver, createFcrdnsVerifier, type FcrdnsVerifier } from '../utils/fcrdns.js'
+import { type FreshLayerBinding, readFreshTwin, writeFreshTwin } from '../utils/fresh-layer.js'
+
+// ─── Public middleware factory ───
+
+export interface SeoMiddlewareDeps {
+	/** Request classifier. Defaults to one backed by a DoH FCrDNS verifier. */
+	classify?: (request: Request) => Promise<CrawlerClass>
+	/** R2/KV binding for the fresh-twin layer. Required for mode 'middleware' or 'both'. */
+	freshLayer?: FreshLayerBinding
+	/** Assets binding for reading the stale build-time twin on R2 miss. */
+	assets?: { fetch(request: Request): Promise<Response> }
+	/** Consumer's raw auth segment for this request. Defaults to 'anon'. */
+	rawAuthSegment?: (request: Request) => RawAuthSegment
+	/** Fire-and-forget ctx.waitUntil equivalent. Used to schedule background revalidation. */
+	waitUntil?: (promise: Promise<unknown>) => void
+}
+
+export interface SeoMiddlewareContext {
+	request: Request
+	locals?: Record<string, unknown>
+}
+
+export function createSeoMiddleware(
+	options: ResolvedSeoOptions,
+	contentProvider?: ContentProvider,
+	deps: SeoMiddlewareDeps = {},
+) {
+	const aeo = resolveAeoTwins(options.aeoTwins)
+	const classify = deps.classify ?? createDefaultClassifier()
+
+	return async (
+		context: SeoMiddlewareContext,
+		next: () => Promise<Response>,
+	): Promise<Response> => {
+		const { request } = context
+		const accept = request.headers.get('accept') ?? ''
+		const url = new URL(request.url)
+
+		// 1. Classify the request. Expose classification on Astro.locals for the
+		//    consumer's cache-key builder.
+		const crawlerClass: CrawlerClass = await classify(request)
+		const rawAuth = deps.rawAuthSegment?.(request) ?? 'anon'
+		const effectiveAuthSegment: EffectiveAuthSegment = computeEffectiveAuthSegment(
+			crawlerClass,
+			rawAuth,
+		)
+
+		if (context.locals) {
+			context.locals.crawlerClass = crawlerClass
+			context.locals.effectiveAuthSegment = effectiveAuthSegment
+		}
+
+		// 2. LLM training crawlers are 403'd on members items, before any other work.
+		//    We look up the item only when crawlerClass requires it to avoid per-request
+		//    contentProvider calls on the hot path for anonymous requests.
+		const blockLlmTraining = options.crawlerPolicy.blockLlmTrainingCrawlers
+		if (blockLlmTraining && crawlerClass === 'llmTrainingCrawler' && contentProvider) {
+			const item = await findItemForPath(contentProvider, context, url.pathname)
+			if (item?.access === 'members') {
+				return new Response('Forbidden', {
+					status: 403,
+					headers: {
+						'Content-Type': 'text/plain; charset=utf-8',
+						'X-Robots-Tag': 'noindex',
+						...varyHeaders(),
+					},
+				})
+			}
+		}
+
+		// 3. Accept: text/markdown handling (middleware mode of AEO twins).
+		const wantsMarkdown = accept.includes('text/markdown')
+		if (aeo && aeo.mode !== 'static' && wantsMarkdown && contentProvider) {
+			const markdownResponse = await serveAeoMarkdown({
+				request,
+				context,
+				options,
+				contentProvider,
+				freshLayer: deps.freshLayer,
+				assets: deps.assets,
+				waitUntil: deps.waitUntil,
+			})
+			if (markdownResponse) return markdownResponse
+		}
+
+		// 4. Pass through to the next handler (SSR page or static asset route).
+		const response = await next()
+
+		// 5. Wrap response with our observability + policy headers.
+		const newHeaders = new Headers(response.headers)
+
+		// Content-Signal on every response.
+		newHeaders.set('content-signal', buildContentSignalHeader(options))
+
+		// Vary must cover every axis that branches the response:
+		//   - Accept: response body differs on text/markdown
+		//   - User-Agent: crawler class depends on it
+		//   - Cookie: auth segment depends on it
+		//   - CF-Connecting-IP: FCrDNS is keyed on it
+		appendVaryHeaders(newHeaders)
+
+		// Link alternate header on HTML responses when aeoTwins enabled, except for
+		// members items (they have no markdown counterpart at rest).
+		if (aeo && isHtmlResponse(response)) {
+			const linkItem = contentProvider
+				? await findItemForPath(contentProvider, context, url.pathname)
+				: undefined
+			if (!linkItem || linkItem.access !== 'members') {
+				const target =
+					aeo.mode === 'static' || aeo.mode === 'both' ? twinUrlFor(url, aeo) : url.toString()
+				const linkValue = `<${target}>; rel="alternate"; type="text/markdown"`
+				const existing = newHeaders.get('Link')
+				newHeaders.set('Link', existing ? `${existing}, ${linkValue}` : linkValue)
+			}
+		}
+
+		return new Response(response.body, {
+			status: response.status,
+			statusText: response.statusText,
+			headers: newHeaders,
+		})
+	}
+}
+
+// ─── Astro middleware entry ───
+
+let _classifier: FcrdnsVerifier | undefined
+
+export const onRequest = async (
+	context: SeoMiddlewareContext,
+	next: () => Promise<Response>,
+): Promise<Response> => {
+	const config = getConfig()
+	const contentProvider = getContentProvider()
+	const env = getRuntimeEnv(context) as SeoEnv
+	const aeo = resolveAeoTwins(config.aeoTwins)
+
+	// Wire bindings from env when aeoTwins is enabled. Middleware-mode requires R2+DO;
+	// static-only consumers get the lighter path.
+	let freshLayer: FreshLayerBinding | undefined
+	if (env && aeo && aeo.mode !== 'static' && aeo.freshLayer) {
+		const bindingName = aeo.freshLayer.bindingName
+		const impl = (env as unknown as Record<string, unknown>)[bindingName]
+		if (impl) {
+			freshLayer = {
+				type: aeo.freshLayer.type,
+				impl: impl as FreshLayerBinding['impl'],
+				deploymentId: env.CF_VERSION_METADATA?.id ?? 'dev',
+			}
+		}
+	}
+
+	const middleware = createSeoMiddleware(config, contentProvider, {
+		freshLayer,
+		assets: env?.ASSETS,
+		waitUntil: getWaitUntil(context),
+	})
+	return middleware(context, next)
+}
+
+// ─── Internals ───
+
+function createDefaultClassifier() {
+	if (!_classifier) {
+		_classifier = createFcrdnsVerifier(createDohResolver())
+	}
+	const verifier = _classifier
+	return (request: Request) => classifyRequest({ request, fcrdnsVerify: verifier })
+}
+
+function buildContentSignalHeader(config: ResolvedSeoOptions): string {
+	const { aiTrain, search, aiInput } = config.contentSignal
+	return `ai-train=${aiTrain}, search=${search}, ai-input=${aiInput}`
+}
+
+function varyHeaders(): Record<string, string> {
+	return { Vary: 'Accept, User-Agent, Cookie, CF-Connecting-IP' }
+}
+
+function appendVaryHeaders(headers: Headers): void {
+	const existing = headers.get('Vary')
+	const required = 'Accept, User-Agent, Cookie, CF-Connecting-IP'
+	headers.set('Vary', existing ? dedupVary(`${existing}, ${required}`) : required)
+}
+
+function dedupVary(combined: string): string {
+	const parts = combined
+		.split(',')
+		.map((s) => s.trim())
+		.filter(Boolean)
+	return [...new Set(parts)].join(', ')
+}
+
+function isHtmlResponse(response: Response): boolean {
+	const ct = response.headers.get('content-type') ?? ''
+	return ct.includes('text/html')
+}
+
+function twinUrlFor(url: URL, aeo: { twinUrl?: (u: string) => string }): string {
+	const fn = aeo.twinUrl ?? ((u: string) => `${u.replace(/\/+$/, '')}.md`)
+	return fn(url.toString())
+}
+
+async function findItemForPath(
+	contentProvider: ContentProvider,
+	context: SeoMiddlewareContext,
+	pathname: string,
+): Promise<ContentItem | undefined> {
+	// Look up a single item by path; narrow the fetch via `slugs` so CMS providers
+	// can short-circuit the lookup instead of loading the whole catalog.
+	const slug = pathname.replace(/^\/+/, '').split('/').pop() ?? ''
+	try {
+		// Type assertion: APIContext is the Astro request context in real code but our
+		// middleware context is narrower; contentProvider's APIContext parameter is
+		// treated opaquely at this layer.
+		const items = await contentProvider({ type: 'articles', slugs: [slug] }, context as never)
+		return items.find((item) => {
+			try {
+				return new URL(item.url).pathname === pathname
+			} catch {
+				return item.url === pathname
+			}
+		})
+	} catch {
+		return undefined
+	}
+}
+
+// ─── AEO markdown serve path ───
+
+interface ServeMarkdownInput {
+	request: Request
+	context: SeoMiddlewareContext
+	options: ResolvedSeoOptions
+	contentProvider: ContentProvider
+	freshLayer?: FreshLayerBinding
+	assets?: { fetch(request: Request): Promise<Response> }
+	waitUntil?: (promise: Promise<unknown>) => void
+}
+
+async function serveAeoMarkdown(input: ServeMarkdownInput): Promise<Response | null> {
+	const { request, context, options, contentProvider, freshLayer, assets, waitUntil } = input
+	const url = new URL(request.url)
+	const aeo = resolveAeoTwins(options.aeoTwins)
+	if (!aeo) return null
+	const twinPath = urlPathFromTwinUrl(twinUrlFor(url, aeo))
+
+	// Layer 1: fresh layer (R2/KV).
+	if (freshLayer) {
+		const hit = await readFreshTwin(freshLayer, twinPath)
+		if (hit) {
+			return markdownResponse(hit.body, options, hit.contentType)
+		}
+	}
+
+	// Layer 2: stale baseline via ASSETS binding.
+	if (assets) {
+		const assetReq = new Request(`https://assets.local${twinPath}`, { method: 'GET' })
+		const assetResp = await assets.fetch(assetReq)
+		if (assetResp.status === 200) {
+			const body = await assetResp.text()
+			return markdownResponse(body, options)
+		}
+	}
+
+	// Layer 3: fallthrough render.
+	const matched = await findItemForPath(contentProvider, context, url.pathname)
+	if (!matched) {
+		return null // Let the route handler serve HTML as usual.
+	}
+	if (matched.access === 'members') {
+		// Never serve markdown for gated items via negotiation.
+		return null
+	}
+
+	const body = matched.description ?? ''
+	const markdown = generateAeoMarkdown(matched, {
+		publisherName: options.organization.name,
+		schemaType: options.schemaType,
+		content: body,
+		ragChunkMarkers: aeo.ragChunkMarkers,
+		canonical: matched.url,
+		twinUrl: twinUrlFor(url, aeo),
+	})
+
+	// When the fresh layer is wired, return a 503 stub and schedule the write in
+	// the background. 200 OK with thin body would risk Googlebot indexing the
+	// description-only response as canonical and not re-crawling for days.
+	if (waitUntil && freshLayer) {
+		waitUntil(writeFreshTwin(freshLayer, twinPath, markdown).catch(() => {}))
+		return new Response('aeo twin generating', {
+			status: 503,
+			headers: {
+				'Content-Type': 'text/plain; charset=utf-8',
+				'Retry-After': '5',
+				'X-Robots-Tag': 'noindex',
+				'Cache-Control': 'no-store',
+				'content-signal': buildContentSignalHeader(options),
+				...varyHeaders(),
+			},
+		})
+	}
+
+	// Simple mode: no fresh layer wired (tests, static-first consumers without a
+	// Worker DO). Render inline synchronously. Still safe — we're not caching a
+	// thin stub, we're serving the full markdown.
+	return markdownResponse(markdown, options)
+}
+
+function markdownResponse(
+	body: string,
+	options: ResolvedSeoOptions,
+	contentType = 'text/markdown; charset=utf-8',
+): Response {
+	const tokenCount = estimateTokenCount(body)
+	return new Response(body, {
+		headers: {
+			'Content-Type': contentType,
+			'x-markdown-tokens': String(tokenCount),
+			'X-Robots-Tag': 'noindex',
+			'content-signal': buildContentSignalHeader(options),
+			...varyHeaders(),
+		},
+	})
+}
+
+function urlPathFromTwinUrl(twinUrl: string): string {
+	try {
+		return new URL(twinUrl).pathname
+	} catch {
+		return twinUrl.startsWith('/') ? twinUrl : `/${twinUrl}`
+	}
+}
+
+export const _internals = {
+	appendVaryHeaders,
+	dedupVary,
+	twinUrlFor,
+	findItemForPath,
+}