@growth-labs/seo 0.4.2 → 0.5.1

package/src/index.ts

@@ -89,53 +89,65 @@ export default function seo(userOptions: SeoOptions): AstroIntegration {
 
 				const injected: string[] = []
 				const skipped: Array<{ pattern: string; reason: string }> = []
+				const injectedRoutes = options.injectedRoutes
 
-				// ─── Sitemaps (provider-wired gated) ───
-				if (providerWired) {
-					injectRoute({
-						pattern: SITEMAP_INDEX_PATH,
-						entrypoint: resolveEntrypoint('./routes/sitemap-index'),
-						prerender: false,
-					})
-					injected.push(SITEMAP_INDEX_PATH)
+				const injectSeoRoute = (pattern: string, entrypoint: string) => {
 					injectRoute({
-						pattern: '/sitemap-articles.xml',
-						entrypoint: resolveEntrypoint('./routes/sitemap-articles'),
+						pattern,
+						entrypoint: resolveEntrypoint(entrypoint),
 						prerender: false,
 					})
-					injected.push('/sitemap-articles.xml')
-					injectRoute({
-						pattern: '/sitemap-pages.xml',
-						entrypoint: resolveEntrypoint('./routes/sitemap-pages'),
-						prerender: false,
-					})
-					injected.push('/sitemap-pages.xml')
-					injectRoute({
-						pattern: '/sitemap-videos.xml',
-						entrypoint: resolveEntrypoint('./routes/sitemap-videos'),
-						prerender: false,
+					injected.push(pattern)
+				}
+
+				const skipDisabledRoute = (pattern: string, optionName: string) => {
+					skipped.push({
+						pattern,
+						reason: `injectedRoutes.${optionName} is false`,
 					})
-					injected.push('/sitemap-videos.xml')
-					if (options.commerce?.enabled) {
-						injectRoute({
-							pattern: '/sitemap-products.xml',
-							entrypoint: resolveEntrypoint('./routes/sitemap-products'),
-							prerender: false,
-						})
-						injected.push('/sitemap-products.xml')
+				}
+
+				// ─── Sitemaps (provider-wired gated) ───
+				if (providerWired) {
+					if (injectedRoutes.sitemapIndex) {
+						injectSeoRoute(SITEMAP_INDEX_PATH, './routes/sitemap-index')
+					} else {
+						skipDisabledRoute(SITEMAP_INDEX_PATH, 'sitemapIndex')
+					}
+
+					if (injectedRoutes.sitemapArticles) {
+						injectSeoRoute('/sitemap-articles.xml', './routes/sitemap-articles')
+					} else {
+						skipDisabledRoute('/sitemap-articles.xml', 'sitemapArticles')
+					}
+
+					if (injectedRoutes.sitemapPages) {
+						injectSeoRoute('/sitemap-pages.xml', './routes/sitemap-pages')
+					} else {
+						skipDisabledRoute('/sitemap-pages.xml', 'sitemapPages')
+					}
+
+					if (injectedRoutes.sitemapVideos) {
+						injectSeoRoute('/sitemap-videos.xml', './routes/sitemap-videos')
+					} else {
+						skipDisabledRoute('/sitemap-videos.xml', 'sitemapVideos')
+					}
+
+					if (!injectedRoutes.sitemapProducts) {
+						skipDisabledRoute('/sitemap-products.xml', 'sitemapProducts')
+					} else if (options.commerce?.enabled) {
+						injectSeoRoute('/sitemap-products.xml', './routes/sitemap-products')
 					} else {
 						skipped.push({
 							pattern: '/sitemap-products.xml',
 							reason: 'commerce.enabled is false',
 						})
 					}
-					if (options.markdownSitemap && aeo && aeo.mode !== 'middleware') {
-						injectRoute({
-							pattern: '/sitemap-markdown.xml',
-							entrypoint: resolveEntrypoint('./routes/sitemap-markdown'),
-							prerender: false,
-						})
-						injected.push('/sitemap-markdown.xml')
+
+					if (!injectedRoutes.sitemapMarkdown) {
+						skipDisabledRoute('/sitemap-markdown.xml', 'sitemapMarkdown')
+					} else if (options.markdownSitemap && aeo && aeo.mode !== 'middleware') {
+						injectSeoRoute('/sitemap-markdown.xml', './routes/sitemap-markdown')
 					} else if (options.markdownSitemap) {
 						skipped.push({
 							pattern: '/sitemap-markdown.xml',
@@ -313,12 +325,17 @@ export default function seo(userOptions: SeoOptions): AstroIntegration {
 
 				for (const file of htmlFiles) {
 					const html = readFileSync(file, 'utf-8')
+					const relPath = file.replace(outDir, '')
 					const result = validatePage(html, {
 						titleMaxLength: options.validation.titleMaxLength,
 						descriptionMaxLength: options.validation.descriptionMaxLength,
 						heroMinWidth: options.validation.heroMinWidth,
+						pagePath: relPath,
+						requireH1: options.validation.requireH1,
+						requireHeroImage: options.validation.requireHeroImage,
+						requireArticleSchema: options.validation.requireArticleSchema,
+						requireMaxImagePreviewLarge: options.validation.requireMaxImagePreviewLarge,
 					})
-					const relPath = file.replace(outDir, '')
 					for (const error of result.errors) {
 						logger.error(`${relPath}: ${error}`)
 						errorCount++
@@ -352,6 +369,9 @@ export default function seo(userOptions: SeoOptions): AstroIntegration {
 
 				if (errorCount || warningCount) {
 					logger.info(`SEO validation: ${errorCount} errors, ${warningCount} warnings`)
+					if (errorCount > 0) {
+						throw new Error(`[@growth-labs/seo] SEO validation failed with ${errorCount} errors`)
+					}
 				} else {
 					logger.info('SEO validation: all checks passed')
 				}

package/src/middleware/seo.ts

@@ -109,13 +109,18 @@ export function createSeoMiddleware(
 		//   - CF-Connecting-IP: FCrDNS is keyed on it
 		appendVaryHeaders(newHeaders)
 
-		// Link alternate header on HTML responses when aeoTwins enabled, except for
-		// members items (they have no markdown counterpart at rest).
+		// Link alternate header on HTML responses when aeoTwins enabled.
+		//
+		// Emit only when contentProvider returns an item for this URL AND that item
+		// is not members-gated. Previously this also emitted Link headers for URLs
+		// the contentProvider did not know about (hubs, region pages, the home page,
+		// etc.) — those URLs have no twin at rest, so the advertised .md target 404s.
+		// Gating to items-only matches actually-emitted twin paths.
 		if (aeo && isHtmlResponse(response)) {
 			const linkItem = contentProvider
 				? await findItemForPath(contentProvider, context, url.pathname)
 				: undefined
-			if (!linkItem || linkItem.access !== 'members') {
+			if (linkItem && linkItem.access !== 'members') {
 				const target =
 					aeo.mode === 'static' || aeo.mode === 'both' ? twinUrlFor(url, aeo) : url.toString()
 				const linkValue = `<${target}>; rel="alternate"; type="text/markdown"`

package/src/options.ts

@@ -185,6 +185,19 @@ const crawlerPolicySchema = z
 	})
 	.default({})
 
+// ─── Injected route ownership ───
+
+const injectedRoutesSchema = z
+	.object({
+		sitemapIndex: z.boolean().default(true),
+		sitemapArticles: z.boolean().default(true),
+		sitemapPages: z.boolean().default(true),
+		sitemapVideos: z.boolean().default(true),
+		sitemapProducts: z.boolean().default(true),
+		sitemapMarkdown: z.boolean().default(true),
+	})
+	.default({})
+
 // ─── Main schema ───
 
 const siteUrlSchema = z.union([
@@ -217,6 +230,10 @@ export const seoOptionsSchema = z.object({
 	markdownSitemap: z.boolean().default(true),
 	rss: z.boolean().default(false),
 
+	// ─── Injected route ownership ───
+	// Set a route false when the consumer application owns that public path.
+	injectedRoutes: injectedRoutesSchema,
+
 	// ─── AEO twins ───
 	// Boolean form = { mode: 'static' } when true, no twins emitted when false.
 	aeoTwins: z.union([z.boolean(), aeoTwinsObjectSchema]).default(false),
@@ -269,6 +286,10 @@ export const seoOptionsSchema = z.object({
 			heroMinWidth: z.number().default(1200),
 			titleMaxLength: z.number().default(110),
 			descriptionMaxLength: z.number().default(160),
+			requireH1: z.boolean().default(true),
+			requireHeroImage: z.boolean().default(true),
+			requireArticleSchema: z.boolean().default(true),
+			requireMaxImagePreviewLarge: z.boolean().default(true),
 			enabled: z.boolean().default(true),
 		})
 		.default({}),

package/src/routes/aeo-twin.ts

@@ -51,11 +51,35 @@ export const getStaticPaths: GetStaticPaths = async () => {
 	// Emit twins for static + both modes; middleware mode serves on-demand.
 	if (!aeo || aeo.mode === 'middleware' || !contentProvider) return []
 
+	// Twin generation now covers articles + pages + videos + podcasts. Hubs,
+	// region pillars, series indexes, and the home page typically come through
+	// `type: 'pages'`. Each consumer's contentProvider decides which surfaces
+	// belong to which type; if a type is unknown it returns [] and we skip.
+	const TWIN_TYPES = ['articles', 'pages', 'videos', 'podcasts'] as const
+
 	let items: ContentItem[]
 	try {
 		// contentProvider resolves here because this function runs inside Astro's
 		// build pipeline — `astro:content` and other Vite virtual modules work.
-		items = await contentProvider({ type: 'articles' }, {} as never)
+		const batches = await Promise.all(
+			TWIN_TYPES.map((type) =>
+				(contentProvider({ type }, {} as never) as Promise<ContentItem[]>).catch(
+					() => [] as ContentItem[],
+				),
+			),
+		)
+		// Deduplicate by URL — a contentProvider might surface the same item under
+		// multiple types (rare, but cheap to defend against).
+		const seen = new Set<string>()
+		items = []
+		for (const batch of batches) {
+			for (const it of batch) {
+				if (!it || typeof it.url !== 'string') continue
+				if (seen.has(it.url)) continue
+				seen.add(it.url)
+				items.push(it)
+			}
+		}
 	} catch {
 		// Don't fail the build; log nothing here because astro's getStaticPaths
 		// swallows console output. The consumer sees "zero paths emitted" which
@@ -74,7 +98,19 @@ export const getStaticPaths: GetStaticPaths = async () => {
 	for (const item of filtered) {
 		const primaryUrl = twinUrl(item.url)
 		const primaryPath = stripOrigin(primaryUrl).replace(/^\//, '')
-		const body = item.description ?? ''
+		// Paywall-aware twin body. For free items the twin echoes the description
+		// (consumer-provided summary). For gated items we never emit the full body
+		// — twins are public regardless of the HTML paywall — so we keep the same
+		// description-only body and append an explicit "Full analysis is gated"
+		// footer that AI engines can cite without claiming the paid content as
+		// theirs. The HTML page carries the structured-data paywall markup
+		// (`isAccessibleForFree: false` + `hasPart`); the twin's textual boundary
+		// must agree with that signal.
+		const isGated = item.access === 'members' || item.isAccessibleForFree === false
+		const sample = item.description ?? ''
+		const body = isGated
+			? `${sample}${sample ? '\n\n' : ''}---\n\nFull analysis is behind the paywall. Read the gated piece at ${item.url}.`
+			: sample
 
 		const contentHash =
 			stalenessMode === 'content-hash' ? await computeContentHash(item, body) : undefined

package/src/routes/sitemap-index.ts

@@ -14,6 +14,7 @@ export const GET: APIRoute = async (context) => {
 	const sitemaps: SitemapEntry[] = []
 
 	const { site } = config
+	const injectedRoutes = config.injectedRoutes
 
 	// Fetch articles lastmod if possible
 	let articlesLastmod: string | undefined
@@ -22,43 +23,49 @@ export const GET: APIRoute = async (context) => {
 	let productsLastmod: string | undefined
 
 	if (contentProvider) {
-		try {
-			const articles = await contentProvider({ type: 'articles' }, context as any)
-			if (articles.length > 0) {
-				const dates = articles
-					.map((a) => a.dateModified ?? a.datePublished)
-					.filter(Boolean) as string[]
-				if (dates.length > 0) {
-					articlesLastmod = dates.sort().at(-1)
+		if (injectedRoutes.sitemapArticles) {
+			try {
+				const articles = await contentProvider({ type: 'articles' }, context as any)
+				if (articles.length > 0) {
+					const dates = articles
+						.map((a) => a.dateModified ?? a.datePublished)
+						.filter(Boolean) as string[]
+					if (dates.length > 0) {
+						articlesLastmod = dates.sort().at(-1)
+					}
 				}
-			}
-		} catch {}
+			} catch {}
+		}
 
-		try {
-			const pages = await contentProvider({ type: 'pages' }, context as any)
-			if (pages.length > 0) {
-				const dates = pages
-					.map((p) => p.dateModified ?? p.datePublished)
-					.filter(Boolean) as string[]
-				if (dates.length > 0) {
-					pagesLastmod = dates.sort().at(-1)
+		if (injectedRoutes.sitemapPages) {
+			try {
+				const pages = await contentProvider({ type: 'pages' }, context as any)
+				if (pages.length > 0) {
+					const dates = pages
+						.map((p) => p.dateModified ?? p.datePublished)
+						.filter(Boolean) as string[]
+					if (dates.length > 0) {
+						pagesLastmod = dates.sort().at(-1)
+					}
 				}
-			}
-		} catch {}
+			} catch {}
+		}
 
-		try {
-			const videos = await contentProvider({ type: 'videos' }, context as any)
-			if (videos.length > 0) {
-				const dates = videos
-					.map((v) => v.dateModified ?? v.datePublished)
-					.filter(Boolean) as string[]
-				if (dates.length > 0) {
-					videosLastmod = dates.sort().at(-1)
+		if (injectedRoutes.sitemapVideos) {
+			try {
+				const videos = await contentProvider({ type: 'videos' }, context as any)
+				if (videos.length > 0) {
+					const dates = videos
+						.map((v) => v.dateModified ?? v.datePublished)
+						.filter(Boolean) as string[]
+					if (dates.length > 0) {
+						videosLastmod = dates.sort().at(-1)
+					}
 				}
-			}
-		} catch {}
+			} catch {}
+		}
 
-		if (config.commerce?.enabled) {
+		if (config.commerce?.enabled && injectedRoutes.sitemapProducts) {
 			try {
 				const products = await contentProvider({ type: 'products' }, context as any)
 				if (products.length > 0) {
@@ -73,11 +80,17 @@ export const GET: APIRoute = async (context) => {
 		}
 	}
 
-	sitemaps.push({ loc: `${site}/sitemap-articles.xml`, lastmod: articlesLastmod })
-	sitemaps.push({ loc: `${site}/sitemap-pages.xml`, lastmod: pagesLastmod })
-	sitemaps.push({ loc: `${site}/sitemap-videos.xml`, lastmod: videosLastmod })
+	if (injectedRoutes.sitemapArticles) {
+		sitemaps.push({ loc: `${site}/sitemap-articles.xml`, lastmod: articlesLastmod })
+	}
+	if (injectedRoutes.sitemapPages) {
+		sitemaps.push({ loc: `${site}/sitemap-pages.xml`, lastmod: pagesLastmod })
+	}
+	if (injectedRoutes.sitemapVideos) {
+		sitemaps.push({ loc: `${site}/sitemap-videos.xml`, lastmod: videosLastmod })
+	}
 
-	if (config.commerce?.enabled) {
+	if (config.commerce?.enabled && injectedRoutes.sitemapProducts) {
 		sitemaps.push({ loc: `${site}/sitemap-products.xml`, lastmod: productsLastmod })
 	}
 

package/src/types.ts

@@ -117,7 +117,7 @@ export interface ContentItem {
 
 // ─── Content provider ───
 
-export type ContentType = 'articles' | 'pages' | 'videos' | 'products' | 'authors'
+export type ContentType = 'articles' | 'pages' | 'videos' | 'podcasts' | 'products' | 'authors'
 
 export interface ContentProviderParams {
 	type: ContentType

package/src/utils/define-content-provider.ts

@@ -16,6 +16,7 @@ export interface ContentItemByType {
 	articles: ContentItem
 	pages: ContentItem
 	videos: ContentItem & { video: NonNullable<ContentItem['video']> }
+	podcasts: ContentItem
 	products: ContentItem & { product: NonNullable<ContentItem['product']> }
 	authors: ContentItem
 }

package/src/utils/json-ld/video.ts

@@ -3,12 +3,22 @@ import type { ContentItem, JsonLdObject } from '../../types.js'
 export function generateVideoJsonLd(item: ContentItem): JsonLdObject {
 	const video = item.video!
 
+	// Derive isAccessibleForFree: explicit field wins, otherwise derive from access.
+	// Matches the article emitter so paywalled videos carry the same Google-recognized
+	// markup as paywalled articles. Without this, Googlebot has no way to know the
+	// gated playback is intentional and may flag the page as cloaking.
+	const isAccessibleForFree = item.isAccessibleForFree ?? item.access !== 'members'
+
 	const result: JsonLdObject = {
 		'@context': 'https://schema.org',
 		'@type': 'VideoObject',
 		name: item.title,
 		thumbnailUrl: video.thumbnailUrl,
 		duration: video.duration,
+		// Google requires the string form 'True'/'False' for Rich Results when paywall
+		// markup is emitted. Same convention as the article schema.
+		// https://developers.google.com/search/docs/appearance/structured-data/paywalled-content
+		isAccessibleForFree: isAccessibleForFree ? 'True' : 'False',
 	}
 
 	if (item.description) result.description = item.description
@@ -16,5 +26,15 @@ export function generateVideoJsonLd(item: ContentItem): JsonLdObject {
 	if (video.embedUrl) result.embedUrl = video.embedUrl
 	if (item.datePublished) result.uploadDate = item.datePublished
 
+	// hasPart paywall marker — emitted whenever the item is gated and a cssSelector
+	// is configured. Mirrors the article emitter behaviour.
+	if (!isAccessibleForFree && item.paywallCssSelector) {
+		result.hasPart = {
+			'@type': 'WebPageElement',
+			isAccessibleForFree: 'False',
+			cssSelector: item.paywallCssSelector,
+		}
+	}
+
 	return result
 }

package/src/utils/sitemap.ts

@@ -105,9 +105,11 @@ ${entries}
 
 export function generateVideoSitemap(items: ContentItem[]): string {
 	const entries = items
-		.filter((item) => item.video)
+		.filter((item): item is ContentItem & { video: NonNullable<ContentItem['video']> } =>
+			Boolean(item.video && (item.video.contentUrl || item.video.embedUrl)),
+		)
 		.map((item) => {
-			const v = item.video!
+			const v = item.video
 			const contentUrlTag = v.contentUrl
 				? `\n      <video:content_loc>${escapeXml(v.contentUrl)}</video:content_loc>`
 				: ''

package/src/utils/validation.ts

@@ -7,6 +7,11 @@ export interface PageValidationOptions {
 	titleMaxLength: number
 	descriptionMaxLength: number
 	heroMinWidth?: number
+	pagePath?: string
+	requireH1?: boolean
+	requireHeroImage?: boolean
+	requireArticleSchema?: boolean
+	requireMaxImagePreviewLarge?: boolean
 }
 
 /**
@@ -139,7 +144,7 @@ export function validatePage(html: string, options: PageValidationOptions): Vali
 		html.match(/<meta\s[^>]*property=["']og:image["'][^>]*>/i) ??
 		html.match(/<meta\s[^>]*property=og:image[^>]*>/i)
 	if (!ogImageMatch) {
-		warnings.push('Missing og:image meta tag')
+		pushIssue(options.requireHeroImage, errors, warnings, 'Missing hero image og:image meta tag')
 	}
 
 	// og:image:width check against heroMinWidth
@@ -154,18 +159,27 @@ export function validatePage(html: string, options: PageValidationOptions): Vali
 			if (widthContentMatch) {
 				const width = Number(widthContentMatch[1])
 				if (!Number.isNaN(width) && width < options.heroMinWidth) {
-					warnings.push(`Hero image width ${width}px is below minimum ${options.heroMinWidth}px`)
+					pushIssue(
+						options.requireHeroImage,
+						errors,
+						warnings,
+						`Hero image width ${width}px is below minimum ${options.heroMinWidth}px`,
+					)
 				}
+			} else if (options.requireHeroImage) {
+				errors.push(`Hero image width is missing; minimum is ${options.heroMinWidth}px`)
 			}
+		} else if (options.requireHeroImage && ogImageMatch) {
+			errors.push(`Hero image width is missing; minimum is ${options.heroMinWidth}px`)
 		}
 	}
 
 	// H1 checks
 	const h1Matches = html.match(/<h1[\s>]/gi) ?? []
 	if (h1Matches.length === 0) {
-		warnings.push('Missing H1 tag')
+		pushIssue(options.requireH1, errors, warnings, 'Missing H1 tag')
 	} else if (h1Matches.length > 1) {
-		warnings.push(`Multiple H1 tags found (${h1Matches.length})`)
+		pushIssue(options.requireH1, errors, warnings, `Multiple H1 tags found (${h1Matches.length})`)
 	}
 
 	// JSON-LD presence check
@@ -174,9 +188,86 @@ export function validatePage(html: string, options: PageValidationOptions): Vali
 		warnings.push('Missing JSON-LD structured data')
 	}
 
+	if (
+		options.requireArticleSchema &&
+		isLikelyArticlePath(options.pagePath) &&
+		!hasArticleJsonLd(html)
+	) {
+		errors.push('Missing valid Article JSON-LD for article route')
+	}
+
+	if (options.requireMaxImagePreviewLarge && !hasMaxImagePreviewLarge(html)) {
+		errors.push('Missing robots max-image-preview:large directive')
+	}
+
 	return { errors, warnings }
 }
 
+function pushIssue(
+	asError: boolean | undefined,
+	errors: string[],
+	warnings: string[],
+	message: string,
+) {
+	if (asError) errors.push(message)
+	else warnings.push(message)
+}
+
+function isLikelyArticlePath(pagePath: string | undefined): boolean {
+	if (!pagePath) return false
+	return /\/(article|articles|news|story|stories)\//i.test(pagePath)
+}
+
+function hasArticleJsonLd(html: string): boolean {
+	for (const rawJson of extractJsonLdBodies(html)) {
+		try {
+			const parsed = JSON.parse(rawJson) as unknown
+			if (hasArticleType(parsed)) return true
+		} catch {}
+	}
+	return false
+}
+
+function extractJsonLdBodies(html: string): string[] {
+	const bodies: string[] = []
+	const pattern = /<script\s[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi
+	for (const match of html.matchAll(pattern)) {
+		if (match[1]) bodies.push(match[1].trim())
+	}
+	return bodies
+}
+
+function hasArticleType(value: unknown): boolean {
+	if (Array.isArray(value)) return value.some(hasArticleType)
+	if (!isRecord(value)) return false
+
+	const type = value['@type']
+	if (type === 'Article' || type === 'NewsArticle' || type === 'BlogPosting') return true
+	if (
+		Array.isArray(type) &&
+		type.some((item) => item === 'Article' || item === 'NewsArticle' || item === 'BlogPosting')
+	) {
+		return true
+	}
+
+	return hasArticleType(value['@graph'])
+}
+
+function hasMaxImagePreviewLarge(html: string): boolean {
+	const metaTags = html.match(/<meta\s+[^>]*>/gi) ?? []
+	for (const tag of metaTags) {
+		if (getHtmlAttr(tag, 'name')?.toLowerCase() !== 'robots') continue
+		const content = getHtmlAttr(tag, 'content')?.toLowerCase() ?? ''
+		const directives = content.split(',').map((part) => part.trim())
+		if (directives.includes('max-image-preview:large')) return true
+	}
+	return false
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null
+}
+
 function isNoindexMetaRefreshRedirect(html: string): boolean {
 	const metaTags = html.match(/<meta\s+[^>]*>/gi) ?? []
 	const hasRefresh = metaTags.some(