@growth-labs/seo 0.4.0 → 0.4.2

package/src/index.ts

@@ -0,0 +1,380 @@
+import { readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import type { AstroIntegration } from 'astro'
+import { _setConfig, _setContentProvider } from './_internal/state.js'
+import { resolveAeoTwins, type SeoOptions, validatedSeoOptionsSchema } from './options.js'
+import { SITEMAP_INDEX_PATH } from './utils/sitemap.js'
+import { validateJsonLd, validatePage, validatePrerenderedGatedRoutes } from './utils/validation.js'
+import { growthLabsSeoPlugin } from './vite-plugin.js'
+
+function resolveEntrypoint(path: string): string {
+	const ext = typeof import.meta.url === 'string' && import.meta.url.endsWith('.ts') ? '.ts' : '.js'
+	return fileURLToPath(new URL(`${path}${ext}`, import.meta.url))
+}
+
+export default function seo(userOptions: SeoOptions): AstroIntegration {
+	const options = validatedSeoOptionsSchema.parse(userOptions)
+	const aeo = resolveAeoTwins(options.aeoTwins)
+
+	_setConfig(options)
+	if (userOptions.contentProvider) {
+		_setContentProvider(userOptions.contentProvider)
+	}
+
+	// ─── Provider-wired detection ───
+	// A "content source" is either the inline `contentProvider` function (main
+	// Worker only — deprecated) or a `contentProviderModule` specifier (canonical,
+	// survives the Cloudflare prerender Worker). When neither is set, we must not
+	// inject crawler-visible routes, because they would silently serve empty bodies
+	// and robots.txt would advertise the empty sitemap index to crawlers — the
+	// exact footgun 0.3.0 closes.
+	const providerWired =
+		typeof userOptions.contentProvider === 'function' ||
+		(typeof options.contentProviderModule === 'string' && options.contentProviderModule.length > 0)
+
+	return {
+		name: '@growth-labs/seo',
+		hooks: {
+			'astro:config:setup': ({ addMiddleware, injectRoute, updateConfig, logger }) => {
+				updateConfig({
+					vite: {
+						plugins: [
+							growthLabsSeoPlugin({
+								config: options,
+								contentProviderModule: options.contentProviderModule,
+							}),
+						],
+					},
+				})
+
+				addMiddleware({ entrypoint: resolveEntrypoint('./middleware/seo'), order: 'post' })
+
+				// ─── Deprecation warning for inline contentProvider ───
+				if (typeof userOptions.contentProvider === 'function') {
+					logger.warn(
+						'`contentProvider` (inline function) is deprecated and does not survive the ' +
+							'Cloudflare prerender Worker. Move your provider into its own module and ' +
+							'set `contentProviderModule: "/src/lib/content-provider.mjs"` instead. ' +
+							'See packages-docs/seo-contentprovidermodule.md. The inline form will be ' +
+							'removed in the next breaking release.',
+					)
+				}
+
+				// ─── Internally-inconsistent config throws ───
+				// Feature flags that only make sense with a content source. Failing
+				// here is louder than silently skipping the route injection below.
+				if (!providerWired) {
+					const needsProvider: string[] = []
+					if (options.aeoTwins !== false) needsProvider.push('aeoTwins')
+					if (options.rss) needsProvider.push('rss: true')
+					if (options.llmsTxt) needsProvider.push('llmsTxt: true')
+					if (options.llmsFullTxt) needsProvider.push('llmsFullTxt: true')
+					if (options.flexibleSampling.enabled) needsProvider.push('flexibleSampling.enabled: true')
+					if (options.appleNews?.enabled) needsProvider.push('appleNews.enabled: true')
+					if (options.podcast?.enabled) needsProvider.push('podcast.enabled: true')
+					if (options.audioNarration?.asPodcastFeed)
+						needsProvider.push('audioNarration.asPodcastFeed: true')
+					if (options.commerce?.enabled) needsProvider.push('commerce.enabled: true')
+					if (needsProvider.length > 0) {
+						throw new Error(
+							`@growth-labs/seo: ${needsProvider.join(', ')} require a content source, but ` +
+								'neither `contentProvider` nor `contentProviderModule` is set.\n' +
+								'  Set `contentProviderModule: "/src/lib/content-provider.mjs"` whose ' +
+								'default export is the ContentProvider function. See ' +
+								'packages-docs/seo-contentprovidermodule.md for a worked D1-backed example.',
+						)
+					}
+				}
+
+				const injected: string[] = []
+				const skipped: Array<{ pattern: string; reason: string }> = []
+
+				// ─── Sitemaps (provider-wired gated) ───
+				if (providerWired) {
+					injectRoute({
+						pattern: SITEMAP_INDEX_PATH,
+						entrypoint: resolveEntrypoint('./routes/sitemap-index'),
+						prerender: false,
+					})
+					injected.push(SITEMAP_INDEX_PATH)
+					injectRoute({
+						pattern: '/sitemap-articles.xml',
+						entrypoint: resolveEntrypoint('./routes/sitemap-articles'),
+						prerender: false,
+					})
+					injected.push('/sitemap-articles.xml')
+					injectRoute({
+						pattern: '/sitemap-pages.xml',
+						entrypoint: resolveEntrypoint('./routes/sitemap-pages'),
+						prerender: false,
+					})
+					injected.push('/sitemap-pages.xml')
+					injectRoute({
+						pattern: '/sitemap-videos.xml',
+						entrypoint: resolveEntrypoint('./routes/sitemap-videos'),
+						prerender: false,
+					})
+					injected.push('/sitemap-videos.xml')
+					if (options.commerce?.enabled) {
+						injectRoute({
+							pattern: '/sitemap-products.xml',
+							entrypoint: resolveEntrypoint('./routes/sitemap-products'),
+							prerender: false,
+						})
+						injected.push('/sitemap-products.xml')
+					} else {
+						skipped.push({
+							pattern: '/sitemap-products.xml',
+							reason: 'commerce.enabled is false',
+						})
+					}
+					if (options.markdownSitemap && aeo && aeo.mode !== 'middleware') {
+						injectRoute({
+							pattern: '/sitemap-markdown.xml',
+							entrypoint: resolveEntrypoint('./routes/sitemap-markdown'),
+							prerender: false,
+						})
+						injected.push('/sitemap-markdown.xml')
+					} else if (options.markdownSitemap) {
+						skipped.push({
+							pattern: '/sitemap-markdown.xml',
+							reason: aeo ? `aeoTwins.mode='${aeo.mode}'` : 'aeoTwins disabled',
+						})
+					}
+				} else {
+					for (const p of [
+						SITEMAP_INDEX_PATH,
+						'/sitemap-articles.xml',
+						'/sitemap-pages.xml',
+						'/sitemap-videos.xml',
+						'/sitemap-products.xml',
+						'/sitemap-markdown.xml',
+					]) {
+						skipped.push({ pattern: p, reason: 'no contentProviderModule wired' })
+					}
+				}
+
+				// ─── robots.txt ───
+				// Inject only when we have something meaningful to emit: either a provider
+				// (so a Sitemap: line is worth advertising) or additionalRules the consumer
+				// configured. Otherwise the consumer can own /robots.txt themselves; we
+				// would only be shipping a no-op `User-agent: *\nAllow: /` anyway.
+				const robotsHasCustomRules = (options.robots.additionalRules?.length ?? 0) > 0
+				if (providerWired || robotsHasCustomRules) {
+					injectRoute({
+						pattern: '/robots.txt',
+						entrypoint: resolveEntrypoint('./routes/robots'),
+						prerender: false,
+					})
+					injected.push('/robots.txt')
+				} else {
+					skipped.push({
+						pattern: '/robots.txt',
+						reason:
+							'no contentProviderModule wired and no robots.additionalRules — consumer owns it',
+					})
+				}
+
+				// ─── llms.txt / llms-full.txt ───
+				if (options.llmsTxt && providerWired) {
+					injectRoute({
+						pattern: '/llms.txt',
+						entrypoint: resolveEntrypoint('./routes/llms'),
+						prerender: false,
+					})
+					injected.push('/llms.txt')
+				}
+				if (options.llmsFullTxt && providerWired) {
+					injectRoute({
+						pattern: '/llms-full.txt',
+						entrypoint: resolveEntrypoint('./routes/llms-full'),
+						prerender: false,
+					})
+					injected.push('/llms-full.txt')
+				}
+
+				// ─── RSS ───
+				if (options.rss && providerWired) {
+					injectRoute({
+						pattern: '/feed.xml',
+						entrypoint: resolveEntrypoint('./routes/rss'),
+						prerender: false,
+					})
+					injected.push('/feed.xml')
+				}
+
+				// ─── Apple News Publisher RSS ───
+				if (options.appleNews?.enabled && providerWired) {
+					injectRoute({
+						pattern: options.appleNews.feedPath,
+						entrypoint: resolveEntrypoint('./routes/apple-news'),
+						prerender: false,
+					})
+					injected.push(options.appleNews.feedPath)
+				}
+
+				// ─── Podcast ───
+				if (options.podcast?.enabled && providerWired) {
+					injectRoute({
+						pattern: options.podcast.feedPath,
+						entrypoint: resolveEntrypoint('./routes/podcast'),
+						prerender: false,
+					})
+					injected.push(options.podcast.feedPath)
+				}
+
+				// ─── Narrated articles podcast feed ───
+				if (options.audioNarration?.asPodcastFeed && providerWired) {
+					injectRoute({
+						pattern: options.audioNarration.podcastFeedPath,
+						entrypoint: resolveEntrypoint('./routes/podcast-narration'),
+						prerender: false,
+					})
+					injected.push(options.audioNarration.podcastFeedPath)
+				}
+
+				// ─── Revalidation endpoint ───
+				// Only when onDemandRevalidation is enabled; the inconsistency throw above
+				// ensures that requires a provider when aeoTwins is enabled.
+				if (aeo?.onDemandRevalidation) {
+					injectRoute({
+						pattern: '/_seo/revalidate',
+						entrypoint: resolveEntrypoint('./routes/revalidate'),
+						prerender: false,
+					})
+					injected.push('/_seo/revalidate')
+				}
+
+				// ─── AEO twin emission (static + both modes) ───
+				// Injected as a prerender route so contentProvider runs inside Astro's
+				// build pipeline. Raw-Node astro:build:done couldn't resolve Vite-virtual
+				// imports like `astro:content` that consumers commonly use in their
+				// contentProvider implementations.
+				if (aeo && aeo.mode !== 'middleware' && providerWired) {
+					injectRoute({
+						pattern: '/[...aeoPath].md',
+						entrypoint: resolveEntrypoint('./routes/aeo-twin'),
+					})
+					injected.push('/[...aeoPath].md')
+				}
+
+				// ─── Injection summary ───
+				// One line covering injected + skipped routes. This is the crawler-visible
+				// contract consumers inherit; making it loud at build time prevents silent
+				// SEO regressions on bump-only adoption.
+				const skippedStr =
+					skipped.length > 0
+						? ` skipped ${skipped.length}: ${skipped
+								.map((s) => `${s.pattern} (${s.reason})`)
+								.join(', ')}`
+						: ''
+				logger.info(
+					`SEO routes: injected ${injected.length} — ${injected.join(', ') || '(none)'}.${skippedStr}`,
+				)
+			},
+
+			'astro:build:done': async ({ dir, pages, logger }) => {
+				const outDir = fileURLToPath(dir)
+				const contentProvider = userOptions.contentProvider
+
+				// ─── Build-time prerender-gated guard (test 32) ───
+				// Note: we try to load items here to check the guard, but this is
+				// best-effort. If contentProvider uses astro:content or other Vite
+				// virtuals, it will fail at raw-Node import time and we skip silently.
+				// The guard is a defence-in-depth check; the primary guarantee comes
+				// from the consumer-set `export const prerender = false` on gated routes.
+				if (contentProvider && options.flexibleSampling?.enabled) {
+					try {
+						const items = await contentProvider({ type: 'articles' }, {} as never)
+						const prerenderedUrls = new Set(pages.map((p) => `/${p.pathname}`.replace(/\/+/g, '/')))
+						const issues = validatePrerenderedGatedRoutes({ prerenderedUrls, items })
+						if (issues.length > 0) {
+							for (const issue of issues) logger.error(issue.message)
+							throw new Error(
+								`[@growth-labs/seo] ${issues.length} prerendered route(s) serve members-gated items. See errors above.`,
+							)
+						}
+					} catch (err) {
+						// Don't fail the build if the provider can't run from Node context —
+						// the injected prerender route handles the primary emission path.
+						logger.info(
+							`Skipped prerender-gated-content guard (contentProvider not Node-ESM-safe): ${err instanceof Error ? err.message : String(err)}`,
+						)
+					}
+				}
+
+				// ─── Legacy per-page validation ───
+				if (!options.validation.enabled) return
+
+				const htmlFiles = findHtmlFiles(outDir)
+				let errorCount = 0
+				let warningCount = 0
+
+				for (const file of htmlFiles) {
+					const html = readFileSync(file, 'utf-8')
+					const result = validatePage(html, {
+						titleMaxLength: options.validation.titleMaxLength,
+						descriptionMaxLength: options.validation.descriptionMaxLength,
+						heroMinWidth: options.validation.heroMinWidth,
+					})
+					const relPath = file.replace(outDir, '')
+					for (const error of result.errors) {
+						logger.error(`${relPath}: ${error}`)
+						errorCount++
+					}
+					for (const warning of result.warnings) {
+						logger.warn(`${relPath}: ${warning}`)
+						warningCount++
+					}
+
+					const jsonLdMatches = html.matchAll(
+						/<script type="application\/ld\+json">([\s\S]*?)<\/script>/gi,
+					)
+					for (const match of jsonLdMatches) {
+						try {
+							const parsed = JSON.parse(match[1]!)
+							const ldResult = validateJsonLd(parsed)
+							for (const e of ldResult.errors) {
+								logger.error(`${relPath} [JSON-LD]: ${e}`)
+								errorCount++
+							}
+							for (const w of ldResult.warnings) {
+								logger.warn(`${relPath} [JSON-LD]: ${w}`)
+								warningCount++
+							}
+						} catch {
+							logger.error(`${relPath}: Invalid JSON-LD`)
+							errorCount++
+						}
+					}
+				}
+
+				if (errorCount || warningCount) {
+					logger.info(`SEO validation: ${errorCount} errors, ${warningCount} warnings`)
+				} else {
+					logger.info('SEO validation: all checks passed')
+				}
+			},
+		},
+	}
+}
+
+function findHtmlFiles(dir: string): string[] {
+	const results: string[] = []
+	try {
+		const entries = readdirSync(dir, { withFileTypes: true })
+		for (const entry of entries) {
+			const fullPath = join(dir, entry.name)
+			if (entry.isDirectory()) results.push(...findHtmlFiles(fullPath))
+			else if (entry.name.endsWith('.html')) results.push(fullPath)
+		}
+	} catch {}
+	return results
+}
+
+export { getConfig, getContentProvider } from './_internal/state.js'
+export type { ResolvedSeoOptions, SeoOptions } from './options.js'
+export { resolveAeoTwins, seoOptionsSchema, validatedSeoOptionsSchema } from './options.js'
+export { resolveSeoConfig, resolveSiteUrl } from './site-url-core.js'
+export type * from './types.js'