@tanstack/start-plugin-core 1.20.3-alpha.1

package/src/prerender.ts

@@ -0,0 +1,245 @@
+import { promises as fsp } from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { getRollupConfig } from 'nitropack/rollup'
+import { build as buildNitro, createNitro } from 'nitropack'
+import { joinURL, withBase, withoutBase } from 'ufo'
+import { Queue } from './queue'
+import { buildNitroEnvironment } from './nitro/build-nitro'
+import type { ViteBuilder } from 'vite'
+import type { $Fetch, Nitro } from 'nitropack'
+import type { TanStackStartOutputConfig } from './plugin'
+import type { Page } from './schema'
+
+export async function prerender({
+  options,
+  nitro,
+  builder,
+}: {
+  options: TanStackStartOutputConfig
+  nitro: Nitro
+  builder: ViteBuilder
+}) {
+  nitro.logger.info('Prendering pages...')
+
+  // If prerender is enabled but no pages are provided, default to prerendering the root page
+  if (options.prerender?.enabled && !options.pages.length) {
+    options.pages = [
+      {
+        path: '/',
+      },
+    ]
+  }
+
+  const serverEnv = builder.environments['server']
+
+  if (!serverEnv) {
+    throw new Error(`Vite's "server" environment not found`)
+  }
+
+  const prerenderOutputDir = path.resolve(
+    options.root,
+    '.tanstack-start/build/prerenderer',
+  )
+
+  const nodeNitro = await createNitro({
+    ...nitro.options._config,
+    preset: 'nitro-prerender',
+    logLevel: 0,
+    output: {
+      dir: prerenderOutputDir,
+      serverDir: path.resolve(prerenderOutputDir, 'server'),
+      publicDir: path.resolve(prerenderOutputDir, 'public'),
+    },
+  })
+
+  const nodeNitroRollupOptions = getRollupConfig(nodeNitro)
+
+  const build = serverEnv.config.build
+
+  build.outDir = prerenderOutputDir
+
+  build.rollupOptions = {
+    ...build.rollupOptions,
+    ...nodeNitroRollupOptions,
+    output: {
+      ...build.rollupOptions.output,
+      ...nodeNitroRollupOptions.output,
+      sourcemap: undefined,
+    },
+  }
+
+  await buildNitroEnvironment(nodeNitro, () => buildNitro(nodeNitro))
+
+  // Import renderer entry
+  const serverFilename =
+    typeof nodeNitroRollupOptions.output.entryFileNames === 'string'
+      ? nodeNitroRollupOptions.output.entryFileNames
+      : 'index.mjs'
+
+  const serverEntrypoint = path.resolve(
+    path.join(nodeNitro.options.output.serverDir, serverFilename),
+  )
+
+  const { closePrerenderer, localFetch } = (await import(serverEntrypoint)) as {
+    closePrerenderer: () => void
+    localFetch: $Fetch
+  }
+
+  try {
+    // Crawl all pages
+    const pages = await prerenderPages()
+
+    nitro.logger.info(`Prerendered ${pages.length} pages:`)
+    pages.forEach((page) => {
+      nitro.logger.info(`- ${page}`)
+    })
+
+    // TODO: Write the prerendered pages to the output directory
+  } catch (error) {
+    nitro.logger.error(error)
+  } finally {
+    // Ensure server is always closed
+    // server.process.kill()
+    closePrerenderer()
+  }
+
+  function extractLinks(html: string): Array<string> {
+    const linkRegex = /<a[^>]+href=["']([^"']+)["'][^>]*>/g
+    const links: Array<string> = []
+    let match
+
+    while ((match = linkRegex.exec(html)) !== null) {
+      const href = match[1]
+      if (href && (href.startsWith('/') || href.startsWith('./'))) {
+        links.push(href)
+      }
+    }
+
+    return links
+  }
+
+  async function prerenderPages() {
+    const seen = new Set<string>()
+    const retriesByPath = new Map<string, number>()
+    const concurrency = options.prerender?.concurrency ?? os.cpus().length
+    nitro.logger.info(`Concurrency: ${concurrency}`)
+    const queue = new Queue({ concurrency })
+
+    options.pages.forEach((_page) => {
+      let page = _page as Page
+
+      if (typeof _page === 'string') {
+        page = { path: _page }
+      }
+
+      addCrawlPageTask(page)
+    })
+
+    await queue.start()
+
+    return Array.from(seen)
+
+    function addCrawlPageTask(page: Page) {
+      // Was the page already seen?
+      if (seen.has(page.path)) return
+
+      // Add the page to the seen set
+      seen.add(page.path)
+
+      if (page.fromCrawl) {
+        options.pages.push(page)
+      }
+
+      // If not enabled, skip
+      if (!(page.prerender?.enabled ?? true)) return
+
+      // If there is a filter link, check if the page should be prerendered
+      if (options.prerender?.filter && !options.prerender.filter(page)) return
+
+      // Resolve the merged default and page-specific prerender options
+      const prerenderOptions = {
+        ...options.prerender,
+        ...page.prerender,
+      }
+
+      // Add the task
+      queue.add(async () => {
+        nitro.logger.info(`Crawling: ${page.path}`)
+        const retries = retriesByPath.get(page.path) || 0
+        try {
+          // Fetch the route
+          const encodedRoute = encodeURI(page.path)
+
+          const res = await localFetch<Response>(
+            withBase(encodedRoute, nodeNitro.options.baseURL),
+            {
+              headers: { 'x-nitro-prerender': encodedRoute },
+            },
+          )
+
+          if (!res.ok) {
+            throw new Error(`Failed to fetch ${page.path}: ${res.statusText}`)
+          }
+
+          // Guess route type and populate fileName
+          const contentType = res.headers.get('content-type') || ''
+          const isImplicitHTML =
+            !page.path.endsWith('.html') && contentType.includes('html')
+          // &&
+          // !JsonSigRx.test(dataBuff.subarray(0, 32).toString('utf8'))
+          const routeWithIndex = page.path.endsWith('/')
+            ? page.path + 'index'
+            : page.path
+
+          const htmlPath =
+            page.path.endsWith('/') || prerenderOptions.autoSubfolderIndex
+              ? joinURL(page.path, 'index.html')
+              : page.path + '.html'
+
+          const filename = withoutBase(
+            isImplicitHTML ? htmlPath : routeWithIndex,
+            nitro.options.baseURL,
+          )
+
+          const html = await res.text()
+
+          const filepath = path.join(nitro.options.output.publicDir, filename)
+
+          await fsp.mkdir(path.dirname(filepath), {
+            recursive: true,
+          })
+
+          await fsp.writeFile(filepath, html)
+
+          const newPage = await prerenderOptions.onSuccess?.({ page, html })
+
+          if (newPage) {
+            Object.assign(page, newPage)
+          }
+
+          // Find new links
+          if (prerenderOptions.crawlLinks ?? true) {
+            const links = extractLinks(html)
+            for (const link of links) {
+              addCrawlPageTask({ path: link, fromCrawl: true })
+            }
+          }
+        } catch (error) {
+          if (retries < (prerenderOptions.retryCount ?? 0)) {
+            nitro.logger.warn(
+              `Encountered error, retrying: ${page.path} in 500ms`,
+            )
+            await new Promise((resolve) =>
+              setTimeout(resolve, prerenderOptions.retryDelay),
+            )
+            retriesByPath.set(page.path, retries + 1)
+            addCrawlPageTask(page)
+          } else {
+            throw error
+          }
+        }
+      })
+    }
+  }
+}

package/src/queue.ts

@@ -0,0 +1,153 @@
+interface PoolConfig {
+  concurrency?: number
+  started?: boolean
+  tasks?: Array<() => Promise<any>>
+}
+
+const defaultConfig: PoolConfig = {
+  concurrency: 5,
+  started: false,
+  tasks: [],
+}
+
+export class Queue<T> {
+  private onSettles: Array<(res: any, error: any) => void> = []
+  private onErrors: Array<(error: any, task: () => Promise<any>) => void> = []
+  private onSuccesses: Array<(result: any, task: () => Promise<any>) => void> =
+    []
+  private running: boolean
+  private active: Array<() => Promise<any>> = []
+  private pending: Array<() => Promise<any>>
+  private currentConcurrency: number
+
+  constructor(config: PoolConfig = defaultConfig) {
+    const { concurrency, started, tasks } = {
+      ...defaultConfig,
+      ...config,
+    }
+    this.running = started!
+    this.pending = tasks as Array<() => Promise<any>>
+    this.currentConcurrency = concurrency!
+  }
+
+  private tick() {
+    if (!this.running) {
+      return
+    }
+    while (
+      this.active.length < this.currentConcurrency &&
+      this.pending.length
+    ) {
+      const nextFn = this.pending.shift()
+      if (!nextFn) {
+        throw new Error('Found task that is not a function')
+      }
+      this.active.push(nextFn)
+      ;(async () => {
+        let success = false
+        let res!: T
+        let error: any
+        try {
+          res = await nextFn()
+          success = true
+        } catch (e) {
+          error = e
+        }
+        this.active = this.active.filter((d) => d !== nextFn)
+        if (success) {
+          this.onSuccesses.forEach((d) => d(res, nextFn))
+        } else {
+          this.onErrors.forEach((d) => d(error, nextFn))
+        }
+        this.onSettles.forEach((d) => d(res, error))
+        this.tick()
+      })()
+    }
+  }
+
+  add(fn: () => Promise<T> | T, { priority }: { priority?: boolean } = {}) {
+    return new Promise<any>((resolve, reject) => {
+      const task = () =>
+        Promise.resolve(fn())
+          .then((res) => {
+            resolve(res)
+            return res
+          })
+          .catch((err) => {
+            reject(err)
+            throw err
+          })
+      if (priority) {
+        this.pending.unshift(task)
+      } else {
+        this.pending.push(task)
+      }
+      this.tick()
+    })
+  }
+
+  throttle(n: number) {
+    this.currentConcurrency = n
+  }
+
+  onSettled(cb: () => void) {
+    this.onSettles.push(cb)
+    return () => {
+      this.onSettles = this.onSettles.filter((d) => d !== cb)
+    }
+  }
+
+  onError(cb: (error: any, task: () => Promise<any>) => void) {
+    this.onErrors.push(cb)
+    return () => {
+      this.onErrors = this.onErrors.filter((d) => d !== cb)
+    }
+  }
+
+  onSuccess(cb: (result: any, task: () => Promise<any>) => void) {
+    this.onSuccesses.push(cb)
+    return () => {
+      this.onSuccesses = this.onSuccesses.filter((d) => d !== cb)
+    }
+  }
+
+  stop() {
+    this.running = false
+  }
+
+  start() {
+    this.running = true
+    this.tick()
+    return new Promise<void>((resolve) => {
+      this.onSettled(() => {
+        if (this.isSettled()) {
+          resolve()
+        }
+      })
+    })
+  }
+
+  clear() {
+    this.pending = []
+  }
+
+  getActive() {
+    return this.active
+  }
+
+  getPending() {
+    return this.pending
+  }
+
+  getAll() {
+    return [...this.active, ...this.pending]
+  }
+
+  isRunning() {
+    return this.running
+  }
+
+  isSettled() {
+    return !this.active.length && !this.pending.length
+  }
+}

package/src/routesManifestPlugin.ts

@@ -0,0 +1,216 @@
+import { readFileSync } from 'node:fs'
+import path from 'node:path'
+import { joinURL } from 'ufo'
+import { rootRouteId } from '@tanstack/router-core'
+import type {
+  PluginOption,
+  ResolvedConfig,
+  Manifest as ViteManifest,
+  ManifestChunk as ViteManifestChunk,
+} from 'vite'
+import type { Manifest, RouterManagedTag } from '@tanstack/router-core'
+import type { TanStackStartOutputConfig } from './plugin'
+
+export function startManifestPlugin(
+  opts: TanStackStartOutputConfig,
+): PluginOption {
+  let config: ResolvedConfig
+
+  return {
+    name: 'tsr-routes-manifest',
+    enforce: 'pre',
+
+    configResolved(resolvedConfig) {
+      config = resolvedConfig
+    },
+    // configEnvironment(env, envConfig) {
+    //   config = envConfig.
+    // },
+    resolveId(id) {
+      if (id === 'tanstack:start-manifest') {
+        return id
+      }
+      return
+    },
+    load(id) {
+      if (id === 'tanstack:start-manifest') {
+        if (this.environment.config.consumer !== 'server') {
+          // this will ultimately fail the build if the plugin is used outside the server environment
+          // TODO: do we need special handling for `serve`?
+          return `export default {}`
+        }
+        // If we're in development, return a dummy manifest
+
+        if (config.command === 'serve') {
+          return `export const tsrStartManifest = () => ({
+            entry: "$${process.env.TSS_CLIENT_BASE}/",
+            routes: {}
+          })`
+        }
+
+        const clientViteManifestPath = path.resolve(
+          opts.root,
+          '.tanstack-start/build/client-dist/.vite/manifest.json',
+        )
+
+        let viteManifest: ViteManifest
+        try {
+          viteManifest = JSON.parse(
+            readFileSync(clientViteManifestPath, 'utf-8'),
+          )
+        } catch (err) {
+          console.error(err)
+          throw new Error(
+            `Could not find the production client vite manifest at '${clientViteManifestPath}'!`,
+          )
+        }
+
+        const routeTreePath = path.resolve(opts.tsr.generatedRouteTree)
+
+        let routeTreeContent: string
+        try {
+          routeTreeContent = readFileSync(routeTreePath, 'utf-8')
+        } catch (err) {
+          console.error(err)
+          throw new Error(
+            `Could not find the generated route tree at '${routeTreePath}'!`,
+          )
+        }
+
+        // Extract the routesManifest JSON from the route tree file.
+        // It's located between the /* ROUTE_MANIFEST_START and ROUTE_MANIFEST_END */ comment block.
+
+        const routerManifest = JSON.parse(
+          routeTreeContent.match(
+            /\/\* ROUTE_MANIFEST_START([\s\S]*?)ROUTE_MANIFEST_END \*\//,
+          )?.[1] || '{ routes: {} }',
+        ) as Manifest
+
+        const routes = routerManifest.routes
+
+        let entryFile: ViteManifestChunk | undefined
+
+        const filesByRouteFilePath: ViteManifest = Object.fromEntries(
+          Object.entries(viteManifest).map(([k, v]) => {
+            if (v.isEntry) {
+              entryFile = v
+            }
+
+            const rPath = k.split('?')[0]
+
+            return [rPath, v]
+          }, {}),
+        )
+
+        const routesDirectoryFromRoot = path.relative(
+          opts.root,
+          opts.tsr.routesDirectory,
+        )
+
+        // Add preloads to the routes from the vite manifest
+        Object.entries(routes).forEach(([k, v]) => {
+          const file =
+            filesByRouteFilePath[
+              path.join(routesDirectoryFromRoot, v.filePath as string)
+            ]
+
+          if (file) {
+            const preloads = (file.imports ?? []).map((d) =>
+              path.join('/', viteManifest[d]!.file),
+            )
+
+            if (file.file) {
+              preloads.unshift(path.join('/', file.file))
+            }
+
+            const cssFiles = file.css ?? []
+            const cssAssetsList: Array<RouterManagedTag> = cssFiles.map(
+              (cssFile) => ({
+                tag: 'link',
+                attrs: {
+                  rel: 'stylesheet',
+                  href: joinURL('/', cssFile),
+                  type: 'text/css',
+                },
+              }),
+            )
+
+            routes[k] = {
+              ...v,
+              assets: [...(v.assets || []), ...cssAssetsList],
+              preloads,
+            }
+          }
+        })
+
+        if (entryFile) {
+          routes[rootRouteId]!.preloads = [
+            path.join('/', entryFile.file),
+            ...(entryFile.imports?.map((d) =>
+              path.join('/', viteManifest[d]!.file),
+            ) || []),
+          ]
+
+          // Gather all the CSS files from the entry file in
+          // the `css` key and add them to the root route
+          const entryCssFiles = entryFile.css ?? []
+          const entryCssAssetsList: Array<RouterManagedTag> = entryCssFiles.map(
+            (cssFile) => ({
+              tag: 'link',
+              attrs: {
+                rel: 'stylesheet',
+                href: joinURL('/', cssFile),
+                type: 'text/css',
+              },
+            }),
+          )
+
+          routes[rootRouteId]!.assets = [
+            ...(routes[rootRouteId]!.assets || []),
+            ...entryCssAssetsList,
+            {
+              tag: 'script',
+              attrs: {
+                src: joinURL('/', entryFile.file),
+                type: 'module',
+              },
+            },
+          ]
+        }
+
+        const recurseRoute = (
+          route: {
+            preloads?: Array<string>
+            children?: Array<any>
+          },
+          seenPreloads = {} as Record<string, true>,
+        ) => {
+          route.preloads = route.preloads?.filter((preload) => {
+            if (seenPreloads[preload]) {
+              return false
+            }
+            seenPreloads[preload] = true
+            return true
+          })
+
+          if (route.children) {
+            route.children.forEach((child) => {
+              const childRoute = routes[child]!
+              recurseRoute(childRoute, { ...seenPreloads })
+            })
+          }
+        }
+
+        // @ts-expect-error
+        recurseRoute(routes[rootRouteId])
+
+        const routesManifest = {
+          routes,
+        }
+
+        return `export const tsrStartManifest = () => (${JSON.stringify(routesManifest)})`
+      }
+      return
+    },
+  }
+}