@lukoweb/apitogo 0.1.0

package/src/vite/prerender/prerender.ts

@@ -0,0 +1,233 @@
+import { readFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { createIndex, type PagefindIndex } from "pagefind";
+import colors from "picocolors";
+import PiscinaImport from "piscina";
+import type { getRoutesByConfig } from "../../app/main.js";
+import { logger } from "../../cli/common/logger.js";
+import { fileExists } from "../../config/file-exists.js";
+import { getBuildConfig } from "../../config/validators/BuildSchema.js";
+import type { ZudokuConfig } from "../../config/validators/ZudokuConfig.js";
+import { runPluginTransformConfig } from "../../lib/core/transform-config.js";
+import invariant from "../../lib/util/invariant.js";
+import { joinUrl } from "../../lib/util/joinUrl.js";
+import type { MarkdownFileInfo } from "../plugin-markdown-export.js";
+import { isTTY, throttle, writeLine } from "../reporter.js";
+import { generateSitemap } from "../sitemap.js";
+import { routesToPaths, routesToRewrites } from "./utils.js";
+import type { StaticWorkerData, WorkerData } from "./worker.js";
+
+const Piscina = PiscinaImport as unknown as typeof PiscinaImport.default;
+
+export type WorkerResult = {
+  outputPath: string;
+  html: string;
+  statusCode: number;
+  redirect?: { from: string; to: string };
+};
+
+export const getRedirectUrls = (
+  workerResults: WorkerResult[],
+  basePath: string | undefined,
+): Set<string> =>
+  new Set(
+    workerResults.flatMap((r) => {
+      if (!r.redirect) return [];
+      const from = r.redirect.from;
+      return [
+        basePath && from.startsWith(basePath)
+          ? from.slice(basePath.length) || "/"
+          : from,
+      ];
+    }),
+  );
+
+export const prerender = async ({
+  html,
+  dir,
+  basePath = "",
+  serverConfigFilename,
+  writeRedirects = true,
+}: {
+  html: string;
+  dir: string;
+  basePath?: string;
+  serverConfigFilename: string;
+  writeRedirects: boolean;
+}) => {
+  const distDir = path.join(dir, "dist", basePath);
+  const serverConfigPath = pathToFileURL(
+    path.join(distDir, "server", serverConfigFilename),
+  ).href;
+  const entryServerPath = pathToFileURL(
+    path.join(distDir, "server/entry.server.js"),
+  ).href;
+
+  const rawConfig: ZudokuConfig = await import(serverConfigPath).then(
+    (m) => m.default,
+  );
+  const config = await runPluginTransformConfig(rawConfig);
+
+  const buildConfig = await getBuildConfig();
+  const module = await import(entryServerPath);
+  const getRoutes = module.getRoutesByConfig as typeof getRoutesByConfig;
+
+  const routes = getRoutes(config);
+  const paths = routesToPaths(routes);
+  const rewrites = routesToRewrites(routes);
+
+  // Add redirect source paths so they get prerendered as redirects
+  if (config.redirects) {
+    for (const r of config.redirects) {
+      paths.push(joinUrl(r.from));
+    }
+  }
+  const maxThreads =
+    buildConfig?.prerender?.workers ?? Math.floor(os.cpus().length * 0.8);
+
+  const start = performance.now();
+  const LOG_INTERVAL_MS = 30_000; // Log every 30 seconds
+  let lastLogTime = start;
+
+  const writeProgress = throttle(
+    (count: number, total: number, urlPath: string) => {
+      writeLine(`prerendering (${count}/${total}) ${colors.dim(urlPath)}`);
+    },
+  );
+
+  if (!isTTY()) {
+    logger.info(
+      colors.dim(
+        `prerendering ${paths.length} routes using ${maxThreads} workers...`,
+      ),
+    );
+  }
+
+  let completedCount = 0;
+  let pagefindIndex: PagefindIndex | undefined;
+
+  if (config.search?.type === "pagefind") {
+    const { index, errors } = await createIndex();
+    invariant(
+      index,
+      `Failed to create pagefind index: ${JSON.stringify(errors)}`,
+    );
+    pagefindIndex = index;
+  }
+
+  const pool = new Piscina<WorkerData, WorkerResult>({
+    filename: new URL("./worker.js", import.meta.url).href,
+    idleTimeout: 5_000,
+    maxThreads,
+    workerData: {
+      template: html,
+      distDir,
+      serverConfigPath,
+      entryServerPath,
+      writeRedirects,
+    } satisfies StaticWorkerData,
+  });
+
+  const workerResults = await Promise.all(
+    paths.map(async (urlPath) => {
+      const result = await pool.run({ urlPath } satisfies WorkerData);
+
+      if (result.statusCode < 400) {
+        await pagefindIndex?.addHTMLFile({
+          url: urlPath,
+          content: result.html,
+        });
+      }
+
+      completedCount++;
+
+      if (isTTY()) {
+        writeProgress(completedCount, paths.length, urlPath);
+      } else {
+        const now = performance.now();
+        if (now - lastLogTime >= LOG_INTERVAL_MS) {
+          logger.info(
+            colors.blue(
+              `prerendered ${completedCount}/${paths.length} routes using ${maxThreads} workers`,
+            ),
+          );
+          lastLogTime = now;
+        }
+      }
+      return result;
+    }),
+  );
+
+  const pagefindWriteResult = await pagefindIndex?.writeFiles({
+    outputPath: path.join(distDir, "pagefind"),
+  });
+
+  const seconds = ((performance.now() - start) / 1000).toFixed(1);
+
+  const message = `✓ finished prerendering ${paths.length} routes in ${seconds} seconds using ${maxThreads} workers`;
+
+  if (isTTY()) {
+    writeLine(colors.blue(`${message}\n`));
+  } else {
+    logger.info(colors.blue(message));
+  }
+  if (pagefindWriteResult?.outputPath) {
+    logger.info(
+      colors.blue(`✓ pagefind index built: ${pagefindWriteResult.outputPath}`),
+    );
+  }
+
+  const redirectUrls = getRedirectUrls(workerResults, config.basePath);
+
+  await generateSitemap({
+    basePath: config.basePath,
+    outputUrls: paths,
+    config: config.sitemap,
+    baseOutputDir: distDir,
+    redirectUrls,
+  });
+
+  // Generate llms.txt files if markdown export is enabled
+  if (config.docs) {
+    const { DocsConfigSchema } =
+      await import("../../config/validators/ZudokuConfig.js");
+    const { generateLlmsTxtFiles } = await import("../llms.js");
+
+    const docsConfig = DocsConfigSchema.parse(config.docs);
+    const llmsConfig = docsConfig.llms ?? {};
+
+    const markdownInfoPath = path.join(
+      dir,
+      "node_modules/.apitogo/markdown-info.json",
+    );
+    let markdownFileInfos: MarkdownFileInfo[] = [];
+
+    if (await fileExists(markdownInfoPath)) {
+      const markdownInfoContent = await readFile(markdownInfoPath, "utf-8");
+      markdownFileInfos = JSON.parse(markdownInfoContent);
+    }
+
+    if (llmsConfig.llmsTxt || llmsConfig.llmsTxtFull) {
+      await generateLlmsTxtFiles({
+        markdownFileInfos,
+        basePath: config.basePath,
+        outputUrls: paths,
+        baseOutputDir: distDir,
+        siteName: config.site?.title,
+        llmsTxt: llmsConfig.llmsTxt,
+        llmsTxtFull: llmsConfig.llmsTxtFull,
+        redirectUrls,
+      });
+    }
+
+    if (!docsConfig.publishMarkdown) {
+      await Promise.all(
+        markdownFileInfos.map((info) => rm(info.filePath).catch(() => {})),
+      );
+    }
+  }
+
+  return { workerResults, rewrites };
+};

package/src/vite/prerender/utils.ts

@@ -0,0 +1,76 @@
+import type { RouteObject } from "react-router";
+import { joinUrl } from "../../lib/util/joinUrl.js";
+
+const resolveRoutePath = (path: string): string | undefined => {
+  const segments = path.split("/");
+  if (segments.some((s) => s.startsWith(":") && !s.endsWith("?"))) {
+    return undefined;
+  }
+  return segments.filter((s) => !s.startsWith(":")).join("/") || undefined;
+};
+
+const isSkipped = (path: string) => path.includes("*") || /^\d+$/.test(path);
+
+export type RouteRewrite = { source: string; destination: string };
+
+type ResolvedRoute = {
+  path: string;
+  hasStrippedParams: boolean;
+  children: ResolvedRoute[];
+};
+
+const resolveRoutes = (
+  routes: RouteObject[],
+  parentPath = "",
+): ResolvedRoute[] =>
+  routes.flatMap((route) => {
+    if (route.path && isSkipped(route.path)) return [];
+
+    const routePath = route.path ? resolveRoutePath(route.path) : undefined;
+    if (route.path && !routePath) return [];
+
+    const fullPath = routePath
+      ? routePath.startsWith("/")
+        ? routePath
+        : joinUrl(parentPath, routePath)
+      : parentPath;
+
+    const hasStrippedParams =
+      route.path?.split("/").some((s) => s.startsWith(":")) ?? false;
+
+    return [
+      {
+        path: fullPath,
+        hasStrippedParams,
+        children: resolveRoutes(route.children ?? [], fullPath),
+      },
+    ];
+  });
+
+const collectPaths = (
+  resolved: ResolvedRoute[],
+  parentPath: string,
+): string[] =>
+  resolved.flatMap((r) => [
+    ...(r.path !== parentPath ? [r.path] : []),
+    ...collectPaths(r.children, r.path),
+  ]);
+
+const collectRewrites = (resolved: ResolvedRoute[]): RouteRewrite[] =>
+  resolved.flatMap((r) => [
+    ...(r.hasStrippedParams && r.path
+      ? [
+          {
+            source: `${r.path}/(.+)`,
+            destination: r.path === "/" ? "/index.html" : `${r.path}.html`,
+          },
+        ]
+      : []),
+    ...collectRewrites(r.children),
+  ]);
+
+export const routesToPaths = (routes: RouteObject[]): string[] =>
+  collectPaths(resolveRoutes(routes), "");
+
+export const routesToRewrites = (routes: RouteObject[]): RouteRewrite[] =>
+  collectRewrites(resolveRoutes(routes));

package/src/vite/prerender/worker.ts

@@ -0,0 +1,118 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import Piscina from "piscina";
+import { matchPath } from "react-router";
+import { ProtectedRoutesSchema } from "../../config/validators/ProtectedRoutesSchema.js";
+import type { ZudokuConfig } from "../../config/validators/ZudokuConfig.js";
+import { runPluginTransformConfig } from "../../lib/core/transform-config.js";
+import { joinUrl } from "../../lib/util/joinUrl.js";
+import type { WorkerResult } from "./prerender.js";
+
+type EntryServer = typeof import("../../app/entry.server.js");
+
+export type StaticWorkerData = {
+  template: string;
+  distDir: string;
+  serverConfigPath: string;
+  entryServerPath: string;
+  writeRedirects: boolean;
+};
+
+export type WorkerData = { urlPath: string };
+
+const { template, distDir, serverConfigPath, entryServerPath, writeRedirects } =
+  Piscina.workerData as StaticWorkerData;
+
+const server: EntryServer = await import(entryServerPath);
+const rawConfig: ZudokuConfig = await import(serverConfigPath).then(
+  (m) => m.default,
+);
+const config = await runPluginTransformConfig(rawConfig);
+
+const routes = server.getRoutesByConfig(config);
+const { basePath } = config;
+
+const renderPage = async ({ urlPath }: WorkerData): Promise<WorkerResult> => {
+  const filename = urlPath === "/" ? "/index.html" : `${urlPath}.html`;
+  const pathname = joinUrl(basePath, urlPath);
+  const url = joinUrl("http://localhost", pathname);
+  const outputPath = path.join(distDir, filename);
+
+  const request = new Request(url);
+
+  const protectedRoutes = ProtectedRoutesSchema.parse(config.protectedRoutes);
+  const isProtectedRoute = protectedRoutes
+    ? Object.keys(protectedRoutes).some((route) =>
+        matchPath({ path: route, end: true }, urlPath),
+      )
+    : false;
+
+  // Get the main response
+  const response = await server.handleRequest({
+    template,
+    request,
+    routes,
+    basePath,
+  });
+
+  // Check for redirects
+  if ([301, 302, 303, 307, 308].includes(response.status)) {
+    const redirectTo = response.headers.get("Location") ?? "";
+
+    if (writeRedirects) {
+      const redirectHtml = `<!doctype html><script>window.location.href=${JSON.stringify(redirectTo)};</script>`;
+      await fs.mkdir(path.dirname(outputPath), { recursive: true });
+      await fs.writeFile(outputPath, redirectHtml);
+    }
+
+    return {
+      outputPath,
+      redirect: { from: pathname, to: redirectTo },
+      statusCode: response.status,
+      html: "",
+    };
+  }
+
+  if (response.status >= 500) {
+    throw new Error(
+      `SSR failed with status ${response.status} for path: ${urlPath}`,
+    );
+  }
+
+  // Get HTML content for file write
+  const fileContent = response.body ? await response.text() : "";
+
+  // For protected routes, do a second render with protection bypassed for search index
+  let html = fileContent;
+  if (isProtectedRoute) {
+    const bypassRequest = new Request(url);
+    const bypassResponse = await server.handleRequest({
+      template,
+      request: bypassRequest,
+      routes,
+      basePath,
+      bypassProtection: true,
+    });
+
+    if (bypassResponse.status >= 500) {
+      throw new Error(
+        `SSR failed (bypass render) with status ${bypassResponse.status} for path: ${urlPath}`,
+      );
+    }
+
+    html = bypassResponse.body ? await bypassResponse.text() : "";
+  }
+
+  // Write the file
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  await fs.writeFile(outputPath, fileContent);
+
+  return {
+    outputPath,
+    statusCode: response.status,
+    redirect: undefined,
+    html,
+  };
+};
+
+export default renderPage;

package/src/vite/reporter.ts

@@ -0,0 +1,34 @@
+// Source: https://github.com/vitejs/vite/blob/e1fa067e65a11e422b6fd94d4ce916a675ebb9a5/packages/vite/src/node/plugins/reporter.ts#L49
+
+export function writeLine(output: string) {
+  clearLine();
+  if (output.length < process.stdout.columns) {
+    process.stdout.write(output);
+  } else {
+    process.stdout.write(output.substring(0, process.stdout.columns - 1));
+  }
+}
+
+export function isTTY() {
+  return process.stdout.isTTY && !process.env.CI;
+}
+
+function clearLine() {
+  if (isTTY()) {
+    process.stdout.clearLine(0);
+    process.stdout.cursorTo(0);
+  }
+}
+
+// biome-ignore lint/complexity/noBannedTypes: todo
+export function throttle(fn: Function) {
+  let timerHandle: NodeJS.Timeout | null = null;
+  // biome-ignore lint/suspicious/noExplicitAny: Allow any type
+  return (...args: any[]) => {
+    if (timerHandle) return;
+    fn(...args);
+    timerHandle = setTimeout(() => {
+      timerHandle = null;
+    }, 100);
+  };
+}

package/src/vite/shadcn-registry.ts

@@ -0,0 +1,48 @@
+import { z } from "zod";
+
+// Taken and simplified from https://github.com/shadcn-ui/ui/blob/main/packages/shadcn/src/registry/schema.ts
+const registryItemCssSchema = z.record(
+  z.string(),
+  z.lazy(() =>
+    z.union([
+      z.string(),
+      z.record(
+        z.string(),
+        z.union([z.string(), z.record(z.string(), z.string())]),
+      ),
+    ]),
+  ),
+);
+
+export type RegistryItemCss = z.infer<typeof registryItemCssSchema>;
+
+const registryItemSchema = z.object({
+  $schema: z.literal("https://ui.shadcn.com/schema/registry-item.json"),
+  name: z.string(),
+  type: z.enum(["registry:theme", "registry:style"]),
+  cssVars: z
+    .object({
+      theme: z.record(z.string(), z.string()).optional(),
+      light: z.record(z.string(), z.string()).optional(),
+      dark: z.record(z.string(), z.string()).optional(),
+    })
+    .optional(),
+  css: registryItemCssSchema.optional(),
+});
+
+type RegistryItem = z.infer<typeof registryItemSchema>;
+
+export const fetchShadcnRegistryItem = async (
+  url: string,
+): Promise<RegistryItem> => {
+  const response = await fetch(url);
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch shadcn registry item from ${url}`);
+  }
+
+  const data = await response.json();
+  const result = registryItemSchema.parse(data);
+
+  return result;
+};

package/src/vite/sitemap.ts

@@ -0,0 +1,78 @@
+import { createWriteStream, existsSync } from "node:fs";
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import colors from "picocolors";
+import { SitemapStream } from "sitemap";
+import type { ZudokuSiteMapConfig } from "../config/validators/ZudokuConfig.js";
+import { joinUrl } from "../lib/util/joinUrl.js";
+export async function generateSitemap({
+  outputUrls,
+  basePath,
+  baseOutputDir,
+  config,
+  redirectUrls,
+}: {
+  basePath: string | undefined;
+  outputUrls: string[];
+  baseOutputDir: string;
+  config: ZudokuSiteMapConfig;
+  redirectUrls: Set<string>;
+}) {
+  if (!config) {
+    return;
+  }
+
+  const sitemap = new SitemapStream({ hostname: config.siteUrl });
+
+  const outputDir = path.resolve(baseOutputDir, config.outDir ?? "");
+
+  // Ensure the output directory exists
+  if (!existsSync(outputDir)) {
+    await mkdir(outputDir, { recursive: true });
+  }
+
+  const sitemapOutputPath = path.join(outputDir, "sitemap.xml");
+
+  const writeStream = createWriteStream(sitemapOutputPath);
+  sitemap.pipe(writeStream);
+
+  let lastmod: string | undefined;
+  if (config.autoLastmod !== false) {
+    lastmod = new Date().toISOString();
+  }
+
+  const exclude =
+    (typeof config.exclude === "function"
+      ? await config.exclude()
+      : config.exclude) ?? [];
+
+  for (const url of outputUrls) {
+    const shouldExclude =
+      exclude.includes(url) ||
+      url.includes("*") ||
+      /(400|404|500)$/.test(url) ||
+      redirectUrls.has(url);
+
+    if (shouldExclude) continue;
+
+    sitemap.write({
+      url: new URL(joinUrl(basePath, url), config.siteUrl).toString(),
+      lastmod,
+      changefreq: config.changefreq ?? "daily",
+      priority: config.priority ?? 0.7,
+    });
+  }
+
+  sitemap.end();
+
+  // Wait for the write stream to finish
+  await new Promise<void>((resolve, reject) => {
+    writeStream.on("finish", resolve);
+    writeStream.on("error", reject);
+  });
+
+  // biome-ignore lint/suspicious/noConsole: Logging allowed here
+  console.debug(
+    colors.blue(`✓ wrote sitemap to ${colors.bold(sitemapOutputPath)}`),
+  );
+}

package/src/vite/ssr-templates/cloudflare.ts

@@ -0,0 +1,19 @@
+// @ts-nocheck
+import { Hono } from "hono";
+import { getRoutesByConfig, handleRequest } from "./entry.server.js";
+const config = await import("__CONFIG_FILE__");
+
+// Cloudflare Workers with Static Assets feature
+// Static files served automatically via wrangler.toml: assets = { directory = "./dist/client" }
+
+const template = "__TEMPLATE__";
+const basePath = "__BASE_PATH__";
+const routes = getRoutesByConfig(config.default ?? config);
+
+const app = new Hono();
+
+app.all("*", (c) =>
+  handleRequest({ template, request: c.req.raw, routes, basePath }),
+);
+
+export default app;

package/src/vite/ssr-templates/node.ts

@@ -0,0 +1,25 @@
+// @ts-nocheck
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { serve } from "@hono/node-server";
+import { serveStatic } from "@hono/node-server/serve-static";
+import { Hono } from "hono";
+import { getRoutesByConfig, handleRequest } from "./entry.server.js";
+const config = await import("__CONFIG_FILE__");
+
+const template = "__TEMPLATE__";
+const basePath = "__BASE_PATH__";
+const routes = getRoutesByConfig(config.default ?? config);
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const staticDir = join(__dirname, "..");
+
+const app = new Hono();
+
+app.use("/server/*", (c) => c.notFound());
+app.use("*", serveStatic({ root: staticDir }));
+app.get("*", (c) =>
+  handleRequest({ template, request: c.req.raw, routes, basePath }),
+);
+
+serve({ fetch: app.fetch, port: Number(process.env.PORT || 3000) });

package/src/vite/ssr-templates/vercel.ts

@@ -0,0 +1,22 @@
+// @ts-nocheck
+import { Hono } from "hono";
+import { getRoutesByConfig, handleRequest } from "./entry.server.js";
+const zudokuConfig = await import("__CONFIG_FILE__");
+
+// Vercel Edge Functions
+// Static files served automatically from dist/client via vercel.json rewrites
+
+const template = "__TEMPLATE__";
+const basePath = "__BASE_PATH__";
+const routes = getRoutesByConfig(zudokuConfig.default ?? zudokuConfig);
+
+const app = new Hono();
+
+app.all("*", (c) =>
+  handleRequest({ template, request: c.req.raw, routes, basePath }),
+);
+
+export const GET = (req: Request) => app.fetch(req);
+export const POST = (req: Request) => app.fetch(req);
+
+export const config = { runtime: "edge" };

package/src/vite/zuplo.ts

@@ -0,0 +1,30 @@
+import bs58 from "bs58";
+
+export type ZuploEnvironmentVariables = {
+  __ZUPLO_DEPLOYMENT_NAME?: string;
+  __ZUPLO_LOGGING_ID?: string;
+  __ZUPLO_REMOTE_LOG_TOKEN?: string;
+  __ZUPLO_AUTH_API_JWT?: string;
+  __ZUPLO_EXTERNAL_SERVICE_TOKEN?: string;
+  ZUPLO_ENVIRONMENT_TYPE?: "PRODUCTION" | "DEVELOPMENT" | "STAGING";
+  ZUPLO_PROJECT_ID?: string;
+  ZUPLO_API_KEY_SERVICE_BUCKET_NAME?: string;
+  ZUPLO_SERVICE_BUCKET_ID?: string;
+  __ZUPLO_CONFIG?: string;
+};
+
+export const getZuploSystemConfigurations = (string?: string) => {
+  if (!string) return undefined;
+
+  try {
+    const zuploSystemConfigurations: ZuploEnvironmentVariables = JSON.parse(
+      Buffer.from(bs58.decode(string)).toString("utf8"),
+    );
+
+    return zuploSystemConfigurations;
+  } catch (error) {
+    // biome-ignore lint/suspicious/noConsole: Logging allowed here
+    console.error(error);
+    return undefined;
+  }
+};

package/src/vite-env.d.ts

@@ -0,0 +1,6 @@
+/// <reference types="vite/client" />
+
+interface ImportMetaEnv {
+  readonly IS_ZUPLO: boolean;
+  readonly ZUPLO_BUILD_ID?: string;
+}