@publish.os/nextjs 0.0.1-alpha.2 → 0.0.1-alpha.3

package/dist/kernel/routes/metadata/llms.js

@@ -0,0 +1,206 @@
+// src/kernel/routes/metadata/llms.ts
+import { cacheLife, cacheTag } from "next/cache";
+
+// src/cache/tags.ts
+import { createHash } from "crypto";
+var MAX_TAG_LEN = 256;
+var MAX_SEGMENT_LEN = 80;
+var TAG_PATTERN = /^[a-z0-9:_./-]+$/;
+function sha256(value) {
+  return createHash("sha256").update(value).digest("hex").slice(0, 16);
+}
+function safeSegment(value) {
+  const raw = value.trim().toLowerCase() || "root";
+  const normalized = raw.replace(/\\/g, "/").replace(/\/+/g, "/").replace(/^\//, "").replace(/\/$/, "").replace(/[^a-z0-9:_./-]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
+  const segment = normalized || "root";
+  if (segment.length <= MAX_SEGMENT_LEN) return segment;
+  return `${segment.slice(0, 48)}.${sha256(segment)}`;
+}
+function assertTag(tag) {
+  if (tag.length > MAX_TAG_LEN || !TAG_PATTERN.test(tag)) {
+    return `tag:${sha256(tag)}`;
+  }
+  return tag;
+}
+var tags = {
+  site() {
+    return "site";
+  },
+  pages() {
+    return "pages";
+  },
+  group(group) {
+    return assertTag(`group:${safeSegment(group)}`);
+  },
+  page(group, slug) {
+    const path = Array.isArray(slug) ? slug.join("/") : slug;
+    return assertTag(`page:${safeSegment(group)}:${safeSegment(path)}`);
+  },
+  manifest(contentHash) {
+    return assertTag(`manifest:${safeSegment(contentHash)}`);
+  },
+  sitemap() {
+    return "sitemap";
+  },
+  robots() {
+    return "robots";
+  },
+  llms(scope = "root") {
+    return assertTag(`llms:${safeSegment(scope)}`);
+  },
+  feed(format, scope = "site") {
+    return assertTag(`feed:${safeSegment(format)}:${safeSegment(scope)}`);
+  }
+};
+
+// src/kernel/routes/response.ts
+var CACHE_CONTROL = "public, max-age=300, s-maxage=86400, stale-while-revalidate=3600, must-revalidate";
+var CDN_CACHE_CONTROL = "public, max-age=2592000, stale-while-revalidate=31536000";
+var SHORT_CACHE_CONTROL = "public, max-age=60, s-maxage=600, stale-while-revalidate=300, must-revalidate";
+var SHORT_CDN_CACHE_CONTROL = "public, max-age=300, stale-while-revalidate=3600";
+function cacheControl(policy) {
+  if (!policy || policy === "default") return CACHE_CONTROL;
+  if (policy === "short") return SHORT_CACHE_CONTROL;
+  return policy;
+}
+function cdnCacheControl(policy) {
+  if (!policy || policy === "default") return CDN_CACHE_CONTROL;
+  if (policy === "short") return SHORT_CDN_CACHE_CONTROL;
+  return policy;
+}
+function cachedTextHeaders(contentType, options = {}) {
+  const headers = new Headers(options.headers);
+  headers.set("cache-control", cacheControl(options.cachePolicy));
+  headers.set("cdn-cache-control", cdnCacheControl(options.cachePolicy));
+  headers.set("vercel-cdn-cache-control", cdnCacheControl(options.cachePolicy));
+  headers.set("content-type", contentType);
+  headers.set("x-content-type-options", "nosniff");
+  if (options.etag) headers.set("etag", options.etag);
+  if (options.noindex) headers.set("x-robots-tag", "noindex");
+  return headers;
+}
+function createCachedTextResponse(options) {
+  const status = options.status ?? 200;
+  const method = options.method ?? options.request?.method ?? "GET";
+  const headers = cachedTextHeaders(options.contentType, options);
+  const isNotModified = status === 200 && options.etag && options.request?.headers.get("if-none-match") === options.etag;
+  return new Response(
+    method === "HEAD" || isNotModified ? null : options.body,
+    {
+      headers,
+      status: isNotModified ? 304 : status
+    }
+  );
+}
+
+// src/kernel/routes/metadata/llms.ts
+var DEFAULT_LLMS_FULL_MAX_BYTES = 1e6;
+var TEXT_ENCODER = new TextEncoder();
+function byteLength(value) {
+  return TEXT_ENCODER.encode(value).byteLength;
+}
+function pageInScope(page, options) {
+  if (options.scope === "site" || !options.group) return true;
+  if (options.scope === "self") return page.group === options.group;
+  return page.group === options.group || page.group.startsWith(`${options.group}/`);
+}
+function llmsSummary(manifest) {
+  return manifest.site.description ? `> ${manifest.site.description}` : `> Agent-readable index for ${manifest.site.name}.`;
+}
+async function llmsTxt(manifest, options = {}) {
+  "use cache";
+  cacheLife("page");
+  cacheTag(
+    tags.site(),
+    tags.llms(options.group ?? options.scope ?? "root"),
+    tags.manifest(manifest.contentHash)
+  );
+  const pages = manifest.pages.filter((page) => page.status === "published").filter((page) => pageInScope(page, options));
+  const visiblePages = options.limit ? pages.slice(0, options.limit) : pages;
+  const lines = visiblePages.map((page) => {
+    const description = options.includeDescriptions && page.description ? ` - ${page.description}` : "";
+    return `- [${page.title}](${page.path})${description}`;
+  });
+  const linkLines = (options.links ?? []).map(
+    (link) => `- [${link.title}](${link.path})`
+  );
+  const linkSection = linkLines.length ? `
+
+## Group LLM indexes
+
+${linkLines.join("\n")}` : "";
+  return `# ${manifest.site.name}
+
+${llmsSummary(manifest)}
+
+${lines.join("\n")}${linkSection}
+`;
+}
+function createLlmsTxtRoute(manifest, options = {}) {
+  return async function GET() {
+    return createCachedTextResponse({
+      body: await llmsTxt(manifest, options),
+      contentType: "text/plain; charset=utf-8"
+    });
+  };
+}
+function createPrecomputedLlmsTxtRoute(output) {
+  return function GET() {
+    return createCachedTextResponse({
+      body: output.body,
+      contentType: "text/plain; charset=utf-8",
+      etag: output.etag
+    });
+  };
+}
+async function llmsFullTxt(manifest, artifacts, options = {}) {
+  const maxBytes = options.maxBytes ?? DEFAULT_LLMS_FULL_MAX_BYTES;
+  const header = `# ${manifest.site.name} Full Content
+
+> LLMS index: /llms.txt
+> Sitemap: /sitemap.md
+
+`;
+  const sections = [];
+  let totalBytes = byteLength(header);
+  let included = 0;
+  let truncated = false;
+  for (const artifact of Object.values(artifacts).sort(
+    (a, b) => a.markdownPath.localeCompare(b.markdownPath)
+  )) {
+    const section = `---
+
+${artifact.body.trim()}
+
+`;
+    const sectionBytes = byteLength(section);
+    if (totalBytes + sectionBytes > maxBytes) {
+      truncated = true;
+      break;
+    }
+    sections.push(section);
+    totalBytes += sectionBytes;
+    included += 1;
+  }
+  const footer = truncated ? `
+> Truncated at ${included} of ${Object.keys(artifacts).length} Markdown pages to stay under ${maxBytes} bytes.
+` : "";
+  return `${header}${sections.join("")}${footer}`;
+}
+async function llmsFullTxtResponse(manifest, artifacts, options = {}) {
+  return createCachedTextResponse({
+    body: await llmsFullTxt(manifest, artifacts, options),
+    contentType: "text/plain; charset=utf-8"
+  });
+}
+function createLlmsFullTxtRoute(manifest, artifacts, options = {}) {
+  return async function GET() {
+    return llmsFullTxtResponse(manifest, artifacts, options);
+  };
+}
+export {
+  createLlmsFullTxtRoute,
+  createLlmsTxtRoute,
+  createPrecomputedLlmsTxtRoute,
+  llmsFullTxtResponse
+};