blodemd 0.0.12 → 0.0.14

package/packages/@repo/previewing/src/search-index.ts

@@ -0,0 +1,38 @@
+import type { SiteConfig } from "@repo/models";
+
+import {
+  buildPageMetadataMap,
+  shouldIncludeSearchEntry,
+} from "./content-index.js";
+import type { ContentIndex, SearchIndexItem, UtilityIndex } from "./types.js";
+
+export const buildSearchIndex = (
+  index: ContentIndex,
+  config: SiteConfig,
+  utilityIndex?: UtilityIndex
+): SearchIndexItem[] => {
+  const pageMetadataMap = buildPageMetadataMap(index);
+  const items = new Map<string, SearchIndexItem>();
+
+  for (const page of utilityIndex?.pages ?? []) {
+    items.set(page.slug, {
+      path: page.slug,
+      title: page.title,
+    });
+  }
+
+  for (const entry of index.entries) {
+    if (!shouldIncludeSearchEntry(entry, pageMetadataMap, config)) {
+      continue;
+    }
+
+    const pageMeta = pageMetadataMap.get(entry.slug);
+    items.set(entry.slug, {
+      href: pageMeta?.url,
+      path: entry.slug,
+      title: pageMeta?.sidebarTitle ?? entry.title,
+    });
+  }
+
+  return [...items.values()];
+};

package/packages/@repo/previewing/src/serialization.ts

@@ -0,0 +1,191 @@
+import {
+  PREBUILT_INDEX_PATH,
+  PREBUILT_OPENAPI_INDEX_PATH,
+  PREBUILT_SEARCH_INDEX_PATH,
+  PREBUILT_TOC_INDEX_PATH,
+  PREBUILT_UTILITY_INDEX_PATH,
+} from "./constants.js";
+import type { ContentSource } from "./content-source.js";
+import type {
+  ContentEntry,
+  ContentIndex,
+  PrebuiltOpenApiEntry,
+  SearchIndexItem,
+  TocItem,
+  UtilityIndex,
+  UtilityPage,
+  UtilitySegment,
+} from "./types.js";
+
+interface SerializedContentIndex {
+  version: 1;
+  entries: ContentEntry[];
+  collections: Record<string, ContentEntry[]>;
+}
+
+interface SerializedOpenApiIndex {
+  entries: PrebuiltOpenApiEntry[];
+  version: 1;
+}
+
+interface SerializedSearchIndex {
+  items: SearchIndexItem[];
+  version: 1;
+}
+
+interface SerializedTocIndex {
+  itemsBySlug: Record<string, TocItem[]>;
+  version: 1;
+}
+
+interface SerializedUtilityIndex {
+  description?: string;
+  name: string;
+  pages: UtilityPage[];
+  segments?: UtilitySegment[];
+  slug?: string;
+  version: 1;
+}
+
+export const serializeContentIndex = (index: ContentIndex): string =>
+  JSON.stringify({
+    collections: Object.fromEntries(index.byCollection),
+    entries: index.entries,
+    version: 1,
+  } satisfies SerializedContentIndex);
+
+export const serializeOpenApiIndex = (
+  entries: PrebuiltOpenApiEntry[]
+): string =>
+  JSON.stringify({
+    entries,
+    version: 1,
+  } satisfies SerializedOpenApiIndex);
+
+export const loadPrebuiltContentIndex = async (
+  source: ContentSource
+): Promise<ContentIndex | null> => {
+  try {
+    const raw = await source.readFile(PREBUILT_INDEX_PATH);
+    const data = JSON.parse(raw) as SerializedContentIndex;
+    if (data.version !== 1 || !Array.isArray(data.entries)) {
+      return null;
+    }
+
+    const bySlug = new Map<string, ContentEntry>();
+    const byCollection = new Map<string, ContentEntry[]>();
+
+    for (const entry of data.entries) {
+      bySlug.set(entry.slug, entry);
+    }
+
+    for (const [collectionId, entries] of Object.entries(
+      data.collections ?? {}
+    )) {
+      byCollection.set(collectionId, entries);
+    }
+
+    return {
+      byCollection,
+      bySlug,
+      entries: data.entries,
+      errors: [],
+    };
+  } catch {
+    return null;
+  }
+};
+
+export const loadPrebuiltOpenApiIndex = async (
+  source: ContentSource
+): Promise<PrebuiltOpenApiEntry[] | null> => {
+  try {
+    const raw = await source.readFile(PREBUILT_OPENAPI_INDEX_PATH);
+    const data = JSON.parse(raw) as SerializedOpenApiIndex;
+    if (data.version !== 1 || !Array.isArray(data.entries)) {
+      return null;
+    }
+
+    return data.entries;
+  } catch {
+    return null;
+  }
+};
+
+export const serializeSearchIndex = (items: SearchIndexItem[]): string =>
+  JSON.stringify({
+    items,
+    version: 1,
+  } satisfies SerializedSearchIndex);
+
+export const loadPrebuiltSearchIndex = async (
+  source: ContentSource
+): Promise<SearchIndexItem[] | null> => {
+  try {
+    const raw = await source.readFile(PREBUILT_SEARCH_INDEX_PATH);
+    const data = JSON.parse(raw) as SerializedSearchIndex;
+    if (data.version !== 1 || !Array.isArray(data.items)) {
+      return null;
+    }
+
+    return data.items;
+  } catch {
+    return null;
+  }
+};
+
+export const serializeTocIndex = (
+  itemsBySlug: Map<string, TocItem[]>
+): string =>
+  JSON.stringify({
+    itemsBySlug: Object.fromEntries(itemsBySlug),
+    version: 1,
+  } satisfies SerializedTocIndex);
+
+export const loadPrebuiltTocIndex = async (
+  source: ContentSource
+): Promise<Map<string, TocItem[]> | null> => {
+  try {
+    const raw = await source.readFile(PREBUILT_TOC_INDEX_PATH);
+    const data = JSON.parse(raw) as SerializedTocIndex;
+    if (data.version !== 1 || typeof data.itemsBySlug !== "object") {
+      return null;
+    }
+
+    return new Map(Object.entries(data.itemsBySlug ?? {}));
+  } catch {
+    return null;
+  }
+};
+
+export const serializeUtilityIndex = (index: UtilityIndex): string =>
+  JSON.stringify({
+    ...index,
+    version: 1,
+  } satisfies SerializedUtilityIndex);
+
+export const loadPrebuiltUtilityIndex = async (
+  source: ContentSource
+): Promise<UtilityIndex | null> => {
+  try {
+    const raw = await source.readFile(PREBUILT_UTILITY_INDEX_PATH);
+    const data = JSON.parse(raw) as SerializedUtilityIndex;
+    if (
+      data.version !== 1 ||
+      typeof data.name !== "string" ||
+      !Array.isArray(data.pages)
+    ) {
+      return null;
+    }
+
+    return {
+      description: data.description,
+      name: data.name,
+      pages: data.pages,
+      segments: Array.isArray(data.segments) ? data.segments : undefined,
+      slug: data.slug,
+    };
+  } catch {
+    return null;
+  }
+};

package/packages/@repo/previewing/src/toc-index.ts

@@ -0,0 +1,72 @@
+import { slugify } from "@repo/common";
+
+import { NEWLINE_REGEX } from "./constants.js";
+import type { ContentSource } from "./content-source.js";
+import type { ContentIndex, TocItem } from "./types.js";
+
+const HEADING_REGEX = /^(#{2,4})\s+(.*)$/;
+// Matches ``` and ~~~ fenced blocks, requiring the same fence to close them.
+const TOC_FENCE_REGEX = /(`{3,}|~{3,})[\s\S]*?\1/g;
+
+export const extractToc = (source: string): TocItem[] => {
+  const withoutCode = source.replaceAll(TOC_FENCE_REGEX, "");
+  const lines = withoutCode.split(NEWLINE_REGEX);
+  const toc: TocItem[] = [];
+  const emitted = new Set<string>();
+
+  for (const line of lines) {
+    const match = HEADING_REGEX.exec(line.trim());
+    if (!match) {
+      continue;
+    }
+
+    const [, hashes = "", heading = ""] = match;
+    if (!(hashes && heading)) {
+      continue;
+    }
+
+    const baseId = slugify(heading.trim());
+    // Increment until the id is unique among the ids we have already emitted,
+    // so a suffixed id never collides with an explicit "foo-1" heading.
+    let id = baseId;
+    let count = 1;
+    while (emitted.has(id)) {
+      id = `${baseId}-${count}`;
+      count += 1;
+    }
+    emitted.add(id);
+
+    toc.push({
+      id,
+      level: hashes.length,
+      title: heading.trim(),
+    });
+  }
+
+  return toc;
+};
+
+export const buildTocIndex = async (
+  index: ContentIndex,
+  source: ContentSource
+): Promise<Map<string, TocItem[]>> => {
+  const itemsBySlug = new Map<string, TocItem[]>();
+
+  const results = await Promise.all(
+    index.entries.map(async (entry) => {
+      if (entry.kind !== "entry") {
+        return null;
+      }
+      const rawContent = await source.readFile(entry.relativePath);
+      return { slug: entry.slug, toc: extractToc(rawContent) };
+    })
+  );
+
+  for (const result of results) {
+    if (result) {
+      itemsBySlug.set(result.slug, result.toc);
+    }
+  }
+
+  return itemsBySlug;
+};

package/packages/@repo/previewing/src/types.ts

@@ -0,0 +1,105 @@
+import type {
+  ContentType,
+  DocsOpenApiSource,
+  FrontmatterByType,
+  PageMode,
+  SiteConfig,
+} from "@repo/models";
+import type { OpenApiOperation, OpenApiSpec } from "@repo/prebuild";
+
+export type SiteConfigResult =
+  | { ok: true; config: SiteConfig; warnings: string[] }
+  | { ok: false; errors: string[] };
+
+export type ContentEntry =
+  | {
+      kind: "entry";
+      slug: string;
+      title: string;
+      description?: string;
+      hidden?: boolean;
+      type: ContentType;
+      collectionId: string;
+      sourcePath: string;
+      relativePath: string;
+      frontmatter: FrontmatterByType[ContentType];
+    }
+  | {
+      kind: "index";
+      slug: string;
+      title: string;
+      description?: string;
+      type: ContentType;
+      collectionId: string;
+    };
+
+export interface ContentIndex {
+  entries: ContentEntry[];
+  bySlug: Map<string, ContentEntry>;
+  byCollection: Map<string, ContentEntry[]>;
+  errors: string[];
+}
+
+export interface PageMetadata {
+  title?: string;
+  sidebarTitle?: string;
+  icon?: string;
+  iconType?: string;
+  tag?: string;
+  hidden?: boolean;
+  deprecated?: boolean;
+  url?: string;
+  mode?: PageMode;
+  noindex?: boolean;
+  hideFooterPagination?: boolean;
+  hideApiMarker?: boolean;
+  keywords?: string[];
+}
+
+export interface SearchIndexItem {
+  href?: string;
+  title: string;
+  path: string;
+}
+
+export interface TocItem {
+  id: string;
+  level: number;
+  title: string;
+}
+
+export interface UtilityPage {
+  content: string;
+  description?: string;
+  slug: string;
+  title: string;
+}
+
+export interface UtilitySegment {
+  pageSlugs: string[];
+  slug: string;
+  title: string;
+}
+
+export interface UtilityIndex {
+  description?: string;
+  name: string;
+  pages: UtilityPage[];
+  segments?: UtilitySegment[];
+  slug?: string;
+}
+
+export interface UtilityArtifact {
+  content: string;
+  contentType: string;
+  path: string;
+}
+
+export interface PrebuiltOpenApiEntry {
+  identifier: string;
+  operation: OpenApiOperation;
+  slug: string;
+  source: DocsOpenApiSource;
+  sourceKey: string;
+  spec: OpenApiSpec;
+}

package/packages/@repo/previewing/src/utility-index.ts

@@ -0,0 +1,304 @@
+import { normalizePath, slugify } from "@repo/common";
+import type { SiteConfig } from "@repo/models";
+
+import {
+  getDocsCollectionWithNavigation,
+  getDocsNavigation,
+} from "./config-loader.js";
+import {
+  PREBUILT_UTILITY_LLMS_FULL_PATH,
+  PREBUILT_UTILITY_LLMS_PATH,
+  PREBUILT_UTILITY_LLMS_SEGMENT_PREFIX,
+  PREBUILT_UTILITY_SITEMAP_PATH,
+  PREBUILT_UTILITY_SKILLS_INDEX_PATH,
+  PREBUILT_UTILITY_SKILLS_MD_PREFIX,
+  UTILITY_DOCS_ROOT_TOKEN,
+} from "./constants.js";
+import {
+  buildPageMetadataMap,
+  shouldIncludeSearchEntry,
+} from "./content-index.js";
+import type { ContentSource } from "./content-source.js";
+import {
+  prepareLlmsFullContent,
+  toAgentMarkdown,
+} from "./markdown/agent-markdown.js";
+import {
+  formatMarkdownPage,
+  formatMarkdownPageSection,
+  stripFrontmatter,
+} from "./markdown/format.js";
+import { buildUtilityOpenApiPages } from "./openapi-pages.js";
+import type {
+  ContentIndex,
+  UtilityArtifact,
+  UtilityIndex,
+  UtilityPage,
+  UtilitySegment,
+} from "./types.js";
+
+const collectUtilitySegments = (
+  config: SiteConfig,
+  pages: Map<string, UtilityPage>
+): UtilitySegment[] => {
+  const segments = new Map<string, UtilitySegment>();
+  const addGroup = (group: {
+    group?: string;
+    hidden?: boolean;
+    pages?: string[];
+  }) => {
+    if (group.hidden || !group.group) {
+      return;
+    }
+    const pageSlugs = (group.pages ?? []).filter((page) => pages.has(page));
+    if (pageSlugs.length === 0) {
+      return;
+    }
+    const slug = slugify(group.group);
+    segments.set(slug, { pageSlugs, slug, title: group.group });
+  };
+
+  const navigation = getDocsNavigation(config);
+  for (const group of navigation?.groups ?? []) {
+    addGroup(group);
+  }
+  for (const tab of navigation?.tabs ?? []) {
+    for (const group of tab.groups ?? []) {
+      addGroup(group);
+    }
+  }
+
+  return [...segments.values()];
+};
+
+export const buildUtilityIndex = async (
+  index: ContentIndex,
+  source: ContentSource,
+  config: SiteConfig
+): Promise<UtilityIndex> => {
+  const pageMetadataMap = buildPageMetadataMap(index);
+  const pages = new Map<string, UtilityPage>();
+
+  const contentPages = await Promise.all(
+    index.entries.map(async (entry) => {
+      if (entry.kind !== "entry") {
+        return null;
+      }
+      if (!shouldIncludeSearchEntry(entry, pageMetadataMap, config)) {
+        return null;
+      }
+      const rawContent = await source.readFile(entry.relativePath);
+      return {
+        content: stripFrontmatter(rawContent),
+        description: entry.description,
+        slug: entry.slug,
+        title: entry.title,
+      } satisfies UtilityPage;
+    })
+  );
+  for (const page of contentPages) {
+    if (page) {
+      pages.set(page.slug, page);
+    }
+  }
+
+  for (const page of await buildUtilityOpenApiPages(
+    config,
+    getDocsCollectionWithNavigation(config),
+    source
+  )) {
+    pages.set(page.slug, page);
+  }
+
+  const sortedPages = [...pages.values()];
+  // oxlint-disable-next-line eslint-plugin-unicorn/no-array-sort
+  sortedPages.sort((left, right) => left.slug.localeCompare(right.slug));
+
+  return {
+    description: config.description,
+    name: config.name,
+    pages: sortedPages,
+    segments: collectUtilitySegments(config, pages),
+    slug: config.slug,
+  };
+};
+
+const toUtilityDocPath = (value: string) => {
+  const clean = normalizePath(value);
+  if (!clean || clean === "index") {
+    return "/";
+  }
+  return `/${clean}`;
+};
+
+const toUtilityTemplatedDocUrl = (value: string) =>
+  `${UTILITY_DOCS_ROOT_TOKEN}${toUtilityDocPath(value)}`;
+
+const toUtilityTemplatedMarkdownDocUrl = (value: string) => {
+  const clean = normalizePath(value);
+  if (!clean || clean === "index") {
+    return `${UTILITY_DOCS_ROOT_TOKEN}/index.md`;
+  }
+  return `${UTILITY_DOCS_ROOT_TOKEN}/${clean}.md`;
+};
+
+export const getPrebuiltUtilityLlmPagePath = (slug: string) => {
+  const normalized = normalizePath(slug);
+  return `_utility/llms-pages/${normalized || "index"}.mdx`;
+};
+
+export const buildUtilityArtifacts = (
+  index: UtilityIndex
+): UtilityArtifact[] => {
+  const segments = index.segments ?? [];
+  const segmentLines =
+    segments.length > 0
+      ? [
+          "",
+          "## Segments",
+          ...segments.map(
+            (segment) =>
+              `- [${segment.title}](${UTILITY_DOCS_ROOT_TOKEN}/llms/${segment.slug}.txt)`
+          ),
+        ]
+      : [];
+  const llmsLines = [
+    `# ${index.name}`,
+    index.description ? `> ${index.description}` : null,
+    "",
+    `Sitemap: ${toUtilityTemplatedDocUrl("sitemap.xml")}`,
+    `Full content: ${UTILITY_DOCS_ROOT_TOKEN}/llms-full.txt`,
+    `Skills: ${UTILITY_DOCS_ROOT_TOKEN}/.well-known/skills/index.json`,
+    ...segmentLines,
+    "",
+    "## Docs",
+    ...index.pages.map((page) => {
+      const description = page.description ? `: ${page.description}` : "";
+      return `- [${page.title}](${toUtilityTemplatedMarkdownDocUrl(page.slug)})${description}`;
+    }),
+  ];
+
+  const sitemap = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${index.pages
+  .map(
+    (page) => `  <url><loc>${toUtilityTemplatedDocUrl(page.slug)}</loc></url>`
+  )
+  .join("\n")}
+</urlset>`;
+
+  const llmsFull = index.pages
+    .map((page) =>
+      formatMarkdownPageSection(
+        page.title,
+        toUtilityTemplatedDocUrl(page.slug),
+        prepareLlmsFullContent(page.content, UTILITY_DOCS_ROOT_TOKEN, ""),
+        page.description
+      )
+    )
+    .join("\n\n");
+
+  const skillSlug = index.slug ?? slugify(index.name);
+  const skillDescription =
+    `${index.name} documentation. ${index.description ?? ""}`.trim();
+  const skillsIndex = JSON.stringify(
+    {
+      skills: [
+        {
+          description: skillDescription,
+          files: ["SKILL.md"],
+          name: skillSlug,
+        },
+      ],
+    },
+    null,
+    2
+  );
+
+  const topPages = index.pages.slice(0, 20);
+  const skillMdLines = [
+    "---",
+    `name: ${skillSlug}`,
+    `description: ${skillDescription} Use when working with ${index.name}, answering questions about its features, or helping users follow its guides.`,
+    "---",
+    "",
+    `# ${index.name}`,
+    "",
+    index.description ? `${index.description}\n` : "",
+    "## Documentation",
+    "",
+    `- Full docs index: ${UTILITY_DOCS_ROOT_TOKEN}/llms.txt`,
+    `- Complete docs content: ${UTILITY_DOCS_ROOT_TOKEN}/llms-full.txt`,
+    "- Append `.md` to any page URL for raw markdown",
+    "",
+    "## Key Pages",
+    "",
+    ...topPages.map((page) => {
+      const desc = page.description ? ` - ${page.description}` : "";
+      return `- [${page.title}](${toUtilityTemplatedMarkdownDocUrl(
+        page.slug
+      )})${desc}`;
+    }),
+  ];
+
+  return [
+    {
+      content: sitemap,
+      contentType: "application/xml; charset=utf-8",
+      path: PREBUILT_UTILITY_SITEMAP_PATH,
+    },
+    {
+      content: llmsLines.filter((line) => line !== null).join("\n"),
+      contentType: "text/plain; charset=utf-8",
+      path: PREBUILT_UTILITY_LLMS_PATH,
+    },
+    {
+      content: llmsFull,
+      contentType: "text/plain; charset=utf-8",
+      path: PREBUILT_UTILITY_LLMS_FULL_PATH,
+    },
+    {
+      content: skillsIndex,
+      contentType: "application/json; charset=utf-8",
+      path: PREBUILT_UTILITY_SKILLS_INDEX_PATH,
+    },
+    {
+      content: skillMdLines.filter((line) => line !== null).join("\n"),
+      contentType: "text/markdown; charset=utf-8",
+      path: `${PREBUILT_UTILITY_SKILLS_MD_PREFIX}${skillSlug}/SKILL.md`,
+    },
+    ...segments.map((segment) => {
+      const segmentPages = index.pages.filter((page) =>
+        segment.pageSlugs.includes(page.slug)
+      );
+      const content = [
+        `# ${index.name} - ${segment.title}`,
+        `> Segment of ${UTILITY_DOCS_ROOT_TOKEN}/llms-full.txt`,
+        "",
+        ...segmentPages.map((page) =>
+          formatMarkdownPageSection(
+            page.title,
+            toUtilityTemplatedDocUrl(page.slug),
+            prepareLlmsFullContent(page.content, UTILITY_DOCS_ROOT_TOKEN, ""),
+            page.description
+          )
+        ),
+      ].join("\n\n");
+
+      return {
+        content,
+        contentType: "text/plain; charset=utf-8",
+        path: `${PREBUILT_UTILITY_LLMS_SEGMENT_PREFIX}${segment.slug}.txt`,
+      };
+    }),
+    ...index.pages.map((page) => ({
+      content: formatMarkdownPage(
+        page.title,
+        toAgentMarkdown(page.content),
+        page.description
+      ),
+      contentType: "text/markdown; charset=utf-8",
+      path: getPrebuiltUtilityLlmPagePath(page.slug),
+    })),
+  ];
+};