blodemd 0.1.0 → 0.2.0

package/packages/@repo/previewing/src/search-index.ts

@@ -1,38 +0,0 @@
-import type { SiteConfig } from "@repo/models";
-
-import {
-  buildPageMetadataMap,
-  shouldIncludeSearchEntry,
-} from "./content-index.js";
-import type { ContentIndex, SearchIndexItem, UtilityIndex } from "./types.js";
-
-export const buildSearchIndex = (
-  index: ContentIndex,
-  config: SiteConfig,
-  utilityIndex?: UtilityIndex
-): SearchIndexItem[] => {
-  const pageMetadataMap = buildPageMetadataMap(index);
-  const items = new Map<string, SearchIndexItem>();
-
-  for (const page of utilityIndex?.pages ?? []) {
-    items.set(page.slug, {
-      path: page.slug,
-      title: page.title,
-    });
-  }
-
-  for (const entry of index.entries) {
-    if (!shouldIncludeSearchEntry(entry, pageMetadataMap, config)) {
-      continue;
-    }
-
-    const pageMeta = pageMetadataMap.get(entry.slug);
-    items.set(entry.slug, {
-      href: pageMeta?.url,
-      path: entry.slug,
-      title: pageMeta?.sidebarTitle ?? entry.title,
-    });
-  }
-
-  return [...items.values()];
-};

package/packages/@repo/previewing/src/serialization.ts

@@ -1,191 +0,0 @@
-import {
-  PREBUILT_INDEX_PATH,
-  PREBUILT_OPENAPI_INDEX_PATH,
-  PREBUILT_SEARCH_INDEX_PATH,
-  PREBUILT_TOC_INDEX_PATH,
-  PREBUILT_UTILITY_INDEX_PATH,
-} from "./constants.js";
-import type { ContentSource } from "./content-source.js";
-import type {
-  ContentEntry,
-  ContentIndex,
-  PrebuiltOpenApiEntry,
-  SearchIndexItem,
-  TocItem,
-  UtilityIndex,
-  UtilityPage,
-  UtilitySegment,
-} from "./types.js";
-
-interface SerializedContentIndex {
-  version: 1;
-  entries: ContentEntry[];
-  collections: Record<string, ContentEntry[]>;
-}
-
-interface SerializedOpenApiIndex {
-  entries: PrebuiltOpenApiEntry[];
-  version: 1;
-}
-
-interface SerializedSearchIndex {
-  items: SearchIndexItem[];
-  version: 1;
-}
-
-interface SerializedTocIndex {
-  itemsBySlug: Record<string, TocItem[]>;
-  version: 1;
-}
-
-interface SerializedUtilityIndex {
-  description?: string;
-  name: string;
-  pages: UtilityPage[];
-  segments?: UtilitySegment[];
-  slug?: string;
-  version: 1;
-}
-
-export const serializeContentIndex = (index: ContentIndex): string =>
-  JSON.stringify({
-    collections: Object.fromEntries(index.byCollection),
-    entries: index.entries,
-    version: 1,
-  } satisfies SerializedContentIndex);
-
-export const serializeOpenApiIndex = (
-  entries: PrebuiltOpenApiEntry[]
-): string =>
-  JSON.stringify({
-    entries,
-    version: 1,
-  } satisfies SerializedOpenApiIndex);
-
-export const loadPrebuiltContentIndex = async (
-  source: ContentSource
-): Promise<ContentIndex | null> => {
-  try {
-    const raw = await source.readFile(PREBUILT_INDEX_PATH);
-    const data = JSON.parse(raw) as SerializedContentIndex;
-    if (data.version !== 1 || !Array.isArray(data.entries)) {
-      return null;
-    }
-
-    const bySlug = new Map<string, ContentEntry>();
-    const byCollection = new Map<string, ContentEntry[]>();
-
-    for (const entry of data.entries) {
-      bySlug.set(entry.slug, entry);
-    }
-
-    for (const [collectionId, entries] of Object.entries(
-      data.collections ?? {}
-    )) {
-      byCollection.set(collectionId, entries);
-    }
-
-    return {
-      byCollection,
-      bySlug,
-      entries: data.entries,
-      errors: [],
-    };
-  } catch {
-    return null;
-  }
-};
-
-export const loadPrebuiltOpenApiIndex = async (
-  source: ContentSource
-): Promise<PrebuiltOpenApiEntry[] | null> => {
-  try {
-    const raw = await source.readFile(PREBUILT_OPENAPI_INDEX_PATH);
-    const data = JSON.parse(raw) as SerializedOpenApiIndex;
-    if (data.version !== 1 || !Array.isArray(data.entries)) {
-      return null;
-    }
-
-    return data.entries;
-  } catch {
-    return null;
-  }
-};
-
-export const serializeSearchIndex = (items: SearchIndexItem[]): string =>
-  JSON.stringify({
-    items,
-    version: 1,
-  } satisfies SerializedSearchIndex);
-
-export const loadPrebuiltSearchIndex = async (
-  source: ContentSource
-): Promise<SearchIndexItem[] | null> => {
-  try {
-    const raw = await source.readFile(PREBUILT_SEARCH_INDEX_PATH);
-    const data = JSON.parse(raw) as SerializedSearchIndex;
-    if (data.version !== 1 || !Array.isArray(data.items)) {
-      return null;
-    }
-
-    return data.items;
-  } catch {
-    return null;
-  }
-};
-
-export const serializeTocIndex = (
-  itemsBySlug: Map<string, TocItem[]>
-): string =>
-  JSON.stringify({
-    itemsBySlug: Object.fromEntries(itemsBySlug),
-    version: 1,
-  } satisfies SerializedTocIndex);
-
-export const loadPrebuiltTocIndex = async (
-  source: ContentSource
-): Promise<Map<string, TocItem[]> | null> => {
-  try {
-    const raw = await source.readFile(PREBUILT_TOC_INDEX_PATH);
-    const data = JSON.parse(raw) as SerializedTocIndex;
-    if (data.version !== 1 || typeof data.itemsBySlug !== "object") {
-      return null;
-    }
-
-    return new Map(Object.entries(data.itemsBySlug ?? {}));
-  } catch {
-    return null;
-  }
-};
-
-export const serializeUtilityIndex = (index: UtilityIndex): string =>
-  JSON.stringify({
-    ...index,
-    version: 1,
-  } satisfies SerializedUtilityIndex);
-
-export const loadPrebuiltUtilityIndex = async (
-  source: ContentSource
-): Promise<UtilityIndex | null> => {
-  try {
-    const raw = await source.readFile(PREBUILT_UTILITY_INDEX_PATH);
-    const data = JSON.parse(raw) as SerializedUtilityIndex;
-    if (
-      data.version !== 1 ||
-      typeof data.name !== "string" ||
-      !Array.isArray(data.pages)
-    ) {
-      return null;
-    }
-
-    return {
-      description: data.description,
-      name: data.name,
-      pages: data.pages,
-      segments: Array.isArray(data.segments) ? data.segments : undefined,
-      slug: data.slug,
-    };
-  } catch {
-    return null;
-  }
-};

package/packages/@repo/previewing/src/toc-index.ts

@@ -1,72 +0,0 @@
-import { slugify } from "@repo/common";
-
-import { NEWLINE_REGEX } from "./constants.js";
-import type { ContentSource } from "./content-source.js";
-import type { ContentIndex, TocItem } from "./types.js";
-
-const HEADING_REGEX = /^(#{2,4})\s+(.*)$/;
-// Matches ``` and ~~~ fenced blocks, requiring the same fence to close them.
-const TOC_FENCE_REGEX = /(`{3,}|~{3,})[\s\S]*?\1/g;
-
-export const extractToc = (source: string): TocItem[] => {
-  const withoutCode = source.replaceAll(TOC_FENCE_REGEX, "");
-  const lines = withoutCode.split(NEWLINE_REGEX);
-  const toc: TocItem[] = [];
-  const emitted = new Set<string>();
-
-  for (const line of lines) {
-    const match = HEADING_REGEX.exec(line.trim());
-    if (!match) {
-      continue;
-    }
-
-    const [, hashes = "", heading = ""] = match;
-    if (!(hashes && heading)) {
-      continue;
-    }
-
-    const baseId = slugify(heading.trim());
-    // Increment until the id is unique among the ids we have already emitted,
-    // so a suffixed id never collides with an explicit "foo-1" heading.
-    let id = baseId;
-    let count = 1;
-    while (emitted.has(id)) {
-      id = `${baseId}-${count}`;
-      count += 1;
-    }
-    emitted.add(id);
-
-    toc.push({
-      id,
-      level: hashes.length,
-      title: heading.trim(),
-    });
-  }
-
-  return toc;
-};
-
-export const buildTocIndex = async (
-  index: ContentIndex,
-  source: ContentSource
-): Promise<Map<string, TocItem[]>> => {
-  const itemsBySlug = new Map<string, TocItem[]>();
-
-  const results = await Promise.all(
-    index.entries.map(async (entry) => {
-      if (entry.kind !== "entry") {
-        return null;
-      }
-      const rawContent = await source.readFile(entry.relativePath);
-      return { slug: entry.slug, toc: extractToc(rawContent) };
-    })
-  );
-
-  for (const result of results) {
-    if (result) {
-      itemsBySlug.set(result.slug, result.toc);
-    }
-  }
-
-  return itemsBySlug;
-};

package/packages/@repo/previewing/src/types.ts

@@ -1,105 +0,0 @@
-import type {
-  ContentType,
-  DocsOpenApiSource,
-  FrontmatterByType,
-  PageMode,
-  SiteConfig,
-} from "@repo/models";
-import type { OpenApiOperation, OpenApiSpec } from "@repo/prebuild";
-
-export type SiteConfigResult =
-  | { ok: true; config: SiteConfig; warnings: string[] }
-  | { ok: false; errors: string[] };
-
-export type ContentEntry =
-  | {
-      kind: "entry";
-      slug: string;
-      title: string;
-      description?: string;
-      hidden?: boolean;
-      type: ContentType;
-      collectionId: string;
-      sourcePath: string;
-      relativePath: string;
-      frontmatter: FrontmatterByType[ContentType];
-    }
-  | {
-      kind: "index";
-      slug: string;
-      title: string;
-      description?: string;
-      type: ContentType;
-      collectionId: string;
-    };
-
-export interface ContentIndex {
-  entries: ContentEntry[];
-  bySlug: Map<string, ContentEntry>;
-  byCollection: Map<string, ContentEntry[]>;
-  errors: string[];
-}
-
-export interface PageMetadata {
-  title?: string;
-  sidebarTitle?: string;
-  icon?: string;
-  iconType?: string;
-  tag?: string;
-  hidden?: boolean;
-  deprecated?: boolean;
-  url?: string;
-  mode?: PageMode;
-  noindex?: boolean;
-  hideFooterPagination?: boolean;
-  hideApiMarker?: boolean;
-  keywords?: string[];
-}
-
-export interface SearchIndexItem {
-  href?: string;
-  title: string;
-  path: string;
-}
-
-export interface TocItem {
-  id: string;
-  level: number;
-  title: string;
-}
-
-export interface UtilityPage {
-  content: string;
-  description?: string;
-  slug: string;
-  title: string;
-}
-
-export interface UtilitySegment {
-  pageSlugs: string[];
-  slug: string;
-  title: string;
-}
-
-export interface UtilityIndex {
-  description?: string;
-  name: string;
-  pages: UtilityPage[];
-  segments?: UtilitySegment[];
-  slug?: string;
-}
-
-export interface UtilityArtifact {
-  content: string;
-  contentType: string;
-  path: string;
-}
-
-export interface PrebuiltOpenApiEntry {
-  identifier: string;
-  operation: OpenApiOperation;
-  slug: string;
-  source: DocsOpenApiSource;
-  sourceKey: string;
-  spec: OpenApiSpec;
-}

package/packages/@repo/previewing/src/utility-index.ts

@@ -1,304 +0,0 @@
-import { normalizePath, slugify } from "@repo/common";
-import type { SiteConfig } from "@repo/models";
-
-import {
-  getDocsCollectionWithNavigation,
-  getDocsNavigation,
-} from "./config-loader.js";
-import {
-  PREBUILT_UTILITY_LLMS_FULL_PATH,
-  PREBUILT_UTILITY_LLMS_PATH,
-  PREBUILT_UTILITY_LLMS_SEGMENT_PREFIX,
-  PREBUILT_UTILITY_SITEMAP_PATH,
-  PREBUILT_UTILITY_SKILLS_INDEX_PATH,
-  PREBUILT_UTILITY_SKILLS_MD_PREFIX,
-  UTILITY_DOCS_ROOT_TOKEN,
-} from "./constants.js";
-import {
-  buildPageMetadataMap,
-  shouldIncludeSearchEntry,
-} from "./content-index.js";
-import type { ContentSource } from "./content-source.js";
-import {
-  prepareLlmsFullContent,
-  toAgentMarkdown,
-} from "./markdown/agent-markdown.js";
-import {
-  formatMarkdownPage,
-  formatMarkdownPageSection,
-  stripFrontmatter,
-} from "./markdown/format.js";
-import { buildUtilityOpenApiPages } from "./openapi-pages.js";
-import type {
-  ContentIndex,
-  UtilityArtifact,
-  UtilityIndex,
-  UtilityPage,
-  UtilitySegment,
-} from "./types.js";
-
-const collectUtilitySegments = (
-  config: SiteConfig,
-  pages: Map<string, UtilityPage>
-): UtilitySegment[] => {
-  const segments = new Map<string, UtilitySegment>();
-  const addGroup = (group: {
-    group?: string;
-    hidden?: boolean;
-    pages?: string[];
-  }) => {
-    if (group.hidden || !group.group) {
-      return;
-    }
-    const pageSlugs = (group.pages ?? []).filter((page) => pages.has(page));
-    if (pageSlugs.length === 0) {
-      return;
-    }
-    const slug = slugify(group.group);
-    segments.set(slug, { pageSlugs, slug, title: group.group });
-  };
-
-  const navigation = getDocsNavigation(config);
-  for (const group of navigation?.groups ?? []) {
-    addGroup(group);
-  }
-  for (const tab of navigation?.tabs ?? []) {
-    for (const group of tab.groups ?? []) {
-      addGroup(group);
-    }
-  }
-
-  return [...segments.values()];
-};
-
-export const buildUtilityIndex = async (
-  index: ContentIndex,
-  source: ContentSource,
-  config: SiteConfig
-): Promise<UtilityIndex> => {
-  const pageMetadataMap = buildPageMetadataMap(index);
-  const pages = new Map<string, UtilityPage>();
-
-  const contentPages = await Promise.all(
-    index.entries.map(async (entry) => {
-      if (entry.kind !== "entry") {
-        return null;
-      }
-      if (!shouldIncludeSearchEntry(entry, pageMetadataMap, config)) {
-        return null;
-      }
-      const rawContent = await source.readFile(entry.relativePath);
-      return {
-        content: stripFrontmatter(rawContent),
-        description: entry.description,
-        slug: entry.slug,
-        title: entry.title,
-      } satisfies UtilityPage;
-    })
-  );
-  for (const page of contentPages) {
-    if (page) {
-      pages.set(page.slug, page);
-    }
-  }
-
-  for (const page of await buildUtilityOpenApiPages(
-    config,
-    getDocsCollectionWithNavigation(config),
-    source
-  )) {
-    pages.set(page.slug, page);
-  }
-
-  const sortedPages = [...pages.values()];
-  // oxlint-disable-next-line eslint-plugin-unicorn/no-array-sort
-  sortedPages.sort((left, right) => left.slug.localeCompare(right.slug));
-
-  return {
-    description: config.description,
-    name: config.name,
-    pages: sortedPages,
-    segments: collectUtilitySegments(config, pages),
-    slug: config.slug,
-  };
-};
-
-const toUtilityDocPath = (value: string) => {
-  const clean = normalizePath(value);
-  if (!clean || clean === "index") {
-    return "/";
-  }
-  return `/${clean}`;
-};
-
-const toUtilityTemplatedDocUrl = (value: string) =>
-  `${UTILITY_DOCS_ROOT_TOKEN}${toUtilityDocPath(value)}`;
-
-const toUtilityTemplatedMarkdownDocUrl = (value: string) => {
-  const clean = normalizePath(value);
-  if (!clean || clean === "index") {
-    return `${UTILITY_DOCS_ROOT_TOKEN}/index.md`;
-  }
-  return `${UTILITY_DOCS_ROOT_TOKEN}/${clean}.md`;
-};
-
-export const getPrebuiltUtilityLlmPagePath = (slug: string) => {
-  const normalized = normalizePath(slug);
-  return `_utility/llms-pages/${normalized || "index"}.mdx`;
-};
-
-export const buildUtilityArtifacts = (
-  index: UtilityIndex
-): UtilityArtifact[] => {
-  const segments = index.segments ?? [];
-  const segmentLines =
-    segments.length > 0
-      ? [
-          "",
-          "## Segments",
-          ...segments.map(
-            (segment) =>
-              `- [${segment.title}](${UTILITY_DOCS_ROOT_TOKEN}/llms/${segment.slug}.txt)`
-          ),
-        ]
-      : [];
-  const llmsLines = [
-    `# ${index.name}`,
-    index.description ? `> ${index.description}` : null,
-    "",
-    `Sitemap: ${toUtilityTemplatedDocUrl("sitemap.xml")}`,
-    `Full content: ${UTILITY_DOCS_ROOT_TOKEN}/llms-full.txt`,
-    `Skills: ${UTILITY_DOCS_ROOT_TOKEN}/.well-known/skills/index.json`,
-    ...segmentLines,
-    "",
-    "## Docs",
-    ...index.pages.map((page) => {
-      const description = page.description ? `: ${page.description}` : "";
-      return `- [${page.title}](${toUtilityTemplatedMarkdownDocUrl(page.slug)})${description}`;
-    }),
-  ];
-
-  const sitemap = `<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-${index.pages
-  .map(
-    (page) => `  <url><loc>${toUtilityTemplatedDocUrl(page.slug)}</loc></url>`
-  )
-  .join("\n")}
-</urlset>`;
-
-  const llmsFull = index.pages
-    .map((page) =>
-      formatMarkdownPageSection(
-        page.title,
-        toUtilityTemplatedDocUrl(page.slug),
-        prepareLlmsFullContent(page.content, UTILITY_DOCS_ROOT_TOKEN, ""),
-        page.description
-      )
-    )
-    .join("\n\n");
-
-  const skillSlug = index.slug ?? slugify(index.name);
-  const skillDescription =
-    `${index.name} documentation. ${index.description ?? ""}`.trim();
-  const skillsIndex = JSON.stringify(
-    {
-      skills: [
-        {
-          description: skillDescription,
-          files: ["SKILL.md"],
-          name: skillSlug,
-        },
-      ],
-    },
-    null,
-    2
-  );
-
-  const topPages = index.pages.slice(0, 20);
-  const skillMdLines = [
-    "---",
-    `name: ${skillSlug}`,
-    `description: ${skillDescription} Use when working with ${index.name}, answering questions about its features, or helping users follow its guides.`,
-    "---",
-    "",
-    `# ${index.name}`,
-    "",
-    index.description ? `${index.description}\n` : "",
-    "## Documentation",
-    "",
-    `- Full docs index: ${UTILITY_DOCS_ROOT_TOKEN}/llms.txt`,
-    `- Complete docs content: ${UTILITY_DOCS_ROOT_TOKEN}/llms-full.txt`,
-    "- Append `.md` to any page URL for raw markdown",
-    "",
-    "## Key Pages",
-    "",
-    ...topPages.map((page) => {
-      const desc = page.description ? ` - ${page.description}` : "";
-      return `- [${page.title}](${toUtilityTemplatedMarkdownDocUrl(
-        page.slug
-      )})${desc}`;
-    }),
-  ];
-
-  return [
-    {
-      content: sitemap,
-      contentType: "application/xml; charset=utf-8",
-      path: PREBUILT_UTILITY_SITEMAP_PATH,
-    },
-    {
-      content: llmsLines.filter((line) => line !== null).join("\n"),
-      contentType: "text/plain; charset=utf-8",
-      path: PREBUILT_UTILITY_LLMS_PATH,
-    },
-    {
-      content: llmsFull,
-      contentType: "text/plain; charset=utf-8",
-      path: PREBUILT_UTILITY_LLMS_FULL_PATH,
-    },
-    {
-      content: skillsIndex,
-      contentType: "application/json; charset=utf-8",
-      path: PREBUILT_UTILITY_SKILLS_INDEX_PATH,
-    },
-    {
-      content: skillMdLines.filter((line) => line !== null).join("\n"),
-      contentType: "text/markdown; charset=utf-8",
-      path: `${PREBUILT_UTILITY_SKILLS_MD_PREFIX}${skillSlug}/SKILL.md`,
-    },
-    ...segments.map((segment) => {
-      const segmentPages = index.pages.filter((page) =>
-        segment.pageSlugs.includes(page.slug)
-      );
-      const content = [
-        `# ${index.name} - ${segment.title}`,
-        `> Segment of ${UTILITY_DOCS_ROOT_TOKEN}/llms-full.txt`,
-        "",
-        ...segmentPages.map((page) =>
-          formatMarkdownPageSection(
-            page.title,
-            toUtilityTemplatedDocUrl(page.slug),
-            prepareLlmsFullContent(page.content, UTILITY_DOCS_ROOT_TOKEN, ""),
-            page.description
-          )
-        ),
-      ].join("\n\n");
-
-      return {
-        content,
-        contentType: "text/plain; charset=utf-8",
-        path: `${PREBUILT_UTILITY_LLMS_SEGMENT_PREFIX}${segment.slug}.txt`,
-      };
-    }),
-    ...index.pages.map((page) => ({
-      content: formatMarkdownPage(
-        page.title,
-        toAgentMarkdown(page.content),
-        page.description
-      ),
-      contentType: "text/markdown; charset=utf-8",
-      path: getPrebuiltUtilityLlmPagePath(page.slug),
-    })),
-  ];
-};