@stainless-api/docs 0.1.0-beta.9 → 0.1.0-beta.91

package/stl-docs/proseSearchIndexing.ts

@@ -0,0 +1,606 @@
+import type { AstroIntegration } from 'astro';
+import { readFile } from 'fs/promises';
+import { getProsePages } from '../shared/getProsePages';
+import { getSharedLogger } from '../shared/getSharedLogger';
+import { bold } from '../shared/terminalUtils';
+import * as cheerio from 'cheerio';
+import { toMarkdown } from './proseMarkdown/toMarkdown';
+import { NormalizedStainlessDocsConfig } from './loadStlDocsConfig';
+import { buildProseIndex } from '@stainless-api/docs-search/providers/algolia';
+
+type ContentBlock =
+  | { type: 'header'; tag: string; id: string; text: string }
+  | { type: 'content'; tag: string; text: string }
+  | { type: 'code'; tag: string; language?: string; text: string };
+
+class SectionContext {
+  headers: { level: number; text: string }[] = [];
+  headerId: string | undefined;
+  headerTag: string | undefined;
+  headerText: string | undefined;
+  hasContent = false;
+
+  get(): string | undefined {
+    if (this.headers.length === 0) return;
+    return this.headers.map((h) => h.text).join(' > ');
+  }
+
+  header({ id, tag, text }: { id: string; tag: string; text: string }) {
+    const level = getHeaderLevel(tag);
+    if (level > 0) {
+      while (this.headers.length > 0 && this.headers[this.headers.length - 1]!.level >= level) {
+        this.headers.pop();
+      }
+      this.headers.push({ level, text });
+    }
+    this.headerId = id;
+    this.headerTag = tag;
+    this.headerText = text;
+    this.hasContent = false;
+  }
+}
+
+// Generate a URL-safe ID from header text (e.g., "OpenAPI Config" -> "openapi-config")
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/`/g, '') // Remove backticks
+    .replace(/[^a-z0-9]+/g, '-') // Replace non-alphanumeric with hyphens
+    .replace(/^-|-$/g, ''); // Trim leading/trailing hyphens
+}
+
+// Check if a word ends with a real table cell boundary (| but not escaped \|)
+function isTableCellBoundary(word: string): boolean {
+  return word.endsWith('|') && !word.endsWith('\\|');
+}
+
+/**
+ * Extracts the header level from a tag like "h1", "h2", etc.
+ */
+function getHeaderLevel(tag: string): number {
+  const match = tag.match(/^h(\d)$/);
+  return match ? parseInt(match[1]!, 10) : 0;
+}
+
+// Chunking configuration
+// We target 64-256 tokens per chunk, using ~1.3 tokens/word for English text
+const TOKENS_PER_WORD = 1.3;
+const MIN_TOKENS = 64;
+const MAX_TOKENS = 256;
+const MIN_WORDS = Math.floor(MIN_TOKENS / TOKENS_PER_WORD); // ~49 words
+const MAX_WORDS = Math.floor(MAX_TOKENS / TOKENS_PER_WORD); // ~197 words
+const LINE_BREAK_WORDS = Math.floor((MAX_TOKENS * 0.75) / TOKENS_PER_WORD); // ~148 words
+const SENTENCE_BREAK_WORDS = Math.floor((MAX_TOKENS * 0.875) / TOKENS_PER_WORD); // ~172 words
+
+/**
+ * Chunks text content into segments of 64-256 tokens using word-based boundaries.
+ * Prefers breaking at sentence endings for natural chunk boundaries.
+ */
+function chunkTextByWords(text: string): string[] {
+  const words = text.split(/\s+/).filter((w) => w.length > 0);
+
+  if (words.length <= MAX_WORDS) {
+    return words.length > 0 ? [words.join(' ')] : [];
+  }
+
+  const chunks: string[] = [];
+  let currentChunk: string[] = [];
+
+  for (const word of words) {
+    currentChunk.push(word);
+
+    // Force break at max words
+    if (currentChunk.length >= MAX_WORDS) {
+      chunks.push(currentChunk.join(' '));
+      currentChunk = [];
+      continue;
+    }
+
+    // Prefer breaking at sentence boundaries after threshold
+    if (currentChunk.length >= SENTENCE_BREAK_WORDS && /[.!?]["']?$/.test(word)) {
+      chunks.push(currentChunk.join(' '));
+      currentChunk = [];
+    }
+  }
+
+  if (currentChunk.length > 0) {
+    if (currentChunk.length < MIN_WORDS && chunks.length > 0) {
+      const lastChunk = chunks[chunks.length - 1]!;
+      const mergedWords = lastChunk.split(/\s+/).length + currentChunk.length;
+      if (mergedWords <= MAX_WORDS) {
+        chunks[chunks.length - 1] = lastChunk + ' ' + currentChunk.join(' ');
+      } else {
+        chunks.push(currentChunk.join(' '));
+      }
+    } else {
+      chunks.push(currentChunk.join(' '));
+    }
+  }
+
+  return chunks;
+}
+
+type ContentBlockChunk = {
+  type: 'prose';
+  content: string;
+  headerId?: string;
+  headerTag?: string;
+  tag?: string;
+  language?: string;
+  sectionContext?: string;
+};
+
+/**
+ * Chunks content blocks into segments of 64-256 tokens.
+ *
+ * Chunking strategy:
+ * 1. Break at headers to keep sections isolated
+ * 2. Prefer breaking at line/table boundaries after LINE_BREAK_WORDS (~148 words / ~192 tokens)
+ * 3. Break at sentence endings after SENTENCE_BREAK_WORDS (~172 words / ~224 tokens)
+ * 4. Force break at MAX_WORDS, preferring table row boundaries if available
+ * 5. Section context (header hierarchy) is recorded alongside each chunk for discoverability
+ */
+function chunkByWords(blocks: ContentBlock[]): ContentBlockChunk[] {
+  const chunks: ContentBlockChunk[] = [];
+
+  let currentChunk: string[] = [];
+  const ctx = new SectionContext();
+
+  // Flush current chunk to output. If splitAt is provided, keep words after that index for next chunk.
+  const flushChunk = (splitAt?: number) => {
+    if (currentChunk.length === 0) return;
+
+    const wordsToFlush = splitAt !== undefined ? currentChunk.slice(0, splitAt) : currentChunk;
+    const wordsToKeep = splitAt !== undefined ? currentChunk.slice(splitAt) : [];
+
+    if (wordsToFlush.length > 0) {
+      const chunkText = wordsToFlush.join(' ').trim();
+      const sectionContext = ctx.get();
+
+      chunks.push({
+        type: 'prose',
+        content: chunkText,
+        headerId: ctx.headerId,
+        headerTag: ctx.headerTag,
+        sectionContext: sectionContext || undefined,
+      });
+      ctx.hasContent = true;
+    }
+    currentChunk = wordsToKeep;
+  };
+
+  // Find a table row boundary to break at (between MIN_WORDS and current length)
+  // Returns the index to split at, or undefined if no good boundary found
+  const findTableRowBoundary = (): number | undefined => {
+    for (let i = currentChunk.length - 1; i >= MIN_WORDS; i--) {
+      const word = currentChunk[i]!;
+      const nextWord = currentChunk[i + 1];
+      // A row boundary is where one cell ends (|) and the next row starts (|)
+      if (isTableCellBoundary(word) && nextWord?.startsWith('|')) {
+        return i + 1;
+      }
+    }
+    return undefined;
+  };
+
+  for (const block of blocks) {
+    if (block.type === 'header') {
+      flushChunk();
+      ctx.header(block);
+      continue;
+    }
+
+    // Chunk code blocks separately; they tend to be more important.
+    if (block.type === 'code') {
+      flushChunk();
+      const codeText = block.text.trim();
+      if (codeText) {
+        for (const chunkText of chunkTextByWords(codeText)) {
+          chunks.push({
+            type: 'prose',
+            content: chunkText,
+            headerId: ctx.headerId,
+            tag: 'code',
+            language: block.language,
+            sectionContext: ctx.get(),
+          });
+          ctx.hasContent = true;
+        }
+      }
+      continue;
+    }
+
+    if (block.type !== 'content') continue;
+
+    // Split by newlines first to preserve line boundary information
+    const lines = block.text.split(/\n/);
+    let inCodeBlock = false;
+
+    for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
+      const line = lines[lineIdx]!;
+
+      // Track code block boundaries (standalone fences only)
+      if (/^(`{3,}|~{3,})([a-zA-Z0-9+-]*)?\s*$/.test(line.trim())) {
+        inCodeBlock = !inCodeBlock;
+      }
+
+      // Calculate indentation level (number of leading spaces, treating tabs as 2 spaces)
+      const indentMatch = line.match(/^(\s*)/);
+      const indentLevel = indentMatch ? indentMatch[1]!.replace(/\t/g, '  ').length : 0;
+
+      const words = line.split(/\s+/).filter((w) => w.length > 0);
+      const isLastLine = lineIdx === lines.length - 1;
+
+      for (let wordIdx = 0; wordIdx < words.length; wordIdx++) {
+        const word = words[wordIdx]!;
+        const isEndOfLine = wordIdx === words.length - 1 && !isLastLine;
+
+        if (currentChunk.length >= MAX_WORDS) {
+          flushChunk(findTableRowBoundary());
+        }
+
+        currentChunk.push(word);
+
+        // In code blocks, avoid early flushes to keep blocks together
+        // - Light indentation (2+ spaces): require more words before flushing
+        // - Deep indentation (4+ spaces): skip early flushes entirely
+        const inShallowCode = inCodeBlock && indentLevel >= 2 && indentLevel < 4;
+        const inDeepCode = inCodeBlock && indentLevel >= 4;
+
+        // Flush early at natural break points
+        const len = currentChunk.length;
+        const atTableBreak = len >= LINE_BREAK_WORDS && isTableCellBoundary(word);
+        // Shallow code: only flush at sentence threshold; Deep code: don't flush early
+        const lineBreakThreshold = inShallowCode ? SENTENCE_BREAK_WORDS : LINE_BREAK_WORDS;
+        const atLineBreak = len >= lineBreakThreshold && isEndOfLine && !inDeepCode;
+        const atSentenceBreak = len >= SENTENCE_BREAK_WORDS && /[.!?]["']?$/.test(word) && !inDeepCode;
+        if (atTableBreak || atLineBreak || atSentenceBreak) {
+          flushChunk();
+        }
+      }
+    }
+  }
+
+  flushChunk();
+  return chunks;
+}
+
+/**
+ * Parses markdown into content blocks, identifying headers, content sections, and code blocks.
+ * Code blocks are extracted separately with language metadata for specialized indexing.
+ */
+function parseMarkdown(markdown: string): ContentBlock[] {
+  const blocks: ContentBlock[] = [];
+
+  // Extract title from frontmatter and treat it as h1
+  const frontmatterMatch = markdown.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  if (frontmatterMatch) {
+    const frontmatter = frontmatterMatch[1]!;
+    const titleMatch = frontmatter.match(/^title:\s*(.+)$/m);
+    if (titleMatch) {
+      const title = titleMatch[1]!.trim().replace(/^["']|["']$/g, ''); // Remove quotes if present
+      blocks.push({
+        type: 'header',
+        tag: 'h1',
+        id: slugify(title),
+        text: title,
+      });
+    }
+  }
+
+  // Remove frontmatter
+  const content = markdown.replace(/^---[\s\S]*?---\r?\n*/, '').trim();
+
+  // Split into lines and process
+  const lines = content.split('\n');
+  let currentContent: string[] = [];
+  let inCodeBlock = false;
+  let codeBlockLanguage: string | undefined;
+  let codeBlockContent: string[] = [];
+
+  const flushContent = () => {
+    const text = currentContent.join('\n').trim();
+    if (text) {
+      blocks.push({ type: 'content', tag: 'p', text });
+    }
+    currentContent = [];
+  };
+
+  const flushCodeBlock = () => {
+    if (codeBlockContent.length > 0) {
+      const code = codeBlockContent.join('\n').trim();
+      if (code) {
+        blocks.push({
+          type: 'code',
+          tag: 'code',
+          text: code,
+          language: codeBlockLanguage || undefined,
+        });
+      }
+    }
+    codeBlockContent = [];
+    codeBlockLanguage = undefined;
+  };
+
+  for (const line of lines) {
+    // Track fenced code blocks (``` or ~~~)
+    // Only match standalone markers: ```[language] with nothing else on the line
+    // This avoids matching inline code blocks in table cells like "``` Then content..."
+    const codeBlockMatch = line.match(/^(`{3,}|~{3,})([a-zA-Z0-9+-]*)?\s*$/);
+    if (codeBlockMatch) {
+      if (!inCodeBlock) {
+        flushContent();
+        inCodeBlock = true;
+        codeBlockLanguage = codeBlockMatch[2] || undefined;
+      } else {
+        flushCodeBlock();
+        inCodeBlock = false;
+      }
+      continue;
+    }
+
+    if (inCodeBlock) {
+      codeBlockContent.push(line);
+      continue;
+    }
+
+    // Only match headers outside of code blocks
+    const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
+
+    if (headerMatch) {
+      flushContent();
+      const level = headerMatch[1]!.length;
+      const headerText = headerMatch[2]!.trim();
+      blocks.push({
+        type: 'header',
+        tag: `h${level}`,
+        id: slugify(headerText),
+        text: headerText,
+      });
+      continue;
+    }
+
+    currentContent.push(line);
+  }
+
+  flushCodeBlock();
+  flushContent();
+  return blocks;
+}
+
+export type IndexEntry = {
+  chunk: { id: string; index: number; total: number };
+  id: string;
+  tag: string;
+  content: string;
+  language?: string;
+  sectionContext?: string;
+};
+
+/**
+ * Extracts and chunks markdown content for search indexing.
+ * Yields prose and code chunks with section context and language metadata.
+ */
+export function* indexMarkdown(markdown: string): Generator<IndexEntry> {
+  const blocks = parseMarkdown(markdown);
+  const chunks = chunkByWords(blocks);
+  const documentId = crypto.randomUUID();
+
+  for (const [index, chunk] of chunks.entries()) {
+    yield {
+      id: chunk.headerId ?? '',
+      tag: chunk.tag ?? chunk.headerTag ?? '',
+      content: chunk.content,
+      ...(chunk.sectionContext ? { sectionContext: chunk.sectionContext } : {}),
+      ...(chunk.language ? { language: chunk.language } : {}),
+      chunk: {
+        id: documentId,
+        index,
+        total: chunks.length,
+      },
+    };
+  }
+}
+
+const DEFAULT_ROOT = 'main';
+const DEFAULT_PATTERN = 'h1, h2, h3, h4, h5, h6, p, li, pre code';
+
+/**
+ * Indexes HTML content for search, with section context and code language extraction.
+ *
+ * Features:
+ * - Tracks header hierarchy to prepend section context (e.g., "Guide > Setup: ...")
+ * - Extracts language metadata from code blocks (class="language-js")
+ * - Uses word-based chunking with sentence boundary detection
+ */
+export function* indexHTML(
+  content: string,
+  root = DEFAULT_ROOT,
+  pattern = DEFAULT_PATTERN,
+): Generator<IndexEntry> {
+  const $ = cheerio.load(content);
+  const matches = $(root).find(pattern);
+
+  const ctx = new SectionContext();
+
+  for (const match of matches) {
+    const tagName = match.tagName.toLowerCase();
+    const rawText = $(match).text().trim();
+
+    if (getHeaderLevel(tagName) > 0) {
+      ctx.header({ id: $(match).attr('id') ?? slugify(rawText), tag: tagName, text: rawText });
+      continue;
+    }
+
+    // Check if this is a code block and extract language
+    const isCode = tagName === 'code' && $(match).parent().is('pre');
+    let language: string | undefined;
+    if (isCode) {
+      const classes = $(match).attr('class') || '';
+      const langMatch = classes.match(/(?:language-|lang-)([a-zA-Z0-9+-]+)/);
+      language = langMatch ? langMatch[1] : undefined;
+    }
+
+    // Build content with section context
+    const sectionContext = ctx.get();
+    const chunks = chunkTextByWords(rawText);
+    const chunkId = crypto.randomUUID();
+
+    for (const [chunkN, chunkText] of chunks.entries()) {
+      yield {
+        id: ctx.headerId ?? $(match).attr('id') ?? chunkId,
+        tag: isCode ? 'code' : tagName,
+        content: chunkText,
+        ...(sectionContext ? { sectionContext } : {}),
+        ...(language && { language }),
+        chunk: {
+          id: chunkId,
+          index: chunkN,
+          total: chunks.length,
+        },
+      };
+      ctx.hasContent = true;
+    }
+  }
+}
+
+export function stainlessDocsAlgoliaProseIndexing({
+  apiReferenceBasePath,
+}: {
+  apiReferenceBasePath: string | null;
+}): AstroIntegration {
+  return {
+    name: 'stl-docs-prose-indexing',
+    hooks: {
+      'astro:build:done': async ({ logger: localLogger, dir }) => {
+        const logger = getSharedLogger({ fallback: localLogger });
+        const outputBasePath = dir.pathname;
+
+        const {
+          PUBLIC_ALGOLIA_APP_ID: appId,
+          PUBLIC_ALGOLIA_INDEX: indexName,
+          PRIVATE_ALGOLIA_WRITE_KEY: algoliaWriteKey,
+        } = process.env;
+
+        if (!appId || !indexName || !algoliaWriteKey) {
+          logger.info('Skipping algolia indexing due to missing environment variables');
+          return;
+        }
+
+        const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
+        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for algolia search`));
+
+        const objects = [];
+        for (const absHtmlPath of pagesToRender) {
+          const content = await readFile(absHtmlPath, 'utf-8');
+          const idx = indexHTML(content);
+          for (const entry of idx)
+            objects.push({
+              ...entry,
+              source: absHtmlPath.slice(outputBasePath.length),
+            });
+        }
+
+        try {
+          await buildProseIndex(appId, `${indexName}-prose`, algoliaWriteKey, objects);
+        } catch (err) {
+          logger.error(`Failed to index prose content: ${err}`);
+        }
+      },
+    },
+  };
+}
+
+export function stainlessDocsVectorProseIndexing(
+  config: NormalizedStainlessDocsConfig,
+  apiReferenceBasePath: string | null,
+): AstroIntegration {
+  return {
+    name: 'stl-docs-prose-indexing',
+    hooks: {
+      'astro:build:done': async ({ logger: localLogger, dir }) => {
+        const logger = getSharedLogger({ fallback: localLogger });
+        const outputBasePath = dir.pathname;
+
+        const stainlessProjectName = config.apiReference?.stainlessProject;
+
+        const {
+          STAINLESS_API_KEY: stainlessApiKey,
+          STAINLESS_DOCS_SITE_ID: stainlessDocsSiteId,
+          STAINLESS_DOCS_REPO_SHA: stainlessDocsRepoSha,
+        } = process.env;
+
+        // Skip indexing if required environment variables are not set
+        if (!stainlessApiKey || !stainlessProjectName || !stainlessDocsSiteId || !stainlessDocsRepoSha) {
+          logger.info(
+            `Skipping vector prose search indexing: required environment/config variables not set, missing: ${[
+              !stainlessApiKey && 'STAINLESS_API_KEY',
+              !stainlessDocsSiteId && 'STAINLESS_DOCS_SITE_ID',
+              !stainlessDocsRepoSha && 'STAINLESS_DOCS_REPO_SHA',
+              !stainlessProjectName && 'stainlessProject in apiReference config',
+            ]
+              .filter(Boolean)
+              .join(', ')}`,
+          );
+          return;
+        }
+
+        const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
+
+        if (pagesToRender.length === 0) {
+          logger.info('No prose pages found to index for vector search');
+          return;
+        }
+
+        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for vector search`));
+
+        const objects: {
+          id: string;
+          tag: string;
+          content: string;
+          language?: string;
+          kind: 'prose';
+          source: string;
+        }[] = [];
+        for (const absHtmlPath of pagesToRender) {
+          const content = await readFile(absHtmlPath, 'utf-8');
+          const markdown = await toMarkdown(content);
+
+          if (markdown) {
+            const idx = indexMarkdown(markdown);
+            for (const { chunk: _, ...entry } of idx)
+              objects.push({
+                ...entry,
+                kind: 'prose',
+                source: absHtmlPath.slice(outputBasePath.length),
+              });
+          }
+        }
+
+        if (objects.length === 0) {
+          logger.info('No prose content extracted to index for vector search');
+          return;
+        }
+
+        logger.info(bold(`Uploading ${objects.length} prose content chunks to stainless docs index`));
+
+        const response = await fetch(
+          `https://api.stainless.com/api/projects/${stainlessProjectName}/docs-sites/${stainlessDocsSiteId}/index`,
+          {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+              Authorization: `Bearer ${stainlessApiKey}`,
+            },
+            body: JSON.stringify({
+              docs_repo_sha: stainlessDocsRepoSha,
+              index: objects,
+            }),
+          },
+        );
+
+        console.log(`docs index API response code ${response.status}: ${await response.text()}`);
+      },
+    },
+  };
+}

package/stl-docs/tabsMiddleware.ts

@@ -3,6 +3,7 @@ import { defineRouteMiddleware } from '@astrojs/starlight/route-data';
 
 import { SPLIT_TABS_ENABLED, TABS } from 'virtual:stl-docs-virtual-module';
 import clsx from 'clsx';
+import path from 'path';
 
 // this fn is loaded in the plugin via addRouteMiddleware
 
@@ -58,7 +59,7 @@ function getTabIndexForSlug(
   match: 'exact' | 'prefix';
 } | null {
   // ↓ exact match eg. slug = "/blog" and there is a link containing "/blog"
-  let tab = linksByTab.get(slug);
+  const tab = linksByTab.get(slug)!;
   if (typeof tab === 'string') {
     return {
       match: 'exact',
@@ -88,18 +89,24 @@ function getNonSplitLinksByTab() {
   const linksByTab = new Map<string, string>();
 
   for (let i = 0; i < TABS.length; i++) {
-    const tab = TABS[i];
+    const tab = TABS[i]!;
     linksByTab.set(tab.link, String(i));
   }
 
   return linksByTab;
 }
 
+export interface StarlightRouteWithStlDocs extends StarlightRouteData {
+  _stlDocs?: {
+    activeTabIndex: number;
+  };
+}
+
 export const onRequest = defineRouteMiddleware(async (context) => {
   // if using content collection schema, use: context.locals.starlightRoute.entry.data.stainlessStarlight
   // this worked without collections but relied on hijacking starlightRoute: context.props.frontmatter.stainlessStarlight
 
-  const slug = `/${context.locals.starlightRoute.id}`; // same as .slug but not deprecated
+  const slug = path.posix.join(import.meta.env.BASE_URL ?? '', context.locals.starlightRoute.id); // same as .slug but not deprecated
 
   /*
     In the index of our starlight plugin, we transform our "tabs" into a plain old Starlight sidebar.
@@ -143,7 +150,8 @@ export const onRequest = defineRouteMiddleware(async (context) => {
   }
 
   // We store the active tab index so we can use it in our nav tabs component
-  context.locals.starlightRoute._stlDocs = {
+  const routeData: StarlightRouteWithStlDocs = context.locals.starlightRoute;
+  routeData._stlDocs = {
     activeTabIndex: activeTabIndex.index,
   };
 
@@ -179,5 +187,6 @@ export const onRequest = defineRouteMiddleware(async (context) => {
 
   matchingGroup?.entries.unshift(...mobileLinks);
 
+  (context.locals._stlStarlightPage ??= {}).fullSidebar = context.locals.starlightRoute.sidebar;
   context.locals.starlightRoute.sidebar = matchingGroup.entries;
 });