@stainless-api/docs 0.1.0-beta.7 → 0.1.0-beta.70

package/stl-docs/proseSearchIndexing.ts

@@ -0,0 +1,450 @@
+import type { AstroIntegration } from 'astro';
+import { readFile } from 'fs/promises';
+import { getProsePages } from '../shared/getProsePages';
+import { getSharedLogger } from '../shared/getSharedLogger';
+import { bold } from '../shared/terminalUtils';
+import * as cheerio from 'cheerio';
+import { toMarkdown } from './proseMarkdown/toMarkdown';
+import { NormalizedStainlessDocsConfig } from './loadStlDocsConfig';
+import { buildProseIndex } from '@stainless-api/docs-search/providers/algolia';
+
+interface ContentBlock {
+  type: 'header' | 'content';
+  tag?: string;
+  id?: string;
+  text: string;
+}
+
+// Chunking configuration
+// We target 64-256 tokens per chunk, using ~1.3 tokens/word for English text
+const TOKENS_PER_WORD = 1.3;
+const MIN_TOKENS = 64;
+const MAX_TOKENS = 256;
+const MIN_WORDS = Math.floor(MIN_TOKENS / TOKENS_PER_WORD); // ~49 words
+const MAX_WORDS = Math.floor(MAX_TOKENS / TOKENS_PER_WORD); // ~197 words
+const LINE_BREAK_WORDS = Math.floor((MAX_TOKENS * 0.75) / TOKENS_PER_WORD); // ~148 words
+const SENTENCE_BREAK_WORDS = Math.floor((MAX_TOKENS * 0.875) / TOKENS_PER_WORD); // ~172 words
+
+// Generate a URL-safe ID from header text (e.g., "OpenAPI Config" -> "openapi-config")
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/`/g, '') // Remove backticks
+    .replace(/[^a-z0-9]+/g, '-') // Replace non-alphanumeric with hyphens
+    .replace(/^-|-$/g, ''); // Trim leading/trailing hyphens
+}
+
+// Check if a word ends with a real table cell boundary (| but not escaped \|)
+function isTableCellBoundary(word: string): boolean {
+  return word.endsWith('|') && !word.endsWith('\\|');
+}
+
+/**
+ * Chunks content blocks into segments of 64-256 tokens.
+ *
+ * Chunking strategy:
+ * 1. Break at headers if chunk has >= MIN_WORDS, otherwise merge with next section
+ * 2. Prefer breaking at line/table boundaries after LINE_BREAK_WORDS (~148 words / ~192 tokens)
+ * 3. Break at sentence endings after SENTENCE_BREAK_WORDS (~172 words / ~224 tokens)
+ * 4. Force break at MAX_WORDS, preferring table row boundaries if available
+ * 5. Header context (id/tag) is preserved for continuation chunks
+ */
+function chunkByWords(blocks: ContentBlock[]): { content: string; headerId?: string; headerTag?: string }[] {
+  const chunks: { content: string; headerId?: string; headerTag?: string }[] = [];
+
+  let currentChunk: string[] = [];
+  let currentHeaderId: string | undefined;
+  let currentHeaderTag: string | undefined;
+
+  // Flush current chunk to output. If splitAt is provided, keep words after that index for next chunk.
+  const flushChunk = (splitAt?: number) => {
+    if (currentChunk.length === 0) return;
+
+    const wordsToFlush = splitAt !== undefined ? currentChunk.slice(0, splitAt) : currentChunk;
+    const wordsToKeep = splitAt !== undefined ? currentChunk.slice(splitAt) : [];
+
+    if (wordsToFlush.length > 0) {
+      chunks.push({
+        content: wordsToFlush.join(' ').trim(),
+        headerId: currentHeaderId,
+        headerTag: currentHeaderTag,
+      });
+    }
+    currentChunk = wordsToKeep;
+  };
+
+  // Find a table row boundary to break at (between MIN_WORDS and current length)
+  // Returns the index to split at, or undefined if no good boundary found
+  const findTableRowBoundary = (): number | undefined => {
+    for (let i = currentChunk.length - 1; i >= MIN_WORDS; i--) {
+      const word = currentChunk[i]!;
+      const nextWord = currentChunk[i + 1];
+      // A row boundary is where one cell ends (|) and the next row starts (|)
+      if (isTableCellBoundary(word) && nextWord?.startsWith('|')) {
+        return i + 1;
+      }
+    }
+    return undefined;
+  };
+
+  for (const block of blocks) {
+    if (block.type === 'header') {
+      // Flush at header boundaries only if chunk meets minimum size
+      // This avoids creating tiny chunks for headers with little content
+      if (currentChunk.length >= MIN_WORDS) {
+        flushChunk();
+      }
+      currentHeaderId = block.id;
+      currentHeaderTag = block.tag;
+      // Include header text at the start of the new chunk
+      currentChunk.push(...block.text.split(/\s+/).filter((w) => w.length > 0));
+      continue;
+    }
+
+    // Split by newlines first to preserve line boundary information
+    const lines = block.text.split(/\n/);
+    let inCodeBlock = false;
+
+    for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
+      const line = lines[lineIdx]!;
+
+      // Track code block boundaries
+      if (/^(`{3,}|~{3,})/.test(line.trim())) {
+        inCodeBlock = !inCodeBlock;
+      }
+
+      // Calculate indentation level (number of leading spaces, treating tabs as 2 spaces)
+      const indentMatch = line.match(/^(\s*)/);
+      const indentLevel = indentMatch ? indentMatch[1]!.replace(/\t/g, '  ').length : 0;
+
+      const words = line.split(/\s+/).filter((w) => w.length > 0);
+      const isLastLine = lineIdx === lines.length - 1;
+
+      for (let wordIdx = 0; wordIdx < words.length; wordIdx++) {
+        const word = words[wordIdx]!;
+        const isEndOfLine = wordIdx === words.length - 1 && !isLastLine;
+
+        if (currentChunk.length >= MAX_WORDS) {
+          flushChunk(findTableRowBoundary());
+        }
+
+        currentChunk.push(word);
+
+        // In code blocks, avoid early flushes to keep blocks together
+        // - Light indentation (2+ spaces): require more words before flushing
+        // - Deep indentation (4+ spaces): skip early flushes entirely
+        const inShallowCode = inCodeBlock && indentLevel >= 2 && indentLevel < 4;
+        const inDeepCode = inCodeBlock && indentLevel >= 4;
+
+        // Flush early at natural break points
+        const len = currentChunk.length;
+        const atTableBreak = len >= LINE_BREAK_WORDS && isTableCellBoundary(word);
+        // Shallow code: only flush at sentence threshold; Deep code: don't flush early
+        const lineBreakThreshold = inShallowCode ? SENTENCE_BREAK_WORDS : LINE_BREAK_WORDS;
+        const atLineBreak = len >= lineBreakThreshold && isEndOfLine && !inDeepCode;
+        const atSentenceBreak = len >= SENTENCE_BREAK_WORDS && /[.!?]["']?$/.test(word) && !inDeepCode;
+        if (atTableBreak || atLineBreak || atSentenceBreak) {
+          flushChunk();
+        }
+      }
+    }
+  }
+
+  flushChunk();
+  return chunks;
+}
+
+/**
+ * Parses markdown into content blocks, identifying headers and content sections.
+ * Tracks fenced code blocks to avoid treating # comments in code as headers.
+ */
+function parseMarkdown(markdown: string): ContentBlock[] {
+  const blocks: ContentBlock[] = [];
+
+  // Extract title from frontmatter and treat it as h1
+  const frontmatterMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
+  if (frontmatterMatch) {
+    const frontmatter = frontmatterMatch[1]!;
+    const titleMatch = frontmatter.match(/^title:\s*(.+)$/m);
+    if (titleMatch) {
+      const title = titleMatch[1]!.trim().replace(/^["']|["']$/g, ''); // Remove quotes if present
+      blocks.push({
+        type: 'header',
+        tag: 'h1',
+        id: slugify(title),
+        text: title,
+      });
+    }
+  }
+
+  // Remove frontmatter
+  const content = markdown.replace(/^---[\s\S]*?---\n*/, '').trim();
+
+  // Split into lines and process
+  const lines = content.split('\n');
+  let currentContent: string[] = [];
+  let inCodeBlock = false;
+
+  const flushContent = () => {
+    const text = currentContent.join('\n').trim();
+    if (text) {
+      blocks.push({ type: 'content', text });
+    }
+    currentContent = [];
+  };
+
+  for (const line of lines) {
+    // Track fenced code blocks (``` or ~~~)
+    // Only match standalone markers: ```[language] with nothing else on the line
+    // This avoids matching inline code blocks in table cells like "``` Then content..."
+    if (/^(`{3,}|~{3,})([a-zA-Z0-9]*)?(\s*)$/.test(line)) {
+      inCodeBlock = !inCodeBlock;
+      currentContent.push(line);
+      continue;
+    }
+
+    // Only match headers outside of code blocks
+    if (!inCodeBlock) {
+      const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
+
+      if (headerMatch) {
+        flushContent();
+        const level = headerMatch[1]!.length;
+        const headerText = headerMatch[2]!.trim();
+        blocks.push({
+          type: 'header',
+          tag: `h${level}`,
+          id: slugify(headerText),
+          text: headerText,
+        });
+        continue;
+      }
+    }
+
+    currentContent.push(line);
+  }
+
+  flushContent();
+  return blocks;
+}
+
+/**
+ * Extracts and chunks markdown content for search indexing.
+ * Yields chunk objects with content, header context, and chunk metadata.
+ */
+export function* indexMarkdown(markdown: string) {
+  const blocks = parseMarkdown(markdown);
+  const chunks = chunkByWords(blocks);
+  const documentId = crypto.randomUUID();
+
+  for (const [index, chunk] of chunks.entries()) {
+    yield {
+      id: chunk.headerId ?? '',
+      tag: chunk.headerTag ?? '',
+      content: chunk.content,
+      chunk: {
+        id: documentId,
+        index,
+        total: chunks.length,
+      },
+    };
+  }
+}
+
+function chunkHTMLByWords(content: string, chunkSize: number = 30000, chunkOverlap: number = 10) {
+  if (Buffer.byteLength(content) < chunkSize) return [content];
+
+  const words = content.split(/\s+/);
+  const chunks: string[] = [];
+
+  let currentChunk: string[] = [];
+  let currentSize = 0;
+
+  for (const word of words) {
+    const wordSize = Buffer.byteLength(word + ' ', 'utf-8');
+
+    if (currentSize + wordSize > chunkSize && currentChunk.length > 0) {
+      chunks.push(currentChunk.join(' '));
+
+      const overlapStart = Math.max(0, currentChunk.length - chunkOverlap);
+      currentChunk = currentChunk.slice(overlapStart);
+      currentSize = Buffer.byteLength(currentChunk.join(' '), 'utf-8');
+    }
+
+    currentChunk.push(word);
+    currentSize += wordSize;
+  }
+
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk.join(' '));
+  }
+
+  return chunks;
+}
+
+export function* indexHTML(content: string, root: string, pattern: string) {
+  const $ = cheerio.load(content);
+  const matches = $(root).find(pattern);
+
+  for (const match of matches) {
+    const rawText = $(match).text().trim();
+    const chunks = chunkHTMLByWords(rawText);
+    const chunkId = crypto.randomUUID();
+
+    for (const [chunkN, content] of chunks.entries()) {
+      yield {
+        id: $(match).attr('id'),
+        tag: match.tagName.toLowerCase(),
+        content,
+        chunk: {
+          id: chunkId,
+          index: chunkN,
+          total: chunks.length,
+        },
+      };
+    }
+  }
+}
+
+const root = 'main';
+const pattern = 'h1, h2, h3, h4, h5, h6, p, li';
+
+export function stainlessDocsAlgoliaProseIndexing({
+  apiReferenceBasePath,
+}: {
+  apiReferenceBasePath: string | null;
+}): AstroIntegration {
+  return {
+    name: 'stl-docs-prose-indexing',
+    hooks: {
+      'astro:build:done': async ({ logger: localLogger, dir }) => {
+        const logger = getSharedLogger({ fallback: localLogger });
+        const outputBasePath = dir.pathname;
+
+        const {
+          PUBLIC_ALGOLIA_APP_ID: appId,
+          PUBLIC_ALGOLIA_INDEX: indexName,
+          PRIVATE_ALGOLIA_WRITE_KEY: algoliaWriteKey,
+        } = process.env;
+
+        if (!appId || !indexName || !algoliaWriteKey) {
+          logger.info('Skipping algolia indexing due to missing environment variables');
+          return;
+        }
+
+        const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
+        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for algolia search`));
+
+        const objects = [];
+        for (const absHtmlPath of pagesToRender) {
+          const content = await readFile(absHtmlPath, 'utf-8');
+          const idx = indexHTML(content, root, pattern);
+          for (const entry of idx)
+            objects.push({
+              ...entry,
+              source: absHtmlPath.slice(outputBasePath.length),
+            });
+        }
+
+        try {
+          await buildProseIndex(appId, `${indexName}-prose`, algoliaWriteKey, objects);
+        } catch (err) {
+          logger.error(`Failed to index prose content: ${err}`);
+        }
+      },
+    },
+  };
+}
+
+export function stainlessDocsVectorProseIndexing(
+  config: NormalizedStainlessDocsConfig,
+  apiReferenceBasePath: string | null,
+): AstroIntegration {
+  return {
+    name: 'stl-docs-prose-indexing',
+    hooks: {
+      'astro:build:done': async ({ logger: localLogger, dir }) => {
+        const logger = getSharedLogger({ fallback: localLogger });
+        const outputBasePath = dir.pathname;
+
+        const stainlessProjectName = config.apiReference?.stainlessProject;
+
+        const {
+          STAINLESS_API_KEY: stainlessApiKey,
+          STAINLESS_DOCS_SITE_ID: stainlessDocsSiteId,
+          STAINLESS_DOCS_REPO_SHA: stainlessDocsRepoSha,
+        } = process.env;
+
+        // Skip indexing if required environment variables are not set
+        if (!stainlessApiKey || !stainlessProjectName || !stainlessDocsSiteId || !stainlessDocsRepoSha) {
+          logger.info(
+            `Skipping vector prose search indexing: required environment/config variables not set, missing: ${[
+              !stainlessApiKey && 'STAINLESS_API_KEY',
+              !stainlessDocsSiteId && 'STAINLESS_DOCS_SITE_ID',
+              !stainlessDocsRepoSha && 'STAINLESS_DOCS_REPO_SHA',
+              !stainlessProjectName && 'stainlessProject in apiReference config',
+            ]
+              .filter(Boolean)
+              .join(', ')}`,
+          );
+          return;
+        }
+
+        const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
+
+        if (pagesToRender.length === 0) {
+          logger.info('No prose pages found to index for vector search');
+          return;
+        }
+
+        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for vector search`));
+
+        const objects: {
+          id: string;
+          tag: string;
+          content: string;
+          kind: 'prose';
+          source: string;
+        }[] = [];
+        for (const absHtmlPath of pagesToRender) {
+          const content = await readFile(absHtmlPath, 'utf-8');
+          const markdown = await toMarkdown(content);
+
+          if (markdown) {
+            const idx = indexMarkdown(markdown);
+            for (const { chunk, ...entry } of idx)
+              objects.push({
+                ...entry,
+                kind: 'prose',
+                source: absHtmlPath.slice(outputBasePath.length),
+              });
+          }
+        }
+
+        if (objects.length === 0) {
+          logger.info('No prose content extracted to index for vector search');
+          return;
+        }
+
+        logger.info(bold(`Uploading ${objects.length} prose content chunks to stainless docs index`));
+
+        const response = await fetch(
+          `https://api.stainless.com/api/projects/${stainlessProjectName}/docs-sites/${stainlessDocsSiteId}/index`,
+          {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+              Authorization: `Bearer ${stainlessApiKey}`,
+            },
+            body: JSON.stringify({
+              docs_repo_sha: stainlessDocsRepoSha,
+              index: objects,
+            }),
+          },
+        );
+
+        console.log(`docs index API response code ${response.status}: ${await response.text()}`);
+      },
+    },
+  };
+}

package/stl-docs/tabsMiddleware.ts

@@ -58,7 +58,7 @@ function getTabIndexForSlug(
   match: 'exact' | 'prefix';
 } | null {
   // ↓ exact match eg. slug = "/blog" and there is a link containing "/blog"
-  let tab = linksByTab.get(slug);
+  const tab = linksByTab.get(slug)!;
   if (typeof tab === 'string') {
     return {
       match: 'exact',
@@ -88,13 +88,19 @@ function getNonSplitLinksByTab() {
   const linksByTab = new Map<string, string>();
 
   for (let i = 0; i < TABS.length; i++) {
-    const tab = TABS[i];
+    const tab = TABS[i]!;
     linksByTab.set(tab.link, String(i));
   }
 
   return linksByTab;
 }
 
+export interface StarlightRouteWithStlDocs extends StarlightRouteData {
+  _stlDocs?: {
+    activeTabIndex: number;
+  };
+}
+
 export const onRequest = defineRouteMiddleware(async (context) => {
   // if using content collection schema, use: context.locals.starlightRoute.entry.data.stainlessStarlight
   // this worked without collections but relied on hijacking starlightRoute: context.props.frontmatter.stainlessStarlight
@@ -143,7 +149,8 @@ export const onRequest = defineRouteMiddleware(async (context) => {
   }
 
   // We store the active tab index so we can use it in our nav tabs component
-  context.locals.starlightRoute._stlStarlight = {
+  const routeData: StarlightRouteWithStlDocs = context.locals.starlightRoute;
+  routeData._stlDocs = {
     activeTabIndex: activeTabIndex.index,
   };
 
@@ -179,5 +186,6 @@ export const onRequest = defineRouteMiddleware(async (context) => {
 
   matchingGroup?.entries.unshift(...mobileLinks);
 
+  (context.locals._stlStarlightPage ??= {}).fullSidebar = context.locals.starlightRoute.sidebar;
   context.locals.starlightRoute.sidebar = matchingGroup.entries;
 });