@stainless-api/docs 0.1.0-beta.102 → 0.1.0-beta.104

package/stl-docs/proseSearchIndexing.ts

@@ -4,15 +4,8 @@ import { getProsePages } from '../shared/getProsePages';
 import { getSharedLogger } from '../shared/getSharedLogger';
 import { bold } from '../shared/terminalUtils';
 import * as cheerio from 'cheerio';
-import { toMarkdown } from './proseMarkdown/toMarkdown';
-import { NormalizedStainlessDocsConfig } from './loadStlDocsConfig';
 import { buildProseIndex } from '@stainless-api/docs-search/providers/algolia';
 
-type ContentBlock =
-  | { type: 'header'; tag: string; id: string; text: string }
-  | { type: 'content'; tag: string; text: string }
-  | { type: 'code'; tag: string; language?: string; text: string };
-
 class SectionContext {
   headers: { level: number; text: string }[] = [];
   headerId: string | undefined;
@@ -40,23 +33,14 @@ class SectionContext {
   }
 }
 
-// Generate a URL-safe ID from header text (e.g., "OpenAPI Config" -> "openapi-config")
 function slugify(text: string): string {
   return text
     .toLowerCase()
-    .replace(/`/g, '') // Remove backticks
-    .replace(/[^a-z0-9]+/g, '-') // Replace non-alphanumeric with hyphens
-    .replace(/^-|-$/g, ''); // Trim leading/trailing hyphens
-}
-
-// Check if a word ends with a real table cell boundary (| but not escaped \|)
-function isTableCellBoundary(word: string): boolean {
-  return word.endsWith('|') && !word.endsWith('\\|');
+    .replace(/`/g, '')
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '');
 }
 
-/**
- * Extracts the header level from a tag like "h1", "h2", etc.
- */
 function getHeaderLevel(tag: string): number {
   const match = tag.match(/^h(\d)$/);
   return match ? parseInt(match[1]!, 10) : 0;
@@ -69,7 +53,6 @@ const MIN_TOKENS = 64;
 const MAX_TOKENS = 256;
 const MIN_WORDS = Math.floor(MIN_TOKENS / TOKENS_PER_WORD); // ~49 words
 const MAX_WORDS = Math.floor(MAX_TOKENS / TOKENS_PER_WORD); // ~197 words
-const LINE_BREAK_WORDS = Math.floor((MAX_TOKENS * 0.75) / TOKENS_PER_WORD); // ~148 words
 const SENTENCE_BREAK_WORDS = Math.floor((MAX_TOKENS * 0.875) / TOKENS_PER_WORD); // ~172 words
 
 /**
@@ -120,254 +103,6 @@ function chunkTextByWords(text: string): string[] {
   return chunks;
 }
 
-type ContentBlockChunk = {
-  type: 'prose';
-  content: string;
-  headerId?: string;
-  headerTag?: string;
-  tag?: string;
-  language?: string;
-  sectionContext?: string;
-};
-
-/**
- * Chunks content blocks into segments of 64-256 tokens.
- *
- * Chunking strategy:
- * 1. Break at headers to keep sections isolated
- * 2. Prefer breaking at line/table boundaries after LINE_BREAK_WORDS (~148 words / ~192 tokens)
- * 3. Break at sentence endings after SENTENCE_BREAK_WORDS (~172 words / ~224 tokens)
- * 4. Force break at MAX_WORDS, preferring table row boundaries if available
- * 5. Section context (header hierarchy) is recorded alongside each chunk for discoverability
- */
-function chunkByWords(blocks: ContentBlock[]): ContentBlockChunk[] {
-  const chunks: ContentBlockChunk[] = [];
-
-  let currentChunk: string[] = [];
-  const ctx = new SectionContext();
-
-  // Flush current chunk to output. If splitAt is provided, keep words after that index for next chunk.
-  const flushChunk = (splitAt?: number) => {
-    if (currentChunk.length === 0) return;
-
-    const wordsToFlush = splitAt !== undefined ? currentChunk.slice(0, splitAt) : currentChunk;
-    const wordsToKeep = splitAt !== undefined ? currentChunk.slice(splitAt) : [];
-
-    if (wordsToFlush.length > 0) {
-      const chunkText = wordsToFlush.join(' ').trim();
-      const sectionContext = ctx.get();
-
-      chunks.push({
-        type: 'prose',
-        content: chunkText,
-        headerId: ctx.headerId,
-        headerTag: ctx.headerTag,
-        sectionContext: sectionContext || undefined,
-      });
-      ctx.hasContent = true;
-    }
-    currentChunk = wordsToKeep;
-  };
-
-  // Find a table row boundary to break at (between MIN_WORDS and current length)
-  // Returns the index to split at, or undefined if no good boundary found
-  const findTableRowBoundary = (): number | undefined => {
-    for (let i = currentChunk.length - 1; i >= MIN_WORDS; i--) {
-      const word = currentChunk[i]!;
-      const nextWord = currentChunk[i + 1];
-      // A row boundary is where one cell ends (|) and the next row starts (|)
-      if (isTableCellBoundary(word) && nextWord?.startsWith('|')) {
-        return i + 1;
-      }
-    }
-    return undefined;
-  };
-
-  for (const block of blocks) {
-    if (block.type === 'header') {
-      flushChunk();
-      ctx.header(block);
-      continue;
-    }
-
-    // Chunk code blocks separately; they tend to be more important.
-    if (block.type === 'code') {
-      flushChunk();
-      const codeText = block.text.trim();
-      if (codeText) {
-        for (const chunkText of chunkTextByWords(codeText)) {
-          chunks.push({
-            type: 'prose',
-            content: chunkText,
-            headerId: ctx.headerId,
-            tag: 'code',
-            language: block.language,
-            sectionContext: ctx.get(),
-          });
-          ctx.hasContent = true;
-        }
-      }
-      continue;
-    }
-
-    if (block.type !== 'content') continue;
-
-    // Split by newlines first to preserve line boundary information
-    const lines = block.text.split(/\n/);
-    let inCodeBlock = false;
-
-    for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
-      const line = lines[lineIdx]!;
-
-      // Track code block boundaries (standalone fences only)
-      if (/^(`{3,}|~{3,})([a-zA-Z0-9+-]*)?\s*$/.test(line.trim())) {
-        inCodeBlock = !inCodeBlock;
-      }
-
-      // Calculate indentation level (number of leading spaces, treating tabs as 2 spaces)
-      const indentMatch = line.match(/^(\s*)/);
-      const indentLevel = indentMatch ? indentMatch[1]!.replace(/\t/g, '  ').length : 0;
-
-      const words = line.split(/\s+/).filter((w) => w.length > 0);
-      const isLastLine = lineIdx === lines.length - 1;
-
-      for (let wordIdx = 0; wordIdx < words.length; wordIdx++) {
-        const word = words[wordIdx]!;
-        const isEndOfLine = wordIdx === words.length - 1 && !isLastLine;
-
-        if (currentChunk.length >= MAX_WORDS) {
-          flushChunk(findTableRowBoundary());
-        }
-
-        currentChunk.push(word);
-
-        // In code blocks, avoid early flushes to keep blocks together
-        // - Light indentation (2+ spaces): require more words before flushing
-        // - Deep indentation (4+ spaces): skip early flushes entirely
-        const inShallowCode = inCodeBlock && indentLevel >= 2 && indentLevel < 4;
-        const inDeepCode = inCodeBlock && indentLevel >= 4;
-
-        // Flush early at natural break points
-        const len = currentChunk.length;
-        const atTableBreak = len >= LINE_BREAK_WORDS && isTableCellBoundary(word);
-        // Shallow code: only flush at sentence threshold; Deep code: don't flush early
-        const lineBreakThreshold = inShallowCode ? SENTENCE_BREAK_WORDS : LINE_BREAK_WORDS;
-        const atLineBreak = len >= lineBreakThreshold && isEndOfLine && !inDeepCode;
-        const atSentenceBreak = len >= SENTENCE_BREAK_WORDS && /[.!?]["']?$/.test(word) && !inDeepCode;
-        if (atTableBreak || atLineBreak || atSentenceBreak) {
-          flushChunk();
-        }
-      }
-    }
-  }
-
-  flushChunk();
-  return chunks;
-}
-
-/**
- * Parses markdown into content blocks, identifying headers, content sections, and code blocks.
- * Code blocks are extracted separately with language metadata for specialized indexing.
- */
-function parseMarkdown(markdown: string): ContentBlock[] {
-  const blocks: ContentBlock[] = [];
-
-  // Extract title from frontmatter and treat it as h1
-  const frontmatterMatch = markdown.match(/^---\r?\n([\s\S]*?)\r?\n---/);
-  if (frontmatterMatch) {
-    const frontmatter = frontmatterMatch[1]!;
-    const titleMatch = frontmatter.match(/^title:\s*(.+)$/m);
-    if (titleMatch) {
-      const title = titleMatch[1]!.trim().replace(/^["']|["']$/g, ''); // Remove quotes if present
-      blocks.push({
-        type: 'header',
-        tag: 'h1',
-        id: slugify(title),
-        text: title,
-      });
-    }
-  }
-
-  // Remove frontmatter
-  const content = markdown.replace(/^---[\s\S]*?---\r?\n*/, '').trim();
-
-  // Split into lines and process
-  const lines = content.split('\n');
-  let currentContent: string[] = [];
-  let inCodeBlock = false;
-  let codeBlockLanguage: string | undefined;
-  let codeBlockContent: string[] = [];
-
-  const flushContent = () => {
-    const text = currentContent.join('\n').trim();
-    if (text) {
-      blocks.push({ type: 'content', tag: 'p', text });
-    }
-    currentContent = [];
-  };
-
-  const flushCodeBlock = () => {
-    if (codeBlockContent.length > 0) {
-      const code = codeBlockContent.join('\n').trim();
-      if (code) {
-        blocks.push({
-          type: 'code',
-          tag: 'code',
-          text: code,
-          language: codeBlockLanguage || undefined,
-        });
-      }
-    }
-    codeBlockContent = [];
-    codeBlockLanguage = undefined;
-  };
-
-  for (const line of lines) {
-    // Track fenced code blocks (``` or ~~~)
-    // Only match standalone markers: ```[language] with nothing else on the line
-    // This avoids matching inline code blocks in table cells like "``` Then content..."
-    const codeBlockMatch = line.match(/^(`{3,}|~{3,})([a-zA-Z0-9+-]*)?\s*$/);
-    if (codeBlockMatch) {
-      if (!inCodeBlock) {
-        flushContent();
-        inCodeBlock = true;
-        codeBlockLanguage = codeBlockMatch[2] || undefined;
-      } else {
-        flushCodeBlock();
-        inCodeBlock = false;
-      }
-      continue;
-    }
-
-    if (inCodeBlock) {
-      codeBlockContent.push(line);
-      continue;
-    }
-
-    // Only match headers outside of code blocks
-    const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
-
-    if (headerMatch) {
-      flushContent();
-      const level = headerMatch[1]!.length;
-      const headerText = headerMatch[2]!.trim();
-      blocks.push({
-        type: 'header',
-        tag: `h${level}`,
-        id: slugify(headerText),
-        text: headerText,
-      });
-      continue;
-    }
-
-    currentContent.push(line);
-  }
-
-  flushCodeBlock();
-  flushContent();
-  return blocks;
-}
-
 export type IndexEntry = {
   chunk: { id: string; index: number; total: number };
   id: string;
@@ -377,31 +112,6 @@ export type IndexEntry = {
   sectionContext?: string;
 };
 
-/**
- * Extracts and chunks markdown content for search indexing.
- * Yields prose and code chunks with section context and language metadata.
- */
-export function* indexMarkdown(markdown: string): Generator<IndexEntry> {
-  const blocks = parseMarkdown(markdown);
-  const chunks = chunkByWords(blocks);
-  const documentId = crypto.randomUUID();
-
-  for (const [index, chunk] of chunks.entries()) {
-    yield {
-      id: chunk.headerId ?? '',
-      tag: chunk.tag ?? chunk.headerTag ?? '',
-      content: chunk.content,
-      ...(chunk.sectionContext ? { sectionContext: chunk.sectionContext } : {}),
-      ...(chunk.language ? { language: chunk.language } : {}),
-      chunk: {
-        id: documentId,
-        index,
-        total: chunks.length,
-      },
-    };
-  }
-}
-
 const DEFAULT_ROOT = 'main';
 const DEFAULT_PATTERN = 'h1, h2, h3, h4, h5, h6, p, li, pre code';
 
@@ -510,97 +220,3 @@ export function stainlessDocsAlgoliaProseIndexing({
     },
   };
 }
-
-export function stainlessDocsVectorProseIndexing(
-  config: NormalizedStainlessDocsConfig,
-  apiReferenceBasePath: string | null,
-): AstroIntegration {
-  return {
-    name: 'stl-docs-prose-indexing',
-    hooks: {
-      'astro:build:done': async ({ logger: localLogger, dir }) => {
-        const logger = getSharedLogger({ fallback: localLogger });
-        const outputBasePath = dir.pathname;
-
-        const stainlessProjectName = config.apiReference?.stainlessProject;
-
-        const {
-          STAINLESS_API_KEY: stainlessApiKey,
-          STAINLESS_DOCS_SITE_ID: stainlessDocsSiteId,
-          STAINLESS_DOCS_REPO_SHA: stainlessDocsRepoSha,
-        } = process.env;
-
-        // Skip indexing if required environment variables are not set
-        if (!stainlessApiKey || !stainlessProjectName || !stainlessDocsSiteId || !stainlessDocsRepoSha) {
-          logger.info(
-            `Skipping vector prose search indexing: required environment/config variables not set, missing: ${[
-              !stainlessApiKey && 'STAINLESS_API_KEY',
-              !stainlessDocsSiteId && 'STAINLESS_DOCS_SITE_ID',
-              !stainlessDocsRepoSha && 'STAINLESS_DOCS_REPO_SHA',
-              !stainlessProjectName && 'stainlessProject in apiReference config',
-            ]
-              .filter(Boolean)
-              .join(', ')}`,
-          );
-          return;
-        }
-
-        const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
-
-        if (pagesToRender.length === 0) {
-          logger.info('No prose pages found to index for vector search');
-          return;
-        }
-
-        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for vector search`));
-
-        const objects: {
-          id: string;
-          tag: string;
-          content: string;
-          language?: string;
-          kind: 'prose';
-          source: string;
-        }[] = [];
-        for (const absHtmlPath of pagesToRender) {
-          const content = await readFile(absHtmlPath, 'utf-8');
-          const markdown = await toMarkdown(content);
-
-          if (markdown) {
-            const idx = indexMarkdown(markdown);
-            for (const { chunk: _, ...entry } of idx)
-              objects.push({
-                ...entry,
-                kind: 'prose',
-                source: absHtmlPath.slice(outputBasePath.length),
-              });
-          }
-        }
-
-        if (objects.length === 0) {
-          logger.info('No prose content extracted to index for vector search');
-          return;
-        }
-
-        logger.info(bold(`Uploading ${objects.length} prose content chunks to stainless docs index`));
-
-        const response = await fetch(
-          `https://api.stainless.com/api/projects/${stainlessProjectName}/docs-sites/${stainlessDocsSiteId}/index`,
-          {
-            method: 'POST',
-            headers: {
-              'Content-Type': 'application/json',
-              Authorization: `Bearer ${stainlessApiKey}`,
-            },
-            body: JSON.stringify({
-              docs_repo_sha: stainlessDocsRepoSha,
-              index: objects,
-            }),
-          },
-        );
-
-        console.log(`docs index API response code ${response.status}: ${await response.text()}`);
-      },
-    },
-  };
-}

package/styles/overrides.css

@@ -54,18 +54,6 @@ mobile-starlight-toc {
   }
 }
 
-@media (min-width: 50rem) {
-  starlight-menu-button button {
-    display: none;
-  }
-
-  mobile-starlight-toc {
-    nav {
-      inset-inline-start: calc(var(--sl-content-inline-start, 0));
-    }
-
-    summary {
-      padding: 2rem 2rem;
-    }
-  }
+starlight-menu-button {
+  display: none;
 }