@stainless-api/docs 0.1.0-beta.98 → 1.0.0-beta.140

package/stl-docs/proseSearchIndexing.ts

@@ -1,606 +0,0 @@
-import type { AstroIntegration } from 'astro';
-import { readFile } from 'fs/promises';
-import { getProsePages } from '../shared/getProsePages';
-import { getSharedLogger } from '../shared/getSharedLogger';
-import { bold } from '../shared/terminalUtils';
-import * as cheerio from 'cheerio';
-import { toMarkdown } from './proseMarkdown/toMarkdown';
-import { NormalizedStainlessDocsConfig } from './loadStlDocsConfig';
-import { buildProseIndex } from '@stainless-api/docs-search/providers/algolia';
-
-type ContentBlock =
-  | { type: 'header'; tag: string; id: string; text: string }
-  | { type: 'content'; tag: string; text: string }
-  | { type: 'code'; tag: string; language?: string; text: string };
-
-class SectionContext {
-  headers: { level: number; text: string }[] = [];
-  headerId: string | undefined;
-  headerTag: string | undefined;
-  headerText: string | undefined;
-  hasContent = false;
-
-  get(): string | undefined {
-    if (this.headers.length === 0) return;
-    return this.headers.map((h) => h.text).join(' > ');
-  }
-
-  header({ id, tag, text }: { id: string; tag: string; text: string }) {
-    const level = getHeaderLevel(tag);
-    if (level > 0) {
-      while (this.headers.length > 0 && this.headers[this.headers.length - 1]!.level >= level) {
-        this.headers.pop();
-      }
-      this.headers.push({ level, text });
-    }
-    this.headerId = id;
-    this.headerTag = tag;
-    this.headerText = text;
-    this.hasContent = false;
-  }
-}
-
-// Generate a URL-safe ID from header text (e.g., "OpenAPI Config" -> "openapi-config")
-function slugify(text: string): string {
-  return text
-    .toLowerCase()
-    .replace(/`/g, '') // Remove backticks
-    .replace(/[^a-z0-9]+/g, '-') // Replace non-alphanumeric with hyphens
-    .replace(/^-|-$/g, ''); // Trim leading/trailing hyphens
-}
-
-// Check if a word ends with a real table cell boundary (| but not escaped \|)
-function isTableCellBoundary(word: string): boolean {
-  return word.endsWith('|') && !word.endsWith('\\|');
-}
-
-/**
- * Extracts the header level from a tag like "h1", "h2", etc.
- */
-function getHeaderLevel(tag: string): number {
-  const match = tag.match(/^h(\d)$/);
-  return match ? parseInt(match[1]!, 10) : 0;
-}
-
-// Chunking configuration
-// We target 64-256 tokens per chunk, using ~1.3 tokens/word for English text
-const TOKENS_PER_WORD = 1.3;
-const MIN_TOKENS = 64;
-const MAX_TOKENS = 256;
-const MIN_WORDS = Math.floor(MIN_TOKENS / TOKENS_PER_WORD); // ~49 words
-const MAX_WORDS = Math.floor(MAX_TOKENS / TOKENS_PER_WORD); // ~197 words
-const LINE_BREAK_WORDS = Math.floor((MAX_TOKENS * 0.75) / TOKENS_PER_WORD); // ~148 words
-const SENTENCE_BREAK_WORDS = Math.floor((MAX_TOKENS * 0.875) / TOKENS_PER_WORD); // ~172 words
-
-/**
- * Chunks text content into segments of 64-256 tokens using word-based boundaries.
- * Prefers breaking at sentence endings for natural chunk boundaries.
- */
-function chunkTextByWords(text: string): string[] {
-  const words = text.split(/\s+/).filter((w) => w.length > 0);
-
-  if (words.length <= MAX_WORDS) {
-    return words.length > 0 ? [words.join(' ')] : [];
-  }
-
-  const chunks: string[] = [];
-  let currentChunk: string[] = [];
-
-  for (const word of words) {
-    currentChunk.push(word);
-
-    // Force break at max words
-    if (currentChunk.length >= MAX_WORDS) {
-      chunks.push(currentChunk.join(' '));
-      currentChunk = [];
-      continue;
-    }
-
-    // Prefer breaking at sentence boundaries after threshold
-    if (currentChunk.length >= SENTENCE_BREAK_WORDS && /[.!?]["']?$/.test(word)) {
-      chunks.push(currentChunk.join(' '));
-      currentChunk = [];
-    }
-  }
-
-  if (currentChunk.length > 0) {
-    if (currentChunk.length < MIN_WORDS && chunks.length > 0) {
-      const lastChunk = chunks[chunks.length - 1]!;
-      const mergedWords = lastChunk.split(/\s+/).length + currentChunk.length;
-      if (mergedWords <= MAX_WORDS) {
-        chunks[chunks.length - 1] = lastChunk + ' ' + currentChunk.join(' ');
-      } else {
-        chunks.push(currentChunk.join(' '));
-      }
-    } else {
-      chunks.push(currentChunk.join(' '));
-    }
-  }
-
-  return chunks;
-}
-
-type ContentBlockChunk = {
-  type: 'prose';
-  content: string;
-  headerId?: string;
-  headerTag?: string;
-  tag?: string;
-  language?: string;
-  sectionContext?: string;
-};
-
-/**
- * Chunks content blocks into segments of 64-256 tokens.
- *
- * Chunking strategy:
- * 1. Break at headers to keep sections isolated
- * 2. Prefer breaking at line/table boundaries after LINE_BREAK_WORDS (~148 words / ~192 tokens)
- * 3. Break at sentence endings after SENTENCE_BREAK_WORDS (~172 words / ~224 tokens)
- * 4. Force break at MAX_WORDS, preferring table row boundaries if available
- * 5. Section context (header hierarchy) is recorded alongside each chunk for discoverability
- */
-function chunkByWords(blocks: ContentBlock[]): ContentBlockChunk[] {
-  const chunks: ContentBlockChunk[] = [];
-
-  let currentChunk: string[] = [];
-  const ctx = new SectionContext();
-
-  // Flush current chunk to output. If splitAt is provided, keep words after that index for next chunk.
-  const flushChunk = (splitAt?: number) => {
-    if (currentChunk.length === 0) return;
-
-    const wordsToFlush = splitAt !== undefined ? currentChunk.slice(0, splitAt) : currentChunk;
-    const wordsToKeep = splitAt !== undefined ? currentChunk.slice(splitAt) : [];
-
-    if (wordsToFlush.length > 0) {
-      const chunkText = wordsToFlush.join(' ').trim();
-      const sectionContext = ctx.get();
-
-      chunks.push({
-        type: 'prose',
-        content: chunkText,
-        headerId: ctx.headerId,
-        headerTag: ctx.headerTag,
-        sectionContext: sectionContext || undefined,
-      });
-      ctx.hasContent = true;
-    }
-    currentChunk = wordsToKeep;
-  };
-
-  // Find a table row boundary to break at (between MIN_WORDS and current length)
-  // Returns the index to split at, or undefined if no good boundary found
-  const findTableRowBoundary = (): number | undefined => {
-    for (let i = currentChunk.length - 1; i >= MIN_WORDS; i--) {
-      const word = currentChunk[i]!;
-      const nextWord = currentChunk[i + 1];
-      // A row boundary is where one cell ends (|) and the next row starts (|)
-      if (isTableCellBoundary(word) && nextWord?.startsWith('|')) {
-        return i + 1;
-      }
-    }
-    return undefined;
-  };
-
-  for (const block of blocks) {
-    if (block.type === 'header') {
-      flushChunk();
-      ctx.header(block);
-      continue;
-    }
-
-    // Chunk code blocks separately; they tend to be more important.
-    if (block.type === 'code') {
-      flushChunk();
-      const codeText = block.text.trim();
-      if (codeText) {
-        for (const chunkText of chunkTextByWords(codeText)) {
-          chunks.push({
-            type: 'prose',
-            content: chunkText,
-            headerId: ctx.headerId,
-            tag: 'code',
-            language: block.language,
-            sectionContext: ctx.get(),
-          });
-          ctx.hasContent = true;
-        }
-      }
-      continue;
-    }
-
-    if (block.type !== 'content') continue;
-
-    // Split by newlines first to preserve line boundary information
-    const lines = block.text.split(/\n/);
-    let inCodeBlock = false;
-
-    for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
-      const line = lines[lineIdx]!;
-
-      // Track code block boundaries (standalone fences only)
-      if (/^(`{3,}|~{3,})([a-zA-Z0-9+-]*)?\s*$/.test(line.trim())) {
-        inCodeBlock = !inCodeBlock;
-      }
-
-      // Calculate indentation level (number of leading spaces, treating tabs as 2 spaces)
-      const indentMatch = line.match(/^(\s*)/);
-      const indentLevel = indentMatch ? indentMatch[1]!.replace(/\t/g, '  ').length : 0;
-
-      const words = line.split(/\s+/).filter((w) => w.length > 0);
-      const isLastLine = lineIdx === lines.length - 1;
-
-      for (let wordIdx = 0; wordIdx < words.length; wordIdx++) {
-        const word = words[wordIdx]!;
-        const isEndOfLine = wordIdx === words.length - 1 && !isLastLine;
-
-        if (currentChunk.length >= MAX_WORDS) {
-          flushChunk(findTableRowBoundary());
-        }
-
-        currentChunk.push(word);
-
-        // In code blocks, avoid early flushes to keep blocks together
-        // - Light indentation (2+ spaces): require more words before flushing
-        // - Deep indentation (4+ spaces): skip early flushes entirely
-        const inShallowCode = inCodeBlock && indentLevel >= 2 && indentLevel < 4;
-        const inDeepCode = inCodeBlock && indentLevel >= 4;
-
-        // Flush early at natural break points
-        const len = currentChunk.length;
-        const atTableBreak = len >= LINE_BREAK_WORDS && isTableCellBoundary(word);
-        // Shallow code: only flush at sentence threshold; Deep code: don't flush early
-        const lineBreakThreshold = inShallowCode ? SENTENCE_BREAK_WORDS : LINE_BREAK_WORDS;
-        const atLineBreak = len >= lineBreakThreshold && isEndOfLine && !inDeepCode;
-        const atSentenceBreak = len >= SENTENCE_BREAK_WORDS && /[.!?]["']?$/.test(word) && !inDeepCode;
-        if (atTableBreak || atLineBreak || atSentenceBreak) {
-          flushChunk();
-        }
-      }
-    }
-  }
-
-  flushChunk();
-  return chunks;
-}
-
-/**
- * Parses markdown into content blocks, identifying headers, content sections, and code blocks.
- * Code blocks are extracted separately with language metadata for specialized indexing.
- */
-function parseMarkdown(markdown: string): ContentBlock[] {
-  const blocks: ContentBlock[] = [];
-
-  // Extract title from frontmatter and treat it as h1
-  const frontmatterMatch = markdown.match(/^---\r?\n([\s\S]*?)\r?\n---/);
-  if (frontmatterMatch) {
-    const frontmatter = frontmatterMatch[1]!;
-    const titleMatch = frontmatter.match(/^title:\s*(.+)$/m);
-    if (titleMatch) {
-      const title = titleMatch[1]!.trim().replace(/^["']|["']$/g, ''); // Remove quotes if present
-      blocks.push({
-        type: 'header',
-        tag: 'h1',
-        id: slugify(title),
-        text: title,
-      });
-    }
-  }
-
-  // Remove frontmatter
-  const content = markdown.replace(/^---[\s\S]*?---\r?\n*/, '').trim();
-
-  // Split into lines and process
-  const lines = content.split('\n');
-  let currentContent: string[] = [];
-  let inCodeBlock = false;
-  let codeBlockLanguage: string | undefined;
-  let codeBlockContent: string[] = [];
-
-  const flushContent = () => {
-    const text = currentContent.join('\n').trim();
-    if (text) {
-      blocks.push({ type: 'content', tag: 'p', text });
-    }
-    currentContent = [];
-  };
-
-  const flushCodeBlock = () => {
-    if (codeBlockContent.length > 0) {
-      const code = codeBlockContent.join('\n').trim();
-      if (code) {
-        blocks.push({
-          type: 'code',
-          tag: 'code',
-          text: code,
-          language: codeBlockLanguage || undefined,
-        });
-      }
-    }
-    codeBlockContent = [];
-    codeBlockLanguage = undefined;
-  };
-
-  for (const line of lines) {
-    // Track fenced code blocks (``` or ~~~)
-    // Only match standalone markers: ```[language] with nothing else on the line
-    // This avoids matching inline code blocks in table cells like "``` Then content..."
-    const codeBlockMatch = line.match(/^(`{3,}|~{3,})([a-zA-Z0-9+-]*)?\s*$/);
-    if (codeBlockMatch) {
-      if (!inCodeBlock) {
-        flushContent();
-        inCodeBlock = true;
-        codeBlockLanguage = codeBlockMatch[2] || undefined;
-      } else {
-        flushCodeBlock();
-        inCodeBlock = false;
-      }
-      continue;
-    }
-
-    if (inCodeBlock) {
-      codeBlockContent.push(line);
-      continue;
-    }
-
-    // Only match headers outside of code blocks
-    const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
-
-    if (headerMatch) {
-      flushContent();
-      const level = headerMatch[1]!.length;
-      const headerText = headerMatch[2]!.trim();
-      blocks.push({
-        type: 'header',
-        tag: `h${level}`,
-        id: slugify(headerText),
-        text: headerText,
-      });
-      continue;
-    }
-
-    currentContent.push(line);
-  }
-
-  flushCodeBlock();
-  flushContent();
-  return blocks;
-}
-
-export type IndexEntry = {
-  chunk: { id: string; index: number; total: number };
-  id: string;
-  tag: string;
-  content: string;
-  language?: string;
-  sectionContext?: string;
-};
-
-/**
- * Extracts and chunks markdown content for search indexing.
- * Yields prose and code chunks with section context and language metadata.
- */
-export function* indexMarkdown(markdown: string): Generator<IndexEntry> {
-  const blocks = parseMarkdown(markdown);
-  const chunks = chunkByWords(blocks);
-  const documentId = crypto.randomUUID();
-
-  for (const [index, chunk] of chunks.entries()) {
-    yield {
-      id: chunk.headerId ?? '',
-      tag: chunk.tag ?? chunk.headerTag ?? '',
-      content: chunk.content,
-      ...(chunk.sectionContext ? { sectionContext: chunk.sectionContext } : {}),
-      ...(chunk.language ? { language: chunk.language } : {}),
-      chunk: {
-        id: documentId,
-        index,
-        total: chunks.length,
-      },
-    };
-  }
-}
-
-const DEFAULT_ROOT = 'main';
-const DEFAULT_PATTERN = 'h1, h2, h3, h4, h5, h6, p, li, pre code';
-
-/**
- * Indexes HTML content for search, with section context and code language extraction.
- *
- * Features:
- * - Tracks header hierarchy to prepend section context (e.g., "Guide > Setup: ...")
- * - Extracts language metadata from code blocks (class="language-js")
- * - Uses word-based chunking with sentence boundary detection
- */
-export function* indexHTML(
-  content: string,
-  root = DEFAULT_ROOT,
-  pattern = DEFAULT_PATTERN,
-): Generator<IndexEntry> {
-  const $ = cheerio.load(content);
-  const matches = $(root).find(pattern);
-
-  const ctx = new SectionContext();
-
-  for (const match of matches) {
-    const tagName = match.tagName.toLowerCase();
-    const rawText = $(match).text().trim();
-
-    if (getHeaderLevel(tagName) > 0) {
-      ctx.header({ id: $(match).attr('id') ?? slugify(rawText), tag: tagName, text: rawText });
-      continue;
-    }
-
-    // Check if this is a code block and extract language
-    const isCode = tagName === 'code' && $(match).parent().is('pre');
-    let language: string | undefined;
-    if (isCode) {
-      const classes = $(match).attr('class') || '';
-      const langMatch = classes.match(/(?:language-|lang-)([a-zA-Z0-9+-]+)/);
-      language = langMatch ? langMatch[1] : undefined;
-    }
-
-    // Build content with section context
-    const sectionContext = ctx.get();
-    const chunks = chunkTextByWords(rawText);
-    const chunkId = crypto.randomUUID();
-
-    for (const [chunkN, chunkText] of chunks.entries()) {
-      yield {
-        id: ctx.headerId ?? $(match).attr('id') ?? chunkId,
-        tag: isCode ? 'code' : tagName,
-        content: chunkText,
-        ...(sectionContext ? { sectionContext } : {}),
-        ...(language && { language }),
-        chunk: {
-          id: chunkId,
-          index: chunkN,
-          total: chunks.length,
-        },
-      };
-      ctx.hasContent = true;
-    }
-  }
-}
-
-export function stainlessDocsAlgoliaProseIndexing({
-  apiReferenceBasePath,
-}: {
-  apiReferenceBasePath: string | null;
-}): AstroIntegration {
-  return {
-    name: 'stl-docs-prose-indexing',
-    hooks: {
-      'astro:build:done': async ({ logger: localLogger, dir }) => {
-        const logger = getSharedLogger({ fallback: localLogger });
-        const outputBasePath = dir.pathname;
-
-        const {
-          PUBLIC_ALGOLIA_APP_ID: appId,
-          PUBLIC_ALGOLIA_INDEX: indexName,
-          PRIVATE_ALGOLIA_WRITE_KEY: algoliaWriteKey,
-        } = process.env;
-
-        if (!appId || !indexName || !algoliaWriteKey) {
-          logger.info('Skipping algolia indexing due to missing environment variables');
-          return;
-        }
-
-        const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
-        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for algolia search`));
-
-        const objects = [];
-        for (const absHtmlPath of pagesToRender) {
-          const content = await readFile(absHtmlPath, 'utf-8');
-          const idx = indexHTML(content);
-          for (const entry of idx)
-            objects.push({
-              ...entry,
-              source: absHtmlPath.slice(outputBasePath.length),
-            });
-        }
-
-        try {
-          await buildProseIndex(appId, `${indexName}-prose`, algoliaWriteKey, objects);
-        } catch (err) {
-          logger.error(`Failed to index prose content: ${err}`);
-        }
-      },
-    },
-  };
-}
-
-export function stainlessDocsVectorProseIndexing(
-  config: NormalizedStainlessDocsConfig,
-  apiReferenceBasePath: string | null,
-): AstroIntegration {
-  return {
-    name: 'stl-docs-prose-indexing',
-    hooks: {
-      'astro:build:done': async ({ logger: localLogger, dir }) => {
-        const logger = getSharedLogger({ fallback: localLogger });
-        const outputBasePath = dir.pathname;
-
-        const stainlessProjectName = config.apiReference?.stainlessProject;
-
-        const {
-          STAINLESS_API_KEY: stainlessApiKey,
-          STAINLESS_DOCS_SITE_ID: stainlessDocsSiteId,
-          STAINLESS_DOCS_REPO_SHA: stainlessDocsRepoSha,
-        } = process.env;
-
-        // Skip indexing if required environment variables are not set
-        if (!stainlessApiKey || !stainlessProjectName || !stainlessDocsSiteId || !stainlessDocsRepoSha) {
-          logger.info(
-            `Skipping vector prose search indexing: required environment/config variables not set, missing: ${[
-              !stainlessApiKey && 'STAINLESS_API_KEY',
-              !stainlessDocsSiteId && 'STAINLESS_DOCS_SITE_ID',
-              !stainlessDocsRepoSha && 'STAINLESS_DOCS_REPO_SHA',
-              !stainlessProjectName && 'stainlessProject in apiReference config',
-            ]
-              .filter(Boolean)
-              .join(', ')}`,
-          );
-          return;
-        }
-
-        const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
-
-        if (pagesToRender.length === 0) {
-          logger.info('No prose pages found to index for vector search');
-          return;
-        }
-
-        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for vector search`));
-
-        const objects: {
-          id: string;
-          tag: string;
-          content: string;
-          language?: string;
-          kind: 'prose';
-          source: string;
-        }[] = [];
-        for (const absHtmlPath of pagesToRender) {
-          const content = await readFile(absHtmlPath, 'utf-8');
-          const markdown = await toMarkdown(content);
-
-          if (markdown) {
-            const idx = indexMarkdown(markdown);
-            for (const { chunk: _, ...entry } of idx)
-              objects.push({
-                ...entry,
-                kind: 'prose',
-                source: absHtmlPath.slice(outputBasePath.length),
-              });
-          }
-        }
-
-        if (objects.length === 0) {
-          logger.info('No prose content extracted to index for vector search');
-          return;
-        }
-
-        logger.info(bold(`Uploading ${objects.length} prose content chunks to stainless docs index`));
-
-        const response = await fetch(
-          `https://api.stainless.com/api/projects/${stainlessProjectName}/docs-sites/${stainlessDocsSiteId}/index`,
-          {
-            method: 'POST',
-            headers: {
-              'Content-Type': 'application/json',
-              Authorization: `Bearer ${stainlessApiKey}`,
-            },
-            body: JSON.stringify({
-              docs_repo_sha: stainlessDocsRepoSha,
-              index: objects,
-            }),
-          },
-        );
-
-        console.log(`docs index API response code ${response.status}: ${await response.text()}`);
-      },
-    },
-  };
-}