@stainless-api/docs 0.1.0-beta.99 → 1.0.0-beta.140

package/stl-docs/proseDocSync.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, it } from 'vitest';
+import { batchBySize, MAX_BATCH_BYTES, MAX_BATCH_DOCS } from './proseDocSync';
+
+function makeDoc(docId: string, sizeBytes: number) {
+  return {
+    docId,
+    content: Buffer.alloc(sizeBytes, 'x'),
+    sha256: 'fake',
+    source: `/${docId}`,
+  };
+}
+
+describe('batchBySize', () => {
+  it('returns empty array for no docs', () => {
+    expect(batchBySize([])).toEqual([]);
+  });
+
+  it('puts all docs in one batch when under both limits', () => {
+    const docs = Array.from({ length: 5 }, (_, i) => makeDoc(`doc-${i}`, 100));
+    const batches = batchBySize(docs);
+    expect(batches).toHaveLength(1);
+    expect(batches[0]).toHaveLength(5);
+  });
+
+  it('splits at MAX_BATCH_DOCS', () => {
+    const docs = Array.from({ length: MAX_BATCH_DOCS + 10 }, (_, i) => makeDoc(`doc-${i}`, 10));
+    const batches = batchBySize(docs);
+    expect(batches).toHaveLength(2);
+    expect(batches[0]).toHaveLength(MAX_BATCH_DOCS);
+    expect(batches[1]).toHaveLength(10);
+  });
+
+  it('splits when cumulative size exceeds MAX_BATCH_BYTES', () => {
+    const docSize = 10 * 1024 * 1024; // 10MB each
+    // 4 docs = 40MB > 30MB limit, so should split into [3, 1]
+    const docs = Array.from({ length: 4 }, (_, i) => makeDoc(`doc-${i}`, docSize));
+    const batches = batchBySize(docs);
+    expect(batches).toHaveLength(2);
+    expect(batches[0]).toHaveLength(3);
+    expect(batches[1]).toHaveLength(1);
+  });
+
+  it('handles a single doc larger than MAX_BATCH_BYTES', () => {
+    const docs = [makeDoc('huge', MAX_BATCH_BYTES + 1)];
+    const batches = batchBySize(docs);
+    // Still goes into a batch on its own — we can't split a single doc
+    expect(batches).toHaveLength(1);
+    expect(batches[0]).toHaveLength(1);
+  });
+
+  it('byte limit takes precedence over doc count when hit first', () => {
+    // 2 docs of 20MB each = 40MB > 30MB limit, well under 100 doc limit
+    const docs = [makeDoc('a', 20 * 1024 * 1024), makeDoc('b', 20 * 1024 * 1024)];
+    const batches = batchBySize(docs);
+    expect(batches).toHaveLength(2);
+    expect(batches[0]).toHaveLength(1);
+    expect(batches[1]).toHaveLength(1);
+  });
+
+  it('creates multiple batches for many large docs', () => {
+    const docSize = 8 * 1024 * 1024; // 8MB each
+    // 10 docs * 8MB = 80MB, should split into batches of ~3 (24MB each)
+    const docs = Array.from({ length: 10 }, (_, i) => makeDoc(`doc-${i}`, docSize));
+    const batches = batchBySize(docs);
+    // Every batch should be <= MAX_BATCH_BYTES
+    for (const batch of batches) {
+      const totalBytes = batch.reduce((sum, d) => sum + d.content.byteLength, 0);
+      expect(totalBytes).toBeLessThanOrEqual(MAX_BATCH_BYTES);
+    }
+    // All docs should be accounted for
+    const totalDocs = batches.reduce((sum, b) => sum + b.length, 0);
+    expect(totalDocs).toBe(10);
+  });
+});

package/stl-docs/proseDocSync.ts

@@ -0,0 +1,344 @@
+import type { AstroIntegration, AstroIntegrationLogger } from 'astro';
+import crypto from 'crypto';
+import { readFile } from 'fs/promises';
+import path from 'path';
+import { getProsePages } from '../shared/getProsePages';
+import { getSharedLogger } from '../shared/getSharedLogger';
+import { bold } from '../shared/terminalUtils';
+import { NormalizedStainlessDocsConfig } from './loadStlDocsConfig';
+
+const DOCS_API_BASE_URL = 'https://api.stainlessapi.com';
+
+export const MAX_BATCH_DOCS = 100;
+// Cloud Run has a 32MB request body limit; leave headroom for JSON envelope overhead
+export const MAX_BATCH_BYTES = 30 * 1024 * 1024;
+
+// ─── API client ──────────────────────────────────────────────────────
+
+async function docsApiRequest(
+  method: string,
+  apiPath: string,
+  apiKey: string,
+  body?: object,
+): Promise<Response> {
+  const headers: Record<string, string> = {
+    Authorization: `Bearer ${apiKey}`,
+  };
+  if (body) {
+    headers['Content-Type'] = 'application/json';
+  }
+  return fetch(`${DOCS_API_BASE_URL}${apiPath}`, {
+    method,
+    headers,
+    ...(body ? { body: JSON.stringify(body) } : {}),
+  });
+}
+
+// ─── Manifest & diffing ─────────────────────────────────────────────
+
+type LocalDoc = { content: Buffer; sha256: string; source: string };
+
+function sha256(content: Buffer): string {
+  return crypto.createHash('sha256').update(content).digest('hex');
+}
+
+async function buildLocalManifest(pages: string[], outputBasePath: string): Promise<Map<string, LocalDoc>> {
+  const docs = new Map<string, LocalDoc>();
+  for (const absHtmlPath of pages) {
+    const content = await readFile(absHtmlPath);
+    const docId = path.relative(outputBasePath, absHtmlPath);
+    docs.set(docId, { content, sha256: sha256(content), source: '/' + docId });
+  }
+  return docs;
+}
+
+async function fetchRemoteManifest(
+  docsSiteId: string,
+  project: string,
+  apiKey: string,
+  logger: AstroIntegrationLogger,
+): Promise<Map<string, string>> {
+  try {
+    const response = await docsApiRequest(
+      'GET',
+      `/api/docs-sites/${docsSiteId}/documents?project=${encodeURIComponent(project)}`,
+      apiKey,
+    );
+
+    if (response.ok) {
+      const data = (await response.json()) as {
+        documents: { id: string; content_sha256: string }[];
+      };
+      return new Map(data.documents.map((d) => [d.id, d.content_sha256]));
+    }
+
+    logger.error(`Failed to list remote documents (HTTP ${response.status}): ${await response.text()}`);
+  } catch (err) {
+    logger.error(`Failed to list remote documents: ${err}`);
+  }
+  return new Map();
+}
+
+function diffManifests(
+  localDocs: Map<string, LocalDoc>,
+  remoteDocs: Map<string, string>,
+): { toPut: (LocalDoc & { docId: string })[]; toDelete: string[] } {
+  const toPut: (LocalDoc & { docId: string })[] = [];
+  for (const [docId, local] of localDocs) {
+    if (remoteDocs.get(docId) !== local.sha256) {
+      toPut.push({ docId, ...local });
+    }
+  }
+
+  const toDelete: string[] = [];
+  for (const remoteDocId of remoteDocs.keys()) {
+    if (!localDocs.has(remoteDocId)) {
+      toDelete.push(remoteDocId);
+    }
+  }
+
+  return { toPut, toDelete };
+}
+
+// ─── Batching ────────────────────────────────────────────────────────
+
+export function batchBySize(docs: (LocalDoc & { docId: string })[]): (LocalDoc & { docId: string })[][] {
+  const batches: (LocalDoc & { docId: string })[][] = [];
+  let current: (LocalDoc & { docId: string })[] = [];
+  let currentBytes = 0;
+
+  for (const doc of docs) {
+    const docBytes = doc.content.byteLength;
+
+    if (
+      current.length > 0 &&
+      (current.length >= MAX_BATCH_DOCS || currentBytes + docBytes > MAX_BATCH_BYTES)
+    ) {
+      batches.push(current);
+      current = [];
+      currentBytes = 0;
+    }
+
+    current.push(doc);
+    currentBytes += docBytes;
+  }
+
+  if (current.length > 0) batches.push(current);
+  return batches;
+}
+
+// ─── Import & delete ────────────────────────────────────────────────
+
+type ImportJobResult = { succeeded: number; failed: number; errors: string[] };
+
+async function importDocuments(
+  docs: (LocalDoc & { docId: string })[],
+  docsSiteId: string,
+  project: string,
+  apiKey: string,
+  logger: AstroIntegrationLogger,
+): Promise<ImportJobResult> {
+  const totals: ImportJobResult = { succeeded: 0, failed: 0, errors: [] };
+  const batches = batchBySize(docs);
+
+  for (const batch of batches) {
+    let response: Response;
+    try {
+      response = await docsApiRequest('POST', `/api/docs-sites/${docsSiteId}/documents/import`, apiKey, {
+        project,
+        documents: batch.map(({ docId, content, source }) => ({
+          id: docId,
+          content: content.toString('utf-8'),
+          content_type: 'text/html',
+          source,
+        })),
+      });
+    } catch (err) {
+      logger.error(`Failed to submit import batch: ${err}`);
+      totals.failed += batch.length;
+      continue;
+    }
+
+    if (!response.ok) {
+      logger.error(`Failed to submit import batch (HTTP ${response.status}): ${await response.text()}`);
+      totals.failed += batch.length;
+      continue;
+    }
+
+    const { job_id } = (await response.json()) as { job_id: string };
+    const result = await pollImportJob(docsSiteId, job_id, apiKey, logger);
+    totals.succeeded += result.succeeded;
+    totals.failed += result.failed;
+    totals.errors.push(...result.errors);
+  }
+
+  return totals;
+}
+
+async function pollImportJob(
+  docsSiteId: string,
+  jobId: string,
+  apiKey: string,
+  logger: AstroIntegrationLogger,
+): Promise<ImportJobResult> {
+  const maxWait = 5 * 60_000;
+  const start = Date.now();
+
+  while (Date.now() - start < maxWait) {
+    await new Promise((r) => setTimeout(r, 2_000));
+
+    let response: Response;
+    try {
+      response = await docsApiRequest('GET', `/api/docs-sites/${docsSiteId}/documents/jobs/${jobId}`, apiKey);
+    } catch (err) {
+      logger.error(`Failed to poll import job ${jobId}: ${err}`);
+      continue;
+    }
+
+    if (!response.ok) {
+      logger.error(`Failed to poll import job ${jobId} (HTTP ${response.status}): ${await response.text()}`);
+      continue;
+    }
+
+    const job = (await response.json()) as {
+      status: string;
+      succeeded: number;
+      failed: number;
+      errors: string[] | null;
+    };
+
+    if (job.status === 'queued' || job.status === 'processing') continue;
+
+    return { succeeded: job.succeeded, failed: job.failed, errors: job.errors ?? [] };
+  }
+
+  logger.error(`Import job ${jobId} timed out after ${maxWait / 1000}s`);
+  return { succeeded: 0, failed: 0, errors: [`Job ${jobId} timed out`] };
+}
+
+async function deleteDocuments(
+  docIds: string[],
+  docsSiteId: string,
+  project: string,
+  apiKey: string,
+  logger: AstroIntegrationLogger,
+): Promise<{ succeeded: number; failed: number }> {
+  let succeeded = 0;
+  let failed = 0;
+
+  await Promise.all(
+    docIds.map(async (docId) => {
+      try {
+        const response = await docsApiRequest(
+          'DELETE',
+          `/api/docs-sites/${docsSiteId}/documents?documentId=${encodeURIComponent(docId)}&project=${encodeURIComponent(project)}`,
+          apiKey,
+        );
+        if (response.ok) {
+          succeeded++;
+        } else {
+          logger.error(`Failed to delete ${docId} (HTTP ${response.status}): ${await response.text()}`);
+          failed++;
+        }
+      } catch (err) {
+        logger.error(`Failed to delete ${docId}: ${err}`);
+        failed++;
+      }
+    }),
+  );
+
+  return { succeeded, failed };
+}
+
+// ─── Sync orchestrator ──────────────────────────────────────────────
+
+async function syncProseDocuments(opts: {
+  docsSiteId: string;
+  project: string;
+  apiKey: string;
+  pages: string[];
+  outputBasePath: string;
+  logger: AstroIntegrationLogger;
+}) {
+  const { docsSiteId, project, apiKey, pages, outputBasePath, logger } = opts;
+
+  logger.info(bold(`Syncing ${pages.length} prose pages to docs search index`));
+
+  const localDocs = await buildLocalManifest(pages, outputBasePath);
+  const remoteDocs = await fetchRemoteManifest(docsSiteId, project, apiKey, logger);
+  const { toPut, toDelete } = diffManifests(localDocs, remoteDocs);
+
+  const unchanged = localDocs.size - toPut.length;
+  logger.info(bold(`${toPut.length} to upload, ${toDelete.length} to delete, ${unchanged} unchanged`));
+
+  if (toPut.length === 0 && toDelete.length === 0) {
+    logger.info('Docs search index is up to date');
+    return;
+  }
+
+  const uploaded =
+    toPut.length > 0
+      ? await importDocuments(toPut, docsSiteId, project, apiKey, logger)
+      : { succeeded: 0, failed: 0, errors: [] as string[] };
+
+  const deleted =
+    toDelete.length > 0
+      ? await deleteDocuments(toDelete, docsSiteId, project, apiKey, logger)
+      : { succeeded: 0, failed: 0 };
+
+  for (const err of uploaded.errors) {
+    logger.error(`Import error: ${err}`);
+  }
+
+  const failures = uploaded.failed + deleted.failed;
+  if (failures > 0) {
+    logger.error(
+      `Docs search index sync completed with ${failures} error(s): ${uploaded.succeeded} uploaded, ${deleted.succeeded} deleted`,
+    );
+  } else {
+    logger.info(
+      bold(`Docs search index synced: ${uploaded.succeeded} uploaded, ${deleted.succeeded} deleted`),
+    );
+  }
+}
+
+// ─── Astro integration ──────────────────────────────────────────────
+
+export function stainlessDocsVectorProseIndexing(
+  config: NormalizedStainlessDocsConfig,
+  apiReferenceBasePath: string | null,
+): AstroIntegration {
+  return {
+    name: 'stl-docs-prose-indexing',
+    hooks: {
+      'astro:build:done': async ({ logger: localLogger, dir }) => {
+        const logger = getSharedLogger({ fallback: localLogger });
+        const outputBasePath = dir.pathname;
+
+        const project = config.apiReference?.stainlessProject;
+        const { STAINLESS_API_KEY: apiKey, STAINLESS_DOCS_SITE_ID: docsSiteId } = process.env;
+
+        if (!apiKey || !project || !docsSiteId) {
+          logger.info(
+            `Skipping vector prose search indexing: required environment/config variables not set, missing: ${[
+              !apiKey && 'STAINLESS_API_KEY',
+              !docsSiteId && 'STAINLESS_DOCS_SITE_ID',
+              !project && 'stainlessProject in apiReference config',
+            ]
+              .filter(Boolean)
+              .join(', ')}`,
+          );
+          return;
+        }
+
+        const pages = await getProsePages({ apiReferenceBasePath, outputBasePath });
+        if (pages.length === 0) {
+          logger.info('No prose pages found to index for vector search');
+          return;
+        }
+
+        await syncProseDocuments({ docsSiteId, project, apiKey, pages, outputBasePath, logger });
+      },
+    },
+  };
+}

package/stl-docs/proseMarkdown/proseMarkdownIntegration.ts

@@ -7,29 +7,21 @@ import { bold } from '../../shared/terminalUtils';
 import { getProsePages } from '../../shared/getProsePages';
 
 export function stainlessDocsMarkdownRenderer({
-  enabled,
   apiReferenceBasePath,
 }: {
-  enabled: boolean;
   apiReferenceBasePath: string | null;
 }): AstroIntegration {
   return {
     name: 'stl-docs-md',
     hooks: {
       'astro:config:setup': ({ addMiddleware }) => {
-        if (enabled) {
-          addMiddleware({
-            entrypoint: resolveSrcFile('/stl-docs/proseMarkdown/proseMarkdownMiddleware.ts'),
-            order: 'post',
-          });
-        }
+        addMiddleware({
+          entrypoint: resolveSrcFile('/stl-docs/proseMarkdown/proseMarkdownMiddleware.ts'),
+          order: 'post',
+        });
       },
       'astro:build:done': async ({ logger: localLogger, dir }) => {
         const logger = getSharedLogger({ fallback: localLogger });
-        if (!enabled) {
-          logger.info('Stainless Docs prose Markdown rendering is disabled, skipping...');
-          return;
-        }
         const outputBasePath = dir.pathname;
         const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
 

package/stl-docs/schema-extension.ts

@@ -0,0 +1,12 @@
+import { z } from 'astro/zod';
+
+export const stainlessDocsSchemaExtension = z.object({
+  ogImageOptions: z
+    .object({
+      logo: z.string().optional(),
+      theme: z.enum(['light', 'dark']).default('light'),
+      title: z.string().optional(),
+      description: z.string().optional(),
+    })
+    .optional(),
+});

package/stl-docs/tabsMiddleware.ts

@@ -102,7 +102,7 @@ export interface StarlightRouteWithStlDocs extends StarlightRouteData {
   };
 }
 
-export const onRequest = defineRouteMiddleware(async (context) => {
+export const onRequest = defineRouteMiddleware((context) => {
   // if using content collection schema, use: context.locals.starlightRoute.entry.data.stainlessStarlight
   // this worked without collections but relied on hijacking starlightRoute: context.props.frontmatter.stainlessStarlight
 

package/styles/overrides.css

@@ -54,18 +54,6 @@ mobile-starlight-toc {
   }
 }
 
-@media (min-width: 50rem) {
-  starlight-menu-button button {
-    display: none;
-  }
-
-  mobile-starlight-toc {
-    nav {
-      inset-inline-start: calc(var(--sl-content-inline-start, 0));
-    }
-
-    summary {
-      padding: 2rem 2rem;
-    }
-  }
+starlight-menu-button {
+  display: none;
 }