@stainless-api/docs 0.1.0-beta.52 → 0.1.0-beta.55

package/scripts/vendor_deps.ts

@@ -11,12 +11,12 @@ config();
 
 const envSchema = z.object({
   // absolute path to the Stainless monorepo root
-  STAINLESS_MONOREPO_PATH: z.string(),
+  STAINLESS_ROOT: z.string(),
 });
 
 const env = envSchema.parse(process.env);
 
-console.log(`Using Stainless monorepo at ${env.STAINLESS_MONOREPO_PATH}`);
+console.log(`Using Stainless monorepo at ${env.STAINLESS_ROOT}`);
 
 if (fs.existsSync(DEPS_DIR)) {
   fs.rmSync(DEPS_DIR, { recursive: true });
@@ -25,11 +25,11 @@ if (fs.existsSync(DEPS_DIR)) {
 fs.mkdirSync(DEPS_DIR, { recursive: true });
 
 const previewWorkerPath = path.join(
-  env.STAINLESS_MONOREPO_PATH,
+  env.STAINLESS_ROOT,
   '/packages/preview-worker/dist/stainless/preview.worker.docs.js',
 );
 
-const readmeTemplatePath = path.join(env.STAINLESS_MONOREPO_PATH, '/legacy-dir-root/templates');
+const readmeTemplatePath = path.join(env.STAINLESS_ROOT, '/legacy-dir-root/templates');
 
 fs.copyFileSync(previewWorkerPath, path.join(DEPS_DIR, 'preview.worker.docs.js'));
 

package/stl-docs/index.ts

@@ -22,6 +22,7 @@ import type * as StlDocsVirtualModule from 'virtual:stl-docs-virtual-module';
 import { resolveSrcFile } from '../resolveSrcFile';
 import { stainlessDocsMarkdownRenderer } from './proseMarkdown/proseMarkdownIntegration';
 import { setSharedLogger } from '../shared/getSharedLogger';
+import { stainlessDocsProseIndexing } from './proseSearchIndexing';
 
 export * from '../plugin';
 
@@ -196,7 +197,7 @@ function stainlessDocsIntegration(
       'astro:build:done': ({ dir }) => {
         if (redirects !== null) {
           const stainlessDir = join(dir.pathname, '_stainless');
-          mkdirSync(stainlessDir);
+          mkdirSync(stainlessDir, { recursive: true });
           const outputPath = join(stainlessDir, 'redirects.json');
           writeFileSync(outputPath, JSON.stringify(redirects, null, 2), {
             encoding: 'utf-8',
@@ -242,5 +243,6 @@ export function stainlessDocs(config: StainlessDocsUserConfig) {
     stainlessDocsStarlightIntegration(normalizedConfig),
     stainlessDocsIntegration(normalizedConfig, apiReferenceBasePath),
     stainlessDocsMarkdownRenderer({ enabled: normalizedConfig.enableProseMarkdownRendering }),
+    stainlessDocsProseIndexing(),
   ];
 }

package/stl-docs/proseSearchIndexing.ts

@@ -0,0 +1,113 @@
+import type { AstroIntegration } from 'astro';
+import { readFile } from 'fs/promises';
+import { getSharedLogger } from '../shared/getSharedLogger';
+import { bold } from '../shared/terminalUtils';
+import { buildProseIndex } from '@stainless-api/docs-ui/search/providers/algolia';
+import * as cheerio from 'cheerio';
+
+function chunkByWords(content: string, chunkSize: number = 30000, chunkOverlap: number = 10) {
+  if (Buffer.byteLength(content) < chunkSize) return [content];
+
+  const words = content.split(/\s+/);
+  const chunks: string[] = [];
+
+  let currentChunk: string[] = [];
+  let currentSize = 0;
+
+  for (const word of words) {
+    const wordSize = Buffer.byteLength(word + ' ', 'utf-8');
+
+    if (currentSize + wordSize > chunkSize && currentChunk.length > 0) {
+      chunks.push(currentChunk.join(' '));
+
+      const overlapStart = Math.max(0, currentChunk.length - chunkOverlap);
+      currentChunk = currentChunk.slice(overlapStart);
+      currentSize = Buffer.byteLength(currentChunk.join(' '), 'utf-8');
+    }
+
+    currentChunk.push(word);
+    currentSize += wordSize;
+  }
+
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk.join(' '));
+  }
+
+  return chunks;
+}
+
+export function* indexHTML(content: string, root: string, pattern: string) {
+  const $ = cheerio.load(content);
+  const matches = $(root).find(pattern);
+
+  for (const match of matches) {
+    const rawText = $(match).text().trim();
+    const chunks = chunkByWords(rawText);
+    const chunkId = crypto.randomUUID();
+
+    for (const [chunkN, content] of chunks.entries()) {
+      yield {
+        id: $(match).attr('id'),
+        tag: match.tagName.toLowerCase(),
+        content,
+        chunk: {
+          id: chunkId,
+          index: chunkN,
+          total: chunks.length,
+        },
+      };
+    }
+  }
+}
+
+const root = 'main';
+const pattern = 'h1, h2, h3, h4, h5, h6, p, li';
+
+export function stainlessDocsProseIndexing(): AstroIntegration {
+  return {
+    name: 'stl-docs-prose-indexing',
+    hooks: {
+      'astro:build:done': async ({ assets, logger: localLogger, dir }) => {
+        const logger = getSharedLogger({ fallback: localLogger });
+        const outputBasePath = dir.pathname;
+
+        const {
+          PUBLIC_ALGOLIA_APP_ID: appId,
+          PUBLIC_ALGOLIA_INDEX: indexName,
+          PRIVATE_ALGOLIA_WRITE_KEY: algoliaWriteKey,
+        } = process.env;
+
+        if (!appId || !indexName || !algoliaWriteKey) {
+          logger.info('Skipping algolia indexing due to missing environment variables');
+          return;
+        }
+
+        const starlightPagePatterns = ['/[...slug]'];
+        const pagesToRender = Array.from(assets.entries())
+          .filter(([k]) => starlightPagePatterns.includes(k))
+          .map(([, v]) => v)
+          .flat()
+          .map((v) => v.pathname);
+
+        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for search`));
+
+        const objects = [];
+        for (const absHtmlPath of pagesToRender) {
+          const content = await readFile(absHtmlPath, 'utf-8');
+          const idx = indexHTML(content, root, pattern);
+          for (const entry of idx)
+            objects.push({
+              ...entry,
+              source: absHtmlPath.slice(outputBasePath.length),
+            });
+        }
+
+        try {
+          await buildProseIndex(appId, `${indexName}-prose`, algoliaWriteKey, objects);
+        } catch (err) {
+          logger.error(`Failed to index prose content: ${err}`);
+        }
+      },
+    },
+  };
+}

package/virtual-module.d.ts

@@ -1,3 +1,7 @@
+declare module 'virtual:stl-playground/create' {
+  export const createPlayground: CreatePlayground;
+}
+
 declare module 'virtual:stl-starlight-virtual-module' {
   import type { CreateShikiHighlighterOptions } from '@astrojs/markdown-remark';
   import type { PropertySettingsType } from '@stainless-api/docs-ui/contexts';
@@ -15,6 +19,7 @@ declare module 'virtual:stl-starlight-virtual-module' {
   export const CONTENT_PANEL_LAYOUT: 'double-pane' | 'single-pane';
   export const EXPERIMENTAL_COLLAPSIBLE_SNIPPETS: boolean | undefined;
   export const EXPERIMENTAL_COLLAPSIBLE_METHOD_DESCRIPTIONS: boolean | undefined;
+  export const EXPERIMENTAL_PLAYGROUNDS: boolean | undefined;
   export const PROPERTY_SETTINGS: PropertySettingsType;
   export const MIDDLEWARE: StlStarlightMiddleware;
   export const ENABLE_CONTEXT_MENU: boolean;