@stainless-api/docs 0.1.0-beta.52 → 0.1.0-beta.53

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @stainless-api/docs
 
+## 0.1.0-beta.53
+
+### Minor Changes
+
+- 2b0fec9: Added prose indexing for search
+
+### Patch Changes
+
+- Updated dependencies [2b0fec9]
+  - @stainless-api/docs-ui@0.1.0-beta.44
+
 ## 0.1.0-beta.52
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stainless-api/docs",
-  "version": "0.1.0-beta.52",
+  "version": "0.1.0-beta.53",
   "publishConfig": {
     "access": "public"
   },
@@ -51,8 +51,8 @@
     "unified": "^11.0.5",
     "web-worker": "^1.5.0",
     "yaml": "^2.8.2",
-    "@stainless-api/docs-ui": "0.1.0-beta.43",
-    "@stainless-api/ui-primitives": "0.1.0-beta.32"
+    "@stainless-api/ui-primitives": "0.1.0-beta.32",
+    "@stainless-api/docs-ui": "0.1.0-beta.44"
   },
   "devDependencies": {
     "@astrojs/check": "^0.9.6",
@@ -66,8 +66,8 @@
     "typescript": "5.9.3",
     "vite": "^6.4.1",
     "zod": "^4.1.13",
-    "@stainless/eslint-config": "0.1.0-beta.0",
-    "@stainless/sdk-json": "^0.1.0-beta.0"
+    "@stainless/sdk-json": "^0.1.0-beta.0",
+    "@stainless/eslint-config": "0.1.0-beta.0"
   },
   "scripts": {
     "vendor-deps": "tsx scripts/vendor_deps.ts",

package/stl-docs/index.ts

@@ -22,6 +22,7 @@ import type * as StlDocsVirtualModule from 'virtual:stl-docs-virtual-module';
 import { resolveSrcFile } from '../resolveSrcFile';
 import { stainlessDocsMarkdownRenderer } from './proseMarkdown/proseMarkdownIntegration';
 import { setSharedLogger } from '../shared/getSharedLogger';
+import { stainlessDocsProseIndexing } from './proseSearchIndexing';
 
 export * from '../plugin';
 
@@ -242,5 +243,6 @@ export function stainlessDocs(config: StainlessDocsUserConfig) {
     stainlessDocsStarlightIntegration(normalizedConfig),
     stainlessDocsIntegration(normalizedConfig, apiReferenceBasePath),
     stainlessDocsMarkdownRenderer({ enabled: normalizedConfig.enableProseMarkdownRendering }),
+    stainlessDocsProseIndexing(),
   ];
 }

package/stl-docs/proseSearchIndexing.ts

@@ -0,0 +1,113 @@
+import type { AstroIntegration } from 'astro';
+import { readFile } from 'fs/promises';
+import { getSharedLogger } from '../shared/getSharedLogger';
+import { bold } from '../shared/terminalUtils';
+import { buildProseIndex } from '@stainless-api/docs-ui/search/providers/algolia';
+import * as cheerio from 'cheerio';
+
+function chunkByWords(content: string, chunkSize: number = 30000, chunkOverlap: number = 10) {
+  if (Buffer.byteLength(content) < chunkSize) return [content];
+
+  const words = content.split(/\s+/);
+  const chunks: string[] = [];
+
+  let currentChunk: string[] = [];
+  let currentSize = 0;
+
+  for (const word of words) {
+    const wordSize = Buffer.byteLength(word + ' ', 'utf-8');
+
+    if (currentSize + wordSize > chunkSize && currentChunk.length > 0) {
+      chunks.push(currentChunk.join(' '));
+
+      const overlapStart = Math.max(0, currentChunk.length - chunkOverlap);
+      currentChunk = currentChunk.slice(overlapStart);
+      currentSize = Buffer.byteLength(currentChunk.join(' '), 'utf-8');
+    }
+
+    currentChunk.push(word);
+    currentSize += wordSize;
+  }
+
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk.join(' '));
+  }
+
+  return chunks;
+}
+
+export function* indexHTML(content: string, root: string, pattern: string) {
+  const $ = cheerio.load(content);
+  const matches = $(root).find(pattern);
+
+  for (const match of matches) {
+    const rawText = $(match).text().trim();
+    const chunks = chunkByWords(rawText);
+    const chunkId = crypto.randomUUID();
+
+    for (const [chunkN, content] of chunks.entries()) {
+      yield {
+        id: $(match).attr('id'),
+        tag: match.tagName.toLowerCase(),
+        content,
+        chunk: {
+          id: chunkId,
+          index: chunkN,
+          total: chunks.length,
+        },
+      };
+    }
+  }
+}
+
+const root = 'main';
+const pattern = 'h1, h2, h3, h4, h5, h6, p, li';
+
+export function stainlessDocsProseIndexing(): AstroIntegration {
+  return {
+    name: 'stl-docs-prose-indexing',
+    hooks: {
+      'astro:build:done': async ({ assets, logger: localLogger, dir }) => {
+        const logger = getSharedLogger({ fallback: localLogger });
+        const outputBasePath = dir.pathname;
+
+        const {
+          PUBLIC_ALGOLIA_APP_ID: appId,
+          PUBLIC_ALGOLIA_INDEX: indexName,
+          PRIVATE_ALGOLIA_WRITE_KEY: algoliaWriteKey,
+        } = process.env;
+
+        if (!appId || !indexName || !algoliaWriteKey) {
+          logger.info('Skipping algolia indexing due to missing environment variables');
+          return;
+        }
+
+        const starlightPagePatterns = ['/[...slug]'];
+        const pagesToRender = Array.from(assets.entries())
+          .filter(([k]) => starlightPagePatterns.includes(k))
+          .map(([, v]) => v)
+          .flat()
+          .map((v) => v.pathname);
+
+        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for search`));
+
+        const objects = [];
+        for (const absHtmlPath of pagesToRender) {
+          const content = await readFile(absHtmlPath, 'utf-8');
+          const idx = indexHTML(content, root, pattern);
+          for (const entry of idx)
+            objects.push({
+              ...entry,
+              source: absHtmlPath.slice(outputBasePath.length),
+            });
+        }
+
+        try {
+          await buildProseIndex(appId, `${indexName}-prose`, algoliaWriteKey, objects);
+        } catch (err) {
+          logger.error(`Failed to index prose content: ${err}`);
+        }
+      },
+    },
+  };
+}