@stainless-api/docs 0.1.0-beta.67 → 0.1.0-beta.68

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @stainless-api/docs
 
+## 0.1.0-beta.68
+
+### Patch Changes
+
+- add prose docs vector indexing
+
 ## 0.1.0-beta.67
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stainless-api/docs",
-  "version": "0.1.0-beta.67",
+  "version": "0.1.0-beta.68",
   "publishConfig": {
     "access": "public"
   },

package/stl-docs/index.ts

@@ -22,7 +22,7 @@ import type * as StlDocsVirtualModule from 'virtual:stl-docs-virtual-module';
 import { resolveSrcFile } from '../resolveSrcFile';
 import { stainlessDocsMarkdownRenderer } from './proseMarkdown/proseMarkdownIntegration';
 import { setSharedLogger } from '../shared/getSharedLogger';
-import { stainlessDocsProseIndexing } from './proseSearchIndexing';
+import { stainlessDocsAlgoliaProseIndexing, stainlessDocsVectorProseIndexing } from './proseSearchIndexing';
 
 export * from '../plugin';
 
@@ -271,6 +271,7 @@ export function stainlessDocs(config: StainlessDocsUserConfig) {
       enabled: normalizedConfig.enableProseMarkdownRendering,
       apiReferenceBasePath,
     }),
-    stainlessDocsProseIndexing({ apiReferenceBasePath }),
+    stainlessDocsAlgoliaProseIndexing({ apiReferenceBasePath }),
+    stainlessDocsVectorProseIndexing(normalizedConfig, apiReferenceBasePath),
   ];
 }

package/stl-docs/proseSearchIndexing.ts

@@ -3,8 +3,10 @@ import { readFile } from 'fs/promises';
 import { getProsePages } from '../shared/getProsePages';
 import { getSharedLogger } from '../shared/getSharedLogger';
 import { bold } from '../shared/terminalUtils';
-import { buildProseIndex } from '@stainless-api/docs-search/providers/algolia';
 import * as cheerio from 'cheerio';
+import { toMarkdown } from './proseMarkdown/toMarkdown';
+import { NormalizedStainlessDocsConfig } from './loadStlDocsConfig';
+import { buildProseIndex } from '@stainless-api/docs-search/providers/algolia';
 
 interface ContentBlock {
   type: 'header' | 'content';
@@ -307,7 +309,7 @@ export function* indexHTML(content: string, root: string, pattern: string) {
 const root = 'main';
 const pattern = 'h1, h2, h3, h4, h5, h6, p, li';
 
-export function stainlessDocsProseIndexing({
+export function stainlessDocsAlgoliaProseIndexing({
   apiReferenceBasePath,
 }: {
   apiReferenceBasePath: string | null;
@@ -331,7 +333,7 @@ export function stainlessDocsProseIndexing({
         }
 
         const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
-        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for search`));
+        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for algolia search`));
 
         const objects = [];
         for (const absHtmlPath of pagesToRender) {
@@ -353,3 +355,96 @@ export function stainlessDocsProseIndexing({
     },
   };
 }
+
+export function stainlessDocsVectorProseIndexing(
+  config: NormalizedStainlessDocsConfig,
+  apiReferenceBasePath: string | null,
+): AstroIntegration {
+  return {
+    name: 'stl-docs-prose-indexing',
+    hooks: {
+      'astro:build:done': async ({ logger: localLogger, dir }) => {
+        const logger = getSharedLogger({ fallback: localLogger });
+        const outputBasePath = dir.pathname;
+
+        const stainlessProjectName = config.apiReference?.stainlessProject;
+
+        const {
+          STAINLESS_API_KEY: stainlessApiKey,
+          STAINLESS_DOCS_SITE_ID: stainlessDocsSiteId,
+          STAINLESS_DOCS_REPO_SHA: stainlessDocsRepoSha,
+        } = process.env;
+
+        // Skip indexing if required environment variables are not set
+        if (!stainlessApiKey || !stainlessProjectName || !stainlessDocsSiteId || !stainlessDocsRepoSha) {
+          logger.info(
+            `Skipping vector prose search indexing: required environment/config variables not set, missing: ${[
+              !stainlessApiKey && 'STAINLESS_API_KEY',
+              !stainlessDocsSiteId && 'STAINLESS_DOCS_SITE_ID',
+              !stainlessDocsRepoSha && 'STAINLESS_DOCS_REPO_SHA',
+              !stainlessProjectName && 'stainlessProject in apiReference config',
+            ]
+              .filter(Boolean)
+              .join(', ')}`,
+          );
+          return;
+        }
+
+        const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
+
+        if (pagesToRender.length === 0) {
+          logger.info('No prose pages found to index for vector search');
+          return;
+        }
+
+        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for vector search`));
+
+        const objects: {
+          id: string;
+          tag: string;
+          content: string;
+          kind: 'prose';
+          source: string;
+        }[] = [];
+        for (const absHtmlPath of pagesToRender) {
+          const content = await readFile(absHtmlPath, 'utf-8');
+          const markdown = await toMarkdown(content);
+
+          if (markdown) {
+            const idx = indexMarkdown(markdown);
+            for (const { chunk, ...entry } of idx)
+              objects.push({
+                ...entry,
+                kind: 'prose',
+                source: absHtmlPath.slice(outputBasePath.length),
+              });
+          }
+        }
+
+        if (objects.length === 0) {
+          logger.info('No prose content extracted to index for vector search');
+          return;
+        }
+
+        logger.info(bold(`Uploading ${objects.length} prose content chunks to stainless docs index`));
+
+        const response = await fetch(
+          `https://api.stainless.com/api/projects/${stainlessProjectName}/docs-sites/${stainlessDocsSiteId}/index`,
+          {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+              Authorization: `Bearer ${stainlessApiKey}`,
+            },
+            body: JSON.stringify({
+              docs_repo_sha: stainlessDocsRepoSha,
+              index: objects,
+            }),
+          },
+        );
+
+        console.log(`docs index API response code ${response.status}: ${await response.text()}`);
+      },
+    },
+  };
+}