@stainless-api/docs 0.1.0-beta.66 → 0.1.0-beta.68

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @stainless-api/docs
 
+## 0.1.0-beta.68
+
+### Patch Changes
+
+- add prose docs vector indexing
+
+## 0.1.0-beta.67
+
+### Patch Changes
+
+- a59d5f7: Pass current page context through as “intent”
+- Updated dependencies [e9567b0]
+  - @stainless-api/ui-primitives@0.1.0-beta.40
+  - @stainless-api/docs-ui@0.1.0-beta.53
+  - @stainless-api/docs-search@0.1.0-beta.5
+
 ## 0.1.0-beta.66
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stainless-api/docs",
-  "version": "0.1.0-beta.66",
+  "version": "0.1.0-beta.68",
   "publishConfig": {
     "access": "public"
   },
@@ -53,9 +53,9 @@
     "vite-plugin-prebundle-workers": "^0.2.0",
     "web-worker": "^1.5.0",
     "yaml": "^2.8.2",
-    "@stainless-api/docs-search": "0.1.0-beta.4",
-    "@stainless-api/docs-ui": "0.1.0-beta.52",
-    "@stainless-api/ui-primitives": "0.1.0-beta.39"
+    "@stainless-api/docs-search": "0.1.0-beta.5",
+    "@stainless-api/docs-ui": "0.1.0-beta.53",
+    "@stainless-api/ui-primitives": "0.1.0-beta.40"
   },
   "devDependencies": {
     "@astrojs/check": "^0.9.6",

package/stl-docs/components/AiChatIsland.tsx

@@ -1,8 +1,14 @@
 import type { DocsLanguage } from '@stainless-api/docs-ui/routing';
 import AiChat, { STAINLESS_PROJECT } from 'virtual:stl-docs/components/AiChat.tsx';
 
-export default function AiChatIsland({ currentLanguage }: { currentLanguage: DocsLanguage | undefined }) {
+export default function AiChatIsland({
+  currentLanguage,
+  siteTitle,
+}: {
+  currentLanguage: DocsLanguage | undefined;
+  siteTitle: string | undefined;
+}) {
   if (!AiChat) throw new Error('AiChatIsland was rendered but could not load AiChat component');
   if (!STAINLESS_PROJECT) return null;
-  return <AiChat projectId={STAINLESS_PROJECT} language={currentLanguage} />;
+  return <AiChat projectId={STAINLESS_PROJECT} language={currentLanguage} siteTitle={siteTitle} />;
 }

package/stl-docs/components/PageFrame.astro

@@ -2,6 +2,10 @@
 import Default from '@astrojs/starlight/components/PageFrame.astro';
 import AiChat from 'virtual:stl-docs/components/AiChat.tsx'; // conditionally resolves to null if ai chat module is not injected
 import AiChatIsland from './AiChatIsland.tsx'; // entrypoint for client island can’t be a virtual module
+
+import starlightConfig from 'virtual:starlight/user-config';
+const locale = Astro.currentLocale ?? starlightConfig.defaultLocale.lang;
+const siteTitle = locale && starlightConfig.title[locale];
 ---
 
 <Default>
@@ -10,5 +14,5 @@ import AiChatIsland from './AiChatIsland.tsx'; // entrypoint for client island c
 
   <slot />
 
-  {!!AiChat && <AiChatIsland client:load currentLanguage={Astro.locals.language} />}
+  {!!AiChat && <AiChatIsland client:load currentLanguage={Astro.locals.language} siteTitle={siteTitle} />}
 </Default>

package/stl-docs/index.ts

@@ -22,7 +22,7 @@ import type * as StlDocsVirtualModule from 'virtual:stl-docs-virtual-module';
 import { resolveSrcFile } from '../resolveSrcFile';
 import { stainlessDocsMarkdownRenderer } from './proseMarkdown/proseMarkdownIntegration';
 import { setSharedLogger } from '../shared/getSharedLogger';
-import { stainlessDocsProseIndexing } from './proseSearchIndexing';
+import { stainlessDocsAlgoliaProseIndexing, stainlessDocsVectorProseIndexing } from './proseSearchIndexing';
 
 export * from '../plugin';
 
@@ -271,6 +271,7 @@ export function stainlessDocs(config: StainlessDocsUserConfig) {
       enabled: normalizedConfig.enableProseMarkdownRendering,
       apiReferenceBasePath,
     }),
-    stainlessDocsProseIndexing({ apiReferenceBasePath }),
+    stainlessDocsAlgoliaProseIndexing({ apiReferenceBasePath }),
+    stainlessDocsVectorProseIndexing(normalizedConfig, apiReferenceBasePath),
   ];
 }

package/stl-docs/proseSearchIndexing.ts

@@ -3,8 +3,10 @@ import { readFile } from 'fs/promises';
 import { getProsePages } from '../shared/getProsePages';
 import { getSharedLogger } from '../shared/getSharedLogger';
 import { bold } from '../shared/terminalUtils';
-import { buildProseIndex } from '@stainless-api/docs-search/providers/algolia';
 import * as cheerio from 'cheerio';
+import { toMarkdown } from './proseMarkdown/toMarkdown';
+import { NormalizedStainlessDocsConfig } from './loadStlDocsConfig';
+import { buildProseIndex } from '@stainless-api/docs-search/providers/algolia';
 
 interface ContentBlock {
   type: 'header' | 'content';
@@ -307,7 +309,7 @@ export function* indexHTML(content: string, root: string, pattern: string) {
 const root = 'main';
 const pattern = 'h1, h2, h3, h4, h5, h6, p, li';
 
-export function stainlessDocsProseIndexing({
+export function stainlessDocsAlgoliaProseIndexing({
   apiReferenceBasePath,
 }: {
   apiReferenceBasePath: string | null;
@@ -331,7 +333,7 @@ export function stainlessDocsProseIndexing({
         }
 
         const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
-        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for search`));
+        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for algolia search`));
 
         const objects = [];
         for (const absHtmlPath of pagesToRender) {
@@ -353,3 +355,96 @@ export function stainlessDocsProseIndexing({
     },
   };
 }
+
+export function stainlessDocsVectorProseIndexing(
+  config: NormalizedStainlessDocsConfig,
+  apiReferenceBasePath: string | null,
+): AstroIntegration {
+  return {
+    name: 'stl-docs-prose-indexing',
+    hooks: {
+      'astro:build:done': async ({ logger: localLogger, dir }) => {
+        const logger = getSharedLogger({ fallback: localLogger });
+        const outputBasePath = dir.pathname;
+
+        const stainlessProjectName = config.apiReference?.stainlessProject;
+
+        const {
+          STAINLESS_API_KEY: stainlessApiKey,
+          STAINLESS_DOCS_SITE_ID: stainlessDocsSiteId,
+          STAINLESS_DOCS_REPO_SHA: stainlessDocsRepoSha,
+        } = process.env;
+
+        // Skip indexing if required environment variables are not set
+        if (!stainlessApiKey || !stainlessProjectName || !stainlessDocsSiteId || !stainlessDocsRepoSha) {
+          logger.info(
+            `Skipping vector prose search indexing: required environment/config variables not set, missing: ${[
+              !stainlessApiKey && 'STAINLESS_API_KEY',
+              !stainlessDocsSiteId && 'STAINLESS_DOCS_SITE_ID',
+              !stainlessDocsRepoSha && 'STAINLESS_DOCS_REPO_SHA',
+              !stainlessProjectName && 'stainlessProject in apiReference config',
+            ]
+              .filter(Boolean)
+              .join(', ')}`,
+          );
+          return;
+        }
+
+        const pagesToRender = await getProsePages({ apiReferenceBasePath, outputBasePath });
+
+        if (pagesToRender.length === 0) {
+          logger.info('No prose pages found to index for vector search');
+          return;
+        }
+
+        logger.info(bold(`Indexing ${pagesToRender.length} prose pages for vector search`));
+
+        const objects: {
+          id: string;
+          tag: string;
+          content: string;
+          kind: 'prose';
+          source: string;
+        }[] = [];
+        for (const absHtmlPath of pagesToRender) {
+          const content = await readFile(absHtmlPath, 'utf-8');
+          const markdown = await toMarkdown(content);
+
+          if (markdown) {
+            const idx = indexMarkdown(markdown);
+            for (const { chunk, ...entry } of idx)
+              objects.push({
+                ...entry,
+                kind: 'prose',
+                source: absHtmlPath.slice(outputBasePath.length),
+              });
+          }
+        }
+
+        if (objects.length === 0) {
+          logger.info('No prose content extracted to index for vector search');
+          return;
+        }
+
+        logger.info(bold(`Uploading ${objects.length} prose content chunks to stainless docs index`));
+
+        const response = await fetch(
+          `https://api.stainless.com/api/projects/${stainlessProjectName}/docs-sites/${stainlessDocsSiteId}/index`,
+          {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+              Authorization: `Bearer ${stainlessApiKey}`,
+            },
+            body: JSON.stringify({
+              docs_repo_sha: stainlessDocsRepoSha,
+              index: objects,
+            }),
+          },
+        );
+
+        console.log(`docs index API response code ${response.status}: ${await response.text()}`);
+      },
+    },
+  };
+}

package/virtual-module.d.ts

@@ -56,6 +56,7 @@ declare module 'virtual:stl-docs/components/AiChat.tsx' {
     | import('react').ComponentType<{
         projectId: string;
         language: import('@stainless-api/docs-ui/routing').DocsLanguage | undefined;
+        siteTitle: string | undefined;
       }>
     | null;
   export default AiChatComponent;