@stainless-api/docs 0.1.0-beta.87 → 0.1.0-beta.89

package/plugin/index.ts

@@ -1,27 +1,26 @@
 import react from '@astrojs/react';
 import type { StarlightPlugin } from '@astrojs/starlight/types';
-import type { AstroIntegration, AstroIntegrationLogger } from 'astro';
+import type { AstroIntegration } from 'astro';
 import type { BundledTheme } from 'shiki';
 import { config } from 'dotenv';
-import getPort from 'get-port';
-import { DevSpecServer, startDevServer, type SpecResp } from './cms/server';
 import { buildAlgoliaIndex } from './buildAlgoliaIndex';
 import {
   getAPIReferencePlaceholderItemFromSidebarConfig,
   makePlaceholderItems,
 } from './referencePlaceholderUtils';
-import type {
-  GeneratedSidebarConfig,
-  ReferenceSidebarConfigGenerateOptions,
-  ReferenceSidebarConfigItem,
-} from './cms/sidebar-builder';
+import {
+  SidebarConfigItemsBuilder,
+  toStarlightSidebar,
+  type GeneratedSidebarConfig,
+  type ReferenceSidebarConfigGenerateOptions,
+  type ReferenceSidebarConfigItem,
+} from './sidebar-utils/sidebar-builder';
 import {
   parseStarlightPluginConfig,
   type NormalizedStainlessStarlightConfig,
   type SomeStainlessStarlightUserConfig,
-  type SpecRetrieverConfig,
 } from './loadPluginConfig';
-import { buildVirtualModuleString } from '../shared/virtualModule';
+import { buildVirtualModuleString, makeAsyncVirtualModPlugin } from '../shared/virtualModule';
 import type * as StlStarlightVirtualModule from 'virtual:stl-starlight-virtual-module';
 import path from 'path';
 import fs from 'fs';
@@ -30,6 +29,10 @@ import { resolveSrcFile } from '../resolveSrcFile';
 import { mkdir, writeFile } from 'fs/promises';
 import { fileURLToPath } from 'url';
 import prebundleWorkers from 'vite-plugin-prebundle-workers';
+import { SpecLoader, startSpecLoader } from './specs';
+
+import type * as ReferenceSidebarsVirtualModule from 'virtual:stl-starlight-reference-sidebars';
+import { getDocsLanguages } from './helpers/getDocsLanguages';
 import { generateMissingRouteList } from '@stainless-api/docs-ui/routing';
 
 export { generateAPILink } from './generateAPIReferenceLink';
@@ -88,37 +91,8 @@ export function generateAPIReferenceItems(
   return makePlaceholderItems(id);
 }
 
-function tmpGetCMSServerConfig(specRetrieverConfig: SpecRetrieverConfig) {
-  if (specRetrieverConfig.kind === 'external_spec_server') {
-    throw new Error('External spec server is not yet supported');
-  }
-  if (specRetrieverConfig.kind === 'local_spec_server_with_files') {
-    if (!specRetrieverConfig.apiKey) {
-      throw new Error('API key is required');
-    }
-    return {
-      apiKey: specRetrieverConfig.apiKey,
-      version: specRetrieverConfig.version,
-      devPaths: specRetrieverConfig.devPaths,
-    };
-  }
-
-  if (specRetrieverConfig.kind === 'local_spec_server_with_remote_files') {
-    return {
-      apiKey: specRetrieverConfig.apiKey,
-      version: specRetrieverConfig.version,
-      devPaths: {
-        oasPath: undefined,
-        configPath: undefined,
-      },
-    };
-  }
-  throw new Error('Invalid spec retriever config');
-}
-
 async function stlStarlightAstroIntegration(
   pluginConfig: NormalizedStainlessStarlightConfig,
-  stlStarlightPluginLogger: AstroIntegrationLogger,
 ): Promise<AstroIntegration> {
   const virtualId = `virtual:stl-starlight-virtual-module`;
   // The '\0' prefix tells Vite “this is a virtual module” and prevents it from being resolved again.
@@ -127,11 +101,15 @@ async function stlStarlightAstroIntegration(
   let buildPlaygrounds;
   let astroBase = '/';
 
-  const CMS_PORT = await getPort();
-
-  const { apiKey, version, devPaths } = tmpGetCMSServerConfig(pluginConfig.specRetrieverConfig);
-
-  let cmsServer: DevSpecServer | undefined;
+  let specLoader: SpecLoader | undefined;
+  async function getSpec() {
+    if (!specLoader) throw new Error('Expected spec loader to exist');
+    const result = await specLoader.specPromise;
+    return {
+      spec: result.spec.data,
+      auth: result.spec.auth,
+    };
+  }
 
   let building: Promise<void> | undefined;
   let reportError: ((message: string) => void) | null = null;
@@ -141,17 +119,9 @@ async function stlStarlightAstroIntegration(
     if (!pluginConfig.experimentalPlaygrounds) return;
     if (building) return building;
     return (building = (async () => {
-      const { data: spec, auth } = (await (
-        await fetch(`http://localhost:${CMS_PORT}/retrieve_spec`, {
-          method: 'POST',
-          body: JSON.stringify({}),
-        })
-      ).json()) as SpecResp;
-      if (!spec || !auth) throw new Error('expected spec');
-
-      const langs = (spec.docs?.languages ?? ['http']).filter(
-        (lang) => !pluginConfig.excludeLanguages?.includes(lang),
-      );
+      const { spec, auth } = await getSpec();
+
+      const langs = getDocsLanguages(spec, pluginConfig.excludeLanguages);
 
       await buildPlaygrounds!({
         spec,
@@ -180,28 +150,13 @@ async function stlStarlightAstroIntegration(
         const projectDir = astroConfig.root.pathname;
         astroBase = astroConfig.base;
 
-        const codegenDir = createCodegenDir().pathname;
-
-        let specCacheDirectory: string | null = null;
-        if (command === 'dev') {
-          specCacheDirectory = path.join(codegenDir, 'spec_cache');
-          fs.mkdirSync(specCacheDirectory, { recursive: true });
-        }
-
-        cmsServer = startDevServer({
-          port: CMS_PORT,
-          apiKey: apiKey.value,
-          version,
-          devPaths,
-          logger: stlStarlightPluginLogger,
-          specCacheDirectory,
-          getGeneratedSidebarConfig: (id: number) => {
-            const config = sidebarConfigs.get(id);
-            if (!config) {
-              return null;
-            }
-            return config;
-          },
+        specLoader = await startSpecLoader(pluginConfig, logger, createCodegenDir());
+        specLoader.specPromise.then((specResult) => {
+          if (specResult.result === 'generated') {
+            logger.info(`Generated new SDKJSON`);
+          } else if (specResult.result === 'exists') {
+            logger.info(`Loaded SDKJSON`);
+          }
         });
 
         reportError = (message: string) => logger.error(message);
@@ -240,6 +195,55 @@ async function stlStarlightAstroIntegration(
         updateConfig({
           vite: {
             plugins: [
+              makeAsyncVirtualModPlugin<typeof ReferenceSidebarsVirtualModule>(
+                'virtual:stl-starlight-reference-sidebars',
+                async () => {
+                  const { spec } = await getSpec();
+                  const languages = getDocsLanguages(spec, pluginConfig.excludeLanguages);
+
+                  const sidebars = [...sidebarConfigs.entries()]
+                    // produce all { id, language } combos with the attached config
+                    // flattens to one item per language * id combo
+                    .flatMap(([id, config]) =>
+                      languages.map((language) => ({
+                        id,
+                        config,
+                        language,
+                      })),
+                    )
+                    // produce a sidebar for each
+                    // later we will .find() the sidebar that matches the (id, language)
+                    .map(({ id, config, language }) => {
+                      const configItemsBuilder = new SidebarConfigItemsBuilder(
+                        spec,
+                        language,
+                        config.options,
+                      );
+
+                      let userSidebarConfig = configItemsBuilder.generateItems();
+                      if (config.transformFn) {
+                        const transformedSidebarConfig = config.transformFn(userSidebarConfig, language);
+                        if (transformedSidebarConfig) userSidebarConfig = transformedSidebarConfig;
+                      }
+
+                      return {
+                        id,
+                        language,
+                        // this has to run multpile times because it depends on the
+                        // userSidebarConfig (which is per-id) and the language
+                        entries: toStarlightSidebar({
+                          basePath: path.posix.join(astroBase, pluginConfig.basePath), // TODO, is this right?
+                          spec,
+                          entries: userSidebarConfig,
+                          currentLanguage: language,
+                        }),
+                      };
+                    });
+
+                  return { sidebars };
+                },
+              ),
+              specLoader.vitePlugin,
               {
                 name: 'stl-starlight-vite',
                 buildStart() {
@@ -251,23 +255,7 @@ async function stlStarlightAstroIntegration(
                       .buildPlaygrounds;
                   }
 
-                  for (const filePath of Object.values(devPaths)) {
-                    if (!filePath) {
-                      continue;
-                    }
-                    server.watcher.add(filePath);
-                  }
-
-                  server.watcher.on('change', async (changed) => {
-                    if (Object.values(devPaths).includes(changed)) {
-                      logger.info(`${changed} changed, reloading...`);
-                      cmsServer?.invalidate();
-                      server.hot.send({
-                        type: 'full-reload',
-                        path: '*',
-                      });
-                    }
-                  });
+                  // TODO: eventually - re-add support for watching local input changes (eg. reloading when OAS/config files change)
                 },
                 resolveId(id) {
                   if (id === virtualId) {
@@ -312,9 +300,8 @@ async function stlStarlightAstroIntegration(
                   if (id === resolvedId) {
                     return [
                       buildVirtualModuleString({
-                        BASE_PATH: path.posix.join(astroConfig.base, pluginConfig.basePath),
-                        CMS_PORT,
-                        EXCLUDE_LANGUAGES: ['php', ...pluginConfig.excludeLanguages],
+                        RESOLVED_API_REFERENCE_PATH: path.posix.join(astroConfig.base, pluginConfig.basePath),
+                        EXCLUDE_LANGUAGES: pluginConfig.excludeLanguages,
                         DEFAULT_LANGUAGE: pluginConfig.defaultLanguage,
                         BREADCRUMB_CONFIG: pluginConfig.breadcrumbs,
                         EXPAND_RESOURCES: pluginConfig.expandResources,
@@ -327,7 +314,7 @@ async function stlStarlightAstroIntegration(
                         ENABLE_CONTEXT_MENU: pluginConfig.contextMenu,
                         EXPERIMENTAL_PLAYGROUNDS: !!pluginConfig.experimentalPlaygrounds,
                         EXPERIMENTAL_REQUEST_BUILDER: pluginConfig.experimentalRequestBuilder,
-                        STAINLESS_PROJECT: version.stainlessProject,
+                        STAINLESS_PROJECT: pluginConfig.specInputs.project,
                       } satisfies Omit<typeof StlStarlightVirtualModule, 'MIDDLEWARE'>),
                       vmMiddlewareExport,
                     ].join('\n');
@@ -353,9 +340,6 @@ async function stlStarlightAstroIntegration(
           },
         });
       },
-      'astro:server:done': async () => {
-        await cmsServer?.destroy();
-      },
       'astro:build:start'({ logger }) {
         collectedErrors = [];
         reportError = (message: string) => {
@@ -384,25 +368,16 @@ async function stlStarlightAstroIntegration(
         // Instead of showing a generic 404, we statically generate pages for these routes and mark them
         // in this file so Cloudflare can serve them with a 404 status. These pages display helpful information
         // about the missing method and provide links to SDKs where it is available.
-        const { data: spec } = (await (
-          await fetch(`http://localhost:${CMS_PORT}/retrieve_spec`, {
-            method: 'POST',
-            body: JSON.stringify({}),
-          })
-        ).json()) as SpecResp;
-        if (spec) {
-          const missingRoutes = await generateMissingRouteList({
-            spec,
-            basePath: path.posix.join(astroBase, pluginConfig.basePath),
-          });
-          await mkdir(stainlessDir, { recursive: true });
-          await writeFile(
-            path.join(stainlessDir, 'missing-routes.json'),
-            JSON.stringify(missingRoutes, null, 2),
-          );
-        } else {
-          console.log('Could not retrieve spec for missing routes check');
-        }
+        const { spec } = await getSpec();
+        const missingRoutes = generateMissingRouteList({
+          spec,
+          basePath: path.posix.join(astroBase, pluginConfig.basePath),
+        });
+        await mkdir(stainlessDir, { recursive: true });
+        await writeFile(
+          path.join(stainlessDir, 'missing-routes.json'),
+          JSON.stringify(missingRoutes, null, 2),
+        );
       },
     },
   };
@@ -444,40 +419,41 @@ export function stainlessStarlight(someUserConfig: SomeStainlessStarlightUserCon
           addIntegration(react());
         }
 
-        if ('apiKey' in config.specRetrieverConfig) {
-          if (!config.specRetrieverConfig.apiKey) {
+        if ('apiKey' in config.specInputs) {
+          if (!config.specInputs.apiKey) {
             logger.info(`Stainless credentials not loaded`);
-          } else if (config.specRetrieverConfig.apiKey.source === 'explicit-config') {
+          } else if (config.specInputs.apiKey.source === 'explicit-config') {
             logger.info(`Stainless credentials loaded from user config`);
-          } else if (config.specRetrieverConfig.apiKey.source === 'environment-variable') {
+          } else if (config.specInputs.apiKey.source === 'environment-variable') {
             logger.info('Stainless credentials loaded from `STAINLESS_API_KEY` environment variable');
-          } else if (config.specRetrieverConfig.apiKey.source === 'cli') {
+          } else if (config.specInputs.apiKey.source === 'cli') {
             logger.info('Stainless credentials loaded from `stl` CLI');
           }
         }
 
-        if (
-          command === 'build' &&
-          config.specRetrieverConfig.kind === 'local_spec_server_with_remote_files'
-        ) {
+        if (command === 'build' && config.specInputs.kind === 'stainless_api_inputs') {
           await buildAlgoliaIndex({
-            version: config.specRetrieverConfig.version,
-            apiKey: config.specRetrieverConfig.apiKey.value,
+            stainlessProject: config.specInputs.project,
+            branch: config.specInputs.branch,
+            apiKey: config.specInputs.apiKey.value,
             logger,
           });
         }
 
-        addIntegration(await stlStarlightAstroIntegration(config, logger));
-
         if (starlightConfig.sidebar) {
           // for pagination (https://starlight.astro.build/reference/configuration/#pagination) to work correctly
           // update the placeholder link to be correct
-          const item = getAPIReferencePlaceholderItemFromSidebarConfig(starlightConfig.sidebar);
-          if (item && typeof item === 'object' && 'link' in item) {
-            item.link = config.basePath;
+          for (const placeholder of getAPIReferencePlaceholderItemFromSidebarConfig(
+            starlightConfig.sidebar,
+          )) {
+            if (placeholder && typeof placeholder === 'object' && 'link' in placeholder) {
+              placeholder.link = config.basePath;
+            }
           }
         }
 
+        addIntegration(await stlStarlightAstroIntegration(config));
+
         const expressiveCodeConfig =
           typeof starlightConfig.expressiveCode === 'object' ? starlightConfig.expressiveCode : {};
 

package/plugin/loadPluginConfig.ts

@@ -5,7 +5,6 @@ import { existsSync, readFileSync } from 'fs';
 import type { CreateShikiHighlighterOptions } from '@astrojs/markdown-remark';
 import type { DocsLanguage } from '@stainless-api/docs-ui/routing';
 import type { PropertySettingsType } from '@stainless-api/docs-ui/contexts';
-import type { InputFilePaths } from '../plugin/cms/server';
 import { bold } from '../shared/terminalUtils';
 
 export type AstroCommand = 'dev' | 'build' | 'preview' | 'sync';
@@ -158,7 +157,7 @@ function resolvePath(inputPath: string) {
   return path.resolve(process.cwd(), inputPath);
 }
 
-function getLocalFilePaths(command: AstroCommand): InputFilePaths | null {
+function getLocalFilePaths(command: AstroCommand) {
   if (command !== 'dev') {
     return null;
   }
@@ -185,26 +184,6 @@ export type LoadedApiKey = {
   source: ApiKeySource;
 };
 
-export type SpecRetrieverConfig =
-  | {
-      kind: 'external_spec_server';
-      specServerUrl: string;
-      stainlessProject: null;
-    }
-  | {
-      kind: 'local_spec_server_with_files';
-      stainlessProject: string;
-      devPaths: InputFilePaths;
-      apiKey: LoadedApiKey | null;
-      version: VersionUserConfig;
-    }
-  | {
-      kind: 'local_spec_server_with_remote_files';
-      stainlessProject: string;
-      apiKey: LoadedApiKey;
-      version: VersionUserConfig;
-    };
-
 function parseAuthJson(authJsonStr: string) {
   let json: unknown;
   try {
@@ -252,10 +231,31 @@ function loadApiKey(configValue: string | undefined): LoadedApiKey | null {
   return { value: accessToken, source: 'cli' };
 }
 
+export type SDKJSONInputs =
+  | {
+      kind: 'file_inputs';
+      project: string;
+      oasPath: string;
+      configPath: string;
+    }
+  | {
+      kind: 'stainless_api_inputs';
+      project: string;
+      branch: string;
+      apiKey: LoadedApiKey;
+    }
+  | {
+      kind: 'string_inputs';
+      oasStr: string;
+      configStr: string;
+      project: string;
+    };
+
 function normalizeConfig(partial: SomeStainlessStarlightUserConfig, command: AstroCommand) {
   const configWithDefaults = {
     basePath: partial.basePath ?? '/api',
-    excludeLanguages: partial.excludeLanguages ?? [],
+    // TODO: why are we excluding php; do we need to
+    excludeLanguages: ['php' as const, ...(partial.excludeLanguages ?? [])],
     defaultLanguage: partial.defaultLanguage ?? 'http',
     breadcrumbs: {
       includeCurrentPage: partial.breadcrumbs?.includeCurrentPage ?? false,
@@ -285,38 +285,23 @@ function normalizeConfig(partial: SomeStainlessStarlightUserConfig, command: Ast
     experimentalPrerender: partial.experimentalPrerender ?? true,
   };
 
-  function getSpecRetrieverConfig(): SpecRetrieverConfig {
-    if ('externalSpecServerUrl' in partial) {
-      return {
-        kind: 'external_spec_server',
-        specServerUrl: partial.externalSpecServerUrl,
-        stainlessProject: null,
-      };
-    }
-
+  function getSpecInputs(): SDKJSONInputs {
     if (!('stainlessProject' in partial)) {
       throw new Error('You must provide a stainlessProject when using Stainless Starlight');
     }
 
-    const apiKey = loadApiKey(partial.apiKey);
-
-    const version = {
-      stainlessProject: partial.stainlessProject,
-      branch: partial.versions?.[0]?.branch ?? 'main',
-      version: partial.versions?.[0]?.version ?? 'v1',
-    };
-
     const localFilePaths = getLocalFilePaths(command);
     if (localFilePaths) {
       return {
-        kind: 'local_spec_server_with_files',
-        devPaths: localFilePaths,
-        stainlessProject: partial.stainlessProject,
-        apiKey,
-        version,
+        kind: 'file_inputs',
+        project: partial.stainlessProject,
+        oasPath: localFilePaths.oasPath,
+        configPath: localFilePaths.configPath,
       };
     }
 
+    const apiKey = loadApiKey(partial.apiKey);
+
     if (!apiKey) {
       throw new Error(
         [
@@ -329,20 +314,19 @@ function normalizeConfig(partial: SomeStainlessStarlightUserConfig, command: Ast
         ].join('\n'),
       );
     }
-
     return {
-      kind: 'local_spec_server_with_remote_files',
+      kind: 'stainless_api_inputs',
+      project: partial.stainlessProject,
+      branch: partial.versions?.[0]?.branch ?? 'main',
       apiKey,
-      stainlessProject: partial.stainlessProject,
-      version,
     };
   }
 
-  const specRetrieverConfig = getSpecRetrieverConfig();
+  const specInputs = getSpecInputs();
 
   return {
     ...configWithDefaults,
-    specRetrieverConfig,
+    specInputs,
   };
 }
 
@@ -354,7 +338,7 @@ export type NormalizedStainlessStarlightConfig = ReturnType<typeof normalizeConf
 
  We've tried to avoid any config values being optional/undefined. To accomplish this:
  - Any optional config values should have their defaults set here: eg. basePath defaults to /api
- - If a field is only used in certain contexts, we make each context a discriminated union (see SpecRetrieverConfig)
+ - If a field is only used in certain contexts, we make each context a discriminated union (see SDKJSONInputs)
  - We prefer empty arrays over undefined/null
 */
 export function parseStarlightPluginConfig(partial: SomeStainlessStarlightUserConfig, command: AstroCommand) {

package/plugin/react/Routing.tsx

@@ -43,7 +43,7 @@ import {
 import { Dropdown } from '@stainless-api/docs/components';
 
 import {
-  BASE_PATH,
+  RESOLVED_API_REFERENCE_PATH,
   EXCLUDE_LANGUAGES,
   EXPAND_RESOURCES,
   HIGHLIGHT_THEMES,
@@ -282,7 +282,7 @@ export function RenderLibraries({ metadata }: { metadata: SpecMetadata }) {
           language={language}
           version={data.version || ''}
           install={data.install || ''}
-          links={{ repo: data.repo_url || '#', docs: `${BASE_PATH}/${language}` }}
+          links={{ repo: data.repo_url || '#', docs: `${RESOLVED_API_REFERENCE_PATH}/${language}` }}
         />
       ))}
     </ComponentProvider>
@@ -295,7 +295,7 @@ export function RenderSpecOverview({ spec, language }: { spec: SDKJSON.Spec; lan
   return (
     <DocsProvider spec={spec} language={language ?? 'node'}>
       <ComponentProvider components={componentOverrides}>
-        <NavigationProvider basePath={BASE_PATH}>
+        <NavigationProvider basePath={RESOLVED_API_REFERENCE_PATH}>
           <MarkdownProvider render={renderMarkdown} highlight={highlight}>
             <div className={style.Overview}>
               {resources
@@ -351,7 +351,7 @@ export function RenderSpec({
       }}
     >
       <ComponentProvider components={componentOverrides}>
-        <NavigationProvider basePath={BASE_PATH} selectedPath={path}>
+        <NavigationProvider basePath={RESOLVED_API_REFERENCE_PATH} selectedPath={path}>
           <MarkdownProvider render={renderMarkdown} highlight={highlight}>
             {
               <div className="stldocs-root stl-ui-not-prose">
@@ -359,7 +359,7 @@ export function RenderSpec({
                   <SDKBreadcrumbs
                     spec={spec as SDKJSON.Spec}
                     currentPath={currentPath}
-                    basePath={BASE_PATH}
+                    basePath={RESOLVED_API_REFERENCE_PATH}
                     config={BREADCRUMB_CONFIG}
                   />
                   {ENABLE_CONTEXT_MENU && <AIDropdown />}

package/plugin/referencePlaceholderUtils.ts

@@ -21,27 +21,30 @@ export function makePlaceholderItems(id: number) {
 
 type StarlightConfig = Parameters<typeof starlight>[0];
 
-type SidebarConfigEntry = Exclude<StarlightConfig['sidebar'], undefined>[number];
+export type SidebarConfigEntry = Exclude<StarlightConfig['sidebar'], undefined>[number];
 
 export function getAPIReferencePlaceholderItemFromSidebarConfig(
   sidebar: SidebarConfigEntry[],
-): SidebarConfigEntry | null {
-  for (const item of sidebar) {
-    if (typeof item === 'string') {
-      continue;
-    }
-    if ('items' in item) {
-      const found = getAPIReferencePlaceholderItemFromSidebarConfig(item.items);
-      if (found) {
-        return found;
+): SidebarConfigEntry[] {
+  const items: SidebarConfigEntry[] = [];
+
+  function recursiveGetPlaceholderItems(entries: SidebarConfigEntry[]) {
+    for (const item of entries) {
+      if (typeof item === 'string') {
+        continue;
+      }
+      if ('attrs' in item && item.attrs?.about === INTERNAL_REFERENCE_ENTRY_MARKER) {
+        items.push(item);
+      }
+      if ('items' in item) {
+        recursiveGetPlaceholderItems(item.items);
       }
-    }
-    if ('attrs' in item && item.attrs?.about === INTERNAL_REFERENCE_ENTRY_MARKER) {
-      return item;
     }
   }
 
-  return null;
+  recursiveGetPlaceholderItems(sidebar);
+
+  return items;
 }
 
 type SidebarEntry = StarlightRouteData['sidebar'][number];