@stainless-api/docs 0.1.0-beta.11 → 0.1.0-beta.110

package/plugin/index.ts

@@ -1,33 +1,46 @@
 import react from '@astrojs/react';
 import type { StarlightPlugin } from '@astrojs/starlight/types';
 import type { AstroIntegration } from 'astro';
+import type { BundledTheme } from 'shiki';
 import { config } from 'dotenv';
-import getPort from 'get-port';
-import { startDevServer } from './cms/server';
-import { buildAlgoliaIndex } from './buildAlgoliaIndex';
+// import { buildAlgoliaIndex } from './buildAlgoliaIndex';
 import {
   getAPIReferencePlaceholderItemFromSidebarConfig,
   makePlaceholderItems,
 } from './referencePlaceholderUtils';
-import type {
-  GeneratedSidebarConfig,
-  ReferenceSidebarConfigGenerateOptions,
-  ReferenceSidebarConfigItem,
-} from './cms/sidebar-builder';
+import {
+  SidebarConfigItemsBuilder,
+  toStarlightSidebar,
+  type GeneratedSidebarConfig,
+  type ReferenceSidebarConfigGenerateOptions,
+  type ReferenceSidebarConfigItem,
+} from './sidebar-utils/sidebar-builder';
 import {
   parseStarlightPluginConfig,
   type NormalizedStainlessStarlightConfig,
   type SomeStainlessStarlightUserConfig,
-  type SpecRetrieverConfig,
 } from './loadPluginConfig';
-import { buildVirtualModuleString } from '../shared/virtualModule';
+import { buildVirtualModuleString, makeAsyncVirtualModPlugin } from '../shared/virtualModule';
+import type * as StlStarlightVirtualModule from 'virtual:stl-starlight-virtual-module';
 import path from 'path';
 import fs from 'fs';
+import { getSharedLogger } from '../shared/getSharedLogger';
+import { resolveSrcFile } from '../resolveSrcFile';
+import { mkdir, writeFile } from 'fs/promises';
+import { fileURLToPath } from 'url';
+import prebundleWorkers from 'vite-plugin-prebundle-workers';
+import { SpecLoader, startSpecLoader } from './specs';
+
+import type * as ReferenceSidebarsVirtualModule from 'virtual:stl-starlight-reference-sidebars';
+import { generateMissingRouteList } from '@stainless-api/docs-ui/routing';
+import { buildAlgoliaIndex } from './buildAlgoliaIndex';
 
 export { generateAPILink } from './generateAPIReferenceLink';
 export type { ReferenceSidebarConfigItem };
 
-config();
+config({
+  quiet: true,
+});
 
 let sidebarIdCounter = 0;
 
@@ -78,150 +91,217 @@ export function generateAPIReferenceItems(
   return makePlaceholderItems(id);
 }
 
-function tmpGetCMSServerConfig(specRetrieverConfig: SpecRetrieverConfig) {
-  if (specRetrieverConfig.kind === 'external_spec_server') {
-    throw new Error('External spec server is not yet supported');
-  }
-  if (specRetrieverConfig.kind === 'local_spec_server_with_files') {
-    if (!specRetrieverConfig.apiKey) {
-      throw new Error('API key is required');
-    }
-    return {
-      apiKey: specRetrieverConfig.apiKey,
-      version: specRetrieverConfig.version,
-      devPaths: specRetrieverConfig.devPaths,
-    };
-  }
-
-  if (specRetrieverConfig.kind === 'local_spec_server_with_remote_files') {
-    return {
-      apiKey: specRetrieverConfig.apiKey,
-      version: specRetrieverConfig.version,
-      devPaths: {
-        oasPath: undefined,
-        configPath: undefined,
-      },
-    };
-  }
-  throw new Error('Invalid spec retriever config');
-}
-
 async function stlStarlightAstroIntegration(
   pluginConfig: NormalizedStainlessStarlightConfig,
 ): Promise<AstroIntegration> {
   const virtualId = `virtual:stl-starlight-virtual-module`;
   // The '\0' prefix tells Vite “this is a virtual module” and prevents it from being resolved again.
   const resolvedId = `\0${virtualId}`;
+  let playgroundsBase: string | undefined;
+  let buildPlaygrounds;
+  let astroBase = '/';
+
+  let specLoader: SpecLoader | undefined;
+  async function resolveSpecs() {
+    if (!specLoader) throw new Error('Expected spec loader to exist');
+    const result = await specLoader.specPromise;
+    return result.specComposite;
+  }
 
-  const CMS_PORT = await getPort();
+  let building: Promise<void> | undefined;
+  let reportError: ((message: string) => void) | null = null;
+  let collectedErrors: string[] | null = null;
 
-  const { apiKey, version, devPaths } = tmpGetCMSServerConfig(pluginConfig.specRetrieverConfig);
+  function startPlaygroundsBuild(playgroundsCachePath: string) {
+    if (!pluginConfig.experimentalPlaygrounds) return;
+    if (building) return building;
+    return (building = (async () => {
+      const specComposite = await resolveSpecs();
 
-  const cmsServer = await startDevServer({
-    port: CMS_PORT,
-    apiKey,
-    version,
-    devPaths,
-    getGeneratedSidebarConfig: (id: number) => {
-      const config = sidebarConfigs.get(id);
-      if (!config) {
-        return null;
+      if (specComposite.listUniqueSpecs().length > 1) {
+        throw new Error('Multiple specs found. This is not supported for Playgrounds.');
       }
-      return config;
-    },
-  });
+
+      const spec = specComposite.listUniqueSpecs()[0]!.data.sdkJson;
+      const auth = specComposite.listUniqueSpecs()[0]!.data.auth;
+
+      const langs = specComposite.getLanguages();
+
+      await buildPlaygrounds!({
+        spec,
+        langs,
+        auth,
+        playPath: playgroundsCachePath,
+        reportError(message: string) {
+          (reportError ?? console.error)(message);
+        },
+      });
+    })());
+  }
 
   return {
     name: 'stl-starlight-astro',
     hooks: {
-      'astro:config:setup': async ({ injectRoute, updateConfig, logger, command, config: astroConfig }) => {
+      'astro:config:setup': async ({
+        injectRoute,
+        updateConfig,
+        logger: localLogger,
+        command,
+        config: astroConfig,
+        createCodegenDir,
+      }) => {
+        const logger = getSharedLogger({ fallback: localLogger });
         const projectDir = astroConfig.root.pathname;
+        astroBase = astroConfig.base;
 
-        const middlewareFile = path.join(projectDir, 'middleware.stainless.ts');
+        specLoader = await startSpecLoader(pluginConfig, logger, createCodegenDir());
+
+        reportError = (message: string) => logger.error(message);
+
+        if (pluginConfig.experimentalPlaygrounds) {
+          playgroundsBase = pluginConfig.experimentalPlaygrounds.playgroundsBase;
+        }
+
+        const middlewareFileBase = path.join(projectDir, 'middleware.stainless');
+        const middlewareFile = ['.tsx', '.ts']
+          .map((ext) => middlewareFileBase + ext)
+          .find((f) => fs.existsSync(f));
 
         let vmMiddlewareExport = 'export const MIDDLEWARE = {};';
-        if (fs.existsSync(middlewareFile)) {
-          logger.info(`Loading middleware from ${middlewareFile}`);
+        if (middlewareFile) {
+          logger.debug(`Loading middleware from ${middlewareFile}`);
           vmMiddlewareExport = `export { default as MIDDLEWARE } from '${middlewareFile}';`;
         }
 
         injectRoute({
-          pattern: `${pluginConfig.basePath}/[...slug].md`,
-          entrypoint: '@stainless-api/docs/MarkdownRoute',
-          prerender: command === 'build',
+          pattern: `${pluginConfig.basePath}/[...slug]/index.md`,
+          entrypoint: resolveSrcFile('/plugin/routes/markdown.ts'),
+          prerender: pluginConfig.experimentalPrerender ? command === 'build' : false,
         });
 
-        const astroFile = command === 'build' ? 'DocsStaticRoute' : 'DocsRoute';
+        const astroFile = command === 'build' ? 'DocsStatic' : 'Docs';
         injectRoute({
           pattern: `${pluginConfig.basePath}/[...slug]`,
-          // in prod I think this points to @stainless-starlight/components/docs.astro
-          entrypoint: `@stainless-api/docs/${astroFile}`,
-          prerender: command === 'build',
+          entrypoint: resolveSrcFile(`/plugin/routes/${astroFile}.astro`),
+          prerender: pluginConfig.experimentalPrerender ? command === 'build' : false,
         });
 
         injectRoute({
           pattern: pluginConfig.basePath,
-          entrypoint: '@stainless-api/docs/OverviewRoute',
-          prerender: command === 'build',
-        });
-
-        // Remove starlight callout syntax in favor of our own callout component
-        // updateConfig always deeply merges arrays so we need to mutate remarkPlugins directly
-        // in order to remove plugins that starlight added
-        astroConfig.markdown.remarkPlugins = astroConfig.markdown.remarkPlugins.filter((plugin, i, arr) => {
-          if (typeof plugin !== 'function') return true;
-          // remove:
-          // 1. remarkDirective plugin
-          if (plugin.name === 'remarkDirective') return false;
-          // 2. directly followed by remarkAsides plugin (inner function named 'attacher')
-          if (plugin.name === 'attacher') {
-            const prev = arr[i - 1];
-            if (typeof prev === 'function' && prev.name === 'remarkDirective') return false;
-          }
-          // 3. remarkDirectivesRestoration plugin
-          if (plugin.name === 'remarkDirectivesRestoration') return false;
-          return true;
+          entrypoint: resolveSrcFile('/plugin/routes/Overview.astro'),
+          prerender: pluginConfig.experimentalPrerender ? command === 'build' : false,
         });
 
         updateConfig({
           vite: {
-            ssr: {
-              noExternal: ['@stainless-api/ui-primitives'],
-            },
-            optimizeDeps: { include: ['@stainless-api/ui-primitives'] },
             plugins: [
+              makeAsyncVirtualModPlugin<typeof ReferenceSidebarsVirtualModule>(
+                'virtual:stl-starlight-reference-sidebars',
+                async () => {
+                  // we know specLoader exists here
+                  const { specComposite } = await specLoader!.specPromise;
+
+                  const sidebars = [...sidebarConfigs.entries()]
+                    // produce all { id, language } combos with the attached config
+                    // flattens to one item per language * id combo
+                    .flatMap(([id, config]) =>
+                      specComposite.listAllLanguagesAndIncludeSpecs().map((res) => ({
+                        id,
+                        config,
+                        language: res.language,
+                        spec: res.spec.data.sdkJson,
+                      })),
+                    )
+                    // produce a sidebar for each
+                    // later we will .find() the sidebar that matches the (id, language)
+                    .map(({ id, config, language, spec }) => {
+                      const configItemsBuilder = new SidebarConfigItemsBuilder(
+                        spec,
+                        language,
+                        config.options,
+                      );
+
+                      let userSidebarConfig = configItemsBuilder.generateItems();
+                      if (config.transformFn) {
+                        const transformedSidebarConfig = config.transformFn(userSidebarConfig, language);
+                        if (transformedSidebarConfig) userSidebarConfig = transformedSidebarConfig;
+                      }
+
+                      return {
+                        id,
+                        language,
+                        // this has to run multpile times because it depends on the
+                        // userSidebarConfig (which is per-id) and the language
+                        entries: toStarlightSidebar({
+                          basePath: path.posix.join(astroBase, pluginConfig.basePath),
+                          spec,
+                          entries: userSidebarConfig,
+                          currentLanguage: language,
+                        }),
+                      };
+                    });
+
+                  return { sidebars };
+                },
+              ),
+              ...specLoader.vitePlugins,
               {
                 name: 'stl-starlight-vite',
-                configureServer(server) {
-                  for (const filePath of Object.values(devPaths)) {
-                    if (!filePath) {
-                      continue;
-                    }
-                    server.watcher.add(filePath);
+                buildStart() {
+                  building = undefined;
+                },
+                async configureServer(server) {
+                  if (playgroundsBase) {
+                    buildPlaygrounds = (await server.ssrLoadModule(playgroundsBase + '/src/build.ts'))
+                      .buildPlaygrounds;
                   }
 
-                  server.watcher.on('change', async (changed) => {
-                    if (Object.values(devPaths).includes(changed)) {
-                      logger.info(`${changed} changed, reloading...`);
-                      cmsServer.invalidate();
-                      server.hot.send({
-                        type: 'full-reload',
-                        path: '*',
-                      });
-                    }
-                  });
+                  // TODO: eventually - re-add support for watching local input changes (eg. reloading when OAS/config files change)
                 },
                 resolveId(id) {
                   if (id === virtualId) {
                     return resolvedId;
                   }
+                  if (id === 'virtual:stl-playground/unstable-update-language') {
+                    return resolveSrcFile('plugin/languages.ts');
+                  }
+                  if (id === 'virtual:stl-playground/create') {
+                    return fileURLToPath(
+                      new URL(
+                        pluginConfig.experimentalPlaygrounds
+                          ? path.join(playgroundsBase!, '/src/create.tsx')
+                          : './globalJs/create-playground.shim.ts',
+                        import.meta.url,
+                      ),
+                    );
+                  }
+                  if (id === 'virtual:stl-playground/data') {
+                    return fileURLToPath(
+                      new URL(
+                        pluginConfig.experimentalPlaygrounds
+                          ? './globalJs/playground-data.ts'
+                          : './globalJs/playground-data.shim.ts',
+                        import.meta.url,
+                      ),
+                    );
+                  }
+                  if (id.startsWith('virtual:stl-playground/')) {
+                    const result = path.join(
+                      this.environment.getTopLevelConfig().cacheDir,
+                      'stl-playground',
+                      id.slice('virtual:stl-playground/'.length),
+                    );
+                    const config = this.environment.getTopLevelConfig();
+                    return startPlaygroundsBuild(path.join(config.cacheDir, 'stl-playground'))?.then(
+                      () => result,
+                    );
+                  }
                 },
                 load(id) {
                   if (id === resolvedId) {
                     return [
                       buildVirtualModuleString({
-                        BASE_PATH: pluginConfig.basePath,
-                        CMS_PORT,
+                        RESOLVED_API_REFERENCE_PATH: path.posix.join(astroConfig.base, pluginConfig.basePath),
                         EXCLUDE_LANGUAGES: pluginConfig.excludeLanguages,
                         DEFAULT_LANGUAGE: pluginConfig.defaultLanguage,
                         BREADCRUMB_CONFIG: pluginConfig.breadcrumbs,
@@ -229,21 +309,86 @@ async function stlStarlightAstroIntegration(
                         HIGHLIGHT_THEMES: pluginConfig.highlighting.themes,
                         CONTENT_PANEL_LAYOUT: pluginConfig.contentPanel.layout,
                         EXPERIMENTAL_COLLAPSIBLE_SNIPPETS: pluginConfig.experimentalCollapsibleSnippets,
+                        EXPERIMENTAL_COLLAPSIBLE_METHOD_DESCRIPTIONS:
+                          pluginConfig.experimentalCollapsibleMethodDescriptions,
                         PROPERTY_SETTINGS: pluginConfig.propertySettings,
-                        SEARCH: pluginConfig.search,
-                        INCLUDE_AI_DROPDOWN_OPTIONS: pluginConfig.includeAIDropdownOptions,
-                      }),
+                        ENABLE_CONTEXT_MENU: pluginConfig.contextMenu,
+                        EXPERIMENTAL_PLAYGROUNDS: !!pluginConfig.experimentalPlaygrounds,
+                        EXPERIMENTAL_REQUEST_BUILDER: pluginConfig.experimentalRequestBuilder,
+                        STAINLESS_PROJECT: pluginConfig.stainlessProject,
+                      } satisfies Omit<typeof StlStarlightVirtualModule, 'MIDDLEWARE'>),
                       vmMiddlewareExport,
                     ].join('\n');
                   }
                 },
               },
+              prebundleWorkers({
+                include: [
+                  '**/typescript/runner*',
+                  '**/typescript/worker*',
+                  '**/pyright.worker*',
+                  '**/python/pyodide*',
+                ],
+                configureEsBuild(_, opts) {
+                  opts.external ??= [];
+                  opts.external.push('url');
+                  opts.loader ??= {};
+                  opts.loader['.wasm'] = 'dataurl';
+                  return opts;
+                },
+              }),
             ],
           },
         });
       },
-      'astro:server:done': async () => {
-        await cmsServer.destroy();
+      'astro:build:start'({ logger }) {
+        collectedErrors = [];
+        reportError = (message: string) => {
+          logger.error(message);
+          collectedErrors!.push(message);
+        };
+      },
+      'astro:build:done': async ({ dir, logger }) => {
+        const dist = fileURLToPath(dir);
+        const stainlessDir = path.join(dist, '_stainless');
+        await mkdir(stainlessDir, { recursive: true });
+        if (collectedErrors) {
+          for (const error of collectedErrors) {
+            logger.error(error);
+          }
+          collectedErrors = null;
+        }
+
+        const manifest = {
+          astroBase,
+        };
+        await writeFile(path.join(stainlessDir, 'stl-manifest.json'), JSON.stringify(manifest, null, 2));
+
+        const specComposite = await resolveSpecs();
+
+        await buildAlgoliaIndex({
+          specComposite,
+          logger,
+        });
+
+        // Generate a list of missing API routes to enable graceful handling of unimplemented SDK methods.
+        // When users switch languages in the docs, some API methods may not be implemented in the target SDK.
+        // Instead of showing a generic 404, we statically generate pages for these routes and mark them
+        // in this file so Cloudflare can serve them with a 404 status. These pages display helpful information
+        // about the missing method and provide links to SDKs where it is available.
+
+        // TODO: (multi-spec) support multiple specs
+        const spec = specComposite.listUniqueSpecs()[0]!.data.sdkJson;
+
+        const missingRoutes = generateMissingRouteList({
+          spec,
+          basePath: path.posix.join(astroBase, pluginConfig.basePath),
+        });
+        await mkdir(stainlessDir, { recursive: true });
+        await writeFile(
+          path.join(stainlessDir, 'missing-routes.json'),
+          JSON.stringify(missingRoutes, null, 2),
+        );
       },
     },
   };
@@ -260,17 +405,26 @@ export function stainlessStarlight(someUserConfig: SomeStainlessStarlightUserCon
         command,
         config: starlightConfig,
         astroConfig,
-        logger,
+        logger: localLogger,
       }) => {
         if (command !== 'build' && command !== 'dev') {
           return;
         }
 
-        const configParseResult = parseStarlightPluginConfig(someUserConfig, command);
+        const logger = getSharedLogger({ fallback: localLogger });
+
+        const configParseResult = parseStarlightPluginConfig(someUserConfig, {
+          command,
+          base: astroConfig.base,
+        });
         if (configParseResult.result === 'error') {
-          logger.error(configParseResult.message);
+          const errorLines = configParseResult.message.split('\n');
+          for (const line of errorLines) {
+            logger.error(line);
+          }
           process.exit(1);
         }
+
         const config = configParseResult.config;
 
         const isReactLoaded = astroConfig.integrations.find(({ name }) => name === '@astrojs/react');
@@ -279,42 +433,52 @@ export function stainlessStarlight(someUserConfig: SomeStainlessStarlightUserCon
           addIntegration(react());
         }
 
-        if (
-          command === 'build' &&
-          config.specRetrieverConfig.kind === 'local_spec_server_with_remote_files'
-        ) {
-          await buildAlgoliaIndex({
-            version: config.specRetrieverConfig.version,
-            apiKey: config.specRetrieverConfig.apiKey,
-          });
-        }
-
-        addIntegration(await stlStarlightAstroIntegration(config));
+        // TODO: (multi-spec) re-add this? not strictly necessary to merge
+        // if ('apiKey' in config.specInputs) {
+        //   if (!config.specInputs.apiKey) {
+        //     logger.info(`Stainless credentials not loaded`);
+        //   } else if (config.specInputs.apiKey.source === 'explicit-config') {
+        //     logger.info(`Stainless credentials loaded from user config`);
+        //   } else if (config.specInputs.apiKey.source === 'environment-variable') {
+        //     logger.info('Stainless credentials loaded from `STAINLESS_API_KEY` environment variable');
+        //   } else if (config.specInputs.apiKey.source === 'cli') {
+        //     logger.info('Stainless credentials loaded from `stl` CLI');
+        //   }
+        // })
 
         if (starlightConfig.sidebar) {
           // for pagination (https://starlight.astro.build/reference/configuration/#pagination) to work correctly
           // update the placeholder link to be correct
-          const item = getAPIReferencePlaceholderItemFromSidebarConfig(starlightConfig.sidebar);
-          if (item && typeof item === 'object' && 'link' in item) {
-            item.link = config.basePath;
+          for (const placeholder of getAPIReferencePlaceholderItemFromSidebarConfig(
+            starlightConfig.sidebar,
+          )) {
+            if (placeholder && typeof placeholder === 'object' && 'link' in placeholder) {
+              placeholder.link = config.basePath;
+            }
           }
         }
 
-        const currentExpressiveCode =
+        addIntegration(await stlStarlightAstroIntegration(config));
+
+        const expressiveCodeConfig =
           typeof starlightConfig.expressiveCode === 'object' ? starlightConfig.expressiveCode : {};
-        updateConfig({
-          expressiveCode: {
-            ...currentExpressiveCode,
-            themes: [...(currentExpressiveCode.themes || []), 'github-light', 'github-dark'],
-          },
-        });
+
+        const themes = expressiveCodeConfig.themes
+          ? (expressiveCodeConfig.themes as BundledTheme[])
+          : (['github-light', 'github-dark'] as BundledTheme[]);
 
         updateConfig({
           sidebar: starlightConfig.sidebar,
+          ...(expressiveCodeConfig && {
+            expressiveCode: {
+              ...expressiveCodeConfig,
+              themes,
+            },
+          }),
         });
 
         addRouteMiddleware({
-          entrypoint: '@stainless-api/docs/replaceSidebarPlaceholderMiddleware',
+          entrypoint: resolveSrcFile('/plugin/replaceSidebarPlaceholderMiddleware.ts'),
           order: 'post',
         });
       },
@@ -323,5 +487,7 @@ export function stainlessStarlight(someUserConfig: SomeStainlessStarlightUserCon
 }
 
 // Additional exports we want for Stainless <-> docs integration.
-export { parseStainlessPath } from '@stainless-api/docs-ui/src/routing';
-export { renderMarkdown } from '@stainless-api/docs-ui/src/markdown';
+export { parseStainlessPath } from '@stainless-api/docs-ui/routing';
+export { renderMarkdown } from '@stainless-api/docs-ui/markdown';
+
+export { resolveSpec } from './specs/inputResolver';

package/plugin/languages.ts

@@ -1,4 +1,4 @@
-import type { DocsLanguage } from '@stainless-api/docs-ui/src/routing';
+import type { DocsLanguage } from '@stainless-api/docs-ui/routing';
 import KotlinIcon from './assets/languages/kotlin.svg';
 import RubyIcon from './assets/languages/ruby.svg';
 import TerraformIcon from './assets/languages/terraform.svg';
@@ -7,6 +7,9 @@ import PythonIcon from './assets/languages/python.svg';
 import JavaIcon from './assets/languages/java.svg';
 import GoIcon from './assets/languages/go.svg';
 import CurlIcon from './assets/languages/curl.svg';
+import CSharpIcon from './assets/languages/csharp.svg';
+import PHPIcon from './assets/languages/php.svg';
+import CLIIcon from './assets/languages/cli.svg';
 
 export const Languages: Record<
   DocsLanguage,
@@ -29,6 +32,9 @@ export const Languages: Record<
   http: { name: 'HTTP', icon: CurlIcon, alt: 'HTTP logo' },
   terraform: { name: 'Terraform', icon: TerraformIcon, alt: 'Terraform logo' },
   ruby: { name: 'Ruby', icon: RubyIcon, alt: 'Ruby logo' },
+  csharp: { name: 'C#', icon: CSharpIcon, alt: 'C# logo' },
+  cli: { name: 'CLI Tool', icon: CLIIcon, alt: 'CLI logo' },
+  php: { name: 'PHP', icon: PHPIcon, alt: 'PHP logo' },
 };
 
 export function generatePrefix(basePath: string, language: string) {
@@ -44,7 +50,7 @@ export function applyLanguageToLinks(basePath?: string, defaultLanguage?: string
     `[data-stldocs-overview],[data-stldocs-method],a.nav-link[href^='${basePath}']`,
   );
 
-  for (var link of links) {
+  for (const link of links) {
     const href = link.getAttribute('href');
     const prefix = generatePrefix(basePath, language);
     if (href?.startsWith(basePath) && !href?.startsWith(prefix)) {