@stainless-api/docs 0.1.0-beta.93 → 0.1.0-beta.94

package/plugin/react/Routing.tsx

@@ -1,5 +1,6 @@
 import * as React from 'react';
 import { marked, Tokens } from 'marked';
+import { getDocsLanguages } from '../helpers/multiSpec';
 
 import {
   createMarkdownProcessor,
@@ -44,7 +45,6 @@ import { Dropdown } from '@stainless-api/docs/components';
 
 import {
   RESOLVED_API_REFERENCE_PATH,
-  EXCLUDE_LANGUAGES,
   EXPAND_RESOURCES,
   HIGHLIGHT_THEMES,
   BREADCRUMB_CONFIG,
@@ -237,10 +237,8 @@ export function SDKSelectReactComponent({
 }
 
 function SDKRequestTitle({ snippetLanguage }: SDKRequestTitleProps) {
-  const spec = useSpec();
-
   const selected = snippetLanguage.split('.').at(0) as DocsLanguage;
-  const languages = (spec?.docs?.languages ?? ['http']).filter((lang) => !EXCLUDE_LANGUAGES.includes(lang));
+  const languages = getDocsLanguages();
 
   return (
     <SDKSelectReactComponent

package/plugin/routes/Docs.astro

@@ -2,17 +2,20 @@
 import StarlightPage from '@astrojs/starlight/components/StarlightPage.astro';
 import { getReadmeContent, buildPageNavigation, RenderSpec, astroMarkdownRender } from '../react/Routing';
 import { getResourceFromSpec } from '@stainless-api/docs-ui/utils';
-import { parseStainlessPath } from '@stainless-api/docs-ui/routing';
+import { parseRoute, parseStainlessPath } from '@stainless-api/docs-ui/routing';
 import {
   CONTENT_PANEL_LAYOUT,
   MIDDLEWARE,
   EXPERIMENTAL_REQUEST_BUILDER,
+  RESOLVED_API_REFERENCE_PATH,
 } from 'virtual:stl-starlight-virtual-module';
 import { generateDocsRoutes } from '../helpers/generateDocsRoutes';
 import { StainlessIslands } from '../components/StainlessIslands';
 import { getSDKJSONInSSR } from '../specs/fetchSpecSSR';
 
-const spec = await getSDKJSONInSSR();
+const language = parseRoute(RESOLVED_API_REFERENCE_PATH, Astro.url.pathname)?.language;
+
+const spec = await getSDKJSONInSSR(language);
 
 export type Props = Awaited<ReturnType<typeof generateDocsRoutes>>[number]['props'] | Record<string, never>;
 

package/plugin/routes/DocsStatic.astro

@@ -1,12 +1,10 @@
 ---
 import type { GetStaticPaths } from 'astro';
 import Docs from './Docs.astro';
-import { generateDocsRoutes } from '../helpers/generateDocsRoutes';
-import { getSDKJSONInSSR } from '../specs/fetchSpecSSR';
+import { generateAllDocsRoutes } from '../helpers/generateDocsRoutes';
 
 export const getStaticPaths = (async () => {
-  const spec = await getSDKJSONInSSR();
-  const routes = generateDocsRoutes(spec);
+  const routes = await generateAllDocsRoutes();
   return routes;
 }) satisfies GetStaticPaths;
 

package/plugin/routes/Overview.astro

@@ -1,17 +1,31 @@
 ---
 import StarlightPage from '@astrojs/starlight/components/StarlightPage.astro';
 import { EXCLUDE_LANGUAGES } from 'virtual:stl-starlight-virtual-module';
-import type { DocsLanguage } from '@stainless-api/docs-ui/routing';
 import { RenderLibraries, RenderSpecOverview, type SpecMetadata } from '../react/Routing';
 import { getSDKJSONInSSR } from '../specs/fetchSpecSSR';
+import { api } from 'virtual:stainless-apis-manifest';
 
-const spec = await getSDKJSONInSSR();
+const spec = await getSDKJSONInSSR('http');
 
-const languages: DocsLanguage[] = spec.docs!.languages ?? ['http'];
-const metadata: SpecMetadata = languages
-  .filter((language) => !['http', 'terraform'].includes(language) && spec.metadata[language])
-  .filter((language) => !EXCLUDE_LANGUAGES.includes(language))
-  .map<SpecMetadata[number]>((language) => [language, spec.metadata[language]!]);
+const langTargets = api.languages
+  .map((l) => l.language)
+  .filter((l) => !EXCLUDE_LANGUAGES.includes(l))
+  .filter((l) => l !== 'http' && l !== 'terraform');
+
+const langsWithSpecs = await Promise.all(
+  langTargets.map(async (language) => {
+    const spec = await getSDKJSONInSSR(language);
+    return {
+      language,
+      spec,
+    };
+  }),
+);
+
+const metadata: SpecMetadata = langsWithSpecs.map(({ language, spec }) => [
+  language,
+  spec.metadata[language]!,
+]);
 
 // PageTitle override will skip rendering the default Starlight title
 Astro.locals._stlStarlightPage = {

package/plugin/routes/markdown.ts

@@ -7,7 +7,7 @@ import { renderMarkdown } from '@stainless-api/docs-ui/markdown';
 import { parseStainlessPath, type DocsLanguage } from '@stainless-api/docs-ui/routing';
 import type { EnvironmentType } from '@stainless-api/docs-ui/markdown/utils';
 import { PROPERTY_SETTINGS, MIDDLEWARE } from 'virtual:stl-starlight-virtual-module';
-import { generateDocsRoutes } from '../helpers/generateDocsRoutes';
+import { generateAllDocsRoutes } from '../helpers/generateDocsRoutes';
 import { getSDKJSONInSSR } from '../specs/fetchSpecSSR';
 
 type RouteProps = {
@@ -17,12 +17,12 @@ type RouteProps = {
 };
 
 export const getStaticPaths = (async () => {
-  const spec = await getSDKJSONInSSR();
-  return generateDocsRoutes(spec);
+  const paths = await generateAllDocsRoutes();
+  return paths;
 }) satisfies GetStaticPaths;
 
 export const GET: APIRoute<RouteProps> = async ({ props }) => {
-  const spec = await getSDKJSONInSSR();
+  const spec = await getSDKJSONInSSR(props.language);
 
   if (props.kind === 'readme') {
     const readmeContent = await getReadmeContent(spec, props.language);

package/plugin/specs/FileCache.ts

@@ -0,0 +1,99 @@
+import crypto from 'crypto';
+import path from 'path';
+import { readdir, readFile, rm, writeFile } from 'fs/promises';
+
+type CacheResultSource = 'memory' | 'disk' | 'generation';
+
+export type CacheGetResult<T> = {
+  resultSource: CacheResultSource;
+  data: T;
+  filePath: string;
+};
+
+export class FileCache<Inputs extends Record<string, unknown>, Output> {
+  private memoryCache: Map<string, Output> = new Map();
+
+  private cacheDirectory: string | null = null;
+
+  public setCacheDirectory(cacheDirectory: string) {
+    this.cacheDirectory = cacheDirectory;
+  }
+  public getCacheDirectory() {
+    if (!this.cacheDirectory) {
+      console.error(`Tried to retrieve entry from cache, but no cache directory was set.`);
+      process.exit(1);
+    }
+    return this.cacheDirectory;
+  }
+
+  private hashInputs(inputs: Inputs): string {
+    return crypto
+      .createHash('sha256')
+      .update(JSON.stringify(inputs) + this.config.globalHashBits)
+      .digest('hex')
+      .slice(0, 10);
+  }
+
+  private getFileName(hash: string) {
+    return `${hash}.json`;
+  }
+
+  public async cleanupUnusedFiles() {
+    const allFiles = await readdir(this.getCacheDirectory());
+    const usedFiles = Array.from(this.memoryCache.keys()).map((key) => this.getFileName(key));
+    const unusedFiles = allFiles.filter((file) => !usedFiles.includes(file));
+    await Promise.all(unusedFiles.map((file) => rm(path.join(this.getCacheDirectory(), file))));
+    return {
+      deletedCount: unusedFiles.length,
+    };
+  }
+
+  public async get(inputs: Inputs): Promise<CacheGetResult<Output>> {
+    const hash = this.hashInputs(inputs);
+    const filePath = path.join(this.getCacheDirectory(), this.getFileName(hash));
+
+    const memoryCacheResult = this.memoryCache.get(hash);
+    if (memoryCacheResult) {
+      return {
+        resultSource: 'memory',
+        data: memoryCacheResult,
+        filePath,
+      };
+    }
+
+    const getFromFileOrGenerate = async () => {
+      try {
+        const fileContents = await readFile(filePath, 'utf8');
+        return {
+          resultSource: 'disk' as const,
+          data: JSON.parse(fileContents) as Output,
+          filePath,
+        };
+      } catch {
+        const data = await this.config.generate(inputs);
+        await writeFile(filePath, JSON.stringify(data), 'utf8');
+        return {
+          resultSource: 'generation' as const,
+          data,
+          filePath,
+        };
+      }
+    };
+
+    const result = await getFromFileOrGenerate();
+    this.memoryCache.set(hash, result.data);
+    return result;
+  }
+
+  constructor(
+    private config: {
+      /**
+       * Additional information to include in the hash of the inputs.
+       * This is useful for cases where the inputs are the same, but the global state is different.
+       * Eg: The preview worker source can be used here.
+       */
+      globalHashBits: string;
+      generate: (inputs: Inputs) => Promise<Output>;
+    },
+  ) {}
+}

package/plugin/specs/fetchSpecSSR.ts

@@ -1,21 +1,27 @@
 import { readFile } from 'fs/promises';
 
-import { specPath } from 'virtual:stainless-sdk-json-manifest';
+import { api } from 'virtual:stainless-apis-manifest';
 import { SpecWithAuth } from './generateSpec';
+import { DocsLanguage } from '@stainless-api/docs-ui/routing';
 
-let cachedSpecWithAuth: SpecWithAuth | null = null;
+const cachedSpecWithAuth: Record<string, SpecWithAuth> = {};
 
-async function getSpecWithAuthInSSR() {
-  if (cachedSpecWithAuth) {
-    return cachedSpecWithAuth;
+async function getSpecWithAuthInSSR(filePath: string) {
+  if (cachedSpecWithAuth[filePath]) {
+    return cachedSpecWithAuth[filePath];
   }
-  const specStr = await readFile(specPath, 'utf8');
+  const specStr = await readFile(filePath, 'utf8');
   const json = JSON.parse(specStr) as SpecWithAuth;
-  cachedSpecWithAuth = json;
+  cachedSpecWithAuth[filePath] = json;
   return json;
 }
 
-export async function getSDKJSONInSSR() {
-  const specWithAuth = await getSpecWithAuthInSSR();
-  return specWithAuth.data;
+export async function getSDKJSONInSSR(language: DocsLanguage) {
+  const filePath = api.languages.find((l) => l.language === language)?.sdkJSONFilePath;
+  if (!filePath) {
+    throw new Error(`No SDK JSON file path for language: ${language}`);
+  }
+
+  const specWithAuth = await getSpecWithAuthInSSR(filePath);
+  return specWithAuth.sdkJson;
 }

package/plugin/specs/generateSpec.ts

@@ -1,29 +1,54 @@
 import type * as SDKJSON from '@stainless/sdk-json';
 import { Languages } from '@stainless-api/docs-ui/routing';
 import { createSDKJSON, ParsedConfig, parseInputs, transformOAS } from './worker';
+import { LanguageGenerateQuery } from '../loadPluginConfig';
+import { FileCache } from './FileCache';
+import previewWorkerCode from '../vendor/preview.worker.docs.js?raw';
 
-function addAuthToSpec(spec: SDKJSON.Spec, config: ParsedConfig) {
-  const opts = Object.entries(config.client_settings.opts).map(([k, v]) => ({ name: k, ...v }));
-  return {
-    data: spec,
-    auth: spec.security_schemes.map((scheme) => ({
-      ...scheme,
-      opts: opts.filter((opt) => opt.auth?.security_scheme === scheme.name),
-    })),
-  };
+function getLanguagesFromStainlessConfig(config: ParsedConfig): SDKJSON.SpecLanguage[] {
+  // if the Stainless config has a list of docs languages, use that
+  if (config.docs?.languages) {
+    return config.docs.languages;
+  }
+
+  // otherwise, just loop over all targets in the config + use the ones that are not skipped
+  return Object.entries(config.targets)
+    .filter(([name, target]) => {
+      if (!Languages.includes(name)) return false; // not  a valid language
+      if (target.skip) return false; // config says to skip this language
+      return true;
+    })
+    .map(([name]) => name) as SDKJSON.SpecLanguage[];
 }
 
-export type SpecWithAuth = ReturnType<typeof addAuthToSpec>;
+// These inputs contain everything needed to generate a spec
+// Combined with the source of the preview workers, we can make a hash to cache the resulting spec
+export type GenerateSpecRawInputs = {
+  oasStr: string;
+  configStr: string;
+  languageOverrides: LanguageGenerateQuery | null;
+  stainlessProject: string;
+  versionInfo: Record<SDKJSON.SpecLanguage, string> | null;
+};
+
+function applyLanguageOverrides(
+  initialLanguages: SDKJSON.SpecLanguage[],
+  languageOverrides: LanguageGenerateQuery | null,
+) {
+  if (!languageOverrides) return initialLanguages;
+  if (languageOverrides.mode === 'exclude') {
+    return initialLanguages.filter((language) => !languageOverrides.list.includes(language));
+  }
+  return languageOverrides.list;
+}
 
-export async function generateSpecFromStrings({
+async function generateSpecFromStrings({
   oasStr,
   configStr,
-  projectName,
-}: {
-  oasStr: string;
-  configStr: string;
-  projectName: string;
-}) {
+  stainlessProject,
+  languageOverrides,
+  versionInfo,
+}: GenerateSpecRawInputs) {
   const { oas, config } = await parseInputs({
     oas: oasStr,
     config: configStr,
@@ -31,20 +56,57 @@ export async function generateSpecFromStrings({
 
   const transformedOAS = await transformOAS({ oas, config });
 
-  const languages =
-    config.docs?.languages ??
-    (Object.entries(config.targets)
-      .filter(([name, target]) => Languages.includes(name) && !target.skip)
-      .map(([name]) => name) as SDKJSON.SpecLanguage[]);
+  let languagesToGenerate = getLanguagesFromStainlessConfig(config);
+  // by default, we should generate the HTTP spec (unless it's explicitly excluded)
+  if (!languagesToGenerate.includes('http')) {
+    languagesToGenerate.push('http');
+  }
+
+  languagesToGenerate = applyLanguageOverrides(languagesToGenerate, languageOverrides);
 
+  // SDKJSON has weird behavior where it will create a spec with HTTP, even if it's not in the languages list
   const sdkJson = await createSDKJSON({
     oas: transformedOAS,
     config,
-    languages,
-    projectName,
+    // if language overrides are provided, use them, otherwise use the languages from the Stainless config
+    languages: languagesToGenerate,
+    projectName: stainlessProject,
   });
 
-  const specWithAuth = addAuthToSpec(sdkJson, config);
+  let languages = sdkJson.docs?.languages;
 
-  return specWithAuth;
+  if (!languages) {
+    throw new Error(`SDKJSON created without any languages`);
+  }
+
+  // if language overrides are provided, filter the languages to only include the ones that are in the overrides
+  languages = languages.filter((language) => languagesToGenerate.includes(language));
+
+  if (versionInfo) {
+    for (const [lang, version] of Object.entries(versionInfo)) {
+      const meta = sdkJson.metadata[lang as SDKJSON.SpecLanguage];
+      if (meta?.version) meta.version = version;
+    }
+  }
+
+  const opts = Object.entries(config.client_settings.opts).map(([k, v]) => ({ name: k, ...v }));
+  return {
+    sdkJson,
+    languages,
+    auth: sdkJson.security_schemes.map((scheme) => ({
+      ...scheme,
+      opts: opts.filter((opt) => opt.auth?.security_scheme === scheme.name),
+    })),
+  };
 }
+
+export const specCache = new FileCache({
+  generate: generateSpecFromStrings,
+  globalHashBits: previewWorkerCode, // you can change this as a last resort to invalidate the cache
+});
+
+export type SpecCacheResult = Awaited<ReturnType<typeof specCache.get>>;
+
+export type GenerateSpecFn = typeof generateSpecFromStrings;
+
+export type SpecWithAuth = Awaited<ReturnType<GenerateSpecFn>>;