@stainless-api/docs 0.1.0-beta.13 → 0.1.0-beta.130

package/plugin/{cms → sidebar-utils}/sidebar-builder.ts

@@ -1,6 +1,20 @@
-import type * as SDKJSON from '~/lib/json-spec-v2/types';
-import { generateRoute, walkTree, type DocsLanguage } from '@stainless-api/docs-ui/src/routing';
+import type * as SDKJSON from '@stainless/sdk-json';
+import { generateRoute, walkTree, type DocsLanguage } from '@stainless-api/docs-ui/routing';
 import type { StarlightRouteData } from '@astrojs/starlight/route-data';
+import { isResourceEntirelyUnsupported } from '@stainless-api/docs-ui/languages/terraform';
+
+function makeMethodOrResourceKey(entry: SDKJSON.Method | SDKJSON.Resource): string {
+  if (entry.kind === 'http_method') {
+    if (entry.endpoint && entry.endpoint !== '') {
+      return entry.endpoint;
+    }
+    return entry.stainlessPath;
+  }
+  if (entry.kind === 'resource') {
+    return entry.stainlessPath;
+  }
+  throw new Error(`Unknown entry kind ${JSON.stringify(entry)}`);
+}
 
 function isResourceNonEmpty(resource: SDKJSON.Resource) {
   return (
@@ -119,23 +133,12 @@ export type GeneratedSidebarConfig = {
   ) => ReferenceSidebarConfigItem[] | void;
 };
 
-function countKeys(obj?: Record<string, any>) {
-  let o = obj ?? {};
-  return Object.keys(o).length;
-}
-
-function getMethodDeclForLanguage(entry: SDKJSON.Method, spec: SDKJSON.Spec, language: DocsLanguage) {
-  const decls = spec.decls[language] ?? {};
-  const decl = decls[entry.stainlessPath];
-  if (decl !== undefined) {
-    if ('ident' in decl) {
-      return decl;
-    }
-  }
-  return null;
+function countKeys(obj?: Record<string, unknown>) {
+  return Object.keys(obj ?? {}).length;
 }
 
-type MethodDecl = Exclude<ReturnType<typeof getMethodDeclForLanguage>, null>;
+type HasIdent<T> = T extends { ident: unknown } ? T : never;
+type MethodDecl = HasIdent<SDKJSON.LanguageDeclNodes[SDKJSON.SpecLanguage]>;
 
 function makeAPIOverviewPage(): UserSidebarAPIOverviewPage {
   return {
@@ -166,22 +169,26 @@ function pullOutSharedModelsResource(resources: SDKJSON.Resource[]): {
 }
 
 export class SidebarConfigItemsBuilder {
-  private getMethodDeclForLanguage(entry: SDKJSON.Method) {
+  private getMethodDeclForLanguage(entry: SDKJSON.Method): MethodDecl | null {
     const decls = this.spec.decls[this.language] ?? {};
     const decl = decls[entry.stainlessPath];
     if (decl !== undefined) {
-      if ('ident' in decl) {
-        return decl;
+      if ('ident' in decl && decl.ident !== undefined) {
+        return decl as MethodDecl;
       }
     }
     return null;
   }
 
+  private isWebhookResource(resource: SDKJSON.Resource): boolean {
+    return resource.stainlessPath === '(resource) webhooks';
+  }
+
   private toResourceOverviewPage(entry: SDKJSON.Resource): UserSidebarResourceOverviewPage {
     return {
       kind: 'resource_overview_page',
-      label: 'Overview',
-      key: entry.configRef,
+      label: this.isWebhookResource(entry) ? 'Events' : 'Overview',
+      key: makeMethodOrResourceKey(entry),
       badge: undefined,
       metadata: {
         subResourceCount: countKeys(entry.subresources),
@@ -195,7 +202,7 @@ export class SidebarConfigItemsBuilder {
     return {
       kind: 'method_page',
       label: entry.title,
-      key: entry.endpoint,
+      key: makeMethodOrResourceKey(entry),
       badge: undefined,
       metadata: {
         deprecated: Boolean(entry.deprecated),
@@ -208,16 +215,10 @@ export class SidebarConfigItemsBuilder {
     };
   }
 
-  private sortByLabel<T extends UserSidebarConfigItem>(items: T[]) {
-    // sorts in place
-    items.sort((a, b) => {
-      return a.label.localeCompare(b.label);
-    });
-  }
-
   private generateResourceGroup(resource: SDKJSON.Resource, collapsed: boolean): ReferenceSidebarGroup {
     const entries: ReferenceSidebarConfigItem[] = [];
-    if (!this.options?.excludeResourceOverviewPages) {
+    // even if we aren't generating resource overview pages, we want to generate them for webhooks
+    if (!this.options?.excludeResourceOverviewPages || this.isWebhookResource(resource)) {
       entries.push(this.toResourceOverviewPage(resource));
     }
     const methods = Object.values(resource.methods ?? {});
@@ -228,7 +229,6 @@ export class SidebarConfigItemsBuilder {
         methodPages.push(this.toMethodPage(m, langDecl));
       }
     }
-    this.sortByLabel(methodPages);
     entries.push(...methodPages);
 
     const subresources = Object.values(resource.subresources ?? {});
@@ -238,26 +238,55 @@ export class SidebarConfigItemsBuilder {
         subresourceGroups.push(this.generateResourceGroup(sub, true));
       }
     }
-    this.sortByLabel(subresourceGroups);
     entries.push(...subresourceGroups);
 
     return {
       kind: 'group',
       badge: undefined,
       label: resource.title,
-      resourceGroupKey: resource.configRef,
+      resourceGroupKey: makeMethodOrResourceKey(resource),
       entries,
       collapsed,
     };
   }
 
+  private generateTerraformItems(resources: SDKJSON.Resource[]) {
+    const entries: ReferenceSidebarConfigItem[] = [];
+
+    for (const resource of resources) {
+      if (isResourceEntirelyUnsupported(resource, this.spec.decls['terraform'])) {
+        continue;
+      }
+      entries.push({
+        kind: 'resource_overview_page',
+        label: resource.title,
+        key: makeMethodOrResourceKey(resource),
+        badge: undefined,
+        metadata: {
+          subResourceCount: countKeys(resource.subresources),
+          methodCount: countKeys(resource.methods),
+          modelCount: countKeys(resource.models),
+        },
+      });
+    }
+    return entries;
+  }
+
   public generateItems(): ReferenceSidebarConfigItem[] {
     const resourceMap = this.spec.resources;
-    let { resources, sharedModelsResource } = pullOutSharedModelsResource(Object.values(resourceMap ?? {}));
+    const { resources, sharedModelsResource } = pullOutSharedModelsResource(Object.values(resourceMap ?? {}));
 
-    const entries: ReferenceSidebarConfigItem[] = resources.filter(isResourceNonEmpty).map((r) => {
-      return this.generateResourceGroup(r, false);
-    });
+    let entries: ReferenceSidebarConfigItem[];
+
+    if (this.language === 'terraform') {
+      // Handle Terraform specifically
+      // In TF, we only render the top level resource, not the subresources.
+      entries = this.generateTerraformItems(resources);
+    } else {
+      entries = resources.filter(isResourceNonEmpty).map((r) => {
+        return this.generateResourceGroup(r, false);
+      });
+    }
 
     const includeSharedModels = this.options?.includeSharedModels ?? false;
     if (includeSharedModels && sharedModelsResource) {
@@ -276,35 +305,24 @@ export class SidebarConfigItemsBuilder {
   ) {}
 }
 
-export function walkSidebarConfigItems(
-  sidebar: ReferenceSidebarConfigItem[],
-  fn: (item: ReferenceSidebarConfigItem) => void,
-) {
-  for (const item of sidebar) {
-    fn(item);
-    if (item.kind === 'group') {
-      walkSidebarConfigItems(item.entries, fn);
-    }
-  }
-}
-
 type SidebarEntry = StarlightRouteData['sidebar'][number];
 
 // This allows us to be a bit more forgiving to the user.
 // As long as they don't modify the key, we can still find the item.
+
 function getResourceOrMethod(spec: SDKJSON.Spec, endpointOrConfigRef: string) {
   for (const entry of walkTree(spec, false)) {
-    if (entry.data.kind === 'resource' && entry.data.configRef === endpointOrConfigRef) {
-      return entry;
+    if (entry.data.kind === 'model') {
+      continue;
     }
-    if (entry.data.kind === 'http_method' && entry.data.endpoint === endpointOrConfigRef) {
+    if (makeMethodOrResourceKey(entry.data) === endpointOrConfigRef) {
       return entry;
     }
   }
   return null;
 }
 
-export function forceGenerateRoute({
+function forceGenerateRoute({
   basePath,
   stainlessPath,
   language,
@@ -320,24 +338,17 @@ export function forceGenerateRoute({
   return route;
 }
 
-export type BuildSidebarParams = {
+type ToStarlightSidebarParams = {
   basePath: string;
-  currentSlug: string;
-};
-
-type ToStarlightSidebarParams = BuildSidebarParams & {
   spec: SDKJSON.Spec;
   entries: ReferenceSidebarConfigItem[];
-  currentStainlessPath: string;
   currentLanguage: DocsLanguage;
 };
 
 export function toStarlightSidebar({
   basePath,
-  currentSlug,
   spec,
   entries,
-  currentStainlessPath,
   currentLanguage,
 }: ToStarlightSidebarParams): SidebarEntry[] {
   const starlightEntries: SidebarEntry[] = [];
@@ -350,7 +361,7 @@ export function toStarlightSidebar({
         type: 'link',
         href: readmeSlug,
         label: entry.label,
-        isCurrent: currentSlug === readmeSlug,
+        isCurrent: false,
         badge: entry.badge,
         attrs: {
           'data-stldocs-overview': 'readme',
@@ -369,14 +380,12 @@ export function toStarlightSidebar({
         language: currentLanguage,
       });
 
-      const isCurrent = resourceOrMethod.data.stainlessPath === currentStainlessPath;
-
       if (resourceOrMethod.data.kind === 'http_method') {
         starlightEntries.push({
           type: 'link',
           href: route,
           label: entry.label,
-          isCurrent,
+          isCurrent: false,
           badge: entry.badge,
           attrs: {
             'data-stldocs-method': resourceOrMethod.data.httpMethod,
@@ -387,7 +396,7 @@ export function toStarlightSidebar({
           type: 'link',
           href: route,
           label: entry.label,
-          isCurrent,
+          isCurrent: false,
           badge: entry.badge,
           attrs: {
             // TODO: @Ryan: is data.name unique? This is what we used before so I'm not changing, but I am curious.
@@ -398,14 +407,20 @@ export function toStarlightSidebar({
         throw new Error(`Unknown entry kind ${JSON.stringify(entry)}`);
       }
     } else if (entry.kind === 'group') {
+      if (entry.resourceGroupKey) {
+        const resourceOrMethod = getResourceOrMethod(spec, entry.resourceGroupKey);
+        // Skip pushing the group if if the resource it represents is not available in the current language.
+        // This occurs when SDK generation for the current language is skipped in the Stainless config for that resource.
+        if (resourceOrMethod?.data?.kind === 'resource' && !resourceOrMethod?.data?.[currentLanguage]) {
+          continue;
+        }
+      }
       starlightEntries.push({
         type: 'group',
         label: entry.label,
         entries: toStarlightSidebar({
           basePath,
-          currentSlug,
           spec,
-          currentStainlessPath,
           entries: entry.entries,
           currentLanguage,
         }),

package/plugin/specs/FileCache.ts

@@ -0,0 +1,99 @@
+import crypto from 'crypto';
+import path from 'path';
+import { readdir, readFile, rm, writeFile } from 'fs/promises';
+
+type CacheResultSource = 'memory' | 'disk' | 'generation';
+
+type CacheGetResult<T> = {
+  resultSource: CacheResultSource;
+  data: T;
+  filePath: string;
+};
+
+export class FileCache<Inputs extends Record<string, unknown>, Output> {
+  private memoryCache: Map<string, Output> = new Map();
+
+  private cacheDirectory: string | null = null;
+
+  public setCacheDirectory(cacheDirectory: string) {
+    this.cacheDirectory = cacheDirectory;
+  }
+  public getCacheDirectory() {
+    if (!this.cacheDirectory) {
+      console.error(`Tried to retrieve entry from cache, but no cache directory was set.`);
+      process.exit(1);
+    }
+    return this.cacheDirectory;
+  }
+
+  private hashInputs(inputs: Inputs): string {
+    return crypto
+      .createHash('sha256')
+      .update(JSON.stringify(inputs) + this.config.globalHashBits)
+      .digest('hex')
+      .slice(0, 10);
+  }
+
+  private getFileName(hash: string) {
+    return `${hash}.json`;
+  }
+
+  public async cleanupUnusedFiles() {
+    const allFiles = await readdir(this.getCacheDirectory());
+    const usedFiles = Array.from(this.memoryCache.keys()).map((key) => this.getFileName(key));
+    const unusedFiles = allFiles.filter((file) => !usedFiles.includes(file));
+    await Promise.all(unusedFiles.map((file) => rm(path.join(this.getCacheDirectory(), file))));
+    return {
+      deletedCount: unusedFiles.length,
+    };
+  }
+
+  public async get(inputs: Inputs): Promise<CacheGetResult<Output>> {
+    const hash = this.hashInputs(inputs);
+    const filePath = path.join(this.getCacheDirectory(), this.getFileName(hash));
+
+    const memoryCacheResult = this.memoryCache.get(hash);
+    if (memoryCacheResult) {
+      return {
+        resultSource: 'memory',
+        data: memoryCacheResult,
+        filePath,
+      };
+    }
+
+    const getFromFileOrGenerate = async () => {
+      try {
+        const fileContents = await readFile(filePath, 'utf8');
+        return {
+          resultSource: 'disk' as const,
+          data: JSON.parse(fileContents) as Output,
+          filePath,
+        };
+      } catch {
+        const data = await this.config.generate(inputs);
+        await writeFile(filePath, JSON.stringify(data), 'utf8');
+        return {
+          resultSource: 'generation' as const,
+          data,
+          filePath,
+        };
+      }
+    };
+
+    const result = await getFromFileOrGenerate();
+    this.memoryCache.set(hash, result.data);
+    return result;
+  }
+
+  constructor(
+    private config: {
+      /**
+       * Additional information to include in the hash of the inputs.
+       * This is useful for cases where the inputs are the same, but the global state is different.
+       * Eg: The preview worker source can be used here.
+       */
+      globalHashBits: string;
+      generate: (inputs: Inputs) => Promise<Output>;
+    },
+  ) {}
+}

package/plugin/specs/fetchSpecSSR.ts

@@ -0,0 +1,27 @@
+import { readFile } from 'fs/promises';
+
+import { api } from 'virtual:stainless-apis-manifest';
+import { SpecWithAuth } from './generateSpec';
+import { DocsLanguage } from '@stainless-api/docs-ui/routing';
+
+const cachedSpecWithAuth: Record<string, SpecWithAuth> = {};
+
+async function getSpecWithAuthInSSR(filePath: string) {
+  if (cachedSpecWithAuth[filePath]) {
+    return cachedSpecWithAuth[filePath];
+  }
+  const specStr = await readFile(filePath, 'utf8');
+  const json = JSON.parse(specStr) as SpecWithAuth;
+  cachedSpecWithAuth[filePath] = json;
+  return json;
+}
+
+export async function getSDKJSONInSSR(language: DocsLanguage) {
+  const filePath = api.languages.find((l) => l.language === language)?.sdkJSONFilePath;
+  if (!filePath) {
+    throw new Error(`No SDK JSON file path for language: ${language}`);
+  }
+
+  const specWithAuth = await getSpecWithAuthInSSR(filePath);
+  return specWithAuth.sdkJson;
+}

package/plugin/specs/generateSpec.ts

@@ -0,0 +1,112 @@
+import type * as SDKJSON from '@stainless/sdk-json';
+import { Languages } from '@stainless-api/docs-ui/routing';
+import { createSDKJSON, ParsedConfig, parseInputs, transformOAS } from './worker';
+import { LanguageGenerateQuery } from '../loadPluginConfig';
+import { FileCache } from './FileCache';
+import previewWorkerCode from '../vendor/preview.worker.docs.js?raw';
+
+function getLanguagesFromStainlessConfig(config: ParsedConfig): SDKJSON.SpecLanguage[] {
+  // if the Stainless config has a list of docs languages, use that
+  if (config.docs?.languages) {
+    return config.docs.languages;
+  }
+
+  // otherwise, just loop over all targets in the config + use the ones that are not skipped
+  return Object.entries(config.targets)
+    .filter(([name, target]) => {
+      if (!Languages.includes(name)) return false; // not  a valid language
+      if (target.skip) return false; // config says to skip this language
+      return true;
+    })
+    .map(([name]) => name) as SDKJSON.SpecLanguage[];
+}
+
+// These inputs contain everything needed to generate a spec
+// Combined with the source of the preview workers, we can make a hash to cache the resulting spec
+export type GenerateSpecRawInputs = {
+  oasStr: string;
+  configStr: string;
+  languageOverrides: LanguageGenerateQuery | null;
+  stainlessProject: string;
+  versionInfo: Record<SDKJSON.SpecLanguage, string> | null;
+};
+
+function applyLanguageOverrides(
+  initialLanguages: SDKJSON.SpecLanguage[],
+  languageOverrides: LanguageGenerateQuery | null,
+) {
+  if (!languageOverrides) return initialLanguages;
+  if (languageOverrides.mode === 'exclude') {
+    return initialLanguages.filter((language) => !languageOverrides.list.includes(language));
+  }
+  return languageOverrides.list;
+}
+
+async function generateSpecFromStrings({
+  oasStr,
+  configStr,
+  stainlessProject,
+  languageOverrides,
+  versionInfo,
+}: GenerateSpecRawInputs) {
+  const { oas, config } = await parseInputs({
+    oas: oasStr,
+    config: configStr,
+  });
+
+  const transformedOAS = await transformOAS({ oas, config });
+
+  let languagesToGenerate = getLanguagesFromStainlessConfig(config);
+  // by default, we should generate the HTTP spec (unless it's explicitly excluded)
+  if (!languagesToGenerate.includes('http')) {
+    languagesToGenerate.push('http');
+  }
+
+  languagesToGenerate = applyLanguageOverrides(languagesToGenerate, languageOverrides);
+
+  // SDKJSON has weird behavior where it will create a spec with HTTP, even if it's not in the languages list
+  const sdkJson = await createSDKJSON({
+    oas: transformedOAS,
+    config,
+    // if language overrides are provided, use them, otherwise use the languages from the Stainless config
+    languages: languagesToGenerate,
+    projectName: stainlessProject,
+  });
+
+  let languages = sdkJson.docs?.languages;
+
+  if (!languages) {
+    throw new Error(`SDKJSON created without any languages`);
+  }
+
+  // if language overrides are provided, filter the languages to only include the ones that are in the overrides
+  languages = languages.filter((language) => languagesToGenerate.includes(language));
+
+  if (versionInfo) {
+    for (const [lang, version] of Object.entries(versionInfo)) {
+      const meta = sdkJson.metadata[lang as SDKJSON.SpecLanguage];
+      if (meta?.version) meta.version = version;
+    }
+  }
+
+  const opts = Object.entries(config.client_settings.opts).map(([k, v]) => ({ name: k, ...v }));
+  return {
+    sdkJson,
+    languages,
+    auth: sdkJson.security_schemes.map((scheme) => ({
+      ...scheme,
+      opts: opts.filter((opt) => opt.auth?.security_scheme === scheme.name),
+    })),
+  };
+}
+
+export const specCache = new FileCache({
+  generate: generateSpecFromStrings,
+  globalHashBits: previewWorkerCode, // you can change this as a last resort to invalidate the cache
+});
+
+export type SpecCacheResult = Awaited<ReturnType<typeof specCache.get>>;
+
+export type GenerateSpecFn = typeof generateSpecFromStrings;
+
+export type SpecWithAuth = Awaited<ReturnType<GenerateSpecFn>>;

package/plugin/specs/index.ts

@@ -0,0 +1,132 @@
+import { mkdir } from 'fs/promises';
+import path from 'path';
+
+import type * as VirtualManifestModule from 'virtual:stainless-apis-manifest';
+
+import { makeAsyncVirtualModPlugin } from '../../shared/virtualModule';
+
+import { NormalizedStainlessStarlightConfig } from '../loadPluginConfig';
+
+import { specCache, SpecCacheResult } from './generateSpec';
+import { AstroIntegrationLogger } from 'astro';
+import type * as SDKJSON from '@stainless/sdk-json';
+
+/**
+ * A helper class to manage multiple spec cache results for a single API
+ * An API may have multiple spec cache results if it has multiple languages
+ * Note that one spec may contain multiple languages.
+ * */
+export class SpecComposite {
+  private languages: Set<SDKJSON.SpecLanguage>;
+  private readonly specs: Partial<Record<SDKJSON.SpecLanguage, SpecCacheResult>>;
+
+  public getLanguages() {
+    return Array.from(this.languages);
+  }
+
+  public getByLanguage(language: SDKJSON.SpecLanguage) {
+    const spec = this.specs[language];
+    if (!spec) {
+      throw new Error(`Spec for language ${language} not found`);
+    }
+    return spec;
+  }
+
+  /**
+   * Returns all specs. It will return each spec once, even if it has multiple languages.
+   * */
+  public listUniqueSpecs() {
+    const seen = new Set<SpecCacheResult>();
+    const unique: SpecCacheResult[] = [];
+    for (const spec of Object.values(this.specs)) {
+      if (!seen.has(spec)) {
+        seen.add(spec);
+        unique.push(spec);
+      }
+    }
+    return unique;
+  }
+
+  public listAllLanguagesAndIncludeSpecs() {
+    return this.getLanguages().map((language) => ({
+      language,
+      spec: this.getByLanguage(language),
+    }));
+  }
+
+  constructor(specs: SpecCacheResult[]) {
+    this.languages = new Set<SDKJSON.SpecLanguage>();
+    this.specs = {};
+    for (const spec of specs) {
+      for (const lang of spec.data.languages) {
+        if (this.languages.has(lang)) {
+          throw new Error(`Language appears multiple times in the same API: ${lang}`);
+        }
+        if (lang === 'openapi' || lang === 'sql') continue;
+        this.languages.add(lang);
+        this.specs[lang] = spec;
+      }
+    }
+  }
+}
+
+/** Runs once in the build process */
+export async function startSpecLoader(
+  pluginConfig: NormalizedStainlessStarlightConfig,
+  logger: AstroIntegrationLogger,
+  codegenDir: URL,
+) {
+  const specsDirectory = path.join(codegenDir.pathname, 'specs');
+  await mkdir(specsDirectory, { recursive: true });
+
+  logger.debug(`Setting cache directory to ${specsDirectory}`);
+
+  // 🚨 Important! You cannot call loadSpecs() before setting the cache directory.
+  specCache.setCacheDirectory(specsDirectory);
+
+  async function load() {
+    const specs = await pluginConfig.api.loadSpecs();
+
+    // not awaited since it's just cleanup
+    specCache
+      .cleanupUnusedFiles()
+      .then((result) => {
+        if (result.deletedCount > 0) {
+          logger.info(`Cleaned up ${result.deletedCount} unused spec files`);
+        } else {
+          logger.debug(`No unused spec files to clean up`);
+        }
+      })
+      .catch(() => {
+        logger.warn(`Failed to clean up unused spec files`);
+      });
+
+    return {
+      specComposite: new SpecComposite(specs),
+    };
+  }
+
+  const specPromise = load();
+
+  return {
+    specPromise,
+    // this virtual module only resolves when the spec is generated
+    // this prevents the SSR module from trying to read the spec file before it's generated
+    vitePlugins: [
+      makeAsyncVirtualModPlugin<typeof VirtualManifestModule>('virtual:stainless-apis-manifest', async () => {
+        const api = await specPromise;
+
+        return {
+          api: {
+            languages: api.specComposite.listAllLanguagesAndIncludeSpecs().map((langSpec) => ({
+              language: langSpec.language,
+              sdkJSONFilePath: langSpec.spec.filePath,
+            })),
+          },
+        };
+      }),
+    ],
+  };
+}
+
+export type SpecLoader = Awaited<ReturnType<typeof startSpecLoader>>;