@stainless-api/docs 0.1.0-beta.11 → 0.1.0-beta.110

package/plugin/specs/FileCache.ts

@@ -0,0 +1,99 @@
+import crypto from 'crypto';
+import path from 'path';
+import { readdir, readFile, rm, writeFile } from 'fs/promises';
+
+type CacheResultSource = 'memory' | 'disk' | 'generation';
+
+export type CacheGetResult<T> = {
+  resultSource: CacheResultSource;
+  data: T;
+  filePath: string;
+};
+
+export class FileCache<Inputs extends Record<string, unknown>, Output> {
+  private memoryCache: Map<string, Output> = new Map();
+
+  private cacheDirectory: string | null = null;
+
+  public setCacheDirectory(cacheDirectory: string) {
+    this.cacheDirectory = cacheDirectory;
+  }
+  public getCacheDirectory() {
+    if (!this.cacheDirectory) {
+      console.error(`Tried to retrieve entry from cache, but no cache directory was set.`);
+      process.exit(1);
+    }
+    return this.cacheDirectory;
+  }
+
+  private hashInputs(inputs: Inputs): string {
+    return crypto
+      .createHash('sha256')
+      .update(JSON.stringify(inputs) + this.config.globalHashBits)
+      .digest('hex')
+      .slice(0, 10);
+  }
+
+  private getFileName(hash: string) {
+    return `${hash}.json`;
+  }
+
+  public async cleanupUnusedFiles() {
+    const allFiles = await readdir(this.getCacheDirectory());
+    const usedFiles = Array.from(this.memoryCache.keys()).map((key) => this.getFileName(key));
+    const unusedFiles = allFiles.filter((file) => !usedFiles.includes(file));
+    await Promise.all(unusedFiles.map((file) => rm(path.join(this.getCacheDirectory(), file))));
+    return {
+      deletedCount: unusedFiles.length,
+    };
+  }
+
+  public async get(inputs: Inputs): Promise<CacheGetResult<Output>> {
+    const hash = this.hashInputs(inputs);
+    const filePath = path.join(this.getCacheDirectory(), this.getFileName(hash));
+
+    const memoryCacheResult = this.memoryCache.get(hash);
+    if (memoryCacheResult) {
+      return {
+        resultSource: 'memory',
+        data: memoryCacheResult,
+        filePath,
+      };
+    }
+
+    const getFromFileOrGenerate = async () => {
+      try {
+        const fileContents = await readFile(filePath, 'utf8');
+        return {
+          resultSource: 'disk' as const,
+          data: JSON.parse(fileContents) as Output,
+          filePath,
+        };
+      } catch {
+        const data = await this.config.generate(inputs);
+        await writeFile(filePath, JSON.stringify(data), 'utf8');
+        return {
+          resultSource: 'generation' as const,
+          data,
+          filePath,
+        };
+      }
+    };
+
+    const result = await getFromFileOrGenerate();
+    this.memoryCache.set(hash, result.data);
+    return result;
+  }
+
+  constructor(
+    private config: {
+      /**
+       * Additional information to include in the hash of the inputs.
+       * This is useful for cases where the inputs are the same, but the global state is different.
+       * Eg: The preview worker source can be used here.
+       */
+      globalHashBits: string;
+      generate: (inputs: Inputs) => Promise<Output>;
+    },
+  ) {}
+}

package/plugin/specs/fetchSpecSSR.ts

@@ -0,0 +1,27 @@
+import { readFile } from 'fs/promises';
+
+import { api } from 'virtual:stainless-apis-manifest';
+import { SpecWithAuth } from './generateSpec';
+import { DocsLanguage } from '@stainless-api/docs-ui/routing';
+
+const cachedSpecWithAuth: Record<string, SpecWithAuth> = {};
+
+async function getSpecWithAuthInSSR(filePath: string) {
+  if (cachedSpecWithAuth[filePath]) {
+    return cachedSpecWithAuth[filePath];
+  }
+  const specStr = await readFile(filePath, 'utf8');
+  const json = JSON.parse(specStr) as SpecWithAuth;
+  cachedSpecWithAuth[filePath] = json;
+  return json;
+}
+
+export async function getSDKJSONInSSR(language: DocsLanguage) {
+  const filePath = api.languages.find((l) => l.language === language)?.sdkJSONFilePath;
+  if (!filePath) {
+    throw new Error(`No SDK JSON file path for language: ${language}`);
+  }
+
+  const specWithAuth = await getSpecWithAuthInSSR(filePath);
+  return specWithAuth.sdkJson;
+}

package/plugin/specs/generateSpec.ts

@@ -0,0 +1,112 @@
+import type * as SDKJSON from '@stainless/sdk-json';
+import { Languages } from '@stainless-api/docs-ui/routing';
+import { createSDKJSON, ParsedConfig, parseInputs, transformOAS } from './worker';
+import { LanguageGenerateQuery } from '../loadPluginConfig';
+import { FileCache } from './FileCache';
+import previewWorkerCode from '../vendor/preview.worker.docs.js?raw';
+
+function getLanguagesFromStainlessConfig(config: ParsedConfig): SDKJSON.SpecLanguage[] {
+  // if the Stainless config has a list of docs languages, use that
+  if (config.docs?.languages) {
+    return config.docs.languages;
+  }
+
+  // otherwise, just loop over all targets in the config + use the ones that are not skipped
+  return Object.entries(config.targets)
+    .filter(([name, target]) => {
+      if (!Languages.includes(name)) return false; // not  a valid language
+      if (target.skip) return false; // config says to skip this language
+      return true;
+    })
+    .map(([name]) => name) as SDKJSON.SpecLanguage[];
+}
+
+// These inputs contain everything needed to generate a spec
+// Combined with the source of the preview workers, we can make a hash to cache the resulting spec
+export type GenerateSpecRawInputs = {
+  oasStr: string;
+  configStr: string;
+  languageOverrides: LanguageGenerateQuery | null;
+  stainlessProject: string;
+  versionInfo: Record<SDKJSON.SpecLanguage, string> | null;
+};
+
+function applyLanguageOverrides(
+  initialLanguages: SDKJSON.SpecLanguage[],
+  languageOverrides: LanguageGenerateQuery | null,
+) {
+  if (!languageOverrides) return initialLanguages;
+  if (languageOverrides.mode === 'exclude') {
+    return initialLanguages.filter((language) => !languageOverrides.list.includes(language));
+  }
+  return languageOverrides.list;
+}
+
+async function generateSpecFromStrings({
+  oasStr,
+  configStr,
+  stainlessProject,
+  languageOverrides,
+  versionInfo,
+}: GenerateSpecRawInputs) {
+  const { oas, config } = await parseInputs({
+    oas: oasStr,
+    config: configStr,
+  });
+
+  const transformedOAS = await transformOAS({ oas, config });
+
+  let languagesToGenerate = getLanguagesFromStainlessConfig(config);
+  // by default, we should generate the HTTP spec (unless it's explicitly excluded)
+  if (!languagesToGenerate.includes('http')) {
+    languagesToGenerate.push('http');
+  }
+
+  languagesToGenerate = applyLanguageOverrides(languagesToGenerate, languageOverrides);
+
+  // SDKJSON has weird behavior where it will create a spec with HTTP, even if it's not in the languages list
+  const sdkJson = await createSDKJSON({
+    oas: transformedOAS,
+    config,
+    // if language overrides are provided, use them, otherwise use the languages from the Stainless config
+    languages: languagesToGenerate,
+    projectName: stainlessProject,
+  });
+
+  let languages = sdkJson.docs?.languages;
+
+  if (!languages) {
+    throw new Error(`SDKJSON created without any languages`);
+  }
+
+  // if language overrides are provided, filter the languages to only include the ones that are in the overrides
+  languages = languages.filter((language) => languagesToGenerate.includes(language));
+
+  if (versionInfo) {
+    for (const [lang, version] of Object.entries(versionInfo)) {
+      const meta = sdkJson.metadata[lang as SDKJSON.SpecLanguage];
+      if (meta?.version) meta.version = version;
+    }
+  }
+
+  const opts = Object.entries(config.client_settings.opts).map(([k, v]) => ({ name: k, ...v }));
+  return {
+    sdkJson,
+    languages,
+    auth: sdkJson.security_schemes.map((scheme) => ({
+      ...scheme,
+      opts: opts.filter((opt) => opt.auth?.security_scheme === scheme.name),
+    })),
+  };
+}
+
+export const specCache = new FileCache({
+  generate: generateSpecFromStrings,
+  globalHashBits: previewWorkerCode, // you can change this as a last resort to invalidate the cache
+});
+
+export type SpecCacheResult = Awaited<ReturnType<typeof specCache.get>>;
+
+export type GenerateSpecFn = typeof generateSpecFromStrings;
+
+export type SpecWithAuth = Awaited<ReturnType<GenerateSpecFn>>;

package/plugin/specs/index.ts

@@ -0,0 +1,132 @@
+import { mkdir } from 'fs/promises';
+import path from 'path';
+
+import type * as VirtualManifestModule from 'virtual:stainless-apis-manifest';
+
+import { makeAsyncVirtualModPlugin } from '../../shared/virtualModule';
+
+import { NormalizedStainlessStarlightConfig, ResolvedAPIConfigEntry } from '../loadPluginConfig';
+
+import { specCache, SpecCacheResult } from './generateSpec';
+import { AstroIntegrationLogger } from 'astro';
+import type * as SDKJSON from '@stainless/sdk-json';
+
+export type LoadedAPIConfigEntry = Omit<ResolvedAPIConfigEntry, 'loadSpecs'> & {
+  specs: SpecCacheResult[];
+  languages: SDKJSON.SpecLanguage[];
+};
+
+/**
+ * A helper class to manage multiple spec cache results for a single API
+ * An API may have multiple spec cache results if it has multiple languages
+ * Note that one spec may contain multiple languages.
+ * */
+export class SpecComposite {
+  private languages: Set<SDKJSON.SpecLanguage>;
+  private readonly specs: Partial<Record<SDKJSON.SpecLanguage, SpecCacheResult>>;
+
+  public getLanguages() {
+    return Array.from(this.languages);
+  }
+
+  public getByLanguage(language: SDKJSON.SpecLanguage) {
+    const spec = this.specs[language];
+    if (!spec) {
+      throw new Error(`Spec for language ${language} not found`);
+    }
+    return spec;
+  }
+
+  /**
+   * Returns all specs. It will return each spec once, even if it has multiple languages.
+   * */
+  public listUniqueSpecs() {
+    const seen = new Set<SpecCacheResult>();
+    const unique: SpecCacheResult[] = [];
+    for (const spec of Object.values(this.specs)) {
+      if (!seen.has(spec)) {
+        seen.add(spec);
+        unique.push(spec);
+      }
+    }
+    return unique;
+  }
+
+  public listAllLanguagesAndIncludeSpecs() {
+    return this.getLanguages().map((language) => ({
+      language,
+      spec: this.getByLanguage(language),
+    }));
+  }
+
+  constructor(specs: SpecCacheResult[]) {
+    this.languages = new Set<SDKJSON.SpecLanguage>();
+    this.specs = {};
+    for (const spec of specs) {
+      for (const lang of spec.data.languages) {
+        if (this.languages.has(lang)) {
+          throw new Error(`Language appears multiple times in the same API: ${lang}`);
+        }
+        if (lang === 'openapi' || lang === 'sql') continue;
+        this.languages.add(lang);
+        this.specs[lang] = spec;
+      }
+    }
+  }
+}
+
+/** Runs once in the build process */
+export async function startSpecLoader(
+  pluginConfig: NormalizedStainlessStarlightConfig,
+  logger: AstroIntegrationLogger,
+  codegenDir: URL,
+) {
+  const specsDirectory = path.join(codegenDir.pathname, 'specs');
+  await mkdir(specsDirectory, { recursive: true });
+
+  logger.debug(`Setting cache directory to ${specsDirectory}`);
+
+  // 🚨 Important! You cannot call loadSpecs() before setting the cache directory.
+  specCache.setCacheDirectory(specsDirectory);
+
+  async function load() {
+    const specs = await pluginConfig.api.loadSpecs();
+
+    // not awaited since it's just cleanup
+    specCache.cleanupUnusedFiles().then((result) => {
+      if (result.deletedCount > 0) {
+        logger.info(`Cleaned up ${result.deletedCount} unused spec files`);
+      } else {
+        logger.debug(`No unused spec files to clean up`);
+      }
+    });
+
+    return {
+      specComposite: new SpecComposite(specs),
+    };
+  }
+
+  const specPromise = load();
+
+  return {
+    specPromise,
+    // this virtual module only resolves when the spec is generated
+    // this prevents the SSR module from trying to read the spec file before it's generated
+    vitePlugins: [
+      makeAsyncVirtualModPlugin<typeof VirtualManifestModule>('virtual:stainless-apis-manifest', async () => {
+        const api = await specPromise;
+
+        return {
+          api: {
+            languages: api.specComposite.listAllLanguagesAndIncludeSpecs().map((langSpec) => ({
+              language: langSpec.language,
+              sdkJSONFilePath: langSpec.spec.filePath,
+            })),
+          },
+        };
+      }),
+    ],
+  };
+}
+
+export type SpecLoader = Awaited<ReturnType<typeof startSpecLoader>>;

package/plugin/specs/inputResolver.ts

@@ -0,0 +1,146 @@
+import { DocsLanguage } from '@stainless-api/docs-ui/routing';
+import { readFile } from 'fs/promises';
+import { AstroIntegrationLogger } from 'astro';
+import { bold } from '../../shared/terminalUtils';
+import type { LanguageGenerateQuery, LoadedApiKey } from '../loadPluginConfig';
+import Stainless, { APIError } from '@stainless-api/sdk';
+import { GenerateSpecRawInputs } from './generateSpec';
+
+export type SpecInputResolver = {
+  resolve: (context: { apiKey: LoadedApiKey | null }) => Promise<GenerateSpecRawInputs>;
+  printError: (error: Error, logger: AstroIntegrationLogger) => void;
+};
+
+function fromFiles({
+  oasPath,
+  configPath,
+  languageOverrides,
+  stainlessProject,
+}: {
+  oasPath: string;
+  configPath: string;
+  languageOverrides: LanguageGenerateQuery | null;
+  stainlessProject: string;
+}): SpecInputResolver {
+  return {
+    resolve: async () => {
+      const oasStr = await readFile(oasPath, 'utf8');
+      const configStr = await readFile(configPath, 'utf8');
+      return {
+        oasStr,
+        configStr,
+        versionInfo: null,
+        languageOverrides,
+        stainlessProject,
+      };
+    },
+    printError: (error: Error, logger: AstroIntegrationLogger) => {
+      logger.error(bold('Failed to resolve spec inputs from files:'));
+      logger.error(error.message);
+    },
+  };
+}
+
+async function fetchVersionInfo(project: string, apiKey: string): Promise<Record<DocsLanguage, string>> {
+  const data = await fetch(`https://api.stainless.com/api/projects/${project}/package-versions`, {
+    headers: { Authorization: `Bearer ${apiKey}` },
+  });
+
+  const content = await data.text();
+  return JSON.parse(content) as Record<DocsLanguage, string>;
+}
+
+function redactApiKey(apiKey: string) {
+  return apiKey
+    .split('')
+    .map((char, index) => (index < 10 ? char : '*'))
+    .join('');
+}
+
+function fromStainlessApi(inputs: {
+  stainlessProject: string;
+  branch: string;
+  apiKey?: LoadedApiKey;
+  languageOverrides: LanguageGenerateQuery | null;
+}): SpecInputResolver {
+  let apiKey: string | undefined;
+  return {
+    async resolve(context) {
+      apiKey = context.apiKey?.value ?? inputs.apiKey?.value;
+
+      if (!apiKey) {
+        throw new Error('No API key provided');
+      }
+
+      const client = new Stainless({ apiKey });
+      const configs = await client.projects.configs.retrieve({
+        project: inputs.stainlessProject,
+        branch: inputs.branch,
+        include: 'openapi',
+      });
+      const versionInfo = await fetchVersionInfo(inputs.stainlessProject, apiKey);
+
+      const configYML = Object.values(configs)[0] as { content: unknown };
+      const oasJson = Object.values(configs)[1] as { content: unknown };
+      const oasStr = oasJson['content'];
+      const configStr = configYML['content'];
+
+      if (typeof oasStr !== 'string' || typeof configStr !== 'string') {
+        throw new Error('Received invalid OAS or config from Stainless');
+      }
+
+      return {
+        oasStr,
+        configStr,
+        versionInfo,
+        languageOverrides: inputs.languageOverrides,
+        stainlessProject: inputs.stainlessProject,
+      };
+    },
+    printError: (error: Error, logger: AstroIntegrationLogger) => {
+      if (error instanceof APIError && error.status >= 400 && error.status < 500) {
+        logger.error(`Requested project slug: "${inputs.stainlessProject}"`);
+        if (apiKey) {
+          logger.error(`API key: "${redactApiKey(apiKey)}"`);
+        }
+        logger.error(
+          `This error can usually be corrected by re-authenticating with the Stainless.  Use the CLI (stl auth login) or verify that the Stainless API key you're using can access the project mentioned above.`,
+        );
+      }
+    },
+  };
+}
+
+function fromStrings({
+  oasStr,
+  configStr,
+  languageOverrides,
+  stainlessProject,
+}: {
+  oasStr: string;
+  configStr: string;
+  languageOverrides: LanguageGenerateQuery | null;
+  stainlessProject: string;
+}): SpecInputResolver {
+  return {
+    async resolve() {
+      return {
+        oasStr,
+        configStr,
+        versionInfo: null,
+        languageOverrides,
+        stainlessProject,
+      };
+    },
+    printError(error: Error, logger: AstroIntegrationLogger) {
+      logger.error(bold('Failed to resolve spec inputs from strings:'));
+      logger.error(error.message);
+    },
+  };
+}
+
+export const resolveSpec = {
+  fromFiles,
+  fromStainlessApi,
+  fromStrings,
+} satisfies Record<string, (...args: never[]) => SpecInputResolver>;

package/plugin/{cms → specs}/worker.ts

@@ -1,6 +1,6 @@
 import Worker from 'web-worker';
-import { Languages, type DocsLanguage } from '@stainless-api/docs-ui/src/routing';
-import type * as SDKJSON from '~/lib/json-spec-v2/types';
+import { Languages, type DocsLanguage } from '@stainless-api/docs-ui/routing';
+import type * as SDKJSON from '@stainless/sdk-json';
 import { fileURLToPath } from 'node:url';
 import { dirname, resolve } from 'node:path';
 import fs from 'fs/promises';
@@ -12,7 +12,81 @@ const __dirname = dirname(__filename);
 const workerPath = resolve(__dirname, '..', 'vendor', 'preview.worker.docs.js');
 
 type OpenAPIDocument = Record<string, any>;
-type ParsedConfig = Record<string, any>;
+export type ParsedConfig = {
+  docs:
+    | {
+        title?: string | undefined;
+        favicon?: string | undefined;
+        logo_icon?: string | undefined;
+        search?:
+          | {
+              algolia?:
+                | {
+                    app_id: string;
+                    index_name: string;
+                    search_key: string;
+                  }
+                | undefined;
+            }
+          | undefined;
+        description?: string | undefined;
+        languages?:
+          | ('node' | 'typescript' | 'python' | 'java' | 'kotlin' | 'go' | 'ruby' | 'terraform' | 'http')[]
+          | undefined;
+        snippets?:
+          | {
+              exclude_languages?: string[] | undefined;
+            }
+          | undefined;
+        show_security?: boolean | undefined;
+        show_readme?: boolean | undefined;
+        base_path?: string | undefined;
+        navigation?:
+          | {
+              menubar?:
+                | {
+                    title: string;
+                    icon?: string;
+                    variant?: string;
+                    href?: string | undefined;
+                    page?: string | undefined;
+                  }[]
+                | undefined;
+              sidebar?:
+                | {
+                    title: string;
+                    icon?: string;
+                    variant?: string;
+                    href?: string | undefined;
+                    page?: string | undefined;
+                  }[]
+                | undefined;
+            }
+          | undefined;
+        pages?: unknown;
+        resources?: unknown[] | undefined;
+      }
+    | undefined;
+  targets: Record<string, { skip?: boolean }>;
+  client_settings: {
+    opts: {
+      [x: string]: {
+        type: 'string' | 'number' | 'boolean' | 'null' | 'integer';
+        nullable: boolean;
+        description?: string | undefined;
+        example?: unknown;
+        default?: unknown;
+        read_env?: string | undefined;
+        auth?:
+          | {
+              security_scheme: string;
+              role?: 'value' | 'password' | 'username' | 'client_id' | 'client_secret' | undefined;
+            }
+          | undefined;
+      };
+    };
+  };
+};
 
 function runJob({ type, signal, data }: { type: string; signal?: AbortSignal; data: any }) {
   const stainlessWorker = new Worker(workerPath, {
@@ -22,6 +96,7 @@ function runJob({ type, signal, data }: { type: string; signal?: AbortSignal; da
 
   return new Promise<any>((resolve, reject) => {
     stainlessWorker.addEventListener('error', (e) => {
+      e.preventDefault();
       reject(e);
     });
 
@@ -85,10 +160,12 @@ export async function createSDKJSON({
   oas,
   config,
   languages,
+  projectName,
 }: {
   oas: OpenAPIDocument;
   config: ParsedConfig;
   languages: DocsLanguage[];
+  projectName: string;
 }) {
   const templatePath = resolve(__dirname, '../vendor/templates');
   const readmeLoader = await Promise.all(
@@ -98,7 +175,7 @@ export async function createSDKJSON({
       try {
         const content = await fs.readFile(mdfile);
         return [language, content.toString()];
-      } catch (err) {
+      } catch {
         return [language, null];
       }
     }),
@@ -113,7 +190,7 @@ export async function createSDKJSON({
       config,
       languages,
       transform: false,
-      projectName: '',
+      projectName,
       readmeTemplates,
     },
   });