@stainless-api/docs 0.1.0-beta.138 → 0.1.0-beta.139

package/plugin/specs/defaultSpecLoader.ts

@@ -0,0 +1,192 @@
+import path from 'path';
+import type { SpecLoaderFn, SpecLoaderParams } from './utils';
+import Stainless, { APIError } from '@stainless-api/sdk';
+import { DocsLanguage } from '@stainless-api/docs-ui/routing';
+import { bold } from '../../shared/terminalUtils';
+import { mkdir, readdir, readFile, rm, writeFile } from 'fs/promises';
+import { generateSpecFromStrings, previewWorkerCode } from '@stainless/sdk-json/spec';
+import { Spec, SpecLanguage } from '@stainless/sdk-json';
+import crypto from 'crypto';
+
+function resolvePath(inputPath: string) {
+  return path.resolve(process.cwd(), inputPath);
+}
+
+function getLocalFilePaths() {
+  // eslint-disable-next-line turbo/no-undeclared-env-vars
+  const oasPath = process.env.OPENAPI_PATH;
+  // eslint-disable-next-line turbo/no-undeclared-env-vars
+  const configPath = process.env.STAINLESS_CONFIG_PATH;
+
+  if (!oasPath || !configPath) {
+    return null;
+  }
+
+  return {
+    oasPath: resolvePath(oasPath),
+    configPath: resolvePath(configPath),
+  };
+}
+
+async function fetchVersionInfo(project: string, apiKey: string): Promise<Record<DocsLanguage, string>> {
+  const data = await fetch(`https://api.stainless.com/api/projects/${project}/package-versions`, {
+    headers: { Authorization: `Bearer ${apiKey}` },
+  });
+
+  const content = await data.text();
+  return JSON.parse(content) as Record<DocsLanguage, string>;
+}
+
+function redactApiKey(apiKey: string) {
+  return apiKey
+    .split('')
+    .map((char, index) => (index < 10 ? char : '*'))
+    .join('');
+}
+
+async function loadInputs({ apiKey, logger, stainlessProject, branch }: SpecLoaderParams) {
+  const localFilePaths = getLocalFilePaths();
+
+  if (localFilePaths) {
+    try {
+      const oasStr = await readFile(localFilePaths.oasPath, 'utf8');
+      const configStr = await readFile(localFilePaths.configPath, 'utf8');
+      return {
+        oasStr,
+        configStr,
+        versionInfo: null,
+      };
+    } catch (e) {
+      logger.error(bold('Failed to load spec inputs from files:'));
+      logger.error(e instanceof Error ? e.message : String(e));
+      process.exit(1);
+    }
+  }
+
+  if (!apiKey) {
+    logger.error(
+      [
+        bold(
+          'No Stainless credentials found. Please choose one of the following options to authenticate with Stainless:',
+        ),
+        '- Run `stl auth login` to authenticate via the Stainless CLI',
+        '- Provide a Stainless API key via the `STAINLESS_API_KEY` environment variable (eg. in a .env file)',
+        '- Set the `apiKey` option in the Stainless Docs config',
+      ].join('\n'),
+    );
+    process.exit(1);
+  }
+
+  try {
+    const client = new Stainless({ apiKey });
+    const configs = await client.projects.configs.retrieve({
+      project: stainlessProject,
+      branch: branch,
+      include: 'openapi',
+    });
+    const versionInfo = await fetchVersionInfo(stainlessProject, apiKey);
+
+    const configYML = Object.values(configs)[0] as { content: unknown };
+    const oasJson = Object.values(configs)[1] as { content: unknown };
+    const oasStr = oasJson['content'];
+    const configStr = configYML['content'];
+
+    if (typeof oasStr !== 'string' || typeof configStr !== 'string') {
+      logger.error('Received invalid OAS or config from Stainless');
+      process.exit(1);
+    }
+
+    return {
+      oasStr,
+      configStr,
+      versionInfo,
+    };
+  } catch (e) {
+    if (e instanceof APIError && e.status >= 400 && e.status < 500) {
+      logger.error(`Failed to load requested project slug: "${stainlessProject}"`);
+      if (apiKey) {
+        logger.error(`API key: "${redactApiKey(apiKey)}"`);
+      }
+      logger.error(
+        `This error can usually be corrected by re-authenticating with the Stainless.  Use the CLI (stl auth login) or verify that the Stainless API key you're using can access the project mentioned above.`,
+      );
+    }
+    process.exit(1);
+  }
+}
+
+async function maybeLoadJSONFile<T>(filePath: string): Promise<T | null> {
+  try {
+    const fileContents = await readFile(filePath, 'utf8');
+    return JSON.parse(fileContents) as T;
+  } catch {
+    return null;
+  }
+}
+
+async function cleanupDirectory(directory: string, filesToKeep: string[]) {
+  const allFiles = await readdir(directory);
+  const unusedFiles = allFiles.filter((file) => !filesToKeep.includes(file));
+  await Promise.all(unusedFiles.map((file) => rm(path.join(directory, file))));
+  return {
+    deletedCount: unusedFiles.length,
+  };
+}
+
+export const defaultSpecLoader: SpecLoaderFn = async (params) => {
+  const { createCodegenDir } = params;
+
+  const inputs = await loadInputs(params);
+
+  const specsDirectory = path.join(createCodegenDir().pathname, 'specs2');
+  await mkdir(specsDirectory, { recursive: true });
+
+  const fileName =
+    crypto
+      .createHash('sha256')
+      .update(JSON.stringify(inputs) + previewWorkerCode)
+      .digest('hex')
+      .slice(0, 10) + '.json';
+
+  const filePath = path.join(specsDirectory, fileName);
+
+  const cachedSpec = await maybeLoadJSONFile<{ languages: SpecLanguage[]; sdkJson: Spec }>(filePath);
+  // skip generation since we already have a cached spec
+  if (cachedSpec) {
+    params.logger.info(`Loaded cached spec: ${fileName}`);
+    return [
+      {
+        filePath,
+        languages: cachedSpec.languages,
+        sdkJson: cachedSpec.sdkJson,
+      },
+    ];
+  }
+
+  const result = await generateSpecFromStrings({
+    oasStr: inputs.oasStr,
+    configStr: inputs.configStr,
+    languageOverrides: {
+      mode: 'exclude',
+      list: params.excludeLanguages ?? [],
+    },
+    versionInfo: inputs.versionInfo,
+    stainlessProject: params.stainlessProject,
+  });
+
+  await writeFile(filePath, JSON.stringify(result), 'utf8');
+  params.logger.info(`Generated: ${fileName}`);
+
+  const { deletedCount } = await cleanupDirectory(specsDirectory, [fileName]);
+  if (deletedCount > 0) {
+    params.logger.info(`Cleaned up ${deletedCount} unused spec file(s)`);
+  }
+
+  return [
+    {
+      filePath,
+      languages: result.languages.filter((language) => language !== 'sql' && language !== 'openapi'),
+      sdkJson: result.sdkJson,
+    },
+  ];
+};

package/plugin/specs/fetchSpecSSR.ts

@@ -1,7 +1,7 @@
 import { readFile } from 'fs/promises';
 
 import { api } from 'virtual:stainless-apis-manifest';
-import { SpecWithAuth } from './generateSpec';
+import type { SpecWithAuth } from '@stainless/sdk-json/spec';
 import { DocsLanguage } from '@stainless-api/docs-ui/routing';
 
 const cachedSpecWithAuth: Record<string, SpecWithAuth> = {};

package/plugin/specs/utils.ts

@@ -0,0 +1,86 @@
+import { Spec } from '@stainless/sdk-json';
+import { readFile } from 'fs/promises';
+import { DocsLanguage } from '@stainless-api/docs-ui/routing';
+import { AstroIntegrationLogger } from 'astro';
+
+type PossibleLanguage = NonNullable<NonNullable<NonNullable<Spec['docs']>['languages']>[number]>;
+
+export type SpecLoaderParams = {
+  /**
+   * The slug of your Stainless project.
+   */
+  stainlessProject: string;
+  /**
+   * The branch of your Stainless project.
+   */
+  branch: string;
+  /**
+   * The Stainless API key. This can be used to make requests against the Stainless API.
+   */
+  apiKey: string | null;
+  /**
+   * The languages that the user has explicitly asked to be excluded from the API reference.
+   */
+  excludeLanguages: DocsLanguage[] | null;
+  /**
+   * An Astro logger. This should be used for logging messages.
+   */
+  logger: AstroIntegrationLogger;
+  /**
+   * A function that creates a directory in .astro. See: https://docs.astro.build/en/reference/integrations-reference/#createcodegendir
+   */
+  createCodegenDir: () => URL;
+};
+
+type SpecLoaderResult = {
+  /**
+   * The file path to the loaded spec. The spec MUST be written to a path on disk.
+   * If you are not sure where to place it, create a directory in .astro using the `createCodegenDir` function passed in the parameters of the spec loader function.
+   */
+  filePath: string;
+  /**
+   * The languages that are represented in the spec.
+   */
+  languages: PossibleLanguage[];
+  /**
+   * Optionally, if you already have already the spec in memory, you can provide it. If not provided, the contents of the file at `filePath` will be read.
+   * IMPORTANT: This should be equivalent to the contents of the file at `filePath`. If not, bugs and inconsistencies will arise.
+   */
+  sdkJson?: Spec;
+};
+
+export type SpecLoaderFn = (opts: SpecLoaderParams) => Promise<SpecLoaderResult[]>;
+
+async function readSpecFromFile(filePath: string) {
+  const txt = await readFile(filePath, 'utf8');
+  const json = JSON.parse(txt) as Spec;
+  return json;
+}
+
+export async function loadAllSpecs(specLoaderResultsPromise: Promise<SpecLoaderResult[]>) {
+  const specLoaderResults = await specLoaderResultsPromise;
+  const specs = await Promise.all(
+    specLoaderResults.map(async (result) => {
+      return {
+        filePath: result.filePath,
+        languages: result.languages,
+        sdkJson: result.sdkJson ?? (await readSpecFromFile(result.filePath)),
+      };
+    }),
+  );
+  return specs;
+}
+
+export type LoadedSpecs = Awaited<ReturnType<typeof loadAllSpecs>>;
+
+export function flatSpecsList(specs: LoadedSpecs) {
+  return specs
+    .map((s) =>
+      s.languages.map((language) => ({
+        language,
+        sdkJson: s.sdkJson,
+        filePath: s.filePath,
+      })),
+    )
+    .flat();
+}

package/tsconfig.json

@@ -7,7 +7,7 @@
     ".stl-docs/**/*",
     "./plugin/**/*"
   ],
-  "exclude": ["dist", "**/preview.worker.docs.js"],
+  "exclude": ["dist"],
   "compilerOptions": {
     "jsx": "react-jsx",
     "jsxImportSource": "react"

package/plugin/specs/FileCache.ts

@@ -1,99 +0,0 @@
-import crypto from 'crypto';
-import path from 'path';
-import { readdir, readFile, rm, writeFile } from 'fs/promises';
-
-type CacheResultSource = 'memory' | 'disk' | 'generation';
-
-type CacheGetResult<T> = {
-  resultSource: CacheResultSource;
-  data: T;
-  filePath: string;
-};
-
-export class FileCache<Inputs extends Record<string, unknown>, Output> {
-  private memoryCache: Map<string, Output> = new Map();
-
-  private cacheDirectory: string | null = null;
-
-  public setCacheDirectory(cacheDirectory: string) {
-    this.cacheDirectory = cacheDirectory;
-  }
-  public getCacheDirectory() {
-    if (!this.cacheDirectory) {
-      console.error(`Tried to retrieve entry from cache, but no cache directory was set.`);
-      process.exit(1);
-    }
-    return this.cacheDirectory;
-  }
-
-  private hashInputs(inputs: Inputs): string {
-    return crypto
-      .createHash('sha256')
-      .update(JSON.stringify(inputs) + this.config.globalHashBits)
-      .digest('hex')
-      .slice(0, 10);
-  }
-
-  private getFileName(hash: string) {
-    return `${hash}.json`;
-  }
-
-  public async cleanupUnusedFiles() {
-    const allFiles = await readdir(this.getCacheDirectory());
-    const usedFiles = Array.from(this.memoryCache.keys()).map((key) => this.getFileName(key));
-    const unusedFiles = allFiles.filter((file) => !usedFiles.includes(file));
-    await Promise.all(unusedFiles.map((file) => rm(path.join(this.getCacheDirectory(), file))));
-    return {
-      deletedCount: unusedFiles.length,
-    };
-  }
-
-  public async get(inputs: Inputs): Promise<CacheGetResult<Output>> {
-    const hash = this.hashInputs(inputs);
-    const filePath = path.join(this.getCacheDirectory(), this.getFileName(hash));
-
-    const memoryCacheResult = this.memoryCache.get(hash);
-    if (memoryCacheResult) {
-      return {
-        resultSource: 'memory',
-        data: memoryCacheResult,
-        filePath,
-      };
-    }
-
-    const getFromFileOrGenerate = async () => {
-      try {
-        const fileContents = await readFile(filePath, 'utf8');
-        return {
-          resultSource: 'disk' as const,
-          data: JSON.parse(fileContents) as Output,
-          filePath,
-        };
-      } catch {
-        const data = await this.config.generate(inputs);
-        await writeFile(filePath, JSON.stringify(data), 'utf8');
-        return {
-          resultSource: 'generation' as const,
-          data,
-          filePath,
-        };
-      }
-    };
-
-    const result = await getFromFileOrGenerate();
-    this.memoryCache.set(hash, result.data);
-    return result;
-  }
-
-  constructor(
-    private config: {
-      /**
-       * Additional information to include in the hash of the inputs.
-       * This is useful for cases where the inputs are the same, but the global state is different.
-       * Eg: The preview worker source can be used here.
-       */
-      globalHashBits: string;
-      generate: (inputs: Inputs) => Promise<Output>;
-    },
-  ) {}
-}

package/plugin/specs/generateSpec.ts

@@ -1,109 +0,0 @@
-import type * as SDKJSON from '@stainless/sdk-json';
-import { Languages } from '@stainless-api/docs-ui/routing';
-import { createSDKJSON, ParsedConfig, parseInputs, transformOAS } from './worker';
-import { LanguageGenerateQuery } from '../loadPluginConfig';
-import { FileCache } from './FileCache';
-import previewWorkerCode from '../vendor/preview.worker.docs.js?raw';
-
-function getLanguagesFromStainlessConfig(config: ParsedConfig): SDKJSON.SpecLanguage[] {
-  // if the Stainless config has a list of docs languages, use that
-  if (config.docs?.languages) {
-    return config.docs.languages;
-  }
-
-  // otherwise, just loop over all targets in the config + use the ones that are not skipped
-  return Object.entries(config.targets)
-    .filter(([name, target]) => {
-      if (!Languages.includes(name)) return false; // not  a valid language
-      if (target.skip) return false; // config says to skip this language
-      return true;
-    })
-    .map(([name]) => name) as SDKJSON.SpecLanguage[];
-}
-
-// These inputs contain everything needed to generate a spec
-// Combined with the source of the preview workers, we can make a hash to cache the resulting spec
-export type GenerateSpecRawInputs = {
-  oasStr: string;
-  configStr: string;
-  languageOverrides: LanguageGenerateQuery | null;
-  stainlessProject: string;
-  versionInfo: Record<SDKJSON.SpecLanguage, string> | null;
-};
-
-function applyLanguageOverrides(
-  initialLanguages: SDKJSON.SpecLanguage[],
-  languageOverrides: LanguageGenerateQuery | null,
-) {
-  if (!languageOverrides) return initialLanguages;
-  if (languageOverrides.mode === 'exclude') {
-    return initialLanguages.filter((language) => !languageOverrides.list.includes(language));
-  }
-  return languageOverrides.list;
-}
-
-async function generateSpecFromStrings({
-  oasStr,
-  configStr,
-  stainlessProject,
-  languageOverrides,
-  versionInfo,
-}: GenerateSpecRawInputs) {
-  const { oas, config } = await parseInputs({
-    oas: oasStr,
-    config: configStr,
-  });
-
-  const transformedOAS = await transformOAS({ oas, config });
-
-  let languagesToGenerate = getLanguagesFromStainlessConfig(config);
-  // by default, we should generate the HTTP spec (unless it's explicitly excluded)
-  if (!languagesToGenerate.includes('http')) {
-    languagesToGenerate.push('http');
-  }
-
-  languagesToGenerate = applyLanguageOverrides(languagesToGenerate, languageOverrides);
-
-  // SDKJSON has weird behavior where it will create a spec with HTTP, even if it's not in the languages list
-  const sdkJson = await createSDKJSON({
-    oas: transformedOAS,
-    config,
-    // if language overrides are provided, use them, otherwise use the languages from the Stainless config
-    languages: languagesToGenerate,
-    projectName: stainlessProject,
-  });
-
-  let languages = sdkJson.docs?.languages;
-
-  if (!languages) {
-    throw new Error(`SDKJSON created without any languages`);
-  }
-
-  // if language overrides are provided, filter the languages to only include the ones that are in the overrides
-  languages = languages.filter((language) => languagesToGenerate.includes(language));
-
-  if (versionInfo) {
-    for (const [lang, version] of Object.entries(versionInfo)) {
-      const meta = sdkJson.metadata[lang as SDKJSON.SpecLanguage];
-      if (meta?.install && meta?.version) {
-        meta.install = meta.install.replace(meta.version, version);
-      }
-
-      if (meta?.version) meta.version = version;
-    }
-  }
-
-  return {
-    sdkJson,
-    languages,
-  };
-}
-
-export const specCache = new FileCache({
-  generate: generateSpecFromStrings,
-  globalHashBits: previewWorkerCode, // you can change this as a last resort to invalidate the cache
-});
-
-export type SpecCacheResult = Awaited<ReturnType<typeof specCache.get>>;
-
-export type SpecWithAuth = Awaited<ReturnType<typeof generateSpecFromStrings>>;

package/plugin/specs/index.ts

@@ -1,132 +0,0 @@
-import { mkdir } from 'fs/promises';
-import path from 'path';
-
-import type * as VirtualManifestModule from 'virtual:stainless-apis-manifest';
-
-import { makeAsyncVirtualModPlugin } from '../../shared/virtualModule';
-
-import { NormalizedStainlessStarlightConfig } from '../loadPluginConfig';
-
-import { specCache, SpecCacheResult } from './generateSpec';
-import { AstroIntegrationLogger } from 'astro';
-import type * as SDKJSON from '@stainless/sdk-json';
-
-/**
- * A helper class to manage multiple spec cache results for a single API
- * An API may have multiple spec cache results if it has multiple languages
- * Note that one spec may contain multiple languages.
- * */
-export class SpecComposite {
-  private languages: Set<SDKJSON.SpecLanguage>;
-  private readonly specs: Partial<Record<SDKJSON.SpecLanguage, SpecCacheResult>>;
-
-  public getLanguages() {
-    return Array.from(this.languages);
-  }
-
-  public getByLanguage(language: SDKJSON.SpecLanguage) {
-    const spec = this.specs[language];
-    if (!spec) {
-      throw new Error(`Spec for language ${language} not found`);
-    }
-    return spec;
-  }
-
-  /**
-   * Returns all specs. It will return each spec once, even if it has multiple languages.
-   * */
-  public listUniqueSpecs() {
-    const seen = new Set<SpecCacheResult>();
-    const unique: SpecCacheResult[] = [];
-    for (const spec of Object.values(this.specs)) {
-      if (!seen.has(spec)) {
-        seen.add(spec);
-        unique.push(spec);
-      }
-    }
-    return unique;
-  }
-
-  public listAllLanguagesAndIncludeSpecs() {
-    return this.getLanguages().map((language) => ({
-      language,
-      spec: this.getByLanguage(language),
-    }));
-  }
-
-  constructor(specs: SpecCacheResult[]) {
-    this.languages = new Set<SDKJSON.SpecLanguage>();
-    this.specs = {};
-    for (const spec of specs) {
-      for (const lang of spec.data.languages) {
-        if (this.languages.has(lang)) {
-          throw new Error(`Language appears multiple times in the same API: ${lang}`);
-        }
-        if (lang === 'openapi' || lang === 'sql') continue;
-        this.languages.add(lang);
-        this.specs[lang] = spec;
-      }
-    }
-  }
-}
-
-/** Runs once in the build process */
-export async function startSpecLoader(
-  pluginConfig: NormalizedStainlessStarlightConfig,
-  logger: AstroIntegrationLogger,
-  codegenDir: URL,
-) {
-  const specsDirectory = path.join(codegenDir.pathname, 'specs');
-  await mkdir(specsDirectory, { recursive: true });
-
-  logger.debug(`Setting cache directory to ${specsDirectory}`);
-
-  // 🚨 Important! You cannot call loadSpecs() before setting the cache directory.
-  specCache.setCacheDirectory(specsDirectory);
-
-  async function load() {
-    const specs = await pluginConfig.api.loadSpecs();
-
-    // not awaited since it's just cleanup
-    specCache
-      .cleanupUnusedFiles()
-      .then((result) => {
-        if (result.deletedCount > 0) {
-          logger.info(`Cleaned up ${result.deletedCount} unused spec files`);
-        } else {
-          logger.debug(`No unused spec files to clean up`);
-        }
-      })
-      .catch(() => {
-        logger.warn(`Failed to clean up unused spec files`);
-      });
-
-    return {
-      specComposite: new SpecComposite(specs),
-    };
-  }
-
-  const specPromise = load();
-
-  return {
-    specPromise,
-    // this virtual module only resolves when the spec is generated
-    // this prevents the SSR module from trying to read the spec file before it's generated
-    vitePlugins: [
-      makeAsyncVirtualModPlugin<typeof VirtualManifestModule>('virtual:stainless-apis-manifest', async () => {
-        const api = await specPromise;
-
-        return {
-          api: {
-            languages: api.specComposite.listAllLanguagesAndIncludeSpecs().map((langSpec) => ({
-              language: langSpec.language,
-              sdkJSONFilePath: langSpec.spec.filePath,
-            })),
-          },
-        };
-      }),
-    ],
-  };
-}
-
-export type SpecLoader = Awaited<ReturnType<typeof startSpecLoader>>;