forgeos 0.1.0-alpha.0

package/src/forge/compiler/package-graph/compiler.ts

@@ -0,0 +1,392 @@
+import { join, relative } from "node:path";
+import ts from "typescript";
+import type { Diagnostic } from "../types/diagnostic.ts";
+import type {
+  AnalyzeOptions,
+  Dependency,
+  Entrypoint,
+  PackageApi,
+  PackageGraph,
+} from "../types/package-graph.ts";
+import type { PackageCacheKey } from "../types/lock.ts";
+import type { ResolutionMode } from "../types/runtime.ts";
+import { forgePkgNoTypes } from "../diagnostics/create.ts";
+import { createDiagnostic } from "../diagnostics/create.ts";
+import { stableSortEntrypoints, stableSortPackages } from "../primitives/sort.ts";
+import {
+  PackageCacheStore,
+  forgeCacheDiscardedMessage,
+} from "../cache/store.ts";
+import { buildPackageCacheKey, cacheKeysEqual } from "../cache/key.ts";
+import { runWithConcurrency } from "../cache/scheduler.ts";
+import {
+  GENERATOR_VERSION,
+  PACKAGE_ANALYZER_VERSION,
+  PACKAGE_GRAPH_SCHEMA_VERSION,
+  DEFAULT_PATTERN_EXPANSION_LIMIT,
+} from "./constants.ts";
+import {
+  computeContentChecksum,
+  hashDtsFiles,
+  hashDtsFilesForCache,
+  hashPackageJson,
+  hashPackageJsonForCache,
+} from "./checksum.ts";
+import {
+  discoverSubpathsFromExports,
+  expandPatternSubpaths,
+} from "./exports-discovery.ts";
+import { DtsSignatureExtractor } from "./dts-extractor.ts";
+import { readTextFile } from "./read-file.ts";
+import {
+  resolveEntrypointTypes,
+  resolveTypesPackage,
+  typesPackageName,
+} from "./resolve.ts";
+import {
+  assertPackageApiSecretSafe,
+  defaultSandboxLimits,
+  inspectExports,
+} from "../sandbox/index.ts";
+import type { SandboxBackend } from "../types/runtime.ts";
+
+export interface BuildOptions extends AnalyzeOptions {
+  concurrency?: number;
+  patternExpansionLimit?: number;
+  /** Informational only — does not invalidate per-package cache entries. */
+  lockfileHash?: string;
+}
+
+export interface AnalyzeResult {
+  api: PackageApi;
+  diagnostics: Diagnostic[];
+  cacheHit: boolean;
+}
+
+export interface BuildResult {
+  graph: PackageGraph;
+  diagnostics: Diagnostic[];
+}
+
+export class PackageGraphCompiler {
+  async build(
+    deps: Dependency[],
+    opts: BuildOptions,
+  ): Promise<BuildResult> {
+    const concurrency = Math.max(1, opts.concurrency ?? 4);
+    const cache = new PackageCacheStore(opts.cacheDir);
+    const diagnostics: Diagnostic[] = [];
+
+    const results = await runWithConcurrency(
+      deps,
+      concurrency,
+      async (dep) => this.analyzeWithCache(dep, opts, cache),
+    );
+
+    for (const result of results) {
+      diagnostics.push(...result.diagnostics);
+    }
+
+    const packages = stableSortPackages(results.map((r) => r.api));
+
+    return {
+      graph: {
+        schemaVersion: PACKAGE_GRAPH_SCHEMA_VERSION,
+        generatorVersion: GENERATOR_VERSION,
+        analyzerVersion: PACKAGE_ANALYZER_VERSION,
+        packages,
+      },
+      diagnostics,
+    };
+  }
+
+  async analyze(
+    dep: Dependency,
+    opts: AnalyzeOptions,
+  ): Promise<PackageApi> {
+    const result = await this.analyzeWithCache(
+      dep,
+      opts,
+      new PackageCacheStore(opts.cacheDir),
+    );
+    return result.api;
+  }
+
+  private async analyzeWithCache(
+    dep: Dependency,
+    opts: BuildOptions,
+    cache: PackageCacheStore,
+  ): Promise<AnalyzeResult> {
+    const cacheKey = buildPackageCacheKey({
+      name: dep.name,
+      version: dep.version,
+      packageManager: dep.packageManager,
+      packageIntegrity: dep.packageIntegrity,
+      packageJsonHash: hashPackageJsonForCache(dep.installPath),
+      dtsFilesHash: hashDtsFilesForCache(dep.installPath),
+      analyzerVersion: PACKAGE_ANALYZER_VERSION,
+      typescriptVersion: ts.version,
+      resolutionMode: opts.resolutionMode,
+      recipeVersion: opts.recipeVersion,
+    });
+
+    const lookup = cache.getWithValidation(cacheKey);
+    if ("hit" in lookup) {
+      return { api: lookup.hit, diagnostics: [], cacheHit: true };
+    }
+
+    const diagnostics: Diagnostic[] = [];
+    if ("corrupt" in lookup) {
+      diagnostics.push(
+        createDiagnostic({
+          severity: "warning",
+          code: "FORGE_CACHE_DISCARD",
+          message: forgeCacheDiscardedMessage(),
+        }),
+      );
+    }
+
+    const analyzed = await this.analyzeWithOptionalRuntime(dep, opts, {
+      packageJsonHash: hashPackageJson(dep.installPath),
+      dtsFilesHash: hashDtsFiles(dep.installPath),
+    });
+    diagnostics.push(...analyzed.diagnostics);
+    assertPackageApiSecretSafe(analyzed.api);
+    await cache.put(cacheKey, analyzed.api);
+    return { api: analyzed.api, diagnostics, cacheHit: false };
+  }
+
+  private async analyzeWithOptionalRuntime(
+    dep: Dependency,
+    opts: BuildOptions,
+    packageHashes: { packageJsonHash: string; dtsFilesHash: string },
+  ): Promise<{ api: PackageApi; diagnostics: Diagnostic[] }> {
+    const staticResult = this.analyzeStatic(dep, opts, packageHashes);
+    if (!opts.runtimeInspect) {
+      return staticResult;
+    }
+
+    const backend = resolveSandboxBackend(opts);
+    const sandbox = await inspectExports(
+      dep,
+      defaultSandboxLimits(backend),
+    );
+
+    const diagnostics = [
+      ...staticResult.diagnostics,
+      ...sandbox.diagnostics,
+    ];
+
+    if (!sandbox.runtimeUsed) {
+      return { api: staticResult.api, diagnostics };
+    }
+
+    const api: PackageApi = {
+      ...staticResult.api,
+      source: "static+runtime",
+      runtimeShape: sandbox.shape,
+      contentChecksum: computeContentChecksum(
+        packageHashes.packageJsonHash,
+        packageHashes.dtsFilesHash,
+        {
+          ...staticResult.api,
+          source: "static+runtime",
+          runtimeShape: sandbox.shape,
+        },
+      ),
+    };
+
+    return { api, diagnostics };
+  }
+
+  analyzeStatic(
+    dep: Dependency,
+    opts: BuildOptions,
+    packageHashes?: { packageJsonHash: string; dtsFilesHash: string },
+  ): { api: PackageApi; diagnostics: Diagnostic[] } {
+    const diagnostics: Diagnostic[] = [];
+    const packageJson = readPackageJson(dep.installPath);
+    const patternLimit =
+      opts.patternExpansionLimit ?? DEFAULT_PATTERN_EXPANSION_LIMIT;
+
+    const discovered = discoverSubpathsFromExports(packageJson.exports);
+    const subpathsToAnalyze: Array<{
+      subpath: string;
+      patternBacked: boolean;
+    }> = [];
+
+    for (const item of discovered) {
+      if (item.patternBacked) {
+        const expanded = expandPatternSubpaths(
+          dep.installPath,
+          item.subpath,
+          patternLimit,
+        );
+        for (const sub of expanded) {
+          subpathsToAnalyze.push({ subpath: sub, patternBacked: true });
+        }
+        if (expanded.length === 0) {
+          subpathsToAnalyze.push(item);
+        }
+      } else {
+        subpathsToAnalyze.push(item);
+      }
+    }
+
+    const entrypoints: Entrypoint[] = [];
+    const dtsExtractor = new DtsSignatureExtractor(opts.resolutionMode);
+
+    for (const { subpath, patternBacked } of subpathsToAnalyze) {
+      const entry = this.analyzeEntrypoint(
+        dep,
+        subpath,
+        patternBacked,
+        opts.resolutionMode,
+        diagnostics,
+        dtsExtractor,
+      );
+      entrypoints.push(entry);
+    }
+
+    const sortedEntrypoints = stableSortEntrypoints(entrypoints);
+    const packageJsonHash =
+      packageHashes?.packageJsonHash ?? hashPackageJson(dep.installPath);
+    const dtsFilesHash =
+      packageHashes?.dtsFilesHash ?? hashDtsFiles(dep.installPath);
+
+    const partialApi = {
+      name: dep.name,
+      version: dep.version,
+      packageManager: dep.packageManager,
+      resolutionMode: opts.resolutionMode,
+      entrypoints: sortedEntrypoints,
+      source: "static" as const,
+    };
+
+    const contentChecksum = computeContentChecksum(
+      packageJsonHash,
+      dtsFilesHash,
+      partialApi,
+    );
+
+    return {
+      api: { ...partialApi, contentChecksum },
+      diagnostics,
+    };
+  }
+
+  private analyzeEntrypoint(
+    dep: Dependency,
+    subpath: string,
+    patternBacked: boolean,
+    mode: ResolutionMode,
+    diagnostics: Diagnostic[],
+    dtsExtractor: DtsSignatureExtractor,
+  ): Entrypoint {
+    let resolved = resolveEntrypointTypes(
+      dep.installPath,
+      dep.name,
+      subpath,
+      mode,
+    );
+
+    if (resolved.dtsPath == null) {
+      const typesPackage = typesPackageName(dep.name);
+      const fallbackPath = resolveTypesPackage(
+        typesPackage,
+        dep.installPath,
+        mode,
+      );
+      if (fallbackPath != null) {
+        resolved = { dtsPath: fallbackPath, conditions: ["types"] };
+      }
+    }
+
+    if (resolved.dtsPath == null) {
+      diagnostics.push(forgePkgNoTypes(dep.name, subpath));
+      return {
+        subpath,
+        conditions: resolved.conditions,
+        patternBacked,
+        dtsPath: null,
+        exports: [],
+      };
+    }
+
+    let exports: Entrypoint["exports"];
+    try {
+      exports = dtsExtractor.extract(
+        resolved.dtsPath,
+        dep.name,
+        subpath,
+      );
+    } catch {
+      diagnostics.push(forgePkgNoTypes(dep.name, subpath));
+      return {
+        subpath,
+        conditions: resolved.conditions,
+        patternBacked,
+        dtsPath: portableDtsPath(resolved.dtsPath, dep),
+        exports: [],
+      };
+    }
+
+    if (exports.length === 0) {
+      diagnostics.push(forgePkgNoTypes(dep.name, subpath));
+    }
+
+    return {
+      subpath,
+      conditions: resolved.conditions,
+      patternBacked,
+      dtsPath: portableDtsPath(resolved.dtsPath, dep),
+      exports,
+    };
+  }
+}
+
+function portableDtsPath(dtsPath: string, dep: Pick<Dependency, "installPath" | "name">): string {
+  const normalized = dtsPath.replace(/\\/g, "/");
+  const marker = "/node_modules/";
+  const index = normalized.lastIndexOf(marker);
+  if (index >= 0) {
+    return normalized.slice(index + 1);
+  }
+  if (normalized.startsWith("node_modules/")) {
+    return normalized;
+  }
+  const packageRelative = relative(dep.installPath, dtsPath).replace(/\\/g, "/");
+  if (packageRelative && !packageRelative.startsWith("../") && packageRelative !== "..") {
+    return `packages/${dep.name}/${packageRelative}`;
+  }
+  return normalized;
+}
+
+function readPackageJson(installPath: string): {
+  exports?: unknown;
+} {
+  const raw = readTextFile(join(installPath, "package.json"));
+  return JSON.parse(raw) as { exports?: unknown };
+}
+
+function resolveSandboxBackend(
+  opts: Pick<AnalyzeOptions, "sandboxBackend">,
+): SandboxBackend {
+  return opts.sandboxBackend ?? "none";
+}
+
+export function recomputeFromInputs(
+  dep: Dependency,
+  opts: Pick<BuildOptions, "resolutionMode" | "patternExpansionLimit">,
+): PackageApi {
+  const compiler = new PackageGraphCompiler();
+  return compiler.analyzeStatic(dep, {
+    runtimeInspect: false,
+    resolutionMode: opts.resolutionMode,
+    cacheDir: "",
+    patternExpansionLimit: opts.patternExpansionLimit,
+  }).api;
+}
+
+export { cacheKeysEqual, buildPackageCacheKey };
+export type { PackageCacheKey };

package/src/forge/compiler/package-graph/constants.ts

@@ -0,0 +1,4 @@
+export const PACKAGE_GRAPH_SCHEMA_VERSION = "1.0.0";
+export const PACKAGE_ANALYZER_VERSION = "0.1.1";
+export const GENERATOR_VERSION = "0.0.0";
+export const DEFAULT_PATTERN_EXPANSION_LIMIT = 50;

package/src/forge/compiler/package-graph/dts-extractor.ts

@@ -0,0 +1,142 @@
+import ts from "typescript";
+import type { ExportKind, ExportSignature } from "../types/package-graph.ts";
+import { stableSortExports } from "../primitives/sort.ts";
+import { stubExportClassification } from "./capabilities-stub.ts";
+import { extractExamples, extractJsDoc } from "./jsdoc.ts";
+import { createResolutionCompilerOptions } from "./resolve.ts";
+import type { ResolutionMode } from "../types/runtime.ts";
+export function normalizeSignatureText(text: string): string {
+  return text
+    .replace(/\r\n/g, "\n")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function classifyDeclKind(node: ts.Declaration): ExportKind {
+  if (ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node)) {
+    return "function";
+  }
+  if (ts.isClassDeclaration(node)) {
+    return "class";
+  }
+  if (ts.isInterfaceDeclaration(node)) {
+    return "interface";
+  }
+  if (ts.isTypeAliasDeclaration(node)) {
+    return "type";
+  }
+  if (ts.isModuleDeclaration(node)) {
+    return "namespace";
+  }
+  return "const";
+}
+
+function resolveExportName(symbol: ts.Symbol, decl: ts.Declaration): string {
+  const symbolName = symbol.getName();
+  if (symbolName !== "default") {
+    return symbolName;
+  }
+  if (ts.isClassDeclaration(decl) && decl.name != null) {
+    return decl.name.text;
+  }
+  if (ts.isFunctionDeclaration(decl) && decl.name != null) {
+    return decl.name.text;
+  }
+  if (ts.isInterfaceDeclaration(decl) && decl.name != null) {
+    return decl.name.text;
+  }
+  return symbolName;
+}
+
+export class DtsSignatureExtractor {
+  private readonly options: ts.CompilerOptions;
+  private rootFiles: string[] = [];
+  private program: ts.Program | null = null;
+  private checker: ts.TypeChecker | null = null;
+
+  constructor(mode: ResolutionMode) {
+    this.options = createResolutionCompilerOptions(mode);
+  }
+
+  registerDtsPath(dtsPath: string): void {
+    if (!this.rootFiles.includes(dtsPath)) {
+      this.rootFiles.push(dtsPath);
+      this.program = null;
+      this.checker = null;
+    }
+  }
+
+  private ensureProgram(): void {
+    if (this.program !== null && this.checker !== null) {
+      return;
+    }
+    this.program = ts.createProgram(this.rootFiles, this.options);
+    this.checker = this.program.getTypeChecker();
+  }
+
+  extract(
+    dtsPath: string,
+    packageName: string,
+    entrypoint: string,
+  ): ExportSignature[] {
+    this.registerDtsPath(dtsPath);
+    this.ensureProgram();
+    const program = this.program!;
+    const checker = this.checker!;
+    const source = program.getSourceFile(dtsPath);
+    if (source == null) {
+      return [];
+    }
+
+    const moduleSymbol = checker.getSymbolAtLocation(source);
+    if (moduleSymbol == null) {
+      return [];
+    }
+
+    const exportsList = checker.getExportsOfModule(moduleSymbol);
+    const results: ExportSignature[] = [];
+
+    for (const sym of exportsList) {
+      const decls = sym.declarations ?? [];
+      if (decls.length === 0) {
+        continue;
+      }
+
+      const type = checker.getTypeOfSymbolAtLocation(sym, decls[0]!);
+      const callSigs = type.getCallSignatures();
+      const printed =
+        callSigs.length > 0
+          ? callSigs.map((sig) =>
+              normalizeSignatureText(checker.signatureToString(sig)),
+            )
+          : [normalizeSignatureText(checker.typeToString(type))];
+
+      const declarationTexts =
+        decls.length > 1
+          ? decls.map((decl) =>
+              normalizeSignatureText(decl.getText(source).replace(/\s+/g, " ")),
+            )
+          : undefined;
+
+      const jsdoc = extractJsDoc(sym, checker);
+      const exportName = resolveExportName(sym, decls[0]!);
+
+      results.push({
+        name: exportName,
+        kind: classifyDeclKind(decls[0]!),
+        signature: printed[0]!,
+        overloads: printed.length > 1 ? printed.slice(1) : undefined,
+        declarations: declarationTexts,
+        classification: stubExportClassification(
+          packageName,
+          entrypoint,
+          exportName,
+        ),
+        jsdoc,
+        examples: extractExamples(jsdoc),
+      });
+    }
+
+    return stableSortExports(results);
+  }
+}

package/src/forge/compiler/package-graph/exports-discovery.ts

@@ -0,0 +1,84 @@
+import { join, posix } from "node:path";
+import { nodeFileSystem } from "../fs/index.ts";
+import { compareBytes } from "../primitives/compare.ts";
+import { DEFAULT_PATTERN_EXPANSION_LIMIT } from "./constants.ts";
+
+export interface DiscoveredSubpath {
+  subpath: string;
+  patternBacked: boolean;
+}
+
+export function discoverSubpathsFromExports(
+  exportsField: unknown,
+): DiscoveredSubpath[] {
+  if (exportsField == null) {
+    return [{ subpath: ".", patternBacked: false }];
+  }
+
+  if (typeof exportsField === "string") {
+    return [{ subpath: ".", patternBacked: false }];
+  }
+
+  if (typeof exportsField !== "object" || Array.isArray(exportsField)) {
+    return [{ subpath: ".", patternBacked: false }];
+  }
+
+  const entries: DiscoveredSubpath[] = [];
+  for (const key of Object.keys(exportsField as Record<string, unknown>)) {
+    const normalized = key === "." ? "." : key.startsWith("./") ? key : `./${key}`;
+    const patternBacked = normalized.endsWith("/*");
+    entries.push({ subpath: normalized, patternBacked });
+  }
+
+  entries.sort((a, b) => compareBytes(a.subpath, b.subpath));
+  return entries;
+}
+
+export function expandPatternSubpaths(
+  installPath: string,
+  patternSubpath: string,
+  limit: number = DEFAULT_PATTERN_EXPANSION_LIMIT,
+): string[] {
+  if (!patternSubpath.endsWith("/*")) {
+    return [patternSubpath];
+  }
+
+  const prefix = patternSubpath.slice(0, -2);
+  const relativePrefix = prefix.startsWith("./") ? prefix.slice(2) : prefix;
+  const baseDir = join(installPath, relativePrefix);
+
+  let files: string[];
+  try {
+    files = listFilesRecursive(baseDir);
+  } catch {
+    return [];
+  }
+
+  const expanded: string[] = [];
+  for (const file of files) {
+    const rel = posix.join(prefix, file);
+    expanded.push(rel.startsWith("./") ? rel : `./${rel}`);
+    if (expanded.length >= limit) {
+      break;
+    }
+  }
+
+  expanded.sort(compareBytes);
+  return expanded;
+}
+
+function listFilesRecursive(dir: string, relative = ""): string[] {
+  const files: string[] = [];
+
+  for (const entry of nodeFileSystem.readDir(dir)) {
+    const full = join(dir, entry.name);
+    const rel = relative ? posix.join(relative, entry.name) : entry.name;
+    if (entry.isDirectory) {
+      files.push(...listFilesRecursive(full, rel));
+    } else {
+      files.push(rel);
+    }
+  }
+
+  return files;
+}

package/src/forge/compiler/package-graph/extract-dts.ts

@@ -0,0 +1,32 @@
+import type { ExportSignature } from "../types/package-graph.ts";
+import { compareExports } from "../primitives/sort.ts";
+import { readTextFile } from "./read-file.ts";
+import type { ResolutionMode } from "../types/runtime.ts";
+import {
+  DtsSignatureExtractor,
+  normalizeSignatureText,
+} from "./dts-extractor.ts";
+
+export { normalizeSignatureText };
+
+export function extractDtsSignatures(
+  dtsPath: string,
+  packageName: string,
+  entrypoint: string,
+  mode: ResolutionMode,
+): ExportSignature[] {
+  const extractor = new DtsSignatureExtractor(mode);
+  return extractor.extract(dtsPath, packageName, entrypoint);
+}
+
+export function canParseDtsFile(dtsPath: string, mode: ResolutionMode): boolean {
+  try {
+    readTextFile(dtsPath);
+    const exports = extractDtsSignatures(dtsPath, "probe", ".", mode);
+    return exports.length >= 0;
+  } catch {
+    return false;
+  }
+}
+
+export { compareExports, DtsSignatureExtractor };

package/src/forge/compiler/package-graph/index.ts

@@ -0,0 +1,33 @@
+export {
+  PACKAGE_GRAPH_SCHEMA_VERSION,
+  PACKAGE_ANALYZER_VERSION,
+  DEFAULT_PATTERN_EXPANSION_LIMIT,
+} from "./constants.ts";
+export {
+  PackageGraphCompiler,
+  recomputeFromInputs,
+} from "./compiler.ts";
+export type {
+  BuildOptions,
+  AnalyzeResult,
+  BuildResult,
+} from "./compiler.ts";
+export {
+  resolveEntrypointTypes,
+  moduleSpecifierForSubpath,
+  createResolutionCompilerOptions,
+  typesPackageName,
+} from "./resolve.ts";
+export {
+  discoverSubpathsFromExports,
+  expandPatternSubpaths,
+} from "./exports-discovery.ts";
+export { extractDtsSignatures, normalizeSignatureText } from "./extract-dts.ts";
+export { extractJsDoc, extractExamples } from "./jsdoc.ts";
+export {
+  hashPackageJson,
+  hashDtsFiles,
+  computeContentChecksum,
+} from "./checksum.ts";
+export { setReadFileTracker } from "./read-file.ts";
+export type { ReadFileTracker } from "./read-file.ts";