forgeos 0.1.0-alpha.0

package/src/forge/compiler/app-graph/parser.ts

@@ -0,0 +1,119 @@
+import Parser from "tree-sitter";
+import TypeScript from "tree-sitter-typescript";
+import { createDiagnostic } from "../diagnostics/create.ts";
+import type { Diagnostic } from "../types/diagnostic.ts";
+import type { ForgeSymbol, SourceFile } from "../types/app-graph.ts";
+import { normalizePath } from "../primitives/paths.ts";
+import { extractSymbolsFromTree } from "./extract.ts";
+import type { ParseInvalidationKey, RawSymbol } from "./types.ts";
+import { parseInvalidationKeyEquals } from "./types.ts";
+
+const tsParser = new Parser();
+tsParser.setLanguage(TypeScript.typescript as unknown as Parser.Language);
+
+const tsxParser = new Parser();
+tsxParser.setLanguage(TypeScript.tsx as unknown as Parser.Language);
+
+function parserForFile(path: string): Parser {
+  return path.endsWith(".tsx") ? tsxParser : tsParser;
+}
+
+function priorSymbolsToRaw(symbols: ForgeSymbol[]): RawSymbol[] {
+  return symbols.map((symbol) => ({
+    kind: symbol.kind,
+    name: symbol.name,
+    qualifiedName: symbol.qualifiedName,
+    file: symbol.file,
+    span: symbol.span,
+    exportPath:
+      typeof symbol.meta.exportPath === "string" ? symbol.meta.exportPath : "",
+    sourceSlice:
+      typeof symbol.meta.sourceSlice === "string" ? symbol.meta.sourceSlice : "",
+  }));
+}
+
+function groupPriorSymbolsByFile(
+  priorSymbols: ForgeSymbol[],
+): Map<string, ForgeSymbol[]> {
+  const byFile = new Map<string, ForgeSymbol[]>();
+  for (const symbol of priorSymbols) {
+    const list = byFile.get(symbol.file) ?? [];
+    list.push(symbol);
+    byFile.set(symbol.file, list);
+  }
+  return byFile;
+}
+
+function priorFileContentHash(
+  priorSymbols: ForgeSymbol[],
+  file: string,
+): string | undefined {
+  const match = priorSymbols.find((symbol) => symbol.file === file);
+  if (!match) {
+    return undefined;
+  }
+  // All symbols from the same file share the same slice hash basis; use meta when present.
+  const fromMeta = match.meta.fileContentHash;
+  return typeof fromMeta === "string" ? fromMeta : undefined;
+}
+
+export interface IncrementalParseResult {
+  symbols: RawSymbol[];
+  diagnostics: Diagnostic[];
+}
+
+export function incrementalParse(
+  sources: SourceFile[],
+  priorSymbols: ForgeSymbol[] | undefined,
+  priorInvalidation: ParseInvalidationKey | undefined,
+  currentInvalidation: ParseInvalidationKey,
+): IncrementalParseResult {
+  const diagnostics: Diagnostic[] = [];
+  const allSymbols: RawSymbol[] = [];
+  const globalInvalidated =
+    priorInvalidation === undefined ||
+    !parseInvalidationKeyEquals(priorInvalidation, currentInvalidation);
+
+  const priorByFile = priorSymbols
+    ? groupPriorSymbolsByFile(priorSymbols)
+    : new Map<string, ForgeSymbol[]>();
+
+  for (const source of sources) {
+    const normalizedFile = normalizePath(source.path);
+    const priorForFile = priorByFile.get(normalizedFile);
+    const priorHash = priorSymbols
+      ? priorFileContentHash(priorSymbols, normalizedFile)
+      : undefined;
+
+    const fileUnchanged =
+      !globalInvalidated &&
+      priorForFile !== undefined &&
+      priorHash === source.contentHash;
+
+    if (fileUnchanged) {
+      allSymbols.push(...priorSymbolsToRaw(priorForFile));
+      continue;
+    }
+
+    try {
+      const tree = parserForFile(normalizedFile).parse(source.text);
+      const extracted = extractSymbolsFromTree(
+        tree.rootNode,
+        normalizedFile,
+        source.text,
+      );
+      allSymbols.push(...extracted);
+    } catch {
+      diagnostics.push(
+        createDiagnostic({
+          severity: "warning",
+          code: "FORGE_UNPARSEABLE_FILE",
+          message: `cannot parse source file: ${normalizedFile}`,
+          file: normalizedFile,
+        }),
+      );
+    }
+  }
+
+  return { symbols: allSymbols, diagnostics };
+}

package/src/forge/compiler/app-graph/symbols.ts

@@ -0,0 +1,48 @@
+import { deriveStableSymbolId, hashStable } from "../primitives/hash.ts";
+import type { ForgeSymbol } from "../types/app-graph.ts";
+import type { SourceFile } from "../types/app-graph.ts";
+import type { RawSymbol } from "./types.ts";
+
+export function rawToForgeSymbol(
+  raw: RawSymbol,
+  fileContentHash: string,
+): ForgeSymbol {
+  const id = deriveStableSymbolId({
+    kind: raw.kind,
+    canonicalModulePath: raw.file,
+    qualifiedName: raw.qualifiedName,
+    exportPath: raw.exportPath,
+  });
+
+  const slice = raw.sourceSlice || "";
+  const contentHash = slice.length > 0 ? hashStable(slice) : fileContentHash;
+
+  return {
+    id,
+    kind: raw.kind,
+    name: raw.name,
+    qualifiedName: raw.qualifiedName,
+    file: raw.file,
+    span: raw.span,
+    contentHash,
+    meta: {
+      exportPath: raw.exportPath,
+      fileContentHash,
+      ...(slice.length > 0 ? { sourceSlice: slice } : {}),
+    },
+  };
+}
+
+export function buildForgeSymbols(
+  rawSymbols: RawSymbol[],
+  sources: SourceFile[],
+): ForgeSymbol[] {
+  const hashByFile = new Map<string, string>();
+  for (const source of sources) {
+    hashByFile.set(source.path, source.contentHash);
+  }
+
+  return rawSymbols.map((raw) =>
+    rawToForgeSymbol(raw, hashByFile.get(raw.file) ?? ""),
+  );
+}

package/src/forge/compiler/app-graph/tsconfig-hash.ts

@@ -0,0 +1,62 @@
+import ts from "typescript";
+import { canonicalJson } from "../primitives/serialize.ts";
+import { hashStable } from "../primitives/hash.ts";
+
+const RELEVANT_COMPILER_OPTIONS: (keyof ts.CompilerOptions)[] = [
+  "baseUrl",
+  "paths",
+  "module",
+  "moduleResolution",
+  "target",
+  "jsx",
+  "allowJs",
+  "resolveJsonModule",
+  "esModuleInterop",
+  "strict",
+  "rootDir",
+  "outDir",
+];
+
+export function hashTsCompilerOptions(options: ts.CompilerOptions): string {
+  const relevant: Record<string, unknown> = {};
+  for (const key of RELEVANT_COMPILER_OPTIONS) {
+    const value = options[key];
+    if (value !== undefined) {
+      relevant[key] = value;
+    }
+  }
+  return hashStable(canonicalJson(relevant));
+}
+
+export function loadTsconfig(
+  workspaceRoot: string,
+  tsconfigPath?: string,
+): ts.ParsedCommandLine {
+  const configPath =
+    tsconfigPath ?? ts.findConfigFile(workspaceRoot, ts.sys.fileExists, "tsconfig.json");
+
+  if (!configPath) {
+    return {
+      options: {},
+      fileNames: [],
+      errors: [],
+    };
+  }
+
+  const read = ts.readConfigFile(configPath, ts.sys.readFile);
+  return ts.parseJsonConfigFileContent(
+    read.config,
+    ts.sys,
+    workspaceRoot,
+    undefined,
+    configPath,
+  );
+}
+
+export function hashTsconfigForWorkspace(
+  workspaceRoot: string,
+  tsconfigPath?: string,
+): string {
+  const parsed = loadTsconfig(workspaceRoot, tsconfigPath);
+  return hashTsCompilerOptions(parsed.options);
+}

package/src/forge/compiler/app-graph/types.ts

@@ -0,0 +1,43 @@
+import type { ForgeKind } from "../types/app-graph.ts";
+import { APP_GRAPH_ANALYZER_VERSION } from "./versions.ts";
+
+/** Symbol extracted from tree-sitter before stable-id assignment. */
+export interface RawSymbol {
+  kind: ForgeKind;
+  name: string;
+  qualifiedName: string;
+  file: string;
+  span: { start: number; end: number };
+  exportPath: string;
+  sourceSlice: string;
+}
+
+/** Factors that invalidate cached per-file parse results. */
+export interface ParseInvalidationKey {
+  schemaVersion: string;
+  grammarVersion: string;
+  classifierVersion: string;
+  tsconfigHash: string;
+}
+
+export function parseInvalidationKeyEquals(
+  a: ParseInvalidationKey,
+  b: ParseInvalidationKey,
+): boolean {
+  return (
+    a.schemaVersion === b.schemaVersion &&
+    a.grammarVersion === b.grammarVersion &&
+    a.classifierVersion === b.classifierVersion &&
+    a.tsconfigHash === b.tsconfigHash
+  );
+}
+
+export function buildAnalyzerVersion(key: ParseInvalidationKey): string {
+  return [
+    APP_GRAPH_ANALYZER_VERSION,
+    `schema:${key.schemaVersion}`,
+    `grammar:${key.grammarVersion}`,
+    `classifier:${key.classifierVersion}`,
+    `tsconfig:${key.tsconfigHash}`,
+  ].join("+");
+}

package/src/forge/compiler/app-graph/versions.ts

@@ -0,0 +1,14 @@
+/** AppGraph model schema version. */
+export const APP_GRAPH_SCHEMA_VERSION = "1.0.0";
+
+/** AppGraph analyzer / classifier toolchain version. */
+export const APP_GRAPH_ANALYZER_VERSION = "0.1.0";
+
+/** tree-sitter-typescript grammar package version (invalidates parse cache). */
+export const TREE_SITTER_GRAMMAR_VERSION = "0.23.2";
+
+/** Forge builder API → ForgeKind classifier version. */
+export const FORGE_CLASSIFIER_VERSION = "0.1.1";
+
+/** Generation orchestrator version (recorded on emitted AppGraph). */
+export const GENERATOR_VERSION = "0.0.0";

package/src/forge/compiler/cache/index.ts

@@ -0,0 +1,17 @@
+export {
+  buildPackageCacheKey,
+  serializePackageCacheKey,
+  fingerprintPackageCacheKey,
+  cacheKeysEqual,
+  lockfileHashAffectsCache,
+} from "./key.ts";
+export {
+  PackageCacheStore,
+  forgeCacheDiscardedMessage,
+} from "./store.ts";
+export type { CacheEntry } from "./store.ts";
+export {
+  runWithConcurrency,
+  runWithConcurrencyTracked,
+  ConcurrencyTracker,
+} from "./scheduler.ts";

package/src/forge/compiler/cache/key.ts

@@ -0,0 +1,46 @@
+import type { PackageCacheKey } from "../types/lock.ts";
+import { hashStable } from "../primitives/hash.ts";
+import { canonicalJson } from "../primitives/serialize.ts";
+
+export function buildPackageCacheKey(
+  input: PackageCacheKey,
+): PackageCacheKey {
+  return {
+    name: input.name,
+    version: input.version,
+    packageManager: input.packageManager,
+    ...(input.packageIntegrity !== undefined
+      ? { packageIntegrity: input.packageIntegrity }
+      : {}),
+    packageJsonHash: input.packageJsonHash,
+    dtsFilesHash: input.dtsFilesHash,
+    analyzerVersion: input.analyzerVersion,
+    typescriptVersion: input.typescriptVersion,
+    resolutionMode: input.resolutionMode,
+    ...(input.recipeVersion !== undefined
+      ? { recipeVersion: input.recipeVersion }
+      : {}),
+  };
+}
+
+export function serializePackageCacheKey(key: PackageCacheKey): string {
+  return canonicalJson(key);
+}
+
+export function fingerprintPackageCacheKey(key: PackageCacheKey): string {
+  return hashStable(serializePackageCacheKey(key));
+}
+
+export function cacheKeysEqual(
+  a: PackageCacheKey,
+  b: PackageCacheKey,
+): boolean {
+  return serializePackageCacheKey(a) === serializePackageCacheKey(b);
+}
+
+/**
+ * Global lockfile hash is informational and intentionally excluded from cache keys.
+ */
+export function lockfileHashAffectsCache(_lockfileHash: string): boolean {
+  return false;
+}

package/src/forge/compiler/cache/scheduler.ts

@@ -0,0 +1,72 @@
+export async function runWithConcurrency<T, R>(
+  items: T[],
+  concurrency: number,
+  worker: (item: T, index: number) => Promise<R> | R,
+): Promise<R[]> {
+  const limit = Math.max(1, Math.floor(concurrency));
+  if (items.length === 0) {
+    return [];
+  }
+
+  if (limit === 1) {
+    const sequential: R[] = [];
+    for (let index = 0; index < items.length; index++) {
+      sequential.push(await worker(items[index]!, index));
+    }
+    return sequential;
+  }
+
+  const results = new Array<R>(items.length);
+  let nextIndex = 0;
+
+  async function runWorker(): Promise<void> {
+    while (true) {
+      const index = nextIndex;
+      nextIndex += 1;
+      if (index >= items.length) {
+        return;
+      }
+      results[index] = await worker(items[index]!, index);
+    }
+  }
+
+  const workers = Array.from({ length: Math.min(limit, items.length) }, () =>
+    runWorker(),
+  );
+  await Promise.all(workers);
+  return results;
+}
+
+export class ConcurrencyTracker {
+  current = 0;
+  maxObserved = 0;
+
+  enter(): void {
+    this.current += 1;
+    if (this.current > this.maxObserved) {
+      this.maxObserved = this.current;
+    }
+  }
+
+  leave(): void {
+    this.current -= 1;
+  }
+}
+
+export async function runWithConcurrencyTracked<T, R>(
+  items: T[],
+  concurrency: number,
+  worker: (item: T, index: number) => Promise<R> | R,
+  tracker: ConcurrencyTracker,
+): Promise<R[]> {
+  const limit = Math.max(1, Math.floor(concurrency));
+
+  return runWithConcurrency(items, limit, async (item, index) => {
+    tracker.enter();
+    try {
+      return await worker(item, index);
+    } finally {
+      tracker.leave();
+    }
+  });
+}

package/src/forge/compiler/cache/store.ts

@@ -0,0 +1,78 @@
+import { join } from "node:path";
+import { nodeFileSystem } from "../fs/index.ts";
+import type { PackageApi } from "../types/package-graph.ts";
+import type { PackageCacheKey } from "../types/lock.ts";
+import { canonicalJson } from "../primitives/serialize.ts";
+import {
+  buildPackageCacheKey,
+  cacheKeysEqual,
+  fingerprintPackageCacheKey,
+  serializePackageCacheKey,
+} from "./key.ts";
+
+export interface CacheEntry {
+  key: PackageCacheKey;
+  result: PackageApi;
+}
+
+export class PackageCacheStore {
+  private readonly packagesDir: string;
+
+  constructor(cacheDir: string) {
+    this.packagesDir = join(cacheDir, "packages");
+    nodeFileSystem.mkdirp(this.packagesDir);
+  }
+
+  private entryPath(key: PackageCacheKey): string {
+    return join(this.packagesDir, `${fingerprintPackageCacheKey(key)}.json`);
+  }
+
+  getWithValidation(
+    key: PackageCacheKey,
+  ): { hit: PackageApi } | { miss: true } | { corrupt: true } {
+    const normalizedKey = buildPackageCacheKey(key);
+    const path = this.entryPath(normalizedKey);
+
+    const raw = nodeFileSystem.readText(path);
+    if (raw === null) {
+      return { miss: true };
+    }
+
+    let entry: CacheEntry;
+    try {
+      entry = JSON.parse(raw) as CacheEntry;
+    } catch {
+      return { corrupt: true };
+    }
+
+    if (!cacheKeysEqual(entry.key, normalizedKey)) {
+      return { corrupt: true };
+    }
+
+    return { hit: entry.result };
+  }
+
+  async put(key: PackageCacheKey, result: PackageApi): Promise<void> {
+    const normalizedKey = buildPackageCacheKey(key);
+    const path = this.entryPath(normalizedKey);
+
+    const entry: CacheEntry = {
+      key: normalizedKey,
+      result,
+    };
+
+    nodeFileSystem.writeText(path, serializeCacheEntry(entry));
+  }
+
+  serializeKey(key: PackageCacheKey): string {
+    return serializePackageCacheKey(key);
+  }
+}
+
+function serializeCacheEntry(entry: CacheEntry): string {
+  return `${canonicalJson(entry)}\n`;
+}
+
+export function forgeCacheDiscardedMessage(): string {
+  return "cache entry was discarded due to read or integrity failure; recomputing analysis";
+}

package/src/forge/compiler/classifier/capabilities.ts

@@ -0,0 +1,78 @@
+import type { CapabilitySet } from "../types/capability.ts";
+import type { IntegrationRecipe } from "../types/integration.ts";
+import type { PackageApi } from "../types/package-graph.ts";
+import { capability, emptyCapabilitySet } from "../recipes/helpers.ts";
+import { gatherSignals } from "./signals.ts";
+
+export function detectCapabilities(
+  api: PackageApi,
+  recipe?: IntegrationRecipe,
+): CapabilitySet {
+  if (recipe) {
+    return cloneCapabilitySet(recipe.capabilities);
+  }
+
+  const signals = gatherSignals(api);
+  const caps = emptyCapabilitySet();
+
+  if (signals.usesNetwork) {
+    caps.network = capability<{ egress?: string[] }>(
+      "required",
+      "static",
+      signals.networkEvidence,
+    );
+  } else {
+    caps.network = capability<{ egress?: string[] }>("unknown", "static", [
+      "static analysis cannot prove absence of network access",
+    ]);
+  }
+
+  if (signals.usesFilesystem) {
+    caps.filesystem = capability<{ read?: boolean; write?: boolean }>(
+      "required",
+      "static",
+      signals.filesystemEvidence,
+      {
+        read: true,
+        write: signals.filesystemEvidence.some((e) => /write/i.test(e)),
+      },
+    );
+  } else {
+    caps.filesystem = capability<{ read?: boolean; write?: boolean }>(
+      "unknown",
+      "static",
+      ["static analysis cannot prove absence of filesystem access"],
+    );
+  }
+
+  if (signals.usesProcess) {
+    caps.process = capability("required", "static", signals.processEvidence);
+  } else {
+    caps.process = capability("unknown", "static", [
+      "static analysis cannot prove absence of process access",
+    ]);
+  }
+
+  if (signals.usesNativeAddon) {
+    caps.nativeAddon = capability("required", "static", signals.nativeAddonEvidence);
+  } else {
+    caps.nativeAddon = capability("not-detected", "static", ["no native addon signals"]);
+  }
+
+  caps.lifecycleScripts = capability("not-detected", "rule", [
+    "forge add disables lifecycle scripts by default",
+  ]);
+
+  return caps;
+}
+
+function cloneCapabilitySet(source: CapabilitySet): CapabilitySet {
+  return {
+    network: { ...source.network },
+    filesystem: { ...source.filesystem },
+    process: { ...source.process },
+    nativeAddon: { ...source.nativeAddon },
+    lifecycleScripts: { ...source.lifecycleScripts },
+    secrets: [...source.secrets],
+  };
+}

package/src/forge/compiler/classifier/classify.ts

@@ -0,0 +1,113 @@
+import type { CapabilitySet, SecretRequirement } from "../types/capability.ts";
+import type { RuntimeClassification } from "../types/classification.ts";
+import type { ExportClassification } from "../types/package-graph.ts";
+import type { IntegrationRecipe } from "../types/integration.ts";
+import type { PackageApi } from "../types/package-graph.ts";
+import { compareBytes } from "../primitives/compare.ts";
+import { resolveByPackageName } from "../recipes/registry.ts";
+import { detectCapabilities } from "./capabilities.ts";
+import { partitionContexts } from "./contexts.ts";
+import { detectSecrets } from "./secrets.ts";
+import { gatherSignals } from "./signals.ts";
+
+export interface RuntimeClassifier {
+  classify(api: PackageApi, recipe?: IntegrationRecipe): RuntimeClassification;
+  detectCapabilities(api: PackageApi, recipe?: IntegrationRecipe): CapabilitySet;
+  detectSecrets(api: PackageApi, recipe?: IntegrationRecipe): SecretRequirement[];
+}
+
+function resolveRecipeForApi(
+  api: PackageApi,
+  recipe?: IntegrationRecipe,
+): IntegrationRecipe | undefined {
+  return recipe ?? resolveByPackageName(api.name) ?? undefined;
+}
+
+function classifyExport(
+  api: PackageApi,
+  entrypoint: string,
+  exportName: string,
+  recipe: IntegrationRecipe | undefined,
+  caps: CapabilitySet,
+  alias: string,
+): ExportClassification {
+  const signals = gatherSignals(api);
+  const { compatible, incompatible } = partitionContexts(
+    recipe,
+    caps,
+    signals,
+    api.name,
+  );
+
+  return {
+    alias,
+    packageName: api.name,
+    entrypoint,
+    exportName,
+    compatible,
+    incompatible,
+    capabilities: caps,
+  };
+}
+
+export function classify(
+  api: PackageApi,
+  recipe?: IntegrationRecipe,
+): RuntimeClassification {
+  const resolvedRecipe = resolveRecipeForApi(api, recipe);
+  const caps = detectCapabilities(api, resolvedRecipe);
+  const capsWithSecrets: CapabilitySet = {
+    ...caps,
+    secrets: detectSecrets(api, resolvedRecipe),
+  };
+  const signals = gatherSignals(api);
+  const alias = resolvedRecipe?.alias ?? api.name;
+
+  const { compatible, incompatible, rationale } = partitionContexts(
+    resolvedRecipe,
+    capsWithSecrets,
+    signals,
+    api.name,
+  );
+
+  const perEntrypoint: ExportClassification[] = [];
+  for (const ep of api.entrypoints) {
+    for (const exp of ep.exports) {
+      perEntrypoint.push(
+        classifyExport(api, ep.subpath, exp.name, resolvedRecipe, capsWithSecrets, alias),
+      );
+    }
+    if (ep.exports.length === 0) {
+      perEntrypoint.push(
+        classifyExport(api, ep.subpath, "*", resolvedRecipe, capsWithSecrets, alias),
+      );
+    }
+  }
+
+  if (api.entrypoints.length === 0) {
+    perEntrypoint.push(
+      classifyExport(api, ".", "*", resolvedRecipe, capsWithSecrets, alias),
+    );
+  }
+
+  perEntrypoint.sort((a, b) => {
+    const ep = compareBytes(a.entrypoint, b.entrypoint);
+    if (ep !== 0) return ep;
+    return compareBytes(a.exportName, b.exportName);
+  });
+
+  return {
+    compatible,
+    incompatible,
+    rationale,
+    perEntrypoint,
+  };
+}
+
+export function createRuntimeClassifier(): RuntimeClassifier {
+  return {
+    classify,
+    detectCapabilities,
+    detectSecrets,
+  };
+}