forgeos 0.1.0-alpha.2 → 0.1.0-alpha.21

package/src/forge/compiler/app-graph/extract.ts

@@ -1,5 +1,6 @@
 import type { SyntaxNode } from "tree-sitter";
 import { classifyForgeCallee } from "./classify.ts";
+import type { ForgeKind } from "../types/app-graph.ts";
 import type { RawSymbol } from "./types.ts";
 
 export interface SymbolNameInfo {
@@ -87,7 +88,113 @@ export function extractSymbolsFromTree(
   const symbols: RawSymbol[] = [];
   const visited = new Set<string>();
 
+  function record(
+    kind: ForgeKind,
+    name: string,
+    qualifiedName: string,
+    node: SyntaxNode,
+    exportPath: string,
+  ): void {
+    const key = `${kind}:${qualifiedName}:${node.startIndex}:${node.endIndex}`;
+    if (visited.has(key)) {
+      return;
+    }
+    visited.add(key);
+    symbols.push({
+      kind,
+      name,
+      qualifiedName,
+      file,
+      span: { start: node.startIndex, end: node.endIndex },
+      exportPath,
+      sourceSlice: source.slice(node.startIndex, node.endIndex),
+    });
+  }
+
+  function symbolNameForDeclaration(node: SyntaxNode): string | null {
+    const nameNode = node.childForFieldName("name");
+    if (
+      nameNode?.type === "identifier" ||
+      nameNode?.type === "type_identifier"
+    ) {
+      return nameNode.text;
+    }
+    return null;
+  }
+
+  function codeDeclarationKind(node: SyntaxNode): ForgeKind | null {
+    if (node.type === "function_declaration") return "code.function";
+    if (node.type === "class_declaration") return "code.class";
+    if (node.type === "interface_declaration") return "code.interface";
+    if (node.type === "type_alias_declaration") return "code.type";
+    if (node.type === "enum_declaration") return "code.enum";
+    return null;
+  }
+
+  function isProgramLevel(node: SyntaxNode): boolean {
+    return node.parent?.type === "program" ||
+      (node.parent?.type === "export_statement" && node.parent.parent?.type === "program");
+  }
+
+  function exportPathForNode(node: SyntaxNode): string {
+    return node.parent?.type === "export_statement" ? "export" : "";
+  }
+
+  function resolveVariableInitializerCallee(declarator: SyntaxNode): string | null {
+    const value = declarator.childForFieldName("value");
+    if (value?.type !== "call_expression") {
+      return null;
+    }
+    const functionNode = value.childForFieldName("function");
+    return functionNode?.type === "identifier" ? resolveCalleeName(functionNode) : null;
+  }
+
+  function recordCodeDeclaration(node: SyntaxNode): void {
+    if (!isProgramLevel(node)) {
+      return;
+    }
+    const kind = codeDeclarationKind(node);
+    if (!kind) {
+      return;
+    }
+    const name = symbolNameForDeclaration(node);
+    if (!name) {
+      return;
+    }
+    record(kind, name, name, node, exportPathForNode(node));
+  }
+
+  function recordCodeVariables(node: SyntaxNode): void {
+    if (
+      node.type !== "lexical_declaration" &&
+      node.type !== "variable_declaration"
+    ) {
+      return;
+    }
+    if (!isProgramLevel(node)) {
+      return;
+    }
+
+    for (const declarator of node.namedChildren) {
+      if (declarator.type !== "variable_declarator") {
+        continue;
+      }
+      const nameNode = declarator.childForFieldName("name");
+      if (nameNode?.type !== "identifier") {
+        continue;
+      }
+      const callee = resolveVariableInitializerCallee(declarator);
+      if (callee && classifyForgeCallee(callee)) {
+        continue;
+      }
+      record("code.const", nameNode.text, nameNode.text, declarator, exportPathForNode(node));
+    }
+  }
+
   function visit(node: SyntaxNode): void {
+    recordCodeDeclaration(node);
+    recordCodeVariables(node);
+
     if (node.type === "call_expression") {
       const functionNode = node.childForFieldName("function");
       if (functionNode && functionNode.type === "identifier") {

package/src/forge/compiler/app-graph/forge-apis.ts

@@ -13,6 +13,7 @@ export const FORGE_BUILDER_APIS: Readonly<Record<string, ForgeKind>> = {
   definePolicies: "policy",
   workflow: "workflow",
   agent: "agent",
+  aiTool: "aiTool",
   telemetryEvent: "telemetryEvent",
 };
 

package/src/forge/compiler/app-graph/module-graph.ts

@@ -15,7 +15,6 @@ import type {
 } from "../types/app-graph.ts";
 import type { RuntimeContext } from "../types/runtime.ts";
 import { FORGE_KIND_TO_CONTEXT } from "./forge-apis.ts";
-import { loadTsconfig } from "./tsconfig-hash.ts";
 import type { RawSymbol } from "./types.ts";
 
 export function moduleIdForFile(file: string): string {
@@ -57,19 +56,27 @@ function stableSortModuleNodes(nodes: ModuleNode[]): ModuleNode[] {
 
 function declaredContextsForFile(
   file: string,
-  symbols: RawSymbol[],
+  contextsByFile: Map<string, RuntimeContext[]>,
 ): RuntimeContext[] {
-  const contexts = new Set<RuntimeContext>();
+  return contextsByFile.get(file) ?? [];
+}
+
+function buildDeclaredContextsByFile(symbols: RawSymbol[]): Map<string, RuntimeContext[]> {
+  const mutable = new Map<string, Set<RuntimeContext>>();
   for (const symbol of symbols) {
-    if (symbol.file !== file) {
-      continue;
-    }
     const context = FORGE_KIND_TO_CONTEXT[symbol.kind];
     if (context) {
-      contexts.add(context);
+      const fileContexts = mutable.get(symbol.file) ?? new Set<RuntimeContext>();
+      fileContexts.add(context);
+      mutable.set(symbol.file, fileContexts);
     }
   }
-  return stableSortStrings([...contexts]) as RuntimeContext[];
+
+  const byFile = new Map<string, RuntimeContext[]>();
+  for (const [file, fileContexts] of mutable) {
+    byFile.set(file, stableSortStrings([...fileContexts]) as RuntimeContext[]);
+  }
+  return byFile;
 }
 
 function spanFromNode(node: ts.Node, sourceFile: ts.SourceFile): {
@@ -81,50 +88,37 @@ function spanFromNode(node: ts.Node, sourceFile: ts.SourceFile): {
   return { start, end };
 }
 
-function toWorkspaceRelativePath(
-  absoluteOrRelativePath: string,
-  workspaceRoot: string,
-): string {
-  const normalized = normalizePath(absoluteOrRelativePath);
-  const root = normalizePath(path.resolve(workspaceRoot).replace(/\\/g, "/"));
-  const rootWithSlash = root.endsWith("/") ? root : `${root}/`;
-
-  if (normalized.startsWith(rootWithSlash)) {
-    return normalized.slice(rootWithSlash.length);
-  }
-
-  if (normalized === root) {
-    return "";
-  }
-
-  return normalized;
-}
-
 function resolveLocalTarget(
   specifier: string,
   fromFile: string,
-  program: ts.Program,
-  workspaceRoot: string,
+  sourcePaths: Set<string>,
 ): string | null {
-  const resolved = ts.resolveModuleName(
-    specifier,
-    fromFile,
-    program.getCompilerOptions(),
-    ts.sys,
-  );
-
-  const fileName = resolved.resolvedModule?.resolvedFileName;
-  if (!fileName) {
+  if (!specifier.startsWith(".")) {
     return null;
   }
 
-  return toWorkspaceRelativePath(fileName, workspaceRoot);
+  const base = normalizePath(path.join(path.dirname(fromFile), specifier));
+  const candidates = [
+    base,
+    `${base}.ts`,
+    `${base}.tsx`,
+    `${base}/index.ts`,
+    `${base}/index.tsx`,
+  ];
+  if (/\.[cm]?jsx?$/.test(base)) {
+    const withoutJsExtension = base.replace(/\.[cm]?jsx?$/, "");
+    candidates.push(
+      `${withoutJsExtension}.ts`,
+      `${withoutJsExtension}.tsx`,
+    );
+  }
+
+  return candidates.find((candidate) => sourcePaths.has(candidate)) ?? null;
 }
 
 function collectImports(
   sourceFile: ts.SourceFile,
-  program: ts.Program,
-  workspaceRoot: string,
+  sourcePaths: Set<string>,
 ): { packageImports: PackageImport[]; localImports: LocalImport[] } {
   const packageImports: PackageImport[] = [];
   const localImports: LocalImport[] = [];
@@ -132,13 +126,10 @@ function collectImports(
 
   function recordPackageImport(
     specifier: string,
+    parsed: { packageName: string; subpath: string },
     span: { start: number; end: number },
     importKind: ImportKind,
   ): void {
-    const parsed = parsePackageSpecifier(specifier);
-    if (!parsed) {
-      return;
-    }
     packageImports.push({
       specifier,
       packageName: parsed.packageName,
@@ -155,13 +146,12 @@ function collectImports(
         const span = spanFromNode(node.moduleSpecifier, sourceFile);
         const parsed = parsePackageSpecifier(specifier);
         if (parsed) {
-          recordPackageImport(specifier, span, "static");
+          recordPackageImport(specifier, parsed, span, "static");
         } else {
           const target = resolveLocalTarget(
             specifier,
             fromFile,
-            program,
-            workspaceRoot,
+            sourcePaths,
           );
           if (target) {
             localImports.push({
@@ -183,13 +173,12 @@ function collectImports(
       const span = spanFromNode(node.arguments[0], sourceFile);
       const parsed = parsePackageSpecifier(specifier);
       if (parsed) {
-        recordPackageImport(specifier, span, "dynamic");
+        recordPackageImport(specifier, parsed, span, "dynamic");
       } else {
         const target = resolveLocalTarget(
           specifier,
           fromFile,
-          program,
-          workspaceRoot,
+          sourcePaths,
         );
         if (target) {
           localImports.push({
@@ -211,13 +200,12 @@ function collectImports(
       const span = spanFromNode(node.arguments[0], sourceFile);
       const parsed = parsePackageSpecifier(specifier);
       if (parsed) {
-        recordPackageImport(specifier, span, "require");
+        recordPackageImport(specifier, parsed, span, "require");
       } else {
         const target = resolveLocalTarget(
           specifier,
           fromFile,
-          program,
-          workspaceRoot,
+          sourcePaths,
         );
         if (target) {
           localImports.push({
@@ -238,8 +226,9 @@ function collectImports(
 function sourcesUnchangedSincePrior(
   sources: SourceFile[],
   prior?: AppGraph,
+  canReusePrior = true,
 ): boolean {
-  if (!prior) {
+  if (!prior || !canReusePrior) {
     return false;
   }
   const priorHashes = new Map(Object.entries(prior.sourceHashes ?? {}));
@@ -254,39 +243,46 @@ function sourcesUnchangedSincePrior(
   return true;
 }
 
+function priorModuleNodesByFile(prior?: AppGraph): Map<string, ModuleNode> {
+  return new Map((prior?.moduleGraph.nodes ?? []).map((node) => [node.file, node]));
+}
+
 export function buildModuleGraph(
   sources: SourceFile[],
   rawSymbols: RawSymbol[],
-  workspaceRoot: string,
-  tsconfigPath?: string,
   prior?: AppGraph,
+  canReusePrior = true,
 ): ModuleGraph {
-  if (sourcesUnchangedSincePrior(sources, prior)) {
+  if (sourcesUnchangedSincePrior(sources, prior, canReusePrior)) {
     return prior!.moduleGraph;
   }
 
-  const parsedConfig = loadTsconfig(workspaceRoot, tsconfigPath);
-  const fileNames = sources.map((source) =>
-    path.resolve(workspaceRoot, source.path).replace(/\\/g, "/"),
-  );
-
-  const program = ts.createProgram(fileNames, {
-    ...parsedConfig.options,
-    noEmit: true,
-    skipLibCheck: true,
-  });
+  const priorHashes = new Map(Object.entries(prior?.sourceHashes ?? {}));
+  const priorNodes = priorModuleNodesByFile(prior);
+  const contextsByFile = buildDeclaredContextsByFile(rawSymbols);
+  const sourcePaths = new Set(sources.map((source) => normalizePath(source.path)));
 
   const nodes: ModuleNode[] = [];
 
   for (const source of sources) {
     const normalizedFile = normalizePath(source.path);
-    const absolutePath = fileNames.find(
-      (name) => normalizePath(name).endsWith(normalizedFile),
-    );
+    const priorNode = priorNodes.get(normalizedFile);
+    if (
+      canReusePrior &&
+      priorNode &&
+      priorHashes.get(normalizedFile) === source.contentHash
+    ) {
+      nodes.push(priorNode);
+      continue;
+    }
 
-    const sourceFile = absolutePath
-      ? program.getSourceFile(absolutePath)
-      : undefined;
+    const sourceFile = ts.createSourceFile(
+      normalizedFile,
+      source.text,
+      ts.ScriptTarget.Latest,
+      true,
+      normalizedFile.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS,
+    );
 
     if (!sourceFile) {
       nodes.push({
@@ -294,7 +290,7 @@ export function buildModuleGraph(
         file: normalizedFile,
         directPackageImports: [],
         localImports: [],
-        declaredContexts: declaredContextsForFile(normalizedFile, rawSymbols),
+        declaredContexts: declaredContextsForFile(normalizedFile, contextsByFile),
         effectiveContexts: [],
       });
       continue;
@@ -302,8 +298,7 @@ export function buildModuleGraph(
 
     const { packageImports, localImports } = collectImports(
       sourceFile,
-      program,
-      workspaceRoot,
+      sourcePaths,
     );
 
     nodes.push({
@@ -311,7 +306,7 @@ export function buildModuleGraph(
       file: normalizedFile,
       directPackageImports: packageImports,
       localImports,
-      declaredContexts: declaredContextsForFile(normalizedFile, rawSymbols),
+      declaredContexts: declaredContextsForFile(normalizedFile, contextsByFile),
       effectiveContexts: [],
     });
   }

package/src/forge/compiler/app-graph/parser.ts

@@ -32,31 +32,30 @@ function priorSymbolsToRaw(symbols: ForgeSymbol[]): RawSymbol[] {
   }));
 }
 
+interface PriorSymbolsForFile {
+  contentHash?: string;
+  symbols: ForgeSymbol[];
+}
+
 function groupPriorSymbolsByFile(
   priorSymbols: ForgeSymbol[],
-): Map<string, ForgeSymbol[]> {
-  const byFile = new Map<string, ForgeSymbol[]>();
+): Map<string, PriorSymbolsForFile> {
+  const byFile = new Map<string, PriorSymbolsForFile>();
   for (const symbol of priorSymbols) {
-    const list = byFile.get(symbol.file) ?? [];
-    list.push(symbol);
-    byFile.set(symbol.file, list);
+    const existing = byFile.get(symbol.file);
+    if (existing) {
+      existing.symbols.push(symbol);
+      continue;
+    }
+    const fromMeta = symbol.meta.fileContentHash;
+    byFile.set(symbol.file, {
+      contentHash: typeof fromMeta === "string" ? fromMeta : undefined,
+      symbols: [symbol],
+    });
   }
   return byFile;
 }
 
-function priorFileContentHash(
-  priorSymbols: ForgeSymbol[],
-  file: string,
-): string | undefined {
-  const match = priorSymbols.find((symbol) => symbol.file === file);
-  if (!match) {
-    return undefined;
-  }
-  // All symbols from the same file share the same slice hash basis; use meta when present.
-  const fromMeta = match.meta.fileContentHash;
-  return typeof fromMeta === "string" ? fromMeta : undefined;
-}
-
 export interface IncrementalParseResult {
   symbols: RawSymbol[];
   diagnostics: Diagnostic[];
@@ -65,6 +64,7 @@ export interface IncrementalParseResult {
 export function incrementalParse(
   sources: SourceFile[],
   priorSymbols: ForgeSymbol[] | undefined,
+  priorSourceHashes: Record<string, string> | undefined,
   priorInvalidation: ParseInvalidationKey | undefined,
   currentInvalidation: ParseInvalidationKey,
 ): IncrementalParseResult {
@@ -76,22 +76,22 @@ export function incrementalParse(
 
   const priorByFile = priorSymbols
     ? groupPriorSymbolsByFile(priorSymbols)
-    : new Map<string, ForgeSymbol[]>();
+    : new Map<string, PriorSymbolsForFile>();
 
   for (const source of sources) {
     const normalizedFile = normalizePath(source.path);
     const priorForFile = priorByFile.get(normalizedFile);
-    const priorHash = priorSymbols
-      ? priorFileContentHash(priorSymbols, normalizedFile)
-      : undefined;
+    const priorHash = priorForFile?.contentHash ?? priorSourceHashes?.[normalizedFile];
 
     const fileUnchanged =
       !globalInvalidated &&
-      priorForFile !== undefined &&
       priorHash === source.contentHash;
 
+    if (fileUnchanged && priorForFile) {
+      allSymbols.push(...priorSymbolsToRaw(priorForFile.symbols));
+      continue;
+    }
     if (fileUnchanged) {
-      allSymbols.push(...priorSymbolsToRaw(priorForFile));
       continue;
     }
 

package/src/forge/compiler/app-graph/profile.ts

@@ -0,0 +1,26 @@
+import type { AppGraph } from "../types/app-graph.ts";
+
+export interface AppGraphPhaseTimings {
+  normalizeMs: number;
+  parseMs: number;
+  symbolsMs: number;
+  duplicatesMs: number;
+  moduleGraphMs: number;
+  inputHashMs: number;
+  totalMs: number;
+}
+
+const appGraphProfiles = new WeakMap<AppGraph, AppGraphPhaseTimings>();
+
+export function recordAppGraphProfile(
+  graph: AppGraph,
+  timings: AppGraphPhaseTimings,
+): void {
+  appGraphProfiles.set(graph, timings);
+}
+
+export function getAppGraphProfile(
+  graph: AppGraph,
+): AppGraphPhaseTimings | undefined {
+  return appGraphProfiles.get(graph);
+}

package/src/forge/compiler/app-graph/versions.ts

@@ -8,7 +8,7 @@ export const APP_GRAPH_ANALYZER_VERSION = "0.1.0";
 export const TREE_SITTER_GRAMMAR_VERSION = "0.23.2";
 
 /** Forge builder API → ForgeKind classifier version. */
-export const FORGE_CLASSIFIER_VERSION = "0.1.1";
+export const FORGE_CLASSIFIER_VERSION = "0.1.3";
 
 /** Generation orchestrator version (recorded on emitted AppGraph). */
 export { GENERATOR_VERSION } from "../../version.ts";

package/src/forge/compiler/classifier/capabilities.ts

@@ -2,17 +2,18 @@ import type { CapabilitySet } from "../types/capability.ts";
 import type { IntegrationRecipe } from "../types/integration.ts";
 import type { PackageApi } from "../types/package-graph.ts";
 import { capability, emptyCapabilitySet } from "../recipes/helpers.ts";
-import { gatherSignals } from "./signals.ts";
+import { gatherSignals, type PackageSignals } from "./signals.ts";
 
 export function detectCapabilities(
   api: PackageApi,
   recipe?: IntegrationRecipe,
+  precomputedSignals?: PackageSignals,
 ): CapabilitySet {
   if (recipe) {
     return cloneCapabilitySet(recipe.capabilities);
   }
 
-  const signals = gatherSignals(api);
+  const signals = precomputedSignals ?? gatherSignals(api);
   const caps = emptyCapabilitySet();
 
   if (signals.usesNetwork) {

package/src/forge/compiler/classifier/classify.ts

@@ -8,7 +8,7 @@ import { resolveByPackageName } from "../recipes/registry.ts";
 import { detectCapabilities } from "./capabilities.ts";
 import { partitionContexts } from "./contexts.ts";
 import { detectSecrets } from "./secrets.ts";
-import { gatherSignals } from "./signals.ts";
+import { gatherSignals, type PackageSignals } from "./signals.ts";
 
 export interface RuntimeClassifier {
   classify(api: PackageApi, recipe?: IntegrationRecipe): RuntimeClassification;
@@ -30,8 +30,8 @@ function classifyExport(
   recipe: IntegrationRecipe | undefined,
   caps: CapabilitySet,
   alias: string,
+  signals: PackageSignals,
 ): ExportClassification {
-  const signals = gatherSignals(api);
   const { compatible, incompatible } = partitionContexts(
     recipe,
     caps,
@@ -55,12 +55,12 @@ export function classify(
   recipe?: IntegrationRecipe,
 ): RuntimeClassification {
   const resolvedRecipe = resolveRecipeForApi(api, recipe);
-  const caps = detectCapabilities(api, resolvedRecipe);
+  const signals = gatherSignals(api);
+  const caps = detectCapabilities(api, resolvedRecipe, signals);
   const capsWithSecrets: CapabilitySet = {
     ...caps,
-    secrets: detectSecrets(api, resolvedRecipe),
+    secrets: detectSecrets(api, resolvedRecipe, signals),
   };
-  const signals = gatherSignals(api);
   const alias = resolvedRecipe?.alias ?? api.name;
 
   const { compatible, incompatible, rationale } = partitionContexts(
@@ -74,19 +74,43 @@ export function classify(
   for (const ep of api.entrypoints) {
     for (const exp of ep.exports) {
       perEntrypoint.push(
-        classifyExport(api, ep.subpath, exp.name, resolvedRecipe, capsWithSecrets, alias),
+        classifyExport(
+          api,
+          ep.subpath,
+          exp.name,
+          resolvedRecipe,
+          capsWithSecrets,
+          alias,
+          signals,
+        ),
       );
     }
     if (ep.exports.length === 0) {
       perEntrypoint.push(
-        classifyExport(api, ep.subpath, "*", resolvedRecipe, capsWithSecrets, alias),
+        classifyExport(
+          api,
+          ep.subpath,
+          "*",
+          resolvedRecipe,
+          capsWithSecrets,
+          alias,
+          signals,
+        ),
       );
     }
   }
 
   if (api.entrypoints.length === 0) {
     perEntrypoint.push(
-      classifyExport(api, ".", "*", resolvedRecipe, capsWithSecrets, alias),
+      classifyExport(
+        api,
+        ".",
+        "*",
+        resolvedRecipe,
+        capsWithSecrets,
+        alias,
+        signals,
+      ),
     );
   }
 

package/src/forge/compiler/classifier/secrets.ts

@@ -2,11 +2,12 @@ import type { SecretRequirement } from "../types/capability.ts";
 import type { IntegrationRecipe } from "../types/integration.ts";
 import type { PackageApi } from "../types/package-graph.ts";
 import { secret } from "../recipes/helpers.ts";
-import { gatherSignals } from "./signals.ts";
+import { gatherSignals, type PackageSignals } from "./signals.ts";
 
 export function detectSecrets(
   api: PackageApi,
   recipe?: IntegrationRecipe,
+  precomputedSignals?: PackageSignals,
 ): SecretRequirement[] {
   const found = new Map<string, SecretRequirement>();
 
@@ -16,7 +17,7 @@ export function detectSecrets(
     }
   }
 
-  const signals = gatherSignals(api);
+  const signals = precomputedSignals ?? gatherSignals(api);
   for (const evidence of signals.envSecretEvidence) {
     const envVar = evidence.replace(/^env:/, "");
     if (!found.has(envVar)) {