@flink-app/flink 0.14.3 → 2.0.0-alpha.100

package/src/TypeScriptCompiler.ts

@@ -1,53 +1,186 @@
 import fs, { promises as fsPromises } from "fs";
-import { JSONSchema7 } from "json-schema";
 import { join } from "path";
 import glob from "tiny-glob";
-import { CompletedConfig, createFormatter, createParser, createProgram, Schema, SchemaGenerator } from "ts-json-schema-generator";
-import {
-    ArrayLiteralExpression,
-    DiagnosticCategory,
-    ImportDeclarationStructure,
-    OptionalKind,
-    Project,
-    SourceFile,
-    Symbol,
-    SyntaxKind,
-    ts,
-    Type,
-    TypeReferenceNode,
-    VariableDeclarationKind,
-} from "ts-morph";
+import { ArrayLiteralExpression, DiagnosticCategory, ImportDeclarationStructure, OptionalKind, Project, SourceFile, SyntaxKind, ts } from "ts-morph";
+import { FlinkLogFactory } from "./FlinkLogFactory";
 import { writeJsonFile } from "./FsUtils";
-import { addImports, getDefaultExport, getInterfaceName, getTypeMetadata, getTypesToImport } from "./TypeScriptUtils";
+import { TypeScriptSourceParser } from "./schema-extraction";
 import { getCollectionNameForRepo, getHttpMethodFromHandlerName, getRepoInstanceName } from "./utils";
+import { FlinkCompilerPlugin, loadFlinkConfig } from "./utils/loadFlinkConfig";
+
+const perfLog = FlinkLogFactory.createLogger("flink.perf");
+const initLog = FlinkLogFactory.createLogger("flink.init");
 
 class TypeScriptCompiler {
+    private cwd: string;
     private project: Project;
-    private schemaGenerator?: SchemaGenerator;
     private isEsm: boolean;
+    private schemaGenerator?: ((source: string, options?: any) => any) | null;
 
     /**
-     * Parsed typescript schemas that will be added to intermediate
-     * schemas.ts file.
-     *
-     * This will be written to file in a batch for performance reasons.
+     * Handler schemas collected during parseHandlers, to be generated later
+     */
+    private handlerSchemasToGenerate: {
+        reqSchemaType?: string;
+        resSchemaType?: string;
+        sourceFile: SourceFile;
+    }[] = [];
+
+    /**
+     * Tool schemas collected during parseTools, to be generated later
+     */
+    private toolSchemasToGenerate: {
+        inputSchemaType?: string;
+        outputSchemaType?: string;
+        inputTypeHint?: "void" | "any" | "named";
+        outputTypeHint?: "void" | "any" | "named";
+        sourceFile: SourceFile;
+    }[] = [];
+
+    /**
+     * Pre-segmented source files by type (cached for performance)
+     */
+    private handlerFiles: SourceFile[] = [];
+    private repoFiles: SourceFile[] = [];
+    private toolFiles: SourceFile[] = [];
+    private agentFiles: SourceFile[] = [];
+    private jobFiles: SourceFile[] = [];
+    private serviceFiles: SourceFile[] = [];
+
+    /**
+     * Tool ID registry for agent validation (built during segmentation)
+     */
+    private toolIdRegistry: Set<string> = new Set();
+
+    /**
+     * Compiler plugins loaded from flink.config.js
+     */
+    private compilerPlugins: FlinkCompilerPlugin[] = [];
+    private disableServices = false;
+
+    /**
+     * Extension files collected during segmentation, keyed by generatedFile name
      */
-    private parsedTsSchemas: string[] = [];
+    private extensionFiles: Map<string, SourceFile[]> = new Map();
 
     /**
-     * Imports needed for schemas.ts.
+     * Generates a schema $id from a file path and type name using the same algorithm
+     * as the schema generator's defineId callback.
      *
-     * This will be added to file in a batch for performance reasons.
+     * Examples:
+     * - src/schemas/Car.ts, "Car" → "Car"
+     * - src/schemas/wrappers/PartialLoginReq.ts, "PartialLoginReq" → "wrappers.PartialLoginReq"
+     * - src/schemas/PatchCarSchemas.ts, "PatchCarReq" → "PatchCarSchemas.PatchCarReq"
+     */
+    private filePathToSchemaId(absolutePath: string, typeName: string): string {
+        const path = require("path");
+        const schemaDir = path.join(this.cwd, "src/schemas");
+
+        // Get path relative to src/schemas/
+        const relativePath = path.relative(schemaDir, absolutePath);
+
+        // Remove .ts extension
+        const pathWithoutExt = relativePath.replace(/\.ts$/, "");
+
+        // Split by path separator
+        const parts = pathWithoutExt.split(path.sep);
+        const fileName = parts[parts.length - 1];
+
+        // Root file with matching name: just type name
+        if (parts.length === 1 && fileName === typeName) {
+            return typeName;
+        }
+
+        // Nested file or multiple exports: include path
+        const pathPrefix = parts.join(".");
+        if (fileName === typeName) {
+            return pathPrefix; // Avoid duplication
+        }
+
+        return `${pathPrefix}.${typeName}`;
+    }
+
+    /**
+     * Resolves a type name to its schema $id by looking up its import path.
+     * Returns undefined if the type is not imported from src/schemas/.
      */
-    private tsSchemasSymbolsToImports: Symbol[] = [];
+    private resolveTypeNameToSchemaId(fileText: string, typeName: string, handlerFilePath: string): string | undefined {
+        const path = require("path");
+
+        // Extract import statement for this type
+        const importStmt = TypeScriptSourceParser.extractImportForType(fileText, typeName);
+        if (!importStmt) {
+            perfLog.trace(`Type ${typeName} not found in imports`);
+            return undefined;
+        }
+
+        // Extract module specifier from import
+        // e.g., import { Foo } from "../schemas/Bar" → "../schemas/Bar"
+        const moduleMatch = importStmt.match(/from\s+["']([^"']+)["']/);
+        if (!moduleMatch) {
+            perfLog.trace(`Could not extract module specifier from: ${importStmt}`);
+            return undefined;
+        }
+
+        const moduleSpecifier = moduleMatch[1];
+
+        // Resolve relative import to absolute path
+        const handlerDir = path.dirname(handlerFilePath);
+        const importPath = path.resolve(handlerDir, moduleSpecifier);
+
+        // Add .ts extension if not present
+        const importPathWithExt = importPath.endsWith(".ts") ? importPath : importPath + ".ts";
+
+        // Check if this is from src/schemas/
+        const schemaDir = path.join(this.cwd, "src/schemas");
+        if (!importPathWithExt.startsWith(schemaDir)) {
+            perfLog.trace(`Type ${typeName} is not from src/schemas/ (from ${importPathWithExt})`);
+            return undefined;
+        }
+
+        // Generate schema $id using the same algorithm
+        return this.filePathToSchemaId(importPathWithExt, typeName);
+    }
+
+    constructor(cwd: string) {
+        const path = require("path");
+        // Convert to absolute path to ensure consistent path comparisons
+        this.cwd = path.resolve(cwd);
 
-    constructor(private cwd: string) {
         // Detect if project is using ESM based solely on package.json "type": "module"
-        this.isEsm = this.isEsmProject(cwd);
+        this.isEsm = this.isEsmProject(this.cwd);
+
+        const tsConfigPath = join(this.cwd, "tsconfig.json");
+
+        // Read and inherit important settings from tsconfig.json
+        let userTsConfig: any = {};
+        try {
+            if (fs.existsSync(tsConfigPath)) {
+                const tsConfigContent = fs.readFileSync(tsConfigPath, "utf8");
+                userTsConfig = JSON.parse(tsConfigContent);
+            }
+        } catch (error) {
+            console.warn("Warning: Could not read tsconfig.json:", error);
+        }
 
         const compilerOptions: ts.CompilerOptions = {
             noEmit: false, // We need to emit files
             outDir: join(cwd, "dist"),
+            // Inherit important performance options from user's tsconfig
+            skipLibCheck: userTsConfig.compilerOptions?.skipLibCheck ?? true, // Default to true for performance
+
+            // Performance optimizations for emit phase
+            // Since we already run getPreEmitDiagnostics() for type checking,
+            // we can use faster transpilation-only options
+            isolatedModules: true, // Transpile each file independently (much faster, no cross-file analysis)
+            // removeComments: true, // Smaller output, faster emit
+            importHelpers: false, // Don't import tslib helpers
+
+            // EXPERIMENTAL: Enable incremental compilation
+            // This creates a .tsbuildinfo file to cache type information
+            // Should speed up subsequent builds by avoiding full type checking
+            // incremental: true,
+            // tsBuildInfoFile: join(cwd, "dist", ".tsbuildinfo"),
         };
 
         // Set appropriate module settings based on detected module system
@@ -62,20 +195,326 @@ class TypeScriptCompiler {
             compilerOptions.moduleResolution = ts.ModuleResolutionKind.NodeJs;
         }
 
-        const tsConfigPath = join(cwd, "tsconfig.json");
-        console.log("TypeScript config path:", require("path").resolve(tsConfigPath));
-        console.log("TypeScript version:", ts.version);
+        initLog.info("TypeScript version:", ts.version, "config path", require("path").resolve(tsConfigPath));
+        // console.log("Compiler options:", JSON.stringify(compilerOptions, null, 2));
 
+        const projectStartTime = Date.now();
         this.project = new Project({
             tsConfigFilePath: tsConfigPath,
             compilerOptions,
+            skipAddingFilesFromTsConfig: true, // Don't auto-load all files - we'll add specific directories we need
+            skipFileDependencyResolution: true, // Don't resolve /// <reference> and type dependencies from node_modules
         });
+        const projectTime = Date.now() - projectStartTime;
+        perfLog.debug(`✓ Project initialization completed in ${projectTime}ms`);
+
+        // Load Flink-specific source paths from tsconfig.json if available
+        const additionalSourcePaths = this.getFlinkSourcePaths(tsConfigPath);
+
+        // Load all TypeScript files from src/ directory
+        // This is simpler and faster than recursive import resolution
+        // We exclude test files and declarations to keep it lean
+        const defaultSourcePaths = [
+            join(cwd, "src/**/*.ts"),
+            "!" + join(cwd, "src/**/*.spec.ts"),
+            "!" + join(cwd, "src/**/*.test.ts"),
+            "!" + join(cwd, "src/**/*.d.ts"),
+        ];
+
+        const allSourcePaths = [...defaultSourcePaths, ...additionalSourcePaths];
+
+        const loadStartTime = Date.now();
+        this.project.addSourceFilesAtPaths(allSourcePaths);
+        const loadTime = Date.now() - loadStartTime;
+        const fileCount = this.project.getSourceFiles().length;
+        perfLog.debug(`✓ All source files loaded in ${loadTime}ms (${fileCount} files)`);
+
+        // Load config from flink.config.js
+        const flinkCfg = loadFlinkConfig(this.cwd);
+        this.compilerPlugins = flinkCfg?.compilerPlugins ?? [];
+        this.disableServices = flinkCfg?.disableServices ?? false;
+
+        if (this.compilerPlugins.length > 0) {
+            initLog.info(
+                `Compiler plugins loaded (${this.compilerPlugins.length}):`,
+                this.compilerPlugins.map((p) => `${p.package} → ${p.scanDir}`).join(", ")
+            );
+        } else {
+            initLog.info("No compiler plugins configured");
+        }
+
+        // Segment files by type for efficient processing
+        this.segmentSourceFiles();
 
         console.log("Loaded", this.project.getSourceFiles().length, "source file(s) from", cwd);
         console.log("Module system:", this.isEsm ? "ESM" : "CommonJS");
         console.log("Using module:", compilerOptions.module === ts.ModuleKind.ESNext ? "ESNext" : "CommonJS");
     }
 
+    /**
+     * Loads additional source paths from tsconfig.json's flink configuration.
+     * Allows projects to specify extra directories to include in compilation.
+     *
+     * Example tsconfig.json:
+     * {
+     *   "flink": {
+     *     "sourcePaths": ["src/custom-types/**\/*.ts", "src/utils/**\/*.ts"]
+     *   }
+     * }
+     */
+    private getFlinkSourcePaths(tsConfigPath: string): string[] {
+        try {
+            if (fs.existsSync(tsConfigPath)) {
+                const tsConfigContent = fs.readFileSync(tsConfigPath, "utf8");
+                const tsConfig = JSON.parse(tsConfigContent);
+
+                if (tsConfig.flink && Array.isArray(tsConfig.flink.sourcePaths)) {
+                    console.log("Found Flink-specific source paths:", tsConfig.flink.sourcePaths);
+                    return tsConfig.flink.sourcePaths.map((path: string) => join(this.cwd, path));
+                }
+            }
+        } catch (error) {
+            console.warn("Error reading Flink source paths from tsconfig.json:", error);
+        }
+
+        return [];
+    }
+
+    /**
+     * Initializes the schema generator.
+     *
+     * Note: ts-source-to-json-schema is ESM-only while @flink-app/flink is CommonJS.
+     * We use new Function() to preserve the actual import() in compiled output.
+     * Without this, TypeScript converts import() to require() which can't load ESM.
+     */
+    private async initSchemaGenerator() {
+        if (this.schemaGenerator !== undefined) {
+            return; // Already initialized
+        }
+
+        try {
+            // Preserve dynamic import() in CommonJS output (TypeScript would convert to require())
+            const dynamicImport = new Function("specifier", "return import(specifier)");
+            const module = await dynamicImport("@flink-app/ts-source-to-json-schema");
+
+            if (typeof module.toJsonSchemasFromFile !== "function") {
+                throw new Error("toJsonSchemasFromFile function not found. Please update @flink-app/ts-source-to-json-schema to >= 0.2.0");
+            }
+
+            this.schemaGenerator = module.toJsonSchemasFromFile;
+        } catch (error: any) {
+            console.error(
+                "\n❌ Schema generator could not be loaded:\n" +
+                    `   Error: ${error.message}\n` +
+                    "\n💡 Make sure @flink-app/ts-source-to-json-schema is installed:\n" +
+                    "   npm install @flink-app/ts-source-to-json-schema\n"
+            );
+            process.exit(1);
+        }
+    }
+
+    /**
+     * Recursively resolves and adds imported files to the project.
+     * This ensures that files imported by handlers, schemas, etc. are available for type resolution.
+     *
+     * Handles three types of imports:
+     * 1. Relative imports (./foo, ../bar) - always resolved
+     * 2. Workspace package imports (@mycompany/shared) - resolved if symlinked outside node_modules
+     * 3. External packages (lodash, express) - skipped to avoid loading entire dependency trees
+     */
+    private resolveImportedFiles() {
+        // TODO: Check if this really is needed!
+
+        const processedFiles = new Set<string>();
+        const filesToProcess = [...this.project.getSourceFiles()];
+
+        let totalImports = 0;
+        let skippedImports = 0;
+        let resolvedImports = 0;
+
+        while (filesToProcess.length > 0) {
+            const sourceFile = filesToProcess.pop()!;
+            const filePath = sourceFile.getFilePath();
+
+            if (processedFiles.has(filePath)) {
+                continue;
+            }
+
+            processedFiles.add(filePath);
+
+            // Get all import declarations
+            const importDeclarations = sourceFile.getImportDeclarations();
+
+            for (const importDecl of importDeclarations) {
+                const moduleSpecifier = importDecl.getModuleSpecifierValue();
+                totalImports++;
+
+                // For relative imports (./foo, ../bar), always resolve
+                const isRelativeImport = moduleSpecifier.startsWith(".") || moduleSpecifier.startsWith("/");
+
+                // For package imports, check if it might be a workspace package
+                // Workspace packages are typically scoped (@company/pkg) or match the workspace pattern
+                const isLikelyWorkspacePackage =
+                    !isRelativeImport && (moduleSpecifier.startsWith("@" + this.getWorkspaceScope() + "/") || this.isLikelyWorkspaceImport(moduleSpecifier));
+
+                // Skip resolution for external packages (performance optimization)
+                // Only resolve relative imports and likely workspace packages
+                if (!isRelativeImport && !isLikelyWorkspacePackage) {
+                    skippedImports++;
+                    continue;
+                }
+
+                // Try to resolve the imported file (expensive operation)
+                resolvedImports++;
+                const moduleSourceFile = importDecl.getModuleSpecifierSourceFile();
+
+                if (moduleSourceFile) {
+                    const importedPath = moduleSourceFile.getFilePath();
+
+                    // Double-check it's not in node_modules (for workspace packages)
+                    const isWorkspacePackage = !isRelativeImport && !importedPath.includes("node_modules");
+
+                    // Add to processing queue if it's a relative import or workspace package
+                    if ((isRelativeImport || isWorkspacePackage) && !processedFiles.has(importedPath)) {
+                        filesToProcess.push(moduleSourceFile);
+                    }
+                }
+            }
+        }
+
+        console.log("Resolved imports, total files loaded:", processedFiles.size);
+        console.log(`  Import stats: ${totalImports} total, ${skippedImports} skipped, ${resolvedImports} resolved`);
+    }
+
+    /**
+     * Gets the workspace scope from package.json (e.g., "@mycompany" from "@mycompany/my-app")
+     * Returns empty string if not a scoped package.
+     */
+    private getWorkspaceScope(): string {
+        try {
+            const packageJson = JSON.parse(fs.readFileSync(join(this.cwd, "package.json"), "utf8"));
+            const name = packageJson.name || "";
+            if (name.startsWith("@")) {
+                return name.split("/")[0].substring(1); // Remove @ and get scope
+            }
+        } catch (error) {
+            // Ignore errors
+        }
+        return "";
+    }
+
+    /**
+     * Checks if an import is likely to be a workspace package.
+     * This is a heuristic - we check for common workspace patterns but can't be 100% sure
+     * without actually resolving. This is a performance optimization to avoid resolving
+     * obvious external packages like "express", "lodash", etc.
+     */
+    private isLikelyWorkspaceImport(moduleSpecifier: string): boolean {
+        // If it contains a workspace marker in the name, it might be workspace
+        // Common patterns: starts with project name, contains "internal", etc.
+        // For now, be conservative and return false - only workspace packages with
+        // matching scope will be resolved
+        return false;
+    }
+
+    /**
+     * Segments source files by type for efficient processing.
+     * Performs all path checks and AST inspections in a single pass.
+     *
+     * This runs once during construction and caches results to avoid
+     * multiple full-file iterations during parse methods.
+     */
+    private segmentSourceFiles() {
+        const startTime = Date.now();
+        const allFiles = this.project.getSourceFiles();
+
+        let handlerTime = 0,
+            repoTime = 0,
+            toolTime = 0,
+            agentTime = 0,
+            jobTime = 0,
+            serviceTime = 0;
+
+        for (const sf of allFiles) {
+            const filePath = sf.getFilePath();
+            const fileStartTime = Date.now();
+
+            // Handlers: simple path check
+            if (filePath.includes("src/handlers/")) {
+                this.handlerFiles.push(sf);
+                handlerTime += Date.now() - fileStartTime;
+                continue; // Skip further checks (mutually exclusive)
+            }
+
+            // Repos: simple path check
+            if (filePath.includes("src/repos/")) {
+                this.repoFiles.push(sf);
+                repoTime += Date.now() - fileStartTime;
+                continue;
+            }
+
+            // Tools: path check + fast text-based detection
+            // Check if file contains "FlinkToolProps" without parsing AST (much faster)
+            if (filePath.includes("src/tools/")) {
+                const fileText = sf.getFullText();
+
+                // Fast text check: must have both "export" and ": FlinkToolProps"
+                if (fileText.includes("FlinkToolProps") && fileText.includes("export")) {
+                    this.toolFiles.push(sf);
+
+                    // Extract tool ID using regex (faster than AST inspection)
+                    // Matches: id: "tool-name" or name: "tool-name"
+                    const idMatch = fileText.match(/(?:id|name):\s*["']([^"']+)["']/);
+                    if (idMatch) {
+                        this.toolIdRegistry.add(idMatch[1]);
+                    }
+                }
+                toolTime += Date.now() - fileStartTime;
+                continue;
+            }
+
+            // Agents: convention-based detection (all .ts files in src/agents/)
+            // Trust that files in src/agents/ are valid agents - validate at runtime
+            if (filePath.includes("src/agents/")) {
+                this.agentFiles.push(sf);
+                agentTime += Date.now() - fileStartTime;
+                continue;
+            }
+
+            // Jobs: simple path check
+            if (filePath.includes("src/jobs/")) {
+                this.jobFiles.push(sf);
+                jobTime += Date.now() - fileStartTime;
+                continue;
+            }
+
+            // Services: simple path check (opt-out via flink.config.js disableServices)
+            if (!this.disableServices && filePath.includes("src/services/")) {
+                this.serviceFiles.push(sf);
+                serviceTime += Date.now() - fileStartTime;
+                continue;
+            }
+
+            // Extension dirs from compiler plugins
+            for (const ext of this.compilerPlugins) {
+                if (filePath.includes(ext.scanDir)) {
+                    if (!ext.detectBy || ext.detectBy(sf.getFullText(), filePath)) {
+                        const list = this.extensionFiles.get(ext.generatedFile) ?? [];
+                        list.push(sf);
+                        this.extensionFiles.set(ext.generatedFile, list);
+                    }
+                    break;
+                }
+            }
+        }
+
+        const segmentTime = Date.now() - startTime;
+        perfLog.debug(
+            `✓ File segmentation completed in ${segmentTime}ms ` +
+                `(${this.handlerFiles.length} handlers, ${this.repoFiles.length} repos, ${this.toolFiles.length} tools, ${this.agentFiles.length} agents, ${this.jobFiles.length} jobs, ${this.serviceFiles.length} services)`
+        );
+    }
+
     /**
      * Detects if the project is using ESM (ECMAScript Modules)
      * by checking type in package.json.
@@ -98,16 +537,36 @@ class TypeScriptCompiler {
 
     /**
      * Gets the module specifier for imports, adding .js extension for ESM
+     * Uses fast path calculation instead of ts-morph's getRelativePathAsModuleSpecifierTo
+     * which triggers expensive language service initialization
      */
     private getModuleSpecifier(fromFile: SourceFile, toFile: SourceFile): string {
-        let moduleSpecifier = fromFile.getRelativePathAsModuleSpecifierTo(toFile);
+        const path = require("path");
+
+        // Get directory paths
+        const fromDir = path.dirname(fromFile.getFilePath());
+        const toPath = toFile.getFilePath();
+
+        // Calculate relative path
+        let relativePath = path.relative(fromDir, toPath);
+
+        // Convert to forward slashes (module specifiers use forward slashes)
+        relativePath = relativePath.replace(/\\/g, "/");
 
-        // Add .js extension for ESM imports (only for relative paths)
-        if (this.isEsm && !moduleSpecifier.startsWith("@") && !moduleSpecifier.endsWith(".js")) {
-            moduleSpecifier += ".js";
+        // Remove .ts extension
+        relativePath = relativePath.replace(/\.ts$/, "");
+
+        // Ensure it starts with ./ or ../
+        if (!relativePath.startsWith(".")) {
+            relativePath = "./" + relativePath;
+        }
+
+        // Add .js extension for ESM imports
+        if (this.isEsm) {
+            relativePath += ".js";
         }
 
-        return moduleSpecifier;
+        return relativePath;
     }
 
     /**
@@ -132,10 +591,108 @@ class TypeScriptCompiler {
     }
 
     /**
-     * Emits compiled javascript source to dist folder
+     * Saves all modified source files in a single batch operation.
+     * Call this before emit() to persist all changes to disk.
+     */
+    async saveAllModifiedFiles() {
+        const startTime = Date.now();
+
+        // Only save files in .flink directory (don't persist metadata in source files)
+        const flinkFiles = this.project.getSourceFiles("**/.flink/**/*.ts");
+        const unsavedFlinkFiles = flinkFiles.filter((sf) => !sf.isSaved());
+
+        await Promise.all(unsavedFlinkFiles.map((sf) => sf.save()));
+
+        const saveTime = Date.now() - startTime;
+        perfLog.debug(`✓ Batch save completed in ${saveTime}ms (${unsavedFlinkFiles.length} files)`);
+    }
+
+    /**
+     * Emits compiled javascript source to dist folder using swc (20-50x faster than tsc)
+     */
+    async emitWithSwc() {
+        const emitStartTime = Date.now();
+        const swc = require("@swc/core");
+        const fs = require("fs-extra");
+        const path = require("path");
+
+        // Get all source files from the ts-morph project (includes src/, .flink/, spec/ when entry is a test runner)
+        // Uses in-memory text so generated files (.flink/) work without being saved to disk first
+        const allSourceFiles = this.project.getSourceFiles().filter((sf) => {
+            const filePath = sf.getFilePath();
+            return !filePath.endsWith(".d.ts") && !filePath.includes("/node_modules/");
+        });
+
+        initLog.debug(`Starting swc compilation for ${allSourceFiles.length} source files...`);
+
+        // Transpile all files in parallel using swc
+        await Promise.all(
+            allSourceFiles.map(async (sf) => {
+                const filePath = sf.getFilePath();
+                const code = sf.getFullText();
+                const result = await swc.transform(code, {
+                    filename: filePath,
+                    jsc: {
+                        parser: {
+                            syntax: "typescript",
+                            decorators: true,
+                            tsx: false, // Flink doesn't use JSX
+                        },
+                        target: "es2017", // Async/await support, modern enough for Node 14+
+                        transform: {
+                            legacyDecorator: true,
+                            decoratorMetadata: true, // Required for reflect-metadata
+                        },
+                        keepClassNames: true, // Preserve class names for debugging
+                    },
+                    module: {
+                        type: this.isEsm ? "es6" : "commonjs",
+                    },
+                    sourceMaps: true,
+                });
+
+                // Preserve directory structure: src/foo.ts → dist/src/foo.js
+                const relativePath = path.relative(this.cwd, filePath);
+                const outPath = path.join(this.cwd, "dist", relativePath).replace(/\.ts$/, ".js");
+
+                await fs.ensureDir(path.dirname(outPath));
+                await fs.writeFile(outPath, result.code);
+
+                if (result.map) {
+                    await fs.writeFile(outPath + ".map", result.map);
+                }
+            })
+        );
+
+        const emitTime = Date.now() - emitStartTime;
+        initLog.debug(`✓ Emitted ${allSourceFiles.length} files with swc in ${emitTime}ms (${(emitTime / allSourceFiles.length).toFixed(1)}ms per file)`);
+    }
+
+    /**
+     * Emits compiled javascript source to dist folder (defaults to swc for speed, falls back to tsc)
+     */
+    async emit() {
+        try {
+            require.resolve("@swc/core");
+            return await this.emitWithSwc();
+        } catch {
+            initLog.debug("@swc/core not found, falling back to TypeScript compiler (slower)");
+            return this.emitWithTsc();
+        }
+    }
+
+    /**
+     * Emits compiled javascript source to dist folder using TypeScript compiler (slower, kept for fallback)
      */
-    emit() {
+    emitWithTsc() {
+        const emitStartTime = Date.now();
+        const sourceFileCount = this.project.getSourceFiles().length;
+        perfLog.debug(`Starting TypeScript compilation for ${sourceFileCount} source files...`);
+
         this.project.emitSync();
+
+        const emitTime = Date.now() - emitStartTime;
+        perfLog.debug(`✓ Emitted ${sourceFileCount} files with tsc in ${emitTime}ms (${(emitTime / sourceFileCount).toFixed(1)}ms per file)`);
     }
 
     /**
@@ -179,7 +736,9 @@ class TypeScriptCompiler {
      * Also extract handlers request and response schemas from Handler
      * type arguments.
      */
-    async parseHandlers(excludeDirs: string[] = []) {
+    async parseHandlers() {
+        const startTime = Date.now();
+
         const generatedFile = this.createSourceFile(
             ["generatedHandlers.ts"],
             `// Generated ${new Date()}
@@ -188,19 +747,25 @@ export const handlers = [];
 autoRegisteredHandlers.push(...handlers);
     `
         );
+
         const handlersArr = generatedFile.getVariableDeclarationOrThrow("handlers").getFirstDescendantByKindOrThrow(SyntaxKind.ArrayLiteralExpression);
 
         const handlers = await this.parseHandlerDir(generatedFile, handlersArr);
 
         generatedFile.addImportDeclarations(handlers.imports);
 
-        await generatedFile.save();
+        // Defer save until batch save at end (performance optimization)
 
-        const schemaFilePath = await this.createIntermediateSchemaFile();
+        // Store handler schemas for later batch processing
+        this.handlerSchemasToGenerate = handlers.schemasToGenerate;
 
-        const jsonSchemas = await this.generateAndSaveJsonSchemas(handlers.schemasToGenerate, schemaFilePath);
+        // Cleanup: forget handler file nodes to reduce memory overhead (ts-morph performance optimization)
+        this.handlerFiles.forEach((sf) => {
+            sf.getClasses().forEach((cls) => cls.forget());
+        });
 
-        this.appendSchemasToHandlerSourceFiles(handlers.schemasToGenerate, jsonSchemas);
+        const handlerParseTime = Date.now() - startTime;
+        perfLog.info(`✓ Handler parsing completed in ${handlerParseTime}ms`);
 
         return generatedFile;
     }
@@ -210,75 +775,55 @@ autoRegisteredHandlers.push(...handlers);
      */
     private async parseHandlerDir(generatedFile: SourceFile, handlersArr: ArrayLiteralExpression) {
         const imports: OptionalKind<ImportDeclarationStructure>[] = [];
-        let i = 0;
         const schemasToGenerate: {
             reqSchemaType?: string;
             resSchemaType?: string;
             sourceFile: SourceFile;
         }[] = [];
 
-        for (const sf of this.project.getSourceFiles()) {
-            if (!sf.getFilePath().includes("src/handlers/")) {
-                continue;
-            }
-
-            const isAutoRegister = this.isAutoRegisterableHandler(sf);
+        // Use pre-segmented handler files (no filtering needed)
+        const handlerFiles = this.handlerFiles;
+
+        // Process handlers in parallel using text-based schema extraction
+        const handlerResults = await Promise.all(
+            handlerFiles.map(async (sf, index) => {
+                const startTime = Date.now();
+                const isAutoRegister = this.isAutoRegisterableHandler(sf);
+                const namespaceImport = sf.getBaseNameWithoutExtension().replace(/\./g, "_") + "_" + index;
+                const assumedHttpMethod = getHttpMethodFromHandlerName(sf.getBaseName());
+
+                // Extract schema information using text parsing
+                const schemaTypes = isAutoRegister ? await this.extractSchemasFromHandlerFast(sf.getFilePath()) : undefined;
+
+                perfLog.trace(`${sf.getBaseName()} took ${Date.now() - startTime}ms`);
+
+                return {
+                    sf,
+                    isAutoRegister,
+                    namespaceImport,
+                    assumedHttpMethod,
+                    schemaTypes,
+                    index,
+                };
+            })
+        );
 
-            console.log(`Detected handler ${sf.getBaseName()} ${!isAutoRegister ? "(requires manual registration)" : ""}`);
+        // Now process results sequentially to maintain order and update source files
+        // Collect handler elements first, then add in one batch
+        const handlerElements: string[] = [];
 
-            const namespaceImport = sf.getBaseNameWithoutExtension().replace(/\./g, "_") + "_" + i;
+        for (const result of handlerResults) {
+            const { sf, isAutoRegister, namespaceImport, assumedHttpMethod, schemaTypes } = result;
 
             imports.push({
                 defaultImport: "* as " + namespaceImport,
                 moduleSpecifier: this.getModuleSpecifier(generatedFile, sf),
             });
 
-            const assumedHttpMethod = getHttpMethodFromHandlerName(sf.getBaseName());
-
-            // Only extract schemas for auto-registered handlers
-            const schemaTypes = isAutoRegister ? await this.extractSchemasFromHandlerSourceFile(sf) : undefined;
-
-            // Remove existing metadata variables if they exist (to avoid redeclaration errors)
-            const existingVars = sf.getVariableStatements().filter((vs) => {
-                const varNames = vs.getDeclarations().map((d) => d.getName());
-                return varNames.some((name) => ["__assumedHttpMethod", "__file", "__query", "__params"].includes(name));
-            });
-            existingVars.forEach((v) => v.remove());
-
-            // Append schemas and metadata to source file that will be part of emitted dist bundle (javascript)
-            sf.addVariableStatement({
-                declarationKind: VariableDeclarationKind.Const,
-                isExported: true,
-                declarations: [
-                    {
-                        name: "__assumedHttpMethod",
-                        initializer: `"${assumedHttpMethod || ""}"`,
-                    },
-                    {
-                        name: "__file",
-                        initializer: `"${sf.getBaseName()}"`,
-                    },
-                    {
-                        name: "__query",
-                        initializer: `[${(schemaTypes?.queryMetadata || [])
-                            .map(({ description, name }) => `{description: "${description}", name: "${name}"}`)
-                            .join(",")}]`,
-                    },
-                    {
-                        name: "__params",
-                        initializer: `[${(schemaTypes?.paramsMetadata || [])
-                            .map(({ description, name }) => `{description: "${description}", name: "${name}"}`)
-                            .join(",")}]`,
-                    },
-                ],
-            });
-
             if (isAutoRegister) {
-                handlersArr.insertElement(
-                    i,
-                    `{handler: ${namespaceImport}, assumedHttpMethod: ${assumedHttpMethod ? "HttpMethod." + assumedHttpMethod : undefined}}`
+                handlerElements.push(
+                    `{handler: ${namespaceImport}, assumedHttpMethod: ${assumedHttpMethod ? "HttpMethod." + assumedHttpMethod : undefined}, __file: "${this.getRelativePath(sf)}"}`
                 );
-                i++;
 
                 // Add schemas to generate list
                 if (schemaTypes) {
@@ -287,6 +832,9 @@ autoRegisteredHandlers.push(...handlers);
             }
         }
 
+        // Add all handler elements in one batch operation
+        handlersArr.addElements(handlerElements);
+
         return {
             imports,
             schemasToGenerate,
@@ -294,6 +842,8 @@ autoRegisteredHandlers.push(...handlers);
     }
 
     async parseRepos() {
+        const startTime = Date.now();
+
         const generatedFile = this.createSourceFile(
             ["generatedRepos.ts"],
             `// Generated ${new Date()}
@@ -307,13 +857,13 @@ autoRegisteredHandlers.push(...handlers);
 
         const imports: OptionalKind<ImportDeclarationStructure>[] = [];
 
-        let i = 0;
+        // Use pre-segmented repo files (no filtering needed)
+        const repoFiles = this.repoFiles;
 
-        for (const sf of this.project.getSourceFiles()) {
-            if (!sf.getFilePath().includes("src/repos/")) {
-                continue;
-            }
+        // Collect all repo elements first, then add in one batch
+        const repoElements: string[] = [];
 
+        for (const sf of repoFiles) {
             console.log(`Detected repo ${sf.getBaseName()}`);
 
             imports.push({
@@ -321,531 +871,517 @@ autoRegisteredHandlers.push(...handlers);
                 moduleSpecifier: this.getModuleSpecifier(generatedFile, sf),
             });
 
-            reposArr.insertElement(
-                i,
+            repoElements.push(
                 `{collectionName: "${getCollectionNameForRepo(sf.getBaseName())}", repoInstanceName: "${getRepoInstanceName(
                     sf.getBaseName()
                 )}", Repo: ${sf.getBaseNameWithoutExtension()}}`
             );
-
-            i++;
         }
 
+        // Add all repo elements in one batch operation
+        reposArr.addElements(repoElements);
+
         generatedFile.addImportDeclarations(imports);
 
-        await generatedFile.save();
+        // Defer save until batch save at end (performance optimization)
+
+        // Cleanup: forget repo file nodes to reduce memory overhead (ts-morph performance optimization)
+        this.repoFiles.forEach((sf) => {
+            sf.getClasses().forEach((cls) => cls.forget());
+        });
+
+        const repoParseTime = Date.now() - startTime;
+        initLog.info(`✓ Repo parsing completed in ${repoParseTime}ms (${repoElements.length} repos)`);
 
         return generatedFile;
     }
 
     /**
-     * Generates a start script that will import references to handlers, repos and the
-     * actual Flink app to start.
+     * Scans project for tools and adds those to Flink
+     * "singleton" property `autoRegisteredTools` so they can
+     * be registered during start.
      *
-     * Note that order is of importance so generated metadata are imported and initialized before start of flink app.
-     * Otherwise singletons `autoRegisteredRepos` and `autoRegisteredHandlers` will not have been set.
+     * Also extracts input and output schemas from FlinkTool type arguments
+     * when manual schemas are not provided.
      */
-    async generateStartScript(appEntryScript = "/src/index.ts") {
-        if (!this.project.getSourceFile((sf) => sf.getFilePath().endsWith(appEntryScript))) {
-            console.error(`Cannot find entry script '${appEntryScript}'`);
-            return process.exit(1);
-        }
+    async parseTools() {
+        const startTime = Date.now();
 
-        const sf = this.createSourceFile(
-            ["start.ts"],
+        // Initialize schema generator BEFORE schema extraction
+        await this.initSchemaGenerator();
+
+        const generatedFile = this.createSourceFile(
+            ["generatedTools.ts"],
             `// Generated ${new Date()}
-import "./generatedHandlers${this.isEsm ? ".js" : ""}";
-import "./generatedRepos${this.isEsm ? ".js" : ""}";
-import "./generatedJobs${this.isEsm ? ".js" : ""}";
-import "..${appEntryScript.replace(/\.ts/g, "")}${this.isEsm ? ".js" : ""}";
-export default {}; // Export an empty object to make it a module
-`
+import { autoRegisteredTools } from "@flink-app/flink";
+export const tools = [];
+autoRegisteredTools.push(...tools);
+    `
         );
 
-        await sf.save();
+        const toolsArr = generatedFile.getVariableDeclarationOrThrow("tools").getFirstDescendantByKindOrThrow(SyntaxKind.ArrayLiteralExpression);
 
-        return sf;
-    }
+        const imports: OptionalKind<ImportDeclarationStructure>[] = [];
+        const schemasToGenerate: {
+            inputSchemaType?: string;
+            outputSchemaType?: string;
+            inputTypeHint?: "void" | "any" | "named";
+            outputTypeHint?: "void" | "any" | "named";
+            sourceFile: SourceFile;
+        }[] = [];
 
-    private createSourceFile(filename: string[], contents: string) {
-        return this.project.createSourceFile(join(this.cwd, ".flink", ...filename), contents, {
-            overwrite: true,
-        });
-    }
+        // Use pre-segmented tool files (filtering and AST inspection already done)
+        const toolFiles = this.toolFiles;
+
+        // Process tools synchronously to avoid ts-morph lazy parsing overhead
+        const toolResults: any[] = [];
+        for (let index = 0; index < toolFiles.length; index++) {
+            const sf = toolFiles[index];
+            (function () {
+                // Get file path ONCE and cache it to avoid triggering ts-morph lazy parsing
+                const filePath = sf.getFilePath();
+                const path = require("path");
+                const baseNameWithoutExt = path.basename(filePath, path.extname(filePath));
+
+                const namespaceImport = baseNameWithoutExt.replace(/\./g, "_") + "_" + index;
+
+                // Extract the export name using fast text matching instead of AST parsing
+                // Read file directly from disk to avoid ts-morph's lazy parsing
+                // Matches: export const Tool: FlinkToolProps or export const MyName: FlinkToolProps
+                const fileText = fs.readFileSync(filePath, "utf8");
+
+                const exportMatch = fileText.match(/export\s+const\s+(\w+)\s*:\s*FlinkToolProps/);
+                const toolPropsExportName = exportMatch ? exportMatch[1] : "Tool"; // Default to "Tool" if not found
+
+                toolResults.push({
+                    sf,
+                    filePath, // Cache file path to avoid triggering sf.getFilePath() later
+                    namespaceImport,
+                    toolPropsExportName,
+                    index,
+                });
+            })();
+        }
 
-    /**
-     * Parses handlers `Handler<...>` function and its type arguments to extract
-     * which schemas to use.
-     *
-     * There are multiple ways of defining schema types as valid ts and this
-     * implementation aims to support all of these.
-     *
-     * Some examples of different cases (check spec/mock-project/src/handlers for more):
-     *
-     * ```
-     * // Interface reference
-     * Handler<Ctx, Car>
-     * // Inline type definition with reference to interface
-     * Handler<Ctx, {car: Car}>
-     * // Inline type definition with literal values
-     * Handler<Ctx, {car: {model: string}}>
-     * // Array
-     * Handler<Ctx, Car[]>
-     * // Array with inline type definition
-     * Handler<Ctx, {car: Car}[]>
-     * ```
-     *
-     * Return names of req and/or res schema types.
-     */
-    private async extractSchemasFromHandlerSourceFile(handlerSourceFile: SourceFile) {
-        const defaultExport = getDefaultExport(handlerSourceFile);
+        // Schema extraction phase - separate from metadata collection
+        for (const result of toolResults) {
+            // Extract schema information using text-based parsing
+            const schemaTypes = await this.extractSchemasFromToolFast(result.filePath);
+
+            schemasToGenerate.push({
+                inputSchemaType: schemaTypes?.inputSchemaType,
+                outputSchemaType: schemaTypes?.outputSchemaType,
+                inputTypeHint: schemaTypes?.inputTypeHint,
+                outputTypeHint: schemaTypes?.outputTypeHint,
+                sourceFile: result.sf,
+            });
+        }
 
-        if (defaultExport) {
-            const handlerTypeRef = defaultExport.getFirstDescendantByKindOrThrow(SyntaxKind.TypeReference);
+        const toolParseTime = Date.now() - startTime;
+        initLog.info(`✓ Tool parsing completed in ${toolParseTime}ms (${toolFiles.length} tools)`);
 
-            return this.extractSchemaTypeFromHandler(handlerTypeRef);
-        } else {
-            console.warn(`Handler ${handlerSourceFile.getBaseName()} is missing default exported handler function`);
-        }
-    }
+        // Now process results sequentially to maintain order and update source files
+        // Collect tool elements first, then add in one batch
+        const toolElements: string[] = [];
 
-    /**
-     * Recursively copies an interface and all its dependencies from the same file
-     */
-    private copyInterfaceWithDependencies(interfaceDecl: any, handlerFile: SourceFile): void {
-        const interfaceName = interfaceDecl.getName?.() || interfaceDecl.getFirstChildByKind(SyntaxKind.Identifier)?.getText();
-        if (!interfaceName) return;
+        for (const result of toolResults) {
+            const { sf, namespaceImport, toolPropsExportName, schemaTypes } = result;
 
-        // Check if already copied
-        const existingInterface = this.parsedTsSchemas.find(
-            s => s.includes(`interface ${interfaceName} `) || s.includes(`type ${interfaceName} =`)
-        );
-        if (existingInterface) return;
-
-        // Copy the interface
-        this.parsedTsSchemas.push(interfaceDecl.getText());
-
-        // Find and recursively copy dependencies from the same file
-        // First, find direct type references in this interface
-        const typeRefIdentifiers = interfaceDecl
-            .getDescendantsOfKind(SyntaxKind.TypeReference)
-            .filter((typeRefNode: any) => !!typeRefNode.getFirstChildIfKind(SyntaxKind.Identifier))
-            .map((typeRefNode: any) => typeRefNode.getFirstChildIfKindOrThrow(SyntaxKind.Identifier));
-
-        for (const typeRefIdentifier of typeRefIdentifiers) {
-            const typeSymbol = typeRefIdentifier.getSymbol();
-            if (typeSymbol) {
-                const declaredType = typeSymbol.getDeclaredType();
-                const declaration = declaredType.getSymbol()?.getDeclarations()[0];
-                if (declaration && declaration.getSourceFile() === handlerFile) {
-                    // Same file - recursively copy this dependency
-                    this.copyInterfaceWithDependencies(declaration, handlerFile);
-                } else if (declaration && declaration.getSourceFile() !== handlerFile) {
-                    // Different file - add to imports
-                    const declaredTypeSymbol = declaredType.getSymbol();
-                    if (declaredTypeSymbol) {
-                        this.tsSchemasSymbolsToImports.push(declaredTypeSymbol);
-                    }
-                }
+            imports.push({
+                defaultImport: "* as " + namespaceImport,
+                moduleSpecifier: this.getModuleSpecifier(generatedFile, sf),
+            });
+
+            // Create an object that wraps the namespace and provides normalized access to the tool props
+            // This creates a consistent "Tool" property regardless of the source export name
+            // Example: {Tool: SearchCarsTool_0.MyCustomToolConfig} works even if export was "MyCustomToolConfig"
+            // __file is set here on the registration object (not injected into source files)
+            toolElements.push(`{...${namespaceImport}, Tool: ${namespaceImport}.${toolPropsExportName}, __file: "${this.getRelativePath(sf)}"}`);
+
+            // Add schemas to generate list
+            if (schemaTypes) {
+                schemasToGenerate.push({ ...schemaTypes, sourceFile: sf });
             }
         }
+
+        // Add all tool elements in one batch operation
+        toolsArr.addElements(toolElements);
+
+        generatedFile.addImportDeclarations(imports);
+
+        // Defer save until batch save at end (performance optimization)
+
+        // Store tool schemas for later batch processing
+        this.toolSchemasToGenerate = schemasToGenerate;
+
+        // Cleanup: forget tool file nodes to reduce memory overhead (ts-morph performance optimization)
+        // Note: We avoid calling methods on sf during iteration since we cached all needed data earlier
+        this.toolFiles.forEach((sf) => {
+            // Only forget if we haven't already processed this file
+            const classes = sf.getClasses();
+            classes.forEach((cls) => cls.forget());
+        });
+
+        return generatedFile;
     }
 
-    private async saveIntermediateTsSchema(schema: Type<ts.Type>, handlerFile: SourceFile, suffix: string) {
-        if (schema.isAny()) {
-            return; // 'any' indicates that no schema is used
-        }
+    /**
+     * Scans project for agents and validates tool references.
+     * Agents are classes extending FlinkAgent exported as default.
+     */
+    async parseAgents() {
+        const startTime = Date.now();
 
-        // Check for void and undefined types - these indicate no schema validation needed
-        const schemaText = schema.getText();
-        if (schemaText === 'void' || schemaText === 'undefined') {
-            return;
-        }
+        const generatedFile = this.createSourceFile(
+            ["generatedAgents.ts"],
+            `// Generated ${new Date()}
+import { autoRegisteredAgents } from "@flink-app/flink";
+export const agents = [];
+autoRegisteredAgents.push(...agents);
+    `
+        );
 
-        const handlerFileName = handlerFile.getBaseNameWithoutExtension().replace(/\./g, "_");
-
-        let generatedSchemaInterfaceStr = "";
-
-        const schemaInterfaceName = `${handlerFileName}_${suffix}`;
-
-        if (schema.isInterface()) {
-            /*
-             * Type argument is an interface. This should be normal case when
-             * schema is defined directly for example `Handler<Ctx, Car>`
-             */
-            const schemaSymbol = schema.getSymbolOrThrow();
-            const interfaceName = getInterfaceName(schemaSymbol);
-            const declaration = schemaSymbol.getDeclarations()[0];
-
-            if (declaration.getSourceFile() === handlerFile) {
-                // Interface is declared within handler file
-                generatedSchemaInterfaceStr = `export interface ${schemaInterfaceName} {
-          ${schema
-              .getProperties()
-              .map((p) => p.getValueDeclarationOrThrow().getText())
-              .join("\n")}
-        }`;
-
-                for (const typeToImport of getTypesToImport(declaration)) {
-                    const typeSymbol = typeToImport.getSymbol();
-                    if (typeSymbol) {
-                        const declaredTypeSymbol = typeSymbol.getDeclaredType().getSymbol();
-                        if (declaredTypeSymbol) {
-                            this.tsSchemasSymbolsToImports.push(declaredTypeSymbol);
-                        }
-                    }
-                }
+        const agentsArr = generatedFile.getVariableDeclarationOrThrow("agents").getFirstDescendantByKindOrThrow(SyntaxKind.ArrayLiteralExpression);
 
-                // Also check for utility types with indexed access patterns like Partial<Foo["bar"]>
-                for (const prop of schema.getProperties()) {
-                    const propDecl = prop.getValueDeclaration();
-                    if (propDecl) {
-                        const propText = propDecl.getText();
-                        // Match interface names in patterns like: Partial<InterfaceName["prop"]>
-                        const interfaceNameMatches = propText.match(/\b([A-Z][a-zA-Z0-9]*)\s*\[/g);
-                        if (interfaceNameMatches) {
-                            for (const match of interfaceNameMatches) {
-                                const referencedInterfaceName = match.replace(/\s*\[$/, '').trim();
-                                // Try to find this interface in the handler file
-                                const referencedInterfaceDecl = handlerFile.getInterface(referencedInterfaceName) || handlerFile.getTypeAlias(referencedInterfaceName);
-                                if (referencedInterfaceDecl) {
-                                    // Interface is in same file - copy it and all its dependencies recursively
-                                    this.copyInterfaceWithDependencies(referencedInterfaceDecl, handlerFile);
-                                }
-                            }
-                        }
-                    }
-                }
-            } else {
-                // Interface is imported from other file
-                generatedSchemaInterfaceStr = `export interface ${schemaInterfaceName} extends ${interfaceName} {}`;
-                this.tsSchemasSymbolsToImports.push(schemaSymbol);
-            }
-        } else if (schema.isArray()) {
-            const arrayTypeArg = schema.getTypeArguments()[0];
-            const schemaSymbol = arrayTypeArg.getSymbolOrThrow();
-            const interfaceName = schemaSymbol.getEscapedName();
-            const declaration = schemaSymbol.getDeclarations()[0];
-
-            if (declaration.getSourceFile() !== handlerFile) {
-                generatedSchemaInterfaceStr = `export interface ${schemaInterfaceName} extends Array<${interfaceName}> {}`;
-                this.tsSchemasSymbolsToImports.push(schemaSymbol);
-            } else {
-                if (arrayTypeArg.isInterface()) {
-                    const props = arrayTypeArg
-                        .getProperties()
-                        .map((p) => p.getValueDeclarationOrThrow().getText())
-                        .join(" ");
-
-                    generatedSchemaInterfaceStr = `export interface ${schemaInterfaceName} extends Array<{${props}}> {}`;
-                } else {
-                    generatedSchemaInterfaceStr = `export interface ${schemaInterfaceName} extends Array<${declaration.getText()}> {}`;
-                }
+        const imports: OptionalKind<ImportDeclarationStructure>[] = [];
 
-                for (const typeToImport of getTypesToImport(declaration)) {
-                    const typeSymbol = typeToImport.getSymbol();
-                    if (typeSymbol) {
-                        const declaredTypeSymbol = typeSymbol.getDeclaredType().getSymbol();
-                        if (declaredTypeSymbol) {
-                            this.tsSchemasSymbolsToImports.push(declaredTypeSymbol);
-                        }
-                    }
-                }
-            }
-        } else if (schema.isObject()) {
-            /*
-             * Schema is defined inline, for example `Handler<Ctx, {car: Car}>`
-             * We need extract `{car: Car}` into its own interface and make sure
-             * to import types if needed to
-             */
-
-            // Try to get symbol - it may not exist for utility types (Partial, Omit, Pick, etc.)
-            const schemaSymbol = schema.getSymbol();
-            const declarations = schemaSymbol?.getDeclarations();
-            const declaration = declarations?.[0];
-
-            // Build property signatures using resolved types instead of source text
-            // This ensures generic type parameters are properly expanded
-            const propertySignatures = schema.getProperties().map((prop) => {
-                const propName = prop.getName();
-                const propType = prop.getTypeAtLocation(handlerFile);
-                const propTypeText = propType.getText(undefined, ts.TypeFormatFlags.UseAliasDefinedOutsideCurrentScope);
-
-                // Check if property is optional
-                // For utility types (Omit, Pick, etc.), properties may not have value declarations
-                const valueDeclaration = prop.getValueDeclaration();
-                let isOptional = false;
-
-                if (valueDeclaration) {
-                    // Property has a source declaration (normal case)
-                    isOptional = valueDeclaration.getType().isNullable() || (valueDeclaration.compilerNode as any).questionToken !== undefined;
-                } else {
-                    // Property is synthetic (from utility types like Omit, Pick, etc.)
-                    // Check if the property itself is optional by examining the symbol flags
-                    isOptional = !!(prop.getFlags() & ts.SymbolFlags.Optional);
-                }
+        // Use pre-built tool ID registry (no scan needed)
+        const registeredToolIds = this.toolIdRegistry;
 
-                return `${propName}${isOptional ? "?" : ""}: ${propTypeText}`;
-            });
+        // Collect agent elements first, then add in one batch
+        const agentElements: string[] = [];
 
-            // Extract type references for imports from resolved types
-            for (const prop of schema.getProperties()) {
-                const propType = prop.getTypeAtLocation(handlerFile);
+        // Use pre-segmented agent files (convention-based: all files in src/agents/ are agents)
+        this.agentFiles.forEach((sf, index) => {
+            // Convention-based approach: Trust that all files in src/agents/ are valid agents
+            // Find the first exported class (assume it's the agent)
+            const agentClass = sf.getClasses().find((cls) => cls.isExported() || cls.isDefaultExport());
 
-                // Get symbol for the property type
-                const typeSymbol = propType.getSymbol();
-                if (typeSymbol) {
-                    const typeDeclaration = typeSymbol.getDeclarations()[0];
-                    if (typeDeclaration && typeDeclaration.getSourceFile() !== handlerFile) {
-                        this.tsSchemasSymbolsToImports.push(typeSymbol);
-                    }
-                }
+            // Skip if no exported class found (shouldn't happen, but defensive)
+            if (!agentClass) {
+                perfLog.debug(`⚠ Skipping ${sf.getBaseName()} - no exported class found`);
+                return;
+            }
 
-                // Also check for array element types
-                if (propType.isArray()) {
-                    const elementType = propType.getArrayElementType();
-                    const elementSymbol = elementType?.getSymbol();
-                    if (elementSymbol) {
-                        const elementDeclaration = elementSymbol.getDeclarations()[0];
-                        if (elementDeclaration && elementDeclaration.getSourceFile() !== handlerFile) {
-                            this.tsSchemasSymbolsToImports.push(elementSymbol);
-                        }
-                    }
-                }
+            // Create unique namespace import (same pattern as handlers and tools)
+            const namespaceImport = sf.getBaseNameWithoutExtension().replace(/\./g, "_") + "_" + index;
+
+            // Validate tool references exist (read from class properties)
+            const toolsProperty = agentClass.getProperty("tools");
+            if (toolsProperty) {
+                const initializer = toolsProperty.getInitializer();
+                if (initializer && initializer.getKind() === SyntaxKind.ArrayLiteralExpression) {
+                    const toolsArray = initializer as any;
+                    const toolElements = toolsArray.getElements();
+                    for (const toolElement of toolElements) {
+                        let toolName: string;
+
+                        // Handle string literals, method calls, and identifier references (tool imports)
+                        if (toolElement.getKind() === SyntaxKind.StringLiteral) {
+                            // Direct string: "tool-name"
+                            toolName = toolElement.getText().replace(/['"]/g, "");
+                        } else if (toolElement.getKind() === SyntaxKind.CallExpression) {
+                            // Method call: this.useTool("tool-name")
+                            const args = toolElement.getArguments();
+                            if (args.length > 0 && args[0].getKind() === SyntaxKind.StringLiteral) {
+                                toolName = args[0].getText().replace(/['"]/g, "");
+                            } else {
+                                console.warn(`Agent ${sf.getBaseName()} has non-string tool reference, skipping validation`);
+                                continue;
+                            }
+                        } else if (toolElement.getKind() === SyntaxKind.Identifier) {
+                            // Tool file reference (imported): SearchCarsByBrandTool
+                            // Look up the import to find the actual tool file
+                            const importName = toolElement.getText();
+                            const importDecl = sf.getImportDeclarations().find((imp) => {
+                                const namedImports = imp.getNamedImports();
+                                return namedImports.some((ni) => ni.getName() === importName);
+                            });
+
+                            if (!importDecl) {
+                                // Try namespace import (* as Foo)
+                                const namespaceImport = sf.getImportDeclarations().find((imp) => {
+                                    return imp.getNamespaceImport()?.getText() === importName;
+                                });
+
+                                if (namespaceImport) {
+                                    const moduleSpecifier = namespaceImport.getModuleSpecifierValue();
+                                    // Extract tool ID from the tool file path
+                                    // e.g., "../tools/SearchCarsByBrandTool" -> find in registeredToolIds
+                                    const toolFileName = moduleSpecifier.split("/").pop()?.replace(/\.ts$/, "");
+                                    const matchingTool = Array.from(registeredToolIds).find((id) => {
+                                        // Try to match by searching for the tool ID
+                                        // This is a heuristic - we'll validate it exists
+                                        return true; // Skip validation for imported tools for now
+                                    });
+                                    // Skip validation - tool imports are validated at runtime
+                                    continue;
+                                }
+
+                                console.warn(`Agent ${sf.getBaseName()} references tool "${importName}" but it's not imported, skipping validation`);
+                                continue;
+                            }
 
-                // Check for utility types (Partial, Omit, Pick, etc.) and extract their type arguments
-                // For example: Partial<Foo["bar"]> should extract Foo
-                // Use a pragmatic text-based approach to find interface references in type expressions
-                // Match interface names in patterns like: Partial<InterfaceName["prop"]>, Omit<InterfaceName, "key">, etc.
-                const currentPropTypeText = propType.getText(undefined, ts.TypeFormatFlags.UseAliasDefinedOutsideCurrentScope);
-                const interfaceNameMatches = currentPropTypeText.match(/\b([A-Z][a-zA-Z0-9]*)\s*\[/g);
-                if (interfaceNameMatches) {
-                    for (const match of interfaceNameMatches) {
-                        const interfaceName = match.replace(/\s*\[$/, '').trim();
-                        // Try to find this interface in the handler file
-                        const interfaceDecl = handlerFile.getInterface(interfaceName) || handlerFile.getTypeAlias(interfaceName);
-                        if (interfaceDecl) {
-                            // Interface is in same file - copy it and all its dependencies recursively
-                            this.copyInterfaceWithDependencies(interfaceDecl, handlerFile);
+                            // Skip validation for tool file references - they're validated at runtime
+                            continue;
+                        } else {
+                            console.warn(`Agent ${sf.getBaseName()} has unexpected tool reference format: ${toolElement.getText()}`);
+                            continue;
                         }
-                    }
-                }
 
-                // Also check regular type arguments (for types like Array<Foo>, Promise<Bar>)
-                const typeArgs = propType.getTypeArguments();
-                if (typeArgs && typeArgs.length > 0) {
-                    for (const typeArg of typeArgs) {
-                        const argSymbol = typeArg.getSymbol();
-                        if (argSymbol) {
-                            const argDeclaration = argSymbol.getDeclarations()[0];
-                            if (argDeclaration && argDeclaration.getSourceFile() !== handlerFile) {
-                                this.tsSchemasSymbolsToImports.push(argSymbol);
-                            }
+                        if (!registeredToolIds.has(toolName)) {
+                            console.error(`Agent ${sf.getBaseName()} references tool "${toolName}" which does not exist`);
+                            throw new Error(`Invalid tool reference in agent ${sf.getBaseName()}`);
                         }
                     }
                 }
             }
 
-            // If we have a declaration, check if we need to import any types
-            if (declaration) {
-                for (const typeToImport of getTypesToImport(declaration)) {
-                    const typeSymbol = typeToImport.getSymbol();
-                    if (typeSymbol) {
-                        const declaredTypeSymbol = typeSymbol.getDeclaredType().getSymbol();
-                        if (declaredTypeSymbol) {
-                            this.tsSchemasSymbolsToImports.push(declaredTypeSymbol);
-                        }
-                    }
-                }
+            const className = agentClass.getName();
+            if (!className) {
+                console.error(`Agent class in ${sf.getBaseName()} has no name`);
+                return;
             }
 
-            generatedSchemaInterfaceStr = `export interface ${schemaInterfaceName} { ${propertySignatures.join(";\n")} }`;
-        } else {
-            console.log("[WARN] Unknown schema type", schema.getText());
-        }
-
-        if (generatedSchemaInterfaceStr) {
-            this.parsedTsSchemas.push(generatedSchemaInterfaceStr);
-
-            return schemaInterfaceName;
-        }
-        return;
-    }
-
-    private initJsonSchemaGenerator(schemaFilePath: string) {
-        const tsconfigPath = join(this.cwd, "tsconfig.json");
-        const conf: CompletedConfig = {
-            path: schemaFilePath, // Point to the intermediate schema file
-            expose: "none", // Do not create shared $ref definitions.
-            topRef: false, // Removes the wrapper object around the schema.
-            additionalProperties: false,
-            jsDoc: "basic",
-            sortProps: false,
-            strictTuples: false,
-            minify: false,
-            markdownDescription: false,
-            skipTypeCheck: false,
-            encodeRefs: false,
-            extraTags: [],
-            functions: "fail",
-            discriminatorType: "json-schema",
-            tsconfig: tsconfigPath,
-        };
+            imports.push({
+                defaultImport: "* as " + namespaceImport,
+                moduleSpecifier: this.getModuleSpecifier(generatedFile, sf),
+            });
+
+            // Register the agent class (access default export via namespace)
+            // __file is set here on the registration object (not injected into source files)
+            agentElements.push(`{ default: ${namespaceImport}.default, __file: "${this.getRelativePath(sf)}" }`);
+        });
 
-        console.log("Creating TypeScript program for schema generation:");
-        console.log("  Schema file:", schemaFilePath);
-        console.log("  tsconfig:", tsconfigPath);
+        // Add all agent elements in one batch operation
+        agentsArr.addElements(agentElements);
 
-        // Create a fresh TypeScript Program that includes the schema file
-        // This ensures ts-json-schema-generator can find the types we just generated
-        let program;
-        try {
-            program = createProgram(conf);
-        } catch (error: any) {
-            // Format the error in a more developer-friendly way
-            console.error("\n❌ Schema generation failed due to TypeScript compilation errors:\n");
-
-            if (error.diagnostic && error.diagnostic.relatedInformation) {
-                // Extract and display only the relevant error messages
-                for (const info of error.diagnostic.relatedInformation) {
-                    if (info.file) {
-                        const { line, character } = info.file.getLineAndCharacterOfPosition(info.start);
-                        const fileName = info.file.fileName.replace(this.cwd, ".");
-                        const message = typeof info.messageText === "string" ? info.messageText : info.messageText.messageText;
-
-                        console.error(`  ${fileName}:${line + 1}:${character + 1}`);
-                        console.error(`  ${message}\n`);
-                    }
-                }
-            } else if (error.message) {
-                console.error(`  ${error.message}\n`);
-            }
+        generatedFile.addImportDeclarations(imports);
 
-            console.error("💡 Tip: Fix the TypeScript errors above and try again.\n");
-            process.exit(1);
-        }
+        // Defer save until batch save at end (performance optimization)
 
-        console.log("  TypeScript version:", ts.version);
-        console.log("  Program root files:", program.getRootFileNames().length);
+        // Cleanup: forget agent file nodes to reduce memory overhead (ts-morph performance optimization)
+        this.agentFiles.forEach((sf) => {
+            sf.getClasses().forEach((cls) => cls.forget());
+        });
 
-        const formatter = createFormatter(conf);
-        const parser = createParser(program, conf);
-        const generator = new SchemaGenerator(program, parser, formatter, conf);
+        const agentParseTime = Date.now() - startTime;
+        initLog.info(`✓ Agent parsing completed in ${agentParseTime}ms (${agentElements.length} agents)`);
 
-        return generator;
+        return generatedFile;
     }
 
-    private async generateAndSaveJsonSchemas(schemas: { reqSchemaType?: string; resSchemaType?: string }[], schemaFilePath: string) {
-        // Reset schema generator to use the newly created intermediate schema file
-        this.schemaGenerator = undefined;
+    /**
+     * Generates a start script that will import references to handlers, repos and the
+     * actual Flink app to start.
+     *
+     * Note that order is of importance so generated metadata are imported and initialized before start of flink app.
+     * Otherwise singletons `autoRegisteredRepos` and `autoRegisteredHandlers` will not have been set.
+     */
+    async generateStartScript(appEntryScript = "/src/index.ts") {
+        // Add the entry script to the project if not already loaded
+        const path = require("path");
+        const entryScriptPath = path.resolve(this.cwd, appEntryScript.replace(/^\//, ""));
 
-        const jsonSchemas: Schema[] = [];
+        if (!fs.existsSync(entryScriptPath)) {
+            console.error(`Cannot find entry script '${appEntryScript}' at ${entryScriptPath}`);
+            return process.exit(1);
+        }
 
-        for (const { reqSchemaType, resSchemaType } of schemas) {
-            if (reqSchemaType) {
-                jsonSchemas.push({ definitions: { [reqSchemaType]: this.generateJsonSchema(reqSchemaType, schemaFilePath) } });
-            }
-            if (resSchemaType) {
-                jsonSchemas.push({ definitions: { [resSchemaType]: this.generateJsonSchema(resSchemaType, schemaFilePath) } });
+        // Add to project if not already present
+        if (!this.project.getSourceFile((sf) => sf.getFilePath().endsWith(appEntryScript))) {
+            this.project.addSourceFileAtPath(entryScriptPath);
+
+            // If the entry script is in the spec directory (test runner), also add all spec files
+            // This handles Jasmine test runners that load specs dynamically via glob patterns
+            if (appEntryScript.includes("/spec/")) {
+                const specFiles = await glob(join(this.cwd, "spec/**/*.ts"));
+                this.project.addSourceFilesAtPaths(specFiles);
+                console.log(`Added ${specFiles.length} spec files to compilation`);
             }
+
+            // Resolve any imports from the entry script
+            this.resolveImportedFiles();
         }
 
-        const mergedSchemas = jsonSchemas.reduce(
-            (out, schema) => {
-                if (schema.definitions) {
-                    out.definitions = { ...out.definitions, ...schema.definitions };
-                }
-                return out;
-            },
-            {
-                $schema: "http://json-schema.org/draft-07/schema#",
-                $ref: "#/definitions/Schemas",
-                definitions: {},
-            }
-        );
+        const extensionImports = this.compilerPlugins
+            .map((ext) => `import "./${ext.generatedFile}${this.isEsm ? ".js" : ""}";`)
+            .join("\n");
 
-        const filePath = join(this.cwd, ".flink", "schemas", "schemas.json");
+        const sf = this.createSourceFile(
+            ["start.ts"],
+            `// Generated ${new Date()}
+import "./generatedHandlers${this.isEsm ? ".js" : ""}";
+import "./generatedRepos${this.isEsm ? ".js" : ""}";
+import "./generatedTools${this.isEsm ? ".js" : ""}";
+import "./generatedAgents${this.isEsm ? ".js" : ""}";
+import "./generatedJobs${this.isEsm ? ".js" : ""}";
+import "./generatedServices${this.isEsm ? ".js" : ""}";
+${extensionImports ? extensionImports + "\n" : ""}import "..${appEntryScript.replace(/\.ts/g, "")}${this.isEsm ? ".js" : ""}";
+export default {}; // Export an empty object to make it a module
+`
+        );
 
-        await writeJsonFile(filePath, mergedSchemas);
+        // Defer save until batch save at end (performance optimization)
 
-        this.project.addSourceFileAtPath(filePath);
+        return sf;
+    }
 
-        return mergedSchemas;
+    private createSourceFile(filename: string[], contents: string) {
+        return this.project.createSourceFile(join(this.cwd, ".flink", ...filename), contents, {
+            overwrite: true,
+        });
     }
 
-    private generateJsonSchema(typeName: string, schemaFilePath: string) {
-        if (!this.schemaGenerator) {
-            this.schemaGenerator = this.initJsonSchemaGenerator(schemaFilePath);
-        }
-        return this.schemaGenerator.createSchema(typeName);
-    }
-
-    private async extractSchemaTypeFromHandler(handlerTypeReference: TypeReferenceNode) {
-        // Name of Handler function - should be either `Handler` or `GetHandler`
-        const handlerType = handlerTypeReference.getTypeName().getText();
-
-        // Get type arguments a.k.a. generics which holds schemas such as this `Handler<Ctx, ReqSchema, ResSchema>`
-        const handlerTypeArgs = handlerTypeReference.getType().getAliasTypeArguments();
-
-        let reqSchema: Type<ts.Type> | undefined;
-        let resSchema: Type<ts.Type> | undefined;
-        let params: Type<ts.Type> | undefined;
-        let query: Type<ts.Type> | undefined;
-
-        if (handlerType === "Handler") {
-            // `Handler<Ctx, ReqSchema, ResSchema, Params, Query>`
-            // 0 = Ctx, 1 = Req schema, 2 = Res schema, 3 = Params, 4 = Query
-            reqSchema = handlerTypeArgs[1];
-            resSchema = handlerTypeArgs[2];
-            params = handlerTypeArgs[3];
-            query = handlerTypeArgs[4];
-        } else if (handlerType === "GetHandler") {
-            // `GetHandler<Ctx, ResSchema, Params, Query>`
-            // 0 = Ctx, 1 = Res schema, 2 = Params, 3 = Query
-            resSchema = handlerTypeArgs[1];
-            params = handlerTypeArgs[2];
-            query = handlerTypeArgs[3];
-        } else {
-            throw new Error(`Unknown handler type ${handlerType} in ${handlerTypeReference.getSourceFile().getBaseName()} - should be Handler or GetHandler`);
+    /**
+     * Extracts schema information from a tool file using text-based parsing.
+     * Resolves type names to schema $ids from the schema universe.
+     */
+    private async extractSchemasFromToolFast(filePath: string) {
+        const fileText = fs.readFileSync(filePath, "utf8");
+
+        // Check if this tool already has schemas defined (Zod or JSON Schema)
+        // These tools don't need TypeScript schema extraction
+        const schemaDetection = TypeScriptSourceParser.detectSchemaType(fileText);
+
+        if (schemaDetection.shouldSkipTypeScriptExtraction) {
+            return {
+                inputSchemaType: undefined,
+                outputSchemaType: undefined,
+            };
         }
 
-        const sf = handlerTypeReference.getSourceFile();
+        // Extract FlinkTool type arguments using utility parser
+        const typeArgs = TypeScriptSourceParser.parseFlinkToolTypeArgs(fileText);
 
-        const createReqSchemaPromise = reqSchema
-            ? this.saveIntermediateTsSchema(reqSchema, sf, `${handlerTypeReference.getStartLineNumber()}_ReqSchema`)
-            : Promise.resolve("");
+        if (!typeArgs) {
+            const baseName = require("path").basename(filePath, require("path").extname(filePath));
+            perfLog.trace(`  Tool ${baseName}: Could not parse FlinkTool type arguments`);
+            return {
+                inputSchemaType: undefined,
+                outputSchemaType: undefined,
+            };
+        }
 
-        const createResSchemaPromise = resSchema
-            ? this.saveIntermediateTsSchema(resSchema, sf, `${handlerTypeReference.getStartLineNumber()}_ResSchema`)
-            : Promise.resolve("");
+        const inputTypeName = typeArgs.inputType;
+        const outputTypeName = typeArgs.outputType;
+
+        // Resolve type names to schema $ids
+        let inputSchemaType: string | undefined = undefined;
+        let outputSchemaType: string | undefined = undefined;
+        let inputTypeHint: "void" | "any" | "named" | undefined = undefined;
+        let outputTypeHint: "void" | "any" | "named" | undefined = undefined;
+
+        // Determine input type hint
+        if (inputTypeName.toLowerCase() === "void") {
+            inputTypeHint = "void";
+        } else if (inputTypeName.toLowerCase() === "any") {
+            inputTypeHint = "any";
+        } else if (TypeScriptSourceParser.shouldGenerateSchema(inputTypeName)) {
+            inputTypeHint = "named";
+            inputSchemaType = this.resolveTypeNameToSchemaId(fileText, inputTypeName, filePath);
+            if (!inputSchemaType) {
+                const baseName = require("path").basename(filePath);
+                perfLog.warn(`Tool ${baseName}: Could not resolve input type "${inputTypeName}" to schema $id. Make sure it's exported from src/schemas/`);
+            }
+        }
 
-        const [reqSchemaType, resSchemaType] = await Promise.all([createReqSchemaPromise, createResSchemaPromise]);
+        // Determine output type hint
+        const unwrappedTypeName = TypeScriptSourceParser.unwrapToolResultType(outputTypeName);
+        if (unwrappedTypeName.toLowerCase() === "void") {
+            outputTypeHint = "void";
+        } else if (unwrappedTypeName.toLowerCase() === "any") {
+            outputTypeHint = "any";
+        } else if (TypeScriptSourceParser.shouldGenerateSchema(unwrappedTypeName)) {
+            outputTypeHint = "named";
+            outputSchemaType = this.resolveTypeNameToSchemaId(fileText, unwrappedTypeName, filePath);
+            if (!outputSchemaType) {
+                const baseName = require("path").basename(filePath);
+                perfLog.warn(`Tool ${baseName}: Could not resolve output type "${unwrappedTypeName}" to schema $id. Make sure it's exported from src/schemas/`);
+            }
+        }
 
         return {
-            reqSchemaType,
-            resSchemaType,
-            queryMetadata: getTypeMetadata(query),
-            paramsMetadata: getTypeMetadata(params),
+            inputSchemaType,
+            outputSchemaType,
+            inputTypeHint,
+            outputTypeHint,
         };
     }
 
     /**
-     * Creates generated source file that contains all
-     * TypeScript schemas that has been derived from handlers.
+     * Extracts schema information from a handler file using text-based parsing.
+     * Resolves type names to schema $ids from the schema universe.
      */
-    private async createIntermediateSchemaFile() {
-        const schemaSourceFile = this.createSourceFile(
-            ["schemas", `schemas.ts`],
-            `// Generated ${new Date()}
-${this.parsedTsSchemas.join("\n\n")}`
-        );
+    private async extractSchemasFromHandlerFast(filePath: string) {
+        const fileText = fs.readFileSync(filePath, "utf8");
+
+        // Extract Handler type arguments using utility parser
+        const typeArgs = TypeScriptSourceParser.parseHandlerTypeArgs(fileText);
+
+        if (!typeArgs) {
+            const baseName = require("path").basename(filePath, require("path").extname(filePath));
+            perfLog.trace(`  Handler ${baseName}: Could not parse Handler type arguments`);
+            return {
+                reqSchemaType: undefined,
+                resSchemaType: undefined,
+                queryMetadata: [],
+                paramsMetadata: [],
+            };
+        }
+
+        const reqTypeName = typeArgs.reqType;
+        const resTypeName = typeArgs.resType;
+        const paramsTypeName = typeArgs.paramsType;
+        const queryTypeName = typeArgs.queryType;
 
-        addImports(schemaSourceFile, this.tsSchemasSymbolsToImports);
+        // Resolve type names to schema $ids
+        let reqSchemaType: string | undefined = undefined;
+        let resSchemaType: string | undefined = undefined;
 
-        await schemaSourceFile.save();
+        // Check if handler skips validation — no need to warn about unresolved schema types
+        const isSkipValidation = /ValidationMode\.SkipValidation/.test(fileText);
 
-        // Return the file path so it can be used by ts-json-schema-generator
-        return schemaSourceFile.getFilePath();
+        if (reqTypeName && TypeScriptSourceParser.shouldGenerateSchema(reqTypeName)) {
+            reqSchemaType = this.resolveTypeNameToSchemaId(fileText, reqTypeName, filePath);
+            if (!reqSchemaType && !isSkipValidation) {
+                const baseName = require("path").basename(filePath);
+                perfLog.warn(`Handler ${baseName}: Could not resolve request type "${reqTypeName}" to schema $id. Make sure it's exported from src/schemas/`);
+            }
+        }
+
+        if (resTypeName && TypeScriptSourceParser.shouldGenerateSchema(resTypeName)) {
+            resSchemaType = this.resolveTypeNameToSchemaId(fileText, resTypeName, filePath);
+            if (!resSchemaType && !isSkipValidation) {
+                const baseName = require("path").basename(filePath);
+                perfLog.warn(`Handler ${baseName}: Could not resolve response type "${resTypeName}" to schema $id. Make sure it's exported from src/schemas/`);
+            }
+        }
+
+        // Extract property metadata for params and query
+        let paramsMetadata: any[] = [];
+        let queryMetadata: any[] = [];
+
+        if (paramsTypeName && paramsTypeName !== "any" && paramsTypeName !== "void") {
+            const metadata = TypeScriptSourceParser.extractPropertyMetadata(fileText, paramsTypeName);
+            if (metadata) {
+                paramsMetadata = metadata;
+            }
+        }
+
+        if (queryTypeName && queryTypeName !== "any" && queryTypeName !== "void") {
+            const metadata = TypeScriptSourceParser.extractPropertyMetadata(fileText, queryTypeName);
+            if (metadata) {
+                queryMetadata = metadata;
+            }
+        }
+
+        return {
+            reqSchemaType,
+            resSchemaType,
+            queryMetadata,
+            paramsMetadata,
+        };
     }
 
     /**
@@ -870,60 +1406,167 @@ ${this.parsedTsSchemas.join("\n\n")}`
     }
 
     /**
-     * Appends generated json schemas to handler source files.
+     * Generates JSON schemas for all handlers and tools.
+     * Should be called after parseHandlers() and parseTools() have completed.
      *
-     * @param handlers
-     * @param jsonSchemas
+     * NEW SIMPLIFIED APPROACH:
+     * 1. Generate schema universe from src/schemas/**\/*.ts (includes wrappers/)
+     * 2. Handlers/tools reference schemas by $id
+     * 3. Create manifest with schema universe and references
      */
-    private appendSchemasToHandlerSourceFiles(
-        handlers: {
-            sourceFile: SourceFile;
-            reqSchemaType?: string;
-            resSchemaType?: string;
-        }[],
-        jsonSchemas: JSONSchema7
-    ) {
-        const jsonSchemaDefs = jsonSchemas.definitions || {};
-
-        for (const { sourceFile, reqSchemaType, resSchemaType } of handlers) {
-            if (reqSchemaType && !jsonSchemaDefs[reqSchemaType]) {
-                console.error(`Handler ${sourceFile.getBaseName()} has request schema (${reqSchemaType}) defined, but no JSON schema has been generated`);
-                continue;
+    async generateAllSchemas(): Promise<void> {
+        const schemaGenStartTime = Date.now();
+
+        // Initialize schema generator
+        await this.initSchemaGenerator();
+
+        if (!this.schemaGenerator) {
+            throw new Error("Schema generator not initialized");
+        }
+
+        const path = require("path");
+        const schemaDir = path.join(this.cwd, "src/schemas");
+
+        // Check if schemas directory exists
+        if (!fs.existsSync(schemaDir)) {
+            perfLog.warn("No src/schemas/ directory found. Skipping schema generation.");
+            return;
+        }
+
+        // Find all schema files in src/schemas/ (including subdirectories)
+        const schemaFiles = await glob(path.join(schemaDir, "**/*.ts"));
+
+        perfLog.debug(`Found ${schemaFiles.length} schema files in src/schemas/`);
+
+        // Generate schemas from each file and merge into schema universe
+        let schemaUniverse: Record<string, any> = {};
+
+        for (const schemaFile of schemaFiles) {
+            const absolutePath = path.join(this.cwd, schemaFile);
+
+            try {
+                const fileSchemas = this.schemaGenerator(absolutePath, {
+                    followImports: "local",
+                    schemaVersion: "http://json-schema.org/draft-07/schema#",
+                    includeJSDoc: true,
+                    strictObjects: false,
+                    additionalProperties: undefined,
+                    defineId: (typeName: string, declaration: any, context: any) => {
+                        if (!context) return typeName;
+
+                        // Generate stable $id using same algorithm as resolveTypeNameToSchemaId
+                        return this.filePathToSchemaId(context.absolutePath, typeName);
+                    },
+                });
+
+                // Merge schemas from this file into universe
+                schemaUniverse = { ...schemaUniverse, ...fileSchemas };
+            } catch (error: any) {
+                perfLog.warn(`Failed to generate schemas from ${schemaFile}: ${error.message}`);
             }
+        }
 
-            if (resSchemaType && !jsonSchemaDefs[resSchemaType]) {
-                console.error(`Handler ${sourceFile.getBaseName()} has response schema (${resSchemaType}) defined, but no JSON schema has been generated`);
-                continue;
+        perfLog.debug(`Generated ${Object.keys(schemaUniverse).length} schemas from ${schemaFiles.length} files`);
+        perfLog.debug(`Sample schema $ids: ${Object.keys(schemaUniverse).slice(0, 10).join(", ")}...`);
+
+        // Create manifest with schema universe and handler/tool references
+        await this.generateSchemaManifest(schemaUniverse);
+
+        const schemaGenTime = Date.now() - schemaGenStartTime;
+        initLog.info(
+            `✓ Schema generation completed in ${schemaGenTime}ms ` +
+                `(${Object.keys(schemaUniverse).length} schemas, ${this.handlerSchemasToGenerate.length} handlers, ${this.toolSchemasToGenerate.length} tools)`
+        );
+    }
+
+    /**
+     * Computes relative path from project root for a source file.
+     * This is used consistently for manifest keys and __file exports.
+     */
+    private getRelativePath(sf: SourceFile): string {
+        const filePath = sf.getFilePath();
+        return filePath.startsWith(this.cwd) ? filePath.substring(this.cwd.length + 1) : filePath;
+    }
+
+    /**
+     * Generates schema manifest with schema universe and handler/tool references.
+     */
+    private async generateSchemaManifest(schemaUniverse: Record<string, any>): Promise<void> {
+        const manifest: any = {
+            version: "2.0",
+            generated: new Date().toISOString(),
+            schemas: schemaUniverse,
+            handlers: {} as Record<string, { reqSchemaName?: string; resSchemaName?: string; queryMetadata?: any[]; paramsMetadata?: any[] }>,
+            tools: {} as Record<
+                string,
+                {
+                    inputSchemaName?: string;
+                    outputSchemaName?: string;
+                    inputTypeHint?: "void" | "any" | "named";
+                    outputTypeHint?: "void" | "any" | "named";
+                }
+            >,
+        };
+
+        // Map handlers with their schema references
+        for (const handler of this.handlerSchemasToGenerate) {
+            const relativePath = this.getRelativePath(handler.sourceFile);
+
+            // Validate schema references exist
+            if (handler.reqSchemaType && !schemaUniverse[handler.reqSchemaType]) {
+                perfLog.warn(
+                    `Handler ${handler.sourceFile.getBaseName()} references request schema "${handler.reqSchemaType}" which was not found in schema universe`
+                );
+            }
+            if (handler.resSchemaType && !schemaUniverse[handler.resSchemaType]) {
+                perfLog.warn(
+                    `Handler ${handler.sourceFile.getBaseName()} references response schema "${handler.resSchemaType}" which was not found in schema universe`
+                );
             }
 
-            const reqJsonSchema = JSON.stringify(reqSchemaType ? jsonSchemaDefs[reqSchemaType] : undefined);
-            const resJsonSchema = JSON.stringify(resSchemaType ? jsonSchemaDefs[resSchemaType] : undefined);
+            manifest.handlers[relativePath] = {
+                reqSchemaName: handler.reqSchemaType,
+                resSchemaName: handler.resSchemaType,
+                queryMetadata: (handler as any).queryMetadata || [],
+                paramsMetadata: (handler as any).paramsMetadata || [],
+            };
+        }
 
-            // Remove existing __schemas variable if it exists (to avoid redeclaration errors)
-            const existingSchemas = sourceFile.getVariableStatements().filter((vs) => {
-                const varNames = vs.getDeclarations().map((d) => d.getName());
-                return varNames.includes("__schemas");
-            });
-            existingSchemas.forEach((v) => v.remove());
-
-            sourceFile.addVariableStatement({
-                declarationKind: VariableDeclarationKind.Const,
-                isExported: true,
-                declarations: [
-                    {
-                        name: "__schemas",
-                        type: "any",
-                        initializer: `{ reqSchema: ${reqJsonSchema}, resSchema: ${resJsonSchema} }`,
-                    },
-                ],
-            });
+        // Map tools with their schema references
+        for (const tool of this.toolSchemasToGenerate) {
+            const relativePath = this.getRelativePath(tool.sourceFile);
+
+            // Validate schema references exist
+            if (tool.inputSchemaType && !schemaUniverse[tool.inputSchemaType]) {
+                perfLog.warn(`Tool ${tool.sourceFile.getBaseName()} references input schema "${tool.inputSchemaType}" which was not found in schema universe`);
+            }
+            if (tool.outputSchemaType && !schemaUniverse[tool.outputSchemaType]) {
+                perfLog.warn(
+                    `Tool ${tool.sourceFile.getBaseName()} references output schema "${tool.outputSchemaType}" which was not found in schema universe`
+                );
+            }
+
+            manifest.tools[relativePath] = {
+                inputSchemaName: tool.inputSchemaType,
+                outputSchemaName: tool.outputSchemaType,
+                inputTypeHint: tool.inputTypeHint,
+                outputTypeHint: tool.outputTypeHint,
+            };
         }
+
+        // Write manifest to .flink directory
+        const manifestPath = join(this.cwd, "dist/.flink/schema-manifest.json");
+        await writeJsonFile(manifestPath, manifest, { ensureDir: true });
+
+        perfLog.debug(`Schema manifest written to: ${manifestPath}`);
     }
 
     /**
      * Scans project for jobs so they can be registered during start.
      */
     async parseJobs() {
+        const startTime = Date.now();
+
         const generatedFile = this.createSourceFile(
             ["generatedJobs.ts"],
             `// Generated ${new Date()}
@@ -936,13 +1579,12 @@ autoRegisteredJobs.push(...jobs);
         const jobsArr = generatedFile.getVariableDeclarationOrThrow("jobs").getFirstDescendantByKindOrThrow(SyntaxKind.ArrayLiteralExpression);
 
         const imports: OptionalKind<ImportDeclarationStructure>[] = [];
-        let i = 0;
 
-        for (const sf of this.project.getSourceFiles()) {
-            if (!sf.getFilePath().includes("src/jobs/")) {
-                continue;
-            }
+        // Collect job elements first, then add in one batch
+        const jobElements: string[] = [];
 
+        // Use pre-segmented job files (no filtering needed)
+        this.jobFiles.forEach((sf, i) => {
             console.log(`Detected job ${sf.getBaseName()}`);
 
             const namespaceImport = sf.getBaseNameWithoutExtension().replace(/\./g, "_") + "_" + i;
@@ -952,36 +1594,131 @@ autoRegisteredJobs.push(...jobs);
                 moduleSpecifier: this.getModuleSpecifier(generatedFile, sf),
             });
 
-            // Remove existing __file variable if it exists (to avoid redeclaration errors)
-            const existingFile = sf.getVariableStatements().filter((vs) => {
-                const varNames = vs.getDeclarations().map((d) => d.getName());
-                return varNames.includes("__file");
-            });
-            existingFile.forEach((v) => v.remove());
-
-            // Append metadata to source file that will be part of emitted dist bundle (javascript)
-            sf.addVariableStatement({
-                declarationKind: VariableDeclarationKind.Const,
-                isExported: true,
-                declarations: [
-                    {
-                        name: "__file",
-                        initializer: `"${sf.getBaseName()}"`,
-                    },
-                ],
-            });
+            // __file is set on the registration object (not injected into source files)
+            jobElements.push(`{...${namespaceImport}, __file: "${this.getRelativePath(sf)}"}`);
+        });
+
+        // Add all job elements in one batch operation
+        jobsArr.addElements(jobElements);
+
+        generatedFile.addImportDeclarations(imports);
+
+        // Defer save until batch save at end (performance optimization)
+
+        // Cleanup: forget job file nodes to reduce memory overhead (ts-morph performance optimization)
+        this.jobFiles.forEach((sf) => {
+            sf.getClasses().forEach((cls) => cls.forget());
+        });
+
+        const jobParseTime = Date.now() - startTime;
+        perfLog.info(`✓ Job parsing completed in ${jobParseTime}ms (${jobElements.length} jobs)`);
+
+        return generatedFile;
+    }
+
+    /**
+     * Scans project for services so they can be registered during start.
+     */
+    async parseServices() {
+        if (this.disableServices) {
+            initLog.info("Services disabled via flink.config.js (disableServices: true)");
+        }
 
-            jobsArr.insertElement(i, namespaceImport);
+        const startTime = Date.now();
 
-            i++;
+        const generatedFile = this.createSourceFile(
+            ["generatedServices.ts"],
+            `// Generated ${new Date()}
+import { autoRegisteredServices } from "@flink-app/flink";
+export const services: any[] = [];
+autoRegisteredServices.push(...services);
+    `
+        );
+
+        const servicesArr = generatedFile.getVariableDeclarationOrThrow("services").getFirstDescendantByKindOrThrow(SyntaxKind.ArrayLiteralExpression);
+
+        const imports: OptionalKind<ImportDeclarationStructure>[] = [];
+
+        const serviceElements: string[] = [];
+
+        for (const sf of this.serviceFiles) {
+            console.log(`Detected service ${sf.getBaseName()}`);
+
+            imports.push({
+                defaultImport: sf.getBaseNameWithoutExtension(),
+                moduleSpecifier: this.getModuleSpecifier(generatedFile, sf),
+            });
+
+            serviceElements.push(
+                `{serviceInstanceName: "${getRepoInstanceName(sf.getBaseName())}", Service: ${sf.getBaseNameWithoutExtension()}}`
+            );
         }
 
+        servicesArr.addElements(serviceElements);
+
+        generatedFile.addImportDeclarations(imports);
+
+        // Cleanup: forget service file nodes to reduce memory overhead
+        this.serviceFiles.forEach((sf) => {
+            sf.getClasses().forEach((cls) => cls.forget());
+        });
+
+        const serviceParseTime = Date.now() - startTime;
+        perfLog.info(`✓ Service parsing completed in ${serviceParseTime}ms (${serviceElements.length} services)`);
+
+        return generatedFile;
+    }
+
+    /**
+     * Generates a .flink/generatedXxx.ts file for a single compiler plugin extension.
+     * Mirrors the same namespace-import + spread pattern used by parseJobs.
+     */
+    async parseExtensionDir(ext: FlinkCompilerPlugin): Promise<SourceFile> {
+        const startTime = Date.now();
+        const files = this.extensionFiles.get(ext.generatedFile) ?? [];
+
+        const generatedFile = this.createSourceFile(
+            [`${ext.generatedFile}.ts`],
+            `// Generated ${new Date()}
+import { ${ext.registrationVar} } from "${ext.package}";
+export const items: any[] = [];
+${ext.registrationVar}.push(...items);
+`
+        );
+
+        const itemsArr = generatedFile.getVariableDeclarationOrThrow("items").getFirstDescendantByKindOrThrow(SyntaxKind.ArrayLiteralExpression);
+
+        const imports: OptionalKind<ImportDeclarationStructure>[] = [];
+        const itemElements: string[] = [];
+
+        files.forEach((sf, i) => {
+            const namespaceImport = sf.getBaseNameWithoutExtension().replace(/\./g, "_") + "_" + i;
+            imports.push({
+                defaultImport: "* as " + namespaceImport,
+                moduleSpecifier: this.getModuleSpecifier(generatedFile, sf),
+            });
+            itemElements.push(`{...${namespaceImport}, __file: "${this.getRelativePath(sf)}"}`);
+        });
+
+        itemsArr.addElements(itemElements);
         generatedFile.addImportDeclarations(imports);
 
-        await generatedFile.save();
+        const elapsed = Date.now() - startTime;
+        initLog.info(`✓ Extension dir "${ext.scanDir}" parsed in ${elapsed}ms (${files.length} files)`);
 
         return generatedFile;
     }
+
+    /**
+     * Iterates all compilerPlugins from flink.config.js and generates
+     * a .flink/generatedXxx.ts file for each one.
+     * Call this after parseJobs() and before generateStartScript().
+     */
+    public async parseAllExtensionDirs(): Promise<void> {
+        for (const ext of this.compilerPlugins) {
+            await this.parseExtensionDir(ext);
+        }
+    }
 }
 
 export default TypeScriptCompiler;