@flink-app/flink 1.0.0 → 2.0.0-alpha.48

package/src/TypeScriptCompiler.ts

@@ -69,13 +69,118 @@ class TypeScriptCompiler {
         this.project = new Project({
             tsConfigFilePath: tsConfigPath,
             compilerOptions,
+            skipAddingFilesFromTsConfig: true, // Don't auto-load all files - we'll add specific directories we need
         });
 
+        // Load Flink-specific source paths from tsconfig.json if available
+        const additionalSourcePaths = this.getFlinkSourcePaths(tsConfigPath);
+
+        // Only add source files from directories we actually need to process
+        // This prevents loading all files from tsconfig include patterns and reduces memory usage
+        const defaultSourcePaths = [
+            join(cwd, "src/handlers/**/*.ts"),
+            join(cwd, "src/repos/**/*.ts"),
+            join(cwd, "src/tools/**/*.ts"),
+            join(cwd, "src/agents/**/*.ts"),
+            join(cwd, "src/jobs/**/*.ts"),
+            join(cwd, "src/index.ts"),
+            join(cwd, "src/schemas/**/*.ts"), // Include schemas for type resolution
+        ];
+
+        const allSourcePaths = [...defaultSourcePaths, ...additionalSourcePaths];
+        this.project.addSourceFilesAtPaths(allSourcePaths);
+
+        // Resolve imports: add any files imported by our source files
+        // This ensures schemas that import types from other locations work correctly
+        this.resolveImportedFiles();
+
         console.log("Loaded", this.project.getSourceFiles().length, "source file(s) from", cwd);
         console.log("Module system:", this.isEsm ? "ESM" : "CommonJS");
         console.log("Using module:", compilerOptions.module === ts.ModuleKind.ESNext ? "ESNext" : "CommonJS");
     }
 
+    /**
+     * Loads additional source paths from tsconfig.json's flink configuration.
+     * Allows projects to specify extra directories to include in compilation.
+     *
+     * Example tsconfig.json:
+     * {
+     *   "flink": {
+     *     "sourcePaths": ["src/custom-types/**\/*.ts", "src/utils/**\/*.ts"]
+     *   }
+     * }
+     */
+    private getFlinkSourcePaths(tsConfigPath: string): string[] {
+        try {
+            if (fs.existsSync(tsConfigPath)) {
+                const tsConfigContent = fs.readFileSync(tsConfigPath, "utf8");
+                const tsConfig = JSON.parse(tsConfigContent);
+
+                if (tsConfig.flink && Array.isArray(tsConfig.flink.sourcePaths)) {
+                    console.log("Found Flink-specific source paths:", tsConfig.flink.sourcePaths);
+                    return tsConfig.flink.sourcePaths.map((path: string) => join(this.cwd, path));
+                }
+            }
+        } catch (error) {
+            console.warn("Error reading Flink source paths from tsconfig.json:", error);
+        }
+
+        return [];
+    }
+
+    /**
+     * Recursively resolves and adds imported files to the project.
+     * This ensures that files imported by handlers, schemas, etc. are available for type resolution.
+     *
+     * Handles three types of imports:
+     * 1. Relative imports (./foo, ../bar) - always resolved
+     * 2. Workspace package imports (@mycompany/shared) - resolved if symlinked outside node_modules
+     * 3. External packages (lodash, express) - skipped to avoid loading entire dependency trees
+     */
+    private resolveImportedFiles() {
+        const processedFiles = new Set<string>();
+        const filesToProcess = [...this.project.getSourceFiles()];
+
+        while (filesToProcess.length > 0) {
+            const sourceFile = filesToProcess.pop()!;
+            const filePath = sourceFile.getFilePath();
+
+            if (processedFiles.has(filePath)) {
+                continue;
+            }
+
+            processedFiles.add(filePath);
+
+            // Get all import declarations
+            const importDeclarations = sourceFile.getImportDeclarations();
+
+            for (const importDecl of importDeclarations) {
+                const moduleSpecifier = importDecl.getModuleSpecifierValue();
+
+                // Try to resolve the imported file
+                const moduleSourceFile = importDecl.getModuleSpecifierSourceFile();
+
+                if (moduleSourceFile) {
+                    const importedPath = moduleSourceFile.getFilePath();
+
+                    // For relative imports (./foo, ../bar), always include
+                    const isRelativeImport = moduleSpecifier.startsWith(".") || moduleSpecifier.startsWith("/");
+
+                    // For package imports (@foo/bar, foo), check if resolved path is outside node_modules
+                    // This handles workspace packages that are symlinked (pnpm, yarn workspaces)
+                    const isWorkspacePackage = !isRelativeImport && !importedPath.includes("node_modules");
+
+                    // Skip if it's in node_modules (external package) or already processed
+                    if ((isRelativeImport || isWorkspacePackage) && !processedFiles.has(importedPath)) {
+                        filesToProcess.push(moduleSourceFile);
+                    }
+                }
+            }
+        }
+
+        console.log("Resolved imports, total files loaded:", processedFiles.size);
+    }
+
     /**
      * Detects if the project is using ESM (ECMAScript Modules)
      * by checking type in package.json.
@@ -171,6 +276,54 @@ class TypeScriptCompiler {
         return true;
     }
 
+    /**
+     * Finds a variable declaration by its TypeScript type name.
+     * This allows finding variables based on their type annotation rather than variable name.
+     *
+     * @param sf Source file to search in
+     * @param typeName Name of the type to search for (e.g., "FlinkToolProps", "FlinkAgentProps")
+     * @returns The first matching variable declaration, or undefined if not found
+     */
+    private findVariableDeclarationByType(sf: SourceFile, typeName: string) {
+        // Get all variable declarations from the source file
+        const variableDeclarations = sf.getVariableDeclarations();
+
+        for (const varDecl of variableDeclarations) {
+            // Check if the variable is exported
+            const variableStatement = varDecl.getVariableStatement();
+            if (!variableStatement?.isExported()) {
+                continue;
+            }
+
+            // Check if it's a const declaration
+            if (varDecl.getVariableStatement()?.getDeclarationKind() !== VariableDeclarationKind.Const) {
+                continue;
+            }
+
+            // Get the type node (explicit type annotation)
+            const typeNode = varDecl.getTypeNode();
+            if (!typeNode) {
+                continue;
+            }
+
+            // Check if it's a type reference
+            if (typeNode.getKind() === SyntaxKind.TypeReference) {
+                const typeRef = typeNode as TypeReferenceNode;
+                const typeRefName = typeRef.getTypeName();
+
+                // Handle both simple type references (FlinkToolProps) and qualified names (flink.FlinkToolProps)
+                const typeRefText = typeRefName.getText();
+
+                // Check if the type name matches (either exact match or ends with the type name for qualified references)
+                if (typeRefText === typeName || typeRefText.endsWith(`.${typeName}`)) {
+                    return varDecl;
+                }
+            }
+        }
+
+        return undefined;
+    }
+
     /**
      * Scans project for handlers and add those to Flink
      * "singleton" property `autoRegisteredHandlers` so they can
@@ -338,6 +491,243 @@ autoRegisteredHandlers.push(...handlers);
         return generatedFile;
     }
 
+    /**
+     * Scans project for tools and adds those to Flink
+     * "singleton" property `autoRegisteredTools` so they can
+     * be registered during start.
+     */
+    async parseTools() {
+        const generatedFile = this.createSourceFile(
+            ["generatedTools.ts"],
+            `// Generated ${new Date()}
+import { autoRegisteredTools } from "@flink-app/flink";
+export const tools = [];
+autoRegisteredTools.push(...tools);
+    `
+        );
+
+        const toolsArr = generatedFile.getVariableDeclarationOrThrow("tools").getFirstDescendantByKindOrThrow(SyntaxKind.ArrayLiteralExpression);
+
+        const imports: OptionalKind<ImportDeclarationStructure>[] = [];
+        let i = 0;
+
+        for (const sf of this.project.getSourceFiles()) {
+            if (!sf.getFilePath().includes("src/tools/")) {
+                continue;
+            }
+
+            console.log(`Detected tool ${sf.getBaseName()}`);
+
+            const namespaceImport = sf.getBaseNameWithoutExtension().replace(/\./g, "_") + "_" + i;
+
+            imports.push({
+                defaultImport: "* as " + namespaceImport,
+                moduleSpecifier: this.getModuleSpecifier(generatedFile, sf),
+            });
+
+            // Find the tool props variable (can have any name, but must be of type FlinkToolProps)
+            const toolPropsVar = this.findVariableDeclarationByType(sf, "FlinkToolProps");
+            const toolPropsExportName = toolPropsVar?.getName() || "Tool";
+
+            // Remove existing __file variable if it exists
+            const existingFile = sf.getVariableStatements().filter((vs) => {
+                const varNames = vs.getDeclarations().map((d) => d.getName());
+                return varNames.includes("__file");
+            });
+            existingFile.forEach((v) => v.remove());
+
+            // Append metadata to source file
+            sf.addVariableStatement({
+                declarationKind: VariableDeclarationKind.Const,
+                isExported: true,
+                declarations: [
+                    {
+                        name: "__file",
+                        initializer: `"${sf.getBaseName()}"`,
+                    },
+                ],
+            });
+
+            // Create an object that wraps the namespace and provides named access to the tool props
+            toolsArr.insertElement(i, `{...${namespaceImport}, Tool: ${namespaceImport}.${toolPropsExportName}}`);
+            i++;
+        }
+
+        generatedFile.addImportDeclarations(imports);
+        await generatedFile.save();
+
+        return generatedFile;
+    }
+
+    /**
+     * Scans project for agents and validates tool references.
+     * Agents are declarative (no default function).
+     */
+    async parseAgents() {
+        const generatedFile = this.createSourceFile(
+            ["generatedAgents.ts"],
+            `// Generated ${new Date()}
+import { autoRegisteredAgents } from "@flink-app/flink";
+export const agents = [];
+autoRegisteredAgents.push(...agents);
+    `
+        );
+
+        const agentsArr = generatedFile.getVariableDeclarationOrThrow("agents").getFirstDescendantByKindOrThrow(SyntaxKind.ArrayLiteralExpression);
+
+        const imports: OptionalKind<ImportDeclarationStructure>[] = [];
+
+        // Collect all registered tool ids for validation
+        const registeredToolIds = new Set<string>();
+        for (const sf of this.project.getSourceFiles()) {
+            if (sf.getFilePath().includes("src/tools/")) {
+                const toolExport = this.findVariableDeclarationByType(sf, "FlinkToolProps");
+                if (toolExport) {
+                    const props = toolExport.getFirstDescendantByKind(SyntaxKind.ObjectLiteralExpression);
+                    // Try 'id' first, fallback to 'name' for backwards compatibility
+                    const idProp = props?.getProperty("id") || props?.getProperty("name");
+                    if (idProp) {
+                        const idValue = idProp.getLastChildByKind(SyntaxKind.StringLiteral);
+                        if (idValue) {
+                            registeredToolIds.add(idValue.getLiteralText());
+                        }
+                    }
+                }
+            }
+        }
+
+        let i = 0;
+
+        for (const sf of this.project.getSourceFiles()) {
+            if (!sf.getFilePath().includes("src/agents/")) {
+                continue;
+            }
+
+            // Check if file has class extending FlinkAgent (new pattern)
+            const agentClass = sf.getClasses().find((cls) => {
+                const baseClass = cls.getBaseClass();
+                return baseClass?.getName() === "FlinkAgent";
+            });
+
+            if (!agentClass) {
+                // Skip files without FlinkAgent class
+                continue;
+            }
+
+            console.log(`Detected agent ${sf.getBaseName()}`);
+
+            // Validate tool references exist (read from class properties)
+            const toolsProperty = agentClass.getProperty("tools");
+            if (toolsProperty) {
+                const initializer = toolsProperty.getInitializer();
+                if (initializer && initializer.getKind() === SyntaxKind.ArrayLiteralExpression) {
+                    const toolsArray = initializer as any;
+                    const toolElements = toolsArray.getElements();
+                    for (const toolElement of toolElements) {
+                        let toolName: string;
+
+                        // Handle string literals, method calls, and identifier references (tool imports)
+                        if (toolElement.getKind() === SyntaxKind.StringLiteral) {
+                            // Direct string: "tool-name"
+                            toolName = toolElement.getText().replace(/['"]/g, "");
+                        } else if (toolElement.getKind() === SyntaxKind.CallExpression) {
+                            // Method call: this.useTool("tool-name")
+                            const args = toolElement.getArguments();
+                            if (args.length > 0 && args[0].getKind() === SyntaxKind.StringLiteral) {
+                                toolName = args[0].getText().replace(/['"]/g, "");
+                            } else {
+                                console.warn(`Agent ${sf.getBaseName()} has non-string tool reference, skipping validation`);
+                                continue;
+                            }
+                        } else if (toolElement.getKind() === SyntaxKind.Identifier) {
+                            // Tool file reference (imported): SearchCarsByBrandTool
+                            // Look up the import to find the actual tool file
+                            const importName = toolElement.getText();
+                            const importDecl = sf.getImportDeclarations().find((imp) => {
+                                const namedImports = imp.getNamedImports();
+                                return namedImports.some((ni) => ni.getName() === importName);
+                            });
+
+                            if (!importDecl) {
+                                // Try namespace import (* as Foo)
+                                const namespaceImport = sf.getImportDeclarations().find((imp) => {
+                                    return imp.getNamespaceImport()?.getText() === importName;
+                                });
+
+                                if (namespaceImport) {
+                                    const moduleSpecifier = namespaceImport.getModuleSpecifierValue();
+                                    // Extract tool ID from the tool file path
+                                    // e.g., "../tools/SearchCarsByBrandTool" -> find in registeredToolIds
+                                    const toolFileName = moduleSpecifier.split("/").pop()?.replace(/\.ts$/, "");
+                                    const matchingTool = Array.from(registeredToolIds).find((id) => {
+                                        // Try to match by searching for the tool ID
+                                        // This is a heuristic - we'll validate it exists
+                                        return true; // Skip validation for imported tools for now
+                                    });
+                                    // Skip validation - tool imports are validated at runtime
+                                    continue;
+                                }
+
+                                console.warn(`Agent ${sf.getBaseName()} references tool "${importName}" but it's not imported, skipping validation`);
+                                continue;
+                            }
+
+                            // Skip validation for tool file references - they're validated at runtime
+                            continue;
+                        } else {
+                            console.warn(`Agent ${sf.getBaseName()} has unexpected tool reference format: ${toolElement.getText()}`);
+                            continue;
+                        }
+
+                        if (!registeredToolIds.has(toolName)) {
+                            console.error(`Agent ${sf.getBaseName()} references tool "${toolName}" which does not exist`);
+                            throw new Error(`Invalid tool reference in agent ${sf.getBaseName()}`);
+                        }
+                    }
+                }
+            }
+
+            const className = agentClass.getName();
+            if (!className) {
+                console.error(`Agent class in ${sf.getBaseName()} has no name`);
+                continue;
+            }
+
+            imports.push({
+                defaultImport: className,
+                moduleSpecifier: this.getModuleSpecifier(generatedFile, sf),
+            });
+
+            // Remove existing __file variable
+            const existingFile = sf.getVariableStatements().filter((vs) => {
+                const varNames = vs.getDeclarations().map((d) => d.getName());
+                return varNames.includes("__file");
+            });
+            existingFile.forEach((v) => v.remove());
+
+            // Append metadata
+            sf.addVariableStatement({
+                declarationKind: VariableDeclarationKind.Const,
+                isExported: true,
+                declarations: [
+                    {
+                        name: "__file",
+                        initializer: `"${sf.getBaseName()}"`,
+                    },
+                ],
+            });
+
+            // Register the agent class
+            agentsArr.insertElement(i, `{ default: ${className}, __file: "${sf.getBaseName()}" }`);
+            i++;
+        }
+
+        generatedFile.addImportDeclarations(imports);
+        await generatedFile.save();
+
+        return generatedFile;
+    }
+
     /**
      * Generates a start script that will import references to handlers, repos and the
      * actual Flink app to start.
@@ -356,6 +746,8 @@ autoRegisteredHandlers.push(...handlers);
             `// Generated ${new Date()}
 import "./generatedHandlers${this.isEsm ? ".js" : ""}";
 import "./generatedRepos${this.isEsm ? ".js" : ""}";
+import "./generatedTools${this.isEsm ? ".js" : ""}";
+import "./generatedAgents${this.isEsm ? ".js" : ""}";
 import "./generatedJobs${this.isEsm ? ".js" : ""}";
 import "..${appEntryScript.replace(/\.ts/g, "")}${this.isEsm ? ".js" : ""}";
 export default {}; // Export an empty object to make it a module
@@ -417,9 +809,7 @@ export default {}; // Export an empty object to make it a module
         if (!interfaceName) return;
 
         // Check if already copied
-        const existingInterface = this.parsedTsSchemas.find(
-            s => s.includes(`interface ${interfaceName} `) || s.includes(`type ${interfaceName} =`)
-        );
+        const existingInterface = this.parsedTsSchemas.find((s) => s.includes(`interface ${interfaceName} `) || s.includes(`type ${interfaceName} =`));
         if (existingInterface) return;
 
         // Copy the interface
@@ -458,7 +848,7 @@ export default {}; // Export an empty object to make it a module
 
         // Check for void and undefined types - these indicate no schema validation needed
         const schemaText = schema.getText();
-        if (schemaText === 'void' || schemaText === 'undefined') {
+        if (schemaText === "void" || schemaText === "undefined") {
             return;
         }
 
@@ -505,9 +895,10 @@ export default {}; // Export an empty object to make it a module
                         const interfaceNameMatches = propText.match(/\b([A-Z][a-zA-Z0-9]*)\s*\[/g);
                         if (interfaceNameMatches) {
                             for (const match of interfaceNameMatches) {
-                                const referencedInterfaceName = match.replace(/\s*\[$/, '').trim();
+                                const referencedInterfaceName = match.replace(/\s*\[$/, "").trim();
                                 // Try to find this interface in the handler file
-                                const referencedInterfaceDecl = handlerFile.getInterface(referencedInterfaceName) || handlerFile.getTypeAlias(referencedInterfaceName);
+                                const referencedInterfaceDecl =
+                                    handlerFile.getInterface(referencedInterfaceName) || handlerFile.getTypeAlias(referencedInterfaceName);
                                 if (referencedInterfaceDecl) {
                                     // Interface is in same file - copy it and all its dependencies recursively
                                     this.copyInterfaceWithDependencies(referencedInterfaceDecl, handlerFile);
@@ -621,7 +1012,7 @@ export default {}; // Export an empty object to make it a module
                 const interfaceNameMatches = currentPropTypeText.match(/\b([A-Z][a-zA-Z0-9]*)\s*\[/g);
                 if (interfaceNameMatches) {
                     for (const match of interfaceNameMatches) {
-                        const interfaceName = match.replace(/\s*\[$/, '').trim();
+                        const interfaceName = match.replace(/\s*\[$/, "").trim();
                         // Try to find this interface in the handler file
                         const interfaceDecl = handlerFile.getInterface(interfaceName) || handlerFile.getTypeAlias(interfaceName);
                         if (interfaceDecl) {

package/src/TypeScriptUtils.ts

@@ -214,6 +214,11 @@ export function getTypeMetadata(type: Type<ts.Type>) {
         return [];
     }
 
+    // Handle empty object literal {} (used in streaming handlers)
+    if (type.getText() === "{}") {
+        return [];
+    }
+
     const symbol = getSymbolOrAlias(type);
 
     if (!symbol) {