@flink-app/flink 0.14.3 → 2.0.0-alpha.100

package/dist/src/SchemaCache.js

@@ -0,0 +1,289 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SchemaCache = void 0;
+var crypto_1 = require("crypto");
+var fs_1 = require("fs");
+var path = __importStar(require("path"));
+/**
+ * Manages persistent caching of generated JSON schemas
+ */
+var SchemaCache = /** @class */ (function () {
+    function SchemaCache(projectRoot) {
+        this.projectRoot = projectRoot;
+        this.entries = new Map();
+        this.dirty = false;
+        this.cacheFile = path.join(projectRoot, '.flink', 'schema-cache.json');
+    }
+    /**
+     * Compute SHA-256 hash of file content
+     */
+    SchemaCache.hashFile = function (filePath) {
+        return __awaiter(this, void 0, void 0, function () {
+            var content, error_1;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        _a.trys.push([0, 2, , 3]);
+                        return [4 /*yield*/, fs_1.promises.readFile(filePath, 'utf-8')];
+                    case 1:
+                        content = _a.sent();
+                        return [2 /*return*/, SchemaCache.hashContent(content)];
+                    case 2:
+                        error_1 = _a.sent();
+                        // File doesn't exist or can't be read
+                        return [2 /*return*/, ''];
+                    case 3: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Compute SHA-256 hash of string content
+     */
+    SchemaCache.hashContent = function (content) {
+        return (0, crypto_1.createHash)('sha256').update(content, 'utf-8').digest('hex');
+    };
+    /**
+     * Load cache from disk
+     */
+    SchemaCache.prototype.load = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var flinkDir, data, cache, currentTsVersion, error_2;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        _a.trys.push([0, 3, , 4]);
+                        flinkDir = path.dirname(this.cacheFile);
+                        return [4 /*yield*/, fs_1.promises.mkdir(flinkDir, { recursive: true })];
+                    case 1:
+                        _a.sent();
+                        return [4 /*yield*/, fs_1.promises.readFile(this.cacheFile, 'utf-8')];
+                    case 2:
+                        data = _a.sent();
+                        cache = JSON.parse(data);
+                        // Validate cache version
+                        if (cache.version !== SchemaCache.CACHE_VERSION) {
+                            console.log('[SchemaCache] Cache version mismatch, invalidating cache');
+                            this.entries.clear();
+                            return [2 /*return*/];
+                        }
+                        currentTsVersion = this.getTsVersion();
+                        if (cache.tsVersion !== currentTsVersion) {
+                            console.log('[SchemaCache] TypeScript version changed, invalidating cache');
+                            this.entries.clear();
+                            return [2 /*return*/];
+                        }
+                        // Load entries
+                        this.entries = new Map(Object.entries(cache.entries));
+                        console.log("[SchemaCache] Loaded ".concat(this.entries.size, " cached schemas"));
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_2 = _a.sent();
+                        if (error_2.code !== 'ENOENT') {
+                            console.warn('[SchemaCache] Failed to load cache:', error_2);
+                        }
+                        this.entries.clear();
+                        return [3 /*break*/, 4];
+                    case 4: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Save cache to disk
+     */
+    SchemaCache.prototype.save = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var cache, flinkDir, error_3;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!this.dirty) {
+                            return [2 /*return*/];
+                        }
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 4, , 5]);
+                        cache = {
+                            version: SchemaCache.CACHE_VERSION,
+                            tsVersion: this.getTsVersion(),
+                            generated: new Date().toISOString(),
+                            entries: Object.fromEntries(this.entries),
+                        };
+                        flinkDir = path.dirname(this.cacheFile);
+                        return [4 /*yield*/, fs_1.promises.mkdir(flinkDir, { recursive: true })];
+                    case 2:
+                        _a.sent();
+                        return [4 /*yield*/, fs_1.promises.writeFile(this.cacheFile, JSON.stringify(cache, null, 2), 'utf-8')];
+                    case 3:
+                        _a.sent();
+                        console.log("[SchemaCache] Saved ".concat(this.entries.size, " schemas to cache"));
+                        this.dirty = false;
+                        return [3 /*break*/, 5];
+                    case 4:
+                        error_3 = _a.sent();
+                        console.warn('[SchemaCache] Failed to save cache:', error_3);
+                        return [3 /*break*/, 5];
+                    case 5: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Get cached entry for schema
+     */
+    SchemaCache.prototype.get = function (schemaName) {
+        return this.entries.get(schemaName);
+    };
+    /**
+     * Add or update cache entry
+     */
+    SchemaCache.prototype.set = function (entry) {
+        this.entries.set(entry.schemaName, entry);
+        this.dirty = true;
+    };
+    /**
+     * Remove cache entry
+     */
+    SchemaCache.prototype.delete = function (schemaName) {
+        if (this.entries.delete(schemaName)) {
+            this.dirty = true;
+        }
+    };
+    /**
+     * Clear all cache entries
+     */
+    SchemaCache.prototype.clear = function () {
+        this.entries.clear();
+        this.dirty = true;
+    };
+    /**
+     * Check if schema needs regeneration
+     * Returns { needed: boolean, reason?: string }
+     */
+    SchemaCache.prototype.needsRegeneration = function (schemaName, schemaFile, dependencies) {
+        return __awaiter(this, void 0, void 0, function () {
+            var cached, currentHash, cachedDeps, currentDeps, _i, _a, _b, depPath, depHash, cachedHash;
+            return __generator(this, function (_c) {
+                switch (_c.label) {
+                    case 0:
+                        cached = this.entries.get(schemaName);
+                        // No cache entry
+                        if (!cached) {
+                            return [2 /*return*/, { needed: true, reason: 'no cache entry' }];
+                        }
+                        return [4 /*yield*/, SchemaCache.hashFile(schemaFile)];
+                    case 1:
+                        currentHash = _c.sent();
+                        if (cached.contentHash !== currentHash) {
+                            return [2 /*return*/, { needed: true, reason: 'schema file changed' }];
+                        }
+                        cachedDeps = Object.keys(cached.dependencyHashes).sort();
+                        currentDeps = Array.from(dependencies.keys()).sort();
+                        if (JSON.stringify(cachedDeps) !== JSON.stringify(currentDeps)) {
+                            return [2 /*return*/, { needed: true, reason: 'dependency set changed' }];
+                        }
+                        // Check each dependency hash
+                        for (_i = 0, _a = Array.from(dependencies.entries()); _i < _a.length; _i++) {
+                            _b = _a[_i], depPath = _b[0], depHash = _b[1];
+                            cachedHash = cached.dependencyHashes[depPath];
+                            if (cachedHash !== depHash) {
+                                return [2 /*return*/, { needed: true, reason: "dependency changed: ".concat(depPath) }];
+                            }
+                        }
+                        // Cache is valid
+                        return [2 /*return*/, { needed: false }];
+                }
+            });
+        });
+    };
+    /**
+     * Get TypeScript version (major.minor)
+     */
+    SchemaCache.prototype.getTsVersion = function () {
+        try {
+            var tsPackageJson = require('typescript/package.json');
+            var _a = tsPackageJson.version.split('.'), major = _a[0], minor = _a[1];
+            return "".concat(major, ".").concat(minor);
+        }
+        catch (_b) {
+            return 'unknown';
+        }
+    };
+    /**
+     * Get cache statistics
+     */
+    SchemaCache.prototype.getStats = function () {
+        if (this.entries.size === 0) {
+            return { totalEntries: 0 };
+        }
+        var timestamps = Array.from(this.entries.values()).map(function (e) { return new Date(e.generatedAt).getTime(); });
+        return {
+            totalEntries: this.entries.size,
+            oldestEntry: new Date(Math.min.apply(Math, timestamps)).toISOString(),
+            newestEntry: new Date(Math.max.apply(Math, timestamps)).toISOString(),
+        };
+    };
+    SchemaCache.CACHE_VERSION = '1.0.0';
+    return SchemaCache;
+}());
+exports.SchemaCache = SchemaCache;

package/dist/src/TypeScriptCompiler.d.ts

@@ -1,23 +1,106 @@
 import { SourceFile } from "ts-morph";
+import { FlinkCompilerPlugin } from "./utils/loadFlinkConfig";
 declare class TypeScriptCompiler {
     private cwd;
     private project;
-    private schemaGenerator?;
     private isEsm;
+    private schemaGenerator?;
     /**
-     * Parsed typescript schemas that will be added to intermediate
-     * schemas.ts file.
-     *
-     * This will be written to file in a batch for performance reasons.
+     * Handler schemas collected during parseHandlers, to be generated later
+     */
+    private handlerSchemasToGenerate;
+    /**
+     * Tool schemas collected during parseTools, to be generated later
+     */
+    private toolSchemasToGenerate;
+    /**
+     * Pre-segmented source files by type (cached for performance)
      */
-    private parsedTsSchemas;
+    private handlerFiles;
+    private repoFiles;
+    private toolFiles;
+    private agentFiles;
+    private jobFiles;
+    private serviceFiles;
     /**
-     * Imports needed for schemas.ts.
+     * Tool ID registry for agent validation (built during segmentation)
+     */
+    private toolIdRegistry;
+    /**
+     * Compiler plugins loaded from flink.config.js
+     */
+    private compilerPlugins;
+    private disableServices;
+    /**
+     * Extension files collected during segmentation, keyed by generatedFile name
+     */
+    private extensionFiles;
+    /**
+     * Generates a schema $id from a file path and type name using the same algorithm
+     * as the schema generator's defineId callback.
      *
-     * This will be added to file in a batch for performance reasons.
+     * Examples:
+     * - src/schemas/Car.ts, "Car" → "Car"
+     * - src/schemas/wrappers/PartialLoginReq.ts, "PartialLoginReq" → "wrappers.PartialLoginReq"
+     * - src/schemas/PatchCarSchemas.ts, "PatchCarReq" → "PatchCarSchemas.PatchCarReq"
+     */
+    private filePathToSchemaId;
+    /**
+     * Resolves a type name to its schema $id by looking up its import path.
+     * Returns undefined if the type is not imported from src/schemas/.
      */
-    private tsSchemasSymbolsToImports;
+    private resolveTypeNameToSchemaId;
     constructor(cwd: string);
+    /**
+     * Loads additional source paths from tsconfig.json's flink configuration.
+     * Allows projects to specify extra directories to include in compilation.
+     *
+     * Example tsconfig.json:
+     * {
+     *   "flink": {
+     *     "sourcePaths": ["src/custom-types/**\/*.ts", "src/utils/**\/*.ts"]
+     *   }
+     * }
+     */
+    private getFlinkSourcePaths;
+    /**
+     * Initializes the schema generator.
+     *
+     * Note: ts-source-to-json-schema is ESM-only while @flink-app/flink is CommonJS.
+     * We use new Function() to preserve the actual import() in compiled output.
+     * Without this, TypeScript converts import() to require() which can't load ESM.
+     */
+    private initSchemaGenerator;
+    /**
+     * Recursively resolves and adds imported files to the project.
+     * This ensures that files imported by handlers, schemas, etc. are available for type resolution.
+     *
+     * Handles three types of imports:
+     * 1. Relative imports (./foo, ../bar) - always resolved
+     * 2. Workspace package imports (@mycompany/shared) - resolved if symlinked outside node_modules
+     * 3. External packages (lodash, express) - skipped to avoid loading entire dependency trees
+     */
+    private resolveImportedFiles;
+    /**
+     * Gets the workspace scope from package.json (e.g., "@mycompany" from "@mycompany/my-app")
+     * Returns empty string if not a scoped package.
+     */
+    private getWorkspaceScope;
+    /**
+     * Checks if an import is likely to be a workspace package.
+     * This is a heuristic - we check for common workspace patterns but can't be 100% sure
+     * without actually resolving. This is a performance optimization to avoid resolving
+     * obvious external packages like "express", "lodash", etc.
+     */
+    private isLikelyWorkspaceImport;
+    /**
+     * Segments source files by type for efficient processing.
+     * Performs all path checks and AST inspections in a single pass.
+     *
+     * This runs once during construction and caches results to avoid
+     * multiple full-file iterations during parse methods.
+     */
+    private segmentSourceFiles;
     /**
      * Detects if the project is using ESM (ECMAScript Modules)
      * by checking type in package.json.
@@ -25,6 +108,8 @@ declare class TypeScriptCompiler {
     private isEsmProject;
     /**
      * Gets the module specifier for imports, adding .js extension for ESM
+     * Uses fast path calculation instead of ts-morph's getRelativePathAsModuleSpecifierTo
+     * which triggers expensive language service initialization
      */
     private getModuleSpecifier;
     /**
@@ -33,9 +118,22 @@ declare class TypeScriptCompiler {
      */
     static clean(cwd: string): Promise<string>;
     /**
-     * Emits compiled javascript source to dist folder
+     * Saves all modified source files in a single batch operation.
+     * Call this before emit() to persist all changes to disk.
+     */
+    saveAllModifiedFiles(): Promise<void>;
+    /**
+     * Emits compiled javascript source to dist folder using swc (20-50x faster than tsc)
+     */
+    emitWithSwc(): Promise<void>;
+    /**
+     * Emits compiled javascript source to dist folder (defaults to swc for speed, falls back to tsc)
+     */
+    emit(): Promise<void>;
+    /**
+     * Emits compiled javascript source to dist folder using TypeScript compiler (slower, kept for fallback)
      */
-    emit(): void;
+    emitWithTsc(): void;
     /**
      * Catch any compilation errors. Will return false if any Errors
      * exists. Warnings will be passed thru but logged.
@@ -49,12 +147,26 @@ declare class TypeScriptCompiler {
      * Also extract handlers request and response schemas from Handler
      * type arguments.
      */
-    parseHandlers(excludeDirs?: string[]): Promise<SourceFile>;
+    parseHandlers(): Promise<SourceFile>;
     /**
      * Scan `/src/handlers/*.ts` for handler files and register those.
      */
     private parseHandlerDir;
     parseRepos(): Promise<SourceFile>;
+    /**
+     * Scans project for tools and adds those to Flink
+     * "singleton" property `autoRegisteredTools` so they can
+     * be registered during start.
+     *
+     * Also extracts input and output schemas from FlinkTool type arguments
+     * when manual schemas are not provided.
+     */
+    parseTools(): Promise<SourceFile>;
+    /**
+     * Scans project for agents and validates tool references.
+     * Agents are classes extending FlinkAgent exported as default.
+     */
+    parseAgents(): Promise<SourceFile>;
     /**
      * Generates a start script that will import references to handlers, repos and the
      * actual Flink app to start.
@@ -65,44 +177,15 @@ declare class TypeScriptCompiler {
     generateStartScript(appEntryScript?: string): Promise<SourceFile>;
     private createSourceFile;
     /**
-     * Parses handlers `Handler<...>` function and its type arguments to extract
-     * which schemas to use.
-     *
-     * There are multiple ways of defining schema types as valid ts and this
-     * implementation aims to support all of these.
-     *
-     * Some examples of different cases (check spec/mock-project/src/handlers for more):
-     *
-     * ```
-     * // Interface reference
-     * Handler<Ctx, Car>
-     * // Inline type definition with reference to interface
-     * Handler<Ctx, {car: Car}>
-     * // Inline type definition with literal values
-     * Handler<Ctx, {car: {model: string}}>
-     * // Array
-     * Handler<Ctx, Car[]>
-     * // Array with inline type definition
-     * Handler<Ctx, {car: Car}[]>
-     * ```
-     *
-     * Return names of req and/or res schema types.
-     */
-    private extractSchemasFromHandlerSourceFile;
-    /**
-     * Recursively copies an interface and all its dependencies from the same file
+     * Extracts schema information from a tool file using text-based parsing.
+     * Resolves type names to schema $ids from the schema universe.
      */
-    private copyInterfaceWithDependencies;
-    private saveIntermediateTsSchema;
-    private initJsonSchemaGenerator;
-    private generateAndSaveJsonSchemas;
-    private generateJsonSchema;
-    private extractSchemaTypeFromHandler;
+    private extractSchemasFromToolFast;
     /**
-     * Creates generated source file that contains all
-     * TypeScript schemas that has been derived from handlers.
+     * Extracts schema information from a handler file using text-based parsing.
+     * Resolves type names to schema $ids from the schema universe.
      */
-    private createIntermediateSchemaFile;
+    private extractSchemasFromHandlerFast;
     /**
      * Check if handler source file is up for auto registration by inspecting
      * the Route.
@@ -112,15 +195,42 @@ declare class TypeScriptCompiler {
      */
     private isAutoRegisterableHandler;
     /**
-     * Appends generated json schemas to handler source files.
+     * Generates JSON schemas for all handlers and tools.
+     * Should be called after parseHandlers() and parseTools() have completed.
      *
-     * @param handlers
-     * @param jsonSchemas
+     * NEW SIMPLIFIED APPROACH:
+     * 1. Generate schema universe from src/schemas/**\/*.ts (includes wrappers/)
+     * 2. Handlers/tools reference schemas by $id
+     * 3. Create manifest with schema universe and references
+     */
+    generateAllSchemas(): Promise<void>;
+    /**
+     * Computes relative path from project root for a source file.
+     * This is used consistently for manifest keys and __file exports.
+     */
+    private getRelativePath;
+    /**
+     * Generates schema manifest with schema universe and handler/tool references.
      */
-    private appendSchemasToHandlerSourceFiles;
+    private generateSchemaManifest;
     /**
      * Scans project for jobs so they can be registered during start.
      */
     parseJobs(): Promise<SourceFile>;
+    /**
+     * Scans project for services so they can be registered during start.
+     */
+    parseServices(): Promise<SourceFile>;
+    /**
+     * Generates a .flink/generatedXxx.ts file for a single compiler plugin extension.
+     * Mirrors the same namespace-import + spread pattern used by parseJobs.
+     */
+    parseExtensionDir(ext: FlinkCompilerPlugin): Promise<SourceFile>;
+    /**
+     * Iterates all compilerPlugins from flink.config.js and generates
+     * a .flink/generatedXxx.ts file for each one.
+     * Call this after parseJobs() and before generateStartScript().
+     */
+    parseAllExtensionDirs(): Promise<void>;
 }
 export default TypeScriptCompiler;