@flink-app/flink 2.0.0-alpha.72 → 2.0.0-alpha.74

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @flink-app/flink
 
+## 2.0.0-alpha.74
+
+### Patch Changes
+
+-   feat: add leader election for horizontally scaled job scheduling
+
+    Adds MongoDB-based leader election so that when multiple instances of a Flink app are running, only one (the leader) executes scheduled jobs. If the leader goes down, another instance takes over automatically. Includes dedicated scheduler logger and improved robustness.
+
+-   fix: add updateById method to FlinkRepo for convenient document updates by ID
+
+## 2.0.0-alpha.73
+
+### Patch Changes
+
+-   fix(flink): register static routes before parameterized routes to prevent e.g. GET /jobs/by-tags being matched by GET /jobs/:id
+
 ## 2.0.0-alpha.72
 
 ### Patch Changes

package/dist/src/FlinkApp.d.ts

@@ -12,6 +12,7 @@ import { FlinkContext } from "./FlinkContext";
 import { FlinkError } from "./FlinkErrors";
 import { FlinkRequest, HandlerFile, HttpMethod, QueryParamMetadata, RouteProps } from "./FlinkHttpHandler";
 import { FlinkJobFile } from "./FlinkJob";
+import { LeaderElectionOptions } from "./LeaderElection";
 import { FlinkPlugin } from "./FlinkPlugin";
 import { FlinkRepo } from "./FlinkRepo";
 import { FlinkResponse } from "./FlinkResponse";
@@ -132,6 +133,33 @@ export interface FlinkOptions {
          * Defaults to true.
          */
         enabled?: boolean;
+        /**
+         * Enable leader election for horizontally scaled deployments.
+         *
+         * When enabled, only one instance (the leader) will run scheduled jobs.
+         * If the leader goes down, another instance automatically takes over.
+         *
+         * Requires a database connection (`db` option) since leader election
+         * state is persisted in MongoDB. If no database is configured, a warning
+         * will be logged and jobs will run on all instances (no leader election).
+         *
+         * Set to `true` for default settings, or pass an options object to customize.
+         *
+         * @example
+         * ```ts
+         * // Use defaults (15s lease, 5s heartbeat)
+         * scheduling: { leaderElection: true }
+         *
+         * // Custom settings
+         * scheduling: {
+         *   leaderElection: {
+         *     leaseDurationMs: 30000,
+         *     heartbeatIntervalMs: 10000,
+         *   }
+         * }
+         * ```
+         */
+        leaderElection?: boolean | LeaderElectionOptions;
     };
     /**
      * AI configuration for agents and tools
@@ -260,6 +288,8 @@ export declare class FlinkApp<C extends FlinkContext> {
      */
     private handlerRouteCache;
     scheduler?: ToadScheduler;
+    private allInstanceScheduler?;
+    private leaderElection?;
     private accessLog;
     constructor(opts: FlinkOptions);
     get ctx(): C;
@@ -346,5 +376,7 @@ export declare class FlinkApp<C extends FlinkContext> {
     private authenticate;
     getRegisteredRoutes(): string[];
     private get isSchedulingEnabled();
+    private get leaderElectionConfig();
+    private startLeaderElection;
     private getMongoConnectionOptions;
 }

package/dist/src/FlinkApp.js

@@ -63,6 +63,7 @@ var toad_scheduler_1 = require("toad-scheduler");
 var uuid_1 = require("uuid");
 var FlinkErrors_1 = require("./FlinkErrors");
 var FlinkHttpHandler_1 = require("./FlinkHttpHandler");
+var LeaderElection_1 = require("./LeaderElection");
 var FlinkLog_1 = require("./FlinkLog");
 var FlinkLogFactory_1 = require("./FlinkLogFactory");
 var FlinkRequestContext_1 = require("./FlinkRequestContext");
@@ -71,6 +72,7 @@ var mock_data_generator_1 = __importDefault(require("./mock-data-generator"));
 var utils_1 = require("./utils");
 var initLog = FlinkLogFactory_1.FlinkLogFactory.createLogger("flink.init");
 var perfLog = FlinkLogFactory_1.FlinkLogFactory.createLogger("flink.perf");
+var schedulerLog = FlinkLogFactory_1.FlinkLogFactory.createLogger("flink.scheduler");
 var ajv = new ajv_1.default();
 (0, ajv_formats_1.default)(ajv);
 var defaultCorsOptions = {
@@ -197,11 +199,11 @@ var FlinkApp = /** @class */ (function () {
                     case 5:
                         _e.sent();
                         perfLog.debug("Initialize agents took ".concat(Date.now() - agentInitStartTime, "ms"));
-                        if (this.isSchedulingEnabled) {
+                        if (this.isSchedulingEnabled && !this.leaderElectionConfig) {
                             this.scheduler = new toad_scheduler_1.ToadScheduler();
                         }
-                        else {
-                            initLog.info("🚫 Scheduling is disabled");
+                        else if (!this.isSchedulingEnabled) {
+                            schedulerLog.info("Scheduling is disabled");
                         }
                         if (!this.disableHttpServer) {
                             this.expressApp = (0, express_1.default)();
@@ -254,14 +256,20 @@ var FlinkApp = /** @class */ (function () {
                     case 13:
                         _e.sent();
                         perfLog.debug("Register handlers took ".concat(Date.now() - handlersStartTime, "ms"));
-                        if (!this.isSchedulingEnabled) return [3 /*break*/, 15];
+                        if (!this.isSchedulingEnabled) return [3 /*break*/, 17];
+                        if (!this.leaderElectionConfig) return [3 /*break*/, 15];
+                        return [4 /*yield*/, this.startLeaderElection()];
+                    case 14:
+                        _e.sent();
+                        return [3 /*break*/, 17];
+                    case 15:
                         jobsStartTime = Date.now();
                         return [4 /*yield*/, this.registerAutoRegisterableJobs()];
-                    case 14:
+                    case 16:
                         _e.sent();
                         perfLog.debug("Register jobs took ".concat(Date.now() - jobsStartTime, "ms"));
-                        _e.label = 15;
-                    case 15:
+                        _e.label = 17;
+                    case 17:
                         // Register 404 with slight delay to allow all manually added routes to be added
                         // TODO: Is there a better solution to force this handler to always run last?
                         setTimeout(function () {
@@ -296,12 +304,24 @@ var FlinkApp = /** @class */ (function () {
                 switch (_a.label) {
                     case 0:
                         FlinkLog_1.log.info("🛑 Stopping Flink app...");
-                        if (!this.scheduler) return [3 /*break*/, 2];
-                        return [4 /*yield*/, this.scheduler.stop()];
+                        if (!this.leaderElection) return [3 /*break*/, 2];
+                        return [4 /*yield*/, this.leaderElection.stop()];
                     case 1:
                         _a.sent();
                         _a.label = 2;
                     case 2:
+                        if (!this.scheduler) return [3 /*break*/, 4];
+                        return [4 /*yield*/, this.scheduler.stop()];
+                    case 3:
+                        _a.sent();
+                        _a.label = 4;
+                    case 4:
+                        if (!this.allInstanceScheduler) return [3 /*break*/, 6];
+                        return [4 /*yield*/, this.allInstanceScheduler.stop()];
+                    case 5:
+                        _a.sent();
+                        _a.label = 6;
+                    case 6:
                         if (this.expressServer) {
                             return [2 /*return*/, new Promise(function (resolve, reject) {
                                     var int = setTimeout(function () {
@@ -797,7 +817,17 @@ var FlinkApp = /** @class */ (function () {
                 schemaManifest = this.loadSchemaManifest();
                 schemaCount = schemaManifest.version === "2.0" ? Object.keys(schemaManifest.schemas || {}).length : Object.keys(schemaManifest.definitions || {}).length;
                 FlinkLog_1.log.debug("Registering ".concat(schemaCount, " schemas with AJV (manifest version: ").concat(schemaManifest.version || "1.0", ")"));
-                for (_i = 0, _a = exports.autoRegisteredHandlers.sort(function (a, b) { var _a, _b; return (((_a = a.handler.Route) === null || _a === void 0 ? void 0 : _a.order) || 0) - (((_b = b.handler.Route) === null || _b === void 0 ? void 0 : _b.order) || 0); }); _i < _a.length; _i++) {
+                for (_i = 0, _a = exports.autoRegisteredHandlers.sort(function (a, b) {
+                    var _a, _b, _c, _d, _e, _f;
+                    var orderDiff = (((_a = a.handler.Route) === null || _a === void 0 ? void 0 : _a.order) || 0) - (((_b = b.handler.Route) === null || _b === void 0 ? void 0 : _b.order) || 0);
+                    if (orderDiff !== 0)
+                        return orderDiff;
+                    // Static segments must be registered before parameterized ones to avoid
+                    // Express matching e.g. GET /jobs/by-tags with the /jobs/:id route.
+                    var aHasParam = ((_d = (_c = a.handler.Route) === null || _c === void 0 ? void 0 : _c.path) === null || _d === void 0 ? void 0 : _d.includes("/:")) ? 1 : 0;
+                    var bHasParam = ((_f = (_e = b.handler.Route) === null || _e === void 0 ? void 0 : _e.path) === null || _f === void 0 ? void 0 : _f.includes("/:")) ? 1 : 0;
+                    return aHasParam - bHasParam;
+                }); _i < _a.length; _i++) {
                     _b = _a[_i], handler = _b.handler, assumedHttpMethod = _b.assumedHttpMethod, __file = _b.__file;
                     if (!handler.Route) {
                         FlinkLog_1.log.error("Missing Props in handler ".concat(__file));
@@ -851,7 +881,7 @@ var FlinkApp = /** @class */ (function () {
             });
         });
     };
-    FlinkApp.prototype.registerAutoRegisterableJobs = function () {
+    FlinkApp.prototype.registerAutoRegisterableJobs = function (filter) {
         return __awaiter(this, void 0, void 0, function () {
             var _loop_1, this_1, _i, autoRegisteredJobs_1, _a, jobProps, jobFn, __file;
             var _this = this;
@@ -860,30 +890,33 @@ var FlinkApp = /** @class */ (function () {
                     throw new Error("Scheduler not initialized"); // should never happen
                 }
                 _loop_1 = function (jobProps, jobFn, __file) {
+                    if (filter && !filter(jobProps)) {
+                        return "continue";
+                    }
                     if (jobProps.cron && jobProps.interval) {
-                        FlinkLog_1.log.error("Cannot register job ".concat(jobProps.id, " - both cron and interval are set in ").concat(__file));
+                        schedulerLog.error("Cannot register job ".concat(jobProps.id, " - both cron and interval are set in ").concat(__file));
                         return "continue";
                     }
                     if (jobProps.cron && jobProps.afterDelay) {
-                        FlinkLog_1.log.error("Cannot register job ".concat(jobProps.id, " - both cron and afterDelay are set in ").concat(__file));
+                        schedulerLog.error("Cannot register job ".concat(jobProps.id, " - both cron and afterDelay are set in ").concat(__file));
                         return "continue";
                     }
                     if (jobProps.interval && jobProps.afterDelay) {
-                        FlinkLog_1.log.error("Cannot register job ".concat(jobProps.id, " - both interval and afterDelay are set in ").concat(__file));
+                        schedulerLog.error("Cannot register job ".concat(jobProps.id, " - both interval and afterDelay are set in ").concat(__file));
                         return "continue";
                     }
                     if (this_1.scheduler.existsById(jobProps.id)) {
-                        FlinkLog_1.log.error("Job with id ".concat(jobProps.id, " is already registered, found duplicate in ").concat(__file));
+                        schedulerLog.error("Job with id ".concat(jobProps.id, " is already registered, found duplicate in ").concat(__file));
                         return "continue";
                     }
-                    FlinkLog_1.log.debug("Registering job ".concat(jobProps.id, ": ").concat(JSON.stringify(jobProps), " from ").concat(__file));
+                    schedulerLog.debug("Registering job ".concat(jobProps.id, ": ").concat(JSON.stringify(jobProps), " from ").concat(__file));
                     var task = new toad_scheduler_1.AsyncTask(jobProps.id, function () { return __awaiter(_this, void 0, void 0, function () {
                         return __generator(this, function (_a) {
                             switch (_a.label) {
                                 case 0: return [4 /*yield*/, jobFn({ ctx: this.ctx })];
                                 case 1:
                                     _a.sent();
-                                    FlinkLog_1.log.debug("Job ".concat(jobProps.id, " completed"));
+                                    schedulerLog.debug("Job ".concat(jobProps.id, " completed"));
                                     if (jobProps.afterDelay) {
                                         // afterDelay runs only once, so we remove the job
                                         this.scheduler.removeById(jobProps.id);
@@ -892,7 +925,7 @@ var FlinkApp = /** @class */ (function () {
                             }
                         });
                     }); }, function (err) {
-                        FlinkLog_1.log.error("Job ".concat(jobProps.id, " threw unhandled exception ").concat(err));
+                        schedulerLog.error("Job ".concat(jobProps.id, " threw unhandled exception ").concat(err));
                         console.error(err);
                     });
                     if (jobProps.cron) {
@@ -927,7 +960,7 @@ var FlinkApp = /** @class */ (function () {
                                             return [3 /*break*/, 3];
                                         case 2:
                                             err_2 = _a.sent();
-                                            FlinkLog_1.log.error("Job ".concat(jobProps.id, " threw unhandled exception ").concat(err_2));
+                                            schedulerLog.error("Job ".concat(jobProps.id, " threw unhandled exception ").concat(err_2));
                                             console.error(err_2);
                                             return [3 /*break*/, 3];
                                         case 3: return [2 /*return*/];
@@ -947,7 +980,7 @@ var FlinkApp = /** @class */ (function () {
                         }
                     }
                     else {
-                        FlinkLog_1.log.error("Cannot register job ".concat(jobProps.id, " - no cron, interval or once set in ").concat(__file));
+                        schedulerLog.error("Cannot register job ".concat(jobProps.id, " - no cron, interval or once set in ").concat(__file));
                         return "continue";
                     }
                 };
@@ -1243,6 +1276,78 @@ var FlinkApp = /** @class */ (function () {
         enumerable: false,
         configurable: true
     });
+    Object.defineProperty(FlinkApp.prototype, "leaderElectionConfig", {
+        get: function () {
+            var _a;
+            var opt = (_a = this.schedulingOptions) === null || _a === void 0 ? void 0 : _a.leaderElection;
+            if (!opt)
+                return undefined;
+            return opt === true ? {} : opt;
+        },
+        enumerable: false,
+        configurable: true
+    });
+    FlinkApp.prototype.startLeaderElection = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var hasAllInstanceJobs, opts;
+            var _this = this;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!!this.db) return [3 /*break*/, 2];
+                        schedulerLog.warn("Leader election is enabled but no database is configured. " +
+                            "Leader election requires a MongoDB connection to coordinate between instances. " +
+                            "Either add a database connection via the `db` option, or remove `scheduling.leaderElection` from your config. " +
+                            "Jobs will run on ALL instances without leader election.");
+                        // Fall back to running jobs on all instances
+                        this.scheduler = new toad_scheduler_1.ToadScheduler();
+                        return [4 /*yield*/, this.registerAutoRegisterableJobs()];
+                    case 1:
+                        _a.sent();
+                        return [2 /*return*/];
+                    case 2:
+                        hasAllInstanceJobs = exports.autoRegisteredJobs.some(function (j) { return j.Job.runOnAllInstances; });
+                        if (!hasAllInstanceJobs) return [3 /*break*/, 4];
+                        this.allInstanceScheduler = new toad_scheduler_1.ToadScheduler();
+                        this.scheduler = this.allInstanceScheduler;
+                        return [4 /*yield*/, this.registerAutoRegisterableJobs(function (job) { return !!job.runOnAllInstances; })];
+                    case 3:
+                        _a.sent();
+                        this.scheduler = undefined;
+                        _a.label = 4;
+                    case 4:
+                        opts = this.leaderElectionConfig;
+                        this.leaderElection = new LeaderElection_1.LeaderElection(this.db, opts);
+                        return [4 /*yield*/, this.leaderElection.start(
+                            // onBecameLeader
+                            function () { return __awaiter(_this, void 0, void 0, function () {
+                                return __generator(this, function (_a) {
+                                    switch (_a.label) {
+                                        case 0:
+                                            schedulerLog.info("This instance is now the leader - starting scheduled jobs");
+                                            this.scheduler = new toad_scheduler_1.ToadScheduler();
+                                            return [4 /*yield*/, this.registerAutoRegisterableJobs(function (job) { return !job.runOnAllInstances; })];
+                                        case 1:
+                                            _a.sent();
+                                            return [2 /*return*/];
+                                    }
+                                });
+                            }); }, 
+                            // onLostLeadership
+                            function () {
+                                schedulerLog.info("This instance lost leadership - stopping scheduled jobs");
+                                if (_this.scheduler) {
+                                    _this.scheduler.stop();
+                                    _this.scheduler = undefined;
+                                }
+                            })];
+                    case 5:
+                        _a.sent();
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
     FlinkApp.prototype.getMongoConnectionOptions = function () {
         if (!this.dbOpts) {
             throw new Error("No db configured");

package/dist/src/FlinkJob.d.ts

@@ -31,6 +31,16 @@ export type FlinkJobProps = {
      * retried after the next interval.
      */
     singleton?: boolean;
+    /**
+     * If true, this job will run on all instances regardless of leader election.
+     *
+     * By default, when leader election is enabled, jobs only run on the leader instance.
+     * Set this to true for jobs that should run on every instance, such as
+     * local cache cleanup or instance-specific health checks.
+     *
+     * Has no effect when leader election is not enabled.
+     */
+    runOnAllInstances?: boolean;
 };
 /**
  * Type for Flink job function. This function should be default exported from

package/dist/src/FlinkRepo.d.ts

@@ -22,6 +22,10 @@ export declare abstract class FlinkRepo<C extends FlinkContext, Model extends Do
     create<C = Omit<Model, "_id">>(model: C): Promise<C & {
         _id: string;
     }>;
+    updateById(id: string | ObjectId, model: PartialModel<Model>): Promise<Model | null>;
+    /**
+     * @deprecated Use `updateById` instead. This will be removed in a future major version.
+     */
     updateOne(id: string | ObjectId, model: PartialModel<Model>): Promise<Model | null>;
     updateMany<U = PartialModel<Model>>(query: any, model: U): Promise<number>;
     deleteById(id: string | ObjectId): Promise<number>;

package/dist/src/FlinkRepo.js

@@ -139,7 +139,7 @@ var FlinkRepo = /** @class */ (function () {
             });
         });
     };
-    FlinkRepo.prototype.updateOne = function (id, model) {
+    FlinkRepo.prototype.updateById = function (id, model) {
         return __awaiter(this, void 0, void 0, function () {
             var oid, _id, modelWithoutId, res;
             return __generator(this, function (_a) {
@@ -161,6 +161,16 @@ var FlinkRepo = /** @class */ (function () {
             });
         });
     };
+    /**
+     * @deprecated Use `updateById` instead. This will be removed in a future major version.
+     */
+    FlinkRepo.prototype.updateOne = function (id, model) {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                return [2 /*return*/, this.updateById(id, model)];
+            });
+        });
+    };
     FlinkRepo.prototype.updateMany = function (query, model) {
         return __awaiter(this, void 0, void 0, function () {
             var _a, _id, modelWithoutId, modifiedCount;

package/dist/src/LeaderElection.d.ts

@@ -0,0 +1,45 @@
+import { Db } from "mongodb";
+export interface LeaderElectionOptions {
+    /**
+     * Duration in milliseconds before a leader's lease expires.
+     * If the leader fails to heartbeat within this time, another instance can take over.
+     * @default 15000
+     */
+    leaseDurationMs?: number;
+    /**
+     * Interval in milliseconds between heartbeats sent by the leader.
+     * Should be significantly less than leaseDurationMs (typically 1/3).
+     * @default 5000
+     */
+    heartbeatIntervalMs?: number;
+    /**
+     * Name of the MongoDB collection used for leader election.
+     * @default "_flink_leader"
+     */
+    collectionName?: string;
+}
+export declare class LeaderElection {
+    private instanceId;
+    private _isLeader;
+    private timer;
+    private collection;
+    private leaseDurationMs;
+    private heartbeatIntervalMs;
+    private onBecameLeader?;
+    private onLostLeadership?;
+    private stopped;
+    private transitioning;
+    constructor(db: Db, opts?: LeaderElectionOptions);
+    get isLeader(): boolean;
+    /**
+     * Start the leader election process.
+     * @param onBecameLeader Called when this instance becomes the leader
+     * @param onLostLeadership Called when this instance loses leadership
+     */
+    start(onBecameLeader: () => void | Promise<void>, onLostLeadership: () => void | Promise<void>): Promise<void>;
+    /**
+     * Stop the leader election and release leadership if held.
+     */
+    stop(): Promise<void>;
+    private tryClaimLeadership;
+}