@flink-app/flink 1.0.0 → 2.0.0-alpha.101

package/src/FlinkApp.ts

@@ -9,18 +9,32 @@ import morgan from "morgan";
 import ms from "ms";
 import { AsyncTask, CronJob, SimpleIntervalJob, ToadScheduler } from "toad-scheduler";
 import { v4 } from "uuid";
+import { FlinkAgentFile } from "./ai/FlinkAgent";
+import { FlinkToolFile } from "./ai/FlinkTool";
+import { AgentObserver } from "./ai/FlinkAgent";
+import { LLMAdapter } from "./ai/LLMAdapter";
+import { ToolExecutor } from "./ai/ToolExecutor";
 import { FlinkAuthPlugin } from "./auth/FlinkAuthPlugin";
 import { FlinkContext } from "./FlinkContext";
 import { FlinkError, internalServerError, notFound, unauthorized } from "./FlinkErrors";
 import { FlinkRequest, Handler, HandlerFile, HttpMethod, QueryParamMetadata, RouteProps, ValidationMode } from "./FlinkHttpHandler";
-import { FlinkJobFile } from "./FlinkJob";
+import { FlinkJobFile, FlinkJobProps } from "./FlinkJob";
+import { LeaderElection, LeaderElectionOptions } from "./LeaderElection";
 import { log } from "./FlinkLog";
+import { FlinkLogFactory } from "./FlinkLogFactory";
 import { FlinkPlugin } from "./FlinkPlugin";
 import { FlinkRepo } from "./FlinkRepo";
+import { FlinkService } from "./FlinkService";
 import { FlinkResponse } from "./FlinkResponse";
+import { requestContext } from "./FlinkRequestContext";
+import { StreamWriterFactory } from "./handlers/StreamWriterFactory";
 import generateMockData from "./mock-data-generator";
 import { formatValidationErrors, getPathParams, isError } from "./utils";
 
+const initLog = FlinkLogFactory.createLogger("flink.init");
+const perfLog = FlinkLogFactory.createLogger("flink.perf");
+const schedulerLog = FlinkLogFactory.createLogger("flink.scheduler");
+
 const ajv = new Ajv();
 addFormats(ajv);
 
@@ -45,6 +59,7 @@ export const expressFn = express;
 export const autoRegisteredHandlers: {
     handler: HandlerFile;
     assumedHttpMethod: HttpMethod;
+    __file?: string;
 }[] = [];
 
 /**
@@ -63,7 +78,28 @@ export const autoRegisteredRepos: {
  */
 export const autoRegisteredJobs: FlinkJobFile[] = [];
 
-export interface FlinkOptions {
+/**
+ * This will be populated at compile time when the apps tools
+ * are picked up by TypeScript compiler
+ */
+export const autoRegisteredTools: FlinkToolFile[] = [];
+
+/**
+ * This will be populated at compile time when the apps agents
+ * are picked up by TypeScript compiler
+ */
+export const autoRegisteredAgents: FlinkAgentFile<any, any>[] = [];
+
+/**
+ * This will be populated at compile time when the apps services
+ * are picked up by TypeScript compiler
+ */
+export const autoRegisteredServices: {
+    serviceInstanceName: string;
+    Service: any;
+}[] = [];
+
+export interface FlinkOptions<C extends FlinkContext = FlinkContext> {
     /**
      * Name of application, will only show in logs and in HTTP header.
      */
@@ -156,28 +192,62 @@ export interface FlinkOptions {
          */
         enabled?: boolean;
 
-        // TODO: Implement master auto assignment
-        //     /**
-        //      * If true, the master (the instance if flink app that will run jobs) will be
-        //      * automatically assigned to the first node that starts.
-        //      *
-        //      * Is persisted in database.
-        //      *
-        //      * Will throw and exception if true but no database is configured.
-        //      */
-        //     autoAssignMaster?: boolean;
-
-        //     /**
-        //      * Name of collection to be used for storing master assignment.
-        //      *
-        //      * Defaults to `flink-scheduling`
-        //      */
-        //     autoAssignCollection?: string;
+        /**
+         * Enable leader election for horizontally scaled deployments.
+         *
+         * When enabled, only one instance (the leader) will run scheduled jobs.
+         * If the leader goes down, another instance automatically takes over.
+         *
+         * Requires a database connection (`db` option) since leader election
+         * state is persisted in MongoDB. If no database is configured, a warning
+         * will be logged and jobs will run on all instances (no leader election).
+         *
+         * Set to `true` for default settings, or pass an options object to customize.
+         *
+         * @example
+         * ```ts
+         * // Use defaults (15s lease, 5s heartbeat)
+         * scheduling: { leaderElection: true }
+         *
+         * // Custom settings
+         * scheduling: {
+         *   leaderElection: {
+         *     leaseDurationMs: 30000,
+         *     heartbeatIntervalMs: 10000,
+         *   }
+         * }
+         * ```
+         */
+        leaderElection?: boolean | LeaderElectionOptions;
+    };
+
+    /**
+     * AI configuration for agents and tools
+     * Register LLM adapters with custom IDs (e.g., "anthropic", "openai", "anthropic-eu", etc.)
+     * This allows multiple adapters of the same type with different configurations
+     */
+    ai?: {
+        llms?: { [id: string]: LLMAdapter };
+
+        /**
+         * Global agent observer for app-level tracing, APM, cost accounting, dev tools, etc.
+         *
+         * Fires for every agent execution in the app. Observer callbacks are invoked
+         * fire-and-forget — they may return a Promise but the framework does not await
+         * them, and any thrown/rejected errors are caught and logged without affecting
+         * agent execution.
+         *
+         * Events: `onRun` (pre-loop), `onLlmCall` (per step, pre-adapter call),
+         * `onStep` (per step end), `onFinish` (post-loop, including error path).
+         *
+         * For agent-local business logic (conversation persistence, guardrails) use the
+         * per-agent `beforeRun` / `onStep` / `afterRun` hooks on `FlinkAgent` instead.
+         */
+        observer?: AgentObserver;
     };
 
     /**
      * If true, the HTTP server will be disabled.
-     * Only useful when starting a Flink app for testing purposes.
      */
     disableHttpServer?: boolean;
 
@@ -199,17 +269,26 @@ export interface FlinkOptions {
     };
 
     /**
-     * Optional callback invoked when an error occurs in a handler.
+     * Optional callback invoked when an error occurs while serving a request.
      * The error response and request context are passed for custom
      * error logging or monitoring. This is a side-effect only and
      * will not modify the response flow.
      *
+     * Invoked for:
+     * - Handler-thrown errors (FlinkErrors and unhandled exceptions)
+     * - Request validation failures (400 Bad request)
+     * - Response validation failures (500 Bad response)
+     *
+     * Not invoked for errors in streaming (SSE/NDJSON) handlers, which are
+     * delivered to the client via `stream.error()` instead.
+     *
      * Supports both synchronous and asynchronous callbacks. Any errors
      * thrown or rejected by the callback will be caught and logged
-     * without affecting the error response to the client.
+     * without affecting the error response to the client. Async callbacks
+     * are fire-and-forget — they are not awaited before the response is sent.
      *
      * @param error - The error response with status and error details
-     * @param context - Request context including method, path, and request ID
+     * @param context - Request context including method, path, request ID and the app context
      *
      * @example
      * ```ts
@@ -225,12 +304,14 @@ export interface FlinkOptions {
      *   }
      * }
      *
-     * // Asynchronous callback
+     * // Asynchronous callback using the app context
      * onError: async (error, context) => {
      *   if (error.status >= 500) {
-     *     await monitoringService.reportError({
-     *       error,
-     *       context
+     *     await context.ctx.repos.errorLogRepo.create({
+     *       status: error.status,
+     *       detail: error.error?.detail,
+     *       route: `${context.method} ${context.path}`,
+     *       reqId: context.reqId,
      *     });
      *   }
      * }
@@ -243,6 +324,7 @@ export interface FlinkOptions {
             method: HttpMethod;
             path: string;
             reqId: string;
+            ctx: C;
         }
     ) => void | Promise<void>;
 }
@@ -278,6 +360,8 @@ export class FlinkApp<C extends FlinkContext> {
 
     private _ctx?: C;
     private dbOpts?: FlinkOptions["db"];
+    private schemaManifest?: any;
+    private schemaAjv?: Ajv;
     private debug = false;
     private onDbConnection?: FlinkOptions["onDbConnection"];
 
@@ -290,9 +374,15 @@ export class FlinkApp<C extends FlinkContext> {
     private schedulingOptions?: FlinkOptions["scheduling"];
     private disableHttpServer = false;
     private expressServer: any; // for simplicity, we don't want to import types from express/node here
-    private onError?: FlinkOptions["onError"];
+    private onError?: FlinkOptions<C>["onError"];
 
     private repos: { [x: string]: FlinkRepo<C, any> } = {};
+    private services: { [x: string]: FlinkService<C> } = {};
+
+    private llmAdapters: Map<string, LLMAdapter> = new Map();
+    private agentObserver?: AgentObserver;
+    private tools: { [x: string]: ToolExecutor<C> } = {};
+    private agents: { [x: string]: any } = {}; // FlinkAgent<C> instances
 
     /**
      * Internal cache used to track registered handlers and potentially any overlapping routes
@@ -300,10 +390,17 @@ export class FlinkApp<C extends FlinkContext> {
     private handlerRouteCache = new Map<string, string>();
 
     public scheduler?: ToadScheduler;
+    private allInstanceScheduler?: ToadScheduler;
+    private leaderElection?: LeaderElection;
 
     private accessLog: { enabled: boolean; format: string };
 
-    constructor(opts: FlinkOptions) {
+    constructor(opts: FlinkOptions<C>) {
+        // Load config file and initialize logging
+        const { loadFlinkConfig } = require("./utils/loadFlinkConfig");
+        const flinkConfig = loadFlinkConfig();
+        FlinkLogFactory.configure(flinkConfig?.logging);
+
         this.name = opts.name;
         this.port = opts.port || 3333;
         this.dbOpts = opts.db;
@@ -314,14 +411,23 @@ export class FlinkApp<C extends FlinkContext> {
         this.rawContentTypes = Array.isArray(opts.rawContentTypes)
             ? opts.rawContentTypes
             : typeof opts.rawContentTypes === "string"
-            ? [opts.rawContentTypes]
-            : undefined;
+              ? [opts.rawContentTypes]
+              : undefined;
         this.auth = opts.auth;
         this.jsonOptions = opts.jsonOptions || { limit: "1mb" };
         this.schedulingOptions = opts.scheduling;
         this.disableHttpServer = !!opts.disableHttpServer;
         this.accessLog = { enabled: true, format: "dev", ...opts.accessLog };
         this.onError = opts.onError;
+
+        // Register LLM adapters if configured
+        if (opts.ai?.llms) {
+            // Convert plain object to Map for internal use
+            this.llmAdapters = new Map(Object.entries(opts.ai.llms));
+        }
+
+        // Register global agent observer if configured
+        this.agentObserver = opts.ai?.observer;
     }
 
     get ctx() {
@@ -333,26 +439,35 @@ export class FlinkApp<C extends FlinkContext> {
 
     async start() {
         const startTime = Date.now();
-        let offsetTime = 0;
 
+        const dbStartTime = Date.now();
         await this.initDb();
+        perfLog.debug(`Init db took ${Date.now() - dbStartTime}ms`);
 
-        if (this.debug) {
-            offsetTime = Date.now();
-            log.bgColorLog("cyan", `Init db took ${offsetTime - startTime} ms`);
-        }
-
+        // Build initial context (without agents - they'll be added later)
+        const contextStartTime = Date.now();
         await this.buildContext();
+        perfLog.debug(`Build context took ${Date.now() - contextStartTime}ms`);
 
-        if (this.debug) {
-            log.bgColorLog("cyan", `Build context took ${Date.now() - offsetTime} ms`);
-            offsetTime = Date.now();
-        }
+        // Register tools (needs context for ToolExecutor)
+        const toolsStartTime = Date.now();
+        await this.registerAutoRegisterableTools();
+        perfLog.debug(`Register tools took ${Date.now() - toolsStartTime}ms`);
 
-        if (this.isSchedulingEnabled) {
+        // Register agents (creates agent instances)
+        const agentsStartTime = Date.now();
+        await this.registerAutoRegisterableAgents();
+        perfLog.debug(`Register agents took ${Date.now() - agentsStartTime}ms`);
+
+        // Initialize agents now that context and tools are ready
+        const agentInitStartTime = Date.now();
+        await this.initializeAgents();
+        perfLog.debug(`Initialize agents took ${Date.now() - agentInitStartTime}ms`);
+
+        if (this.isSchedulingEnabled && !this.leaderElectionConfig) {
             this.scheduler = new ToadScheduler();
-        } else {
-            log.info("🚫 Scheduling is disabled");
+        } else if (!this.isSchedulingEnabled) {
+            schedulerLog.info("Scheduling is disabled");
         }
 
         if (!this.disableHttpServer) {
@@ -379,6 +494,7 @@ export class FlinkApp<C extends FlinkContext> {
 
         // TODO: Add better more fine grained control when plugins are initialized, i.e. in what order
 
+        const pluginsStartTime = Date.now();
         for (const plugin of this.plugins) {
             let db;
 
@@ -390,22 +506,23 @@ export class FlinkApp<C extends FlinkContext> {
                 await plugin.init(this, db);
             }
 
-            log.info(`Initialized plugin '${plugin.id}'`);
+            initLog.info(`Initialized plugin '${plugin.id}'`);
+        }
+        if (this.plugins.length > 0) {
+            perfLog.debug(`Initialize plugins took ${Date.now() - pluginsStartTime}ms (${this.plugins.length} plugins)`);
         }
 
+        const handlersStartTime = Date.now();
         await this.registerAutoRegisterableHandlers();
-
-        if (this.debug) {
-            log.bgColorLog("cyan", `Register handlers took ${Date.now() - offsetTime} ms`);
-            offsetTime = Date.now();
-        }
+        perfLog.debug(`Register handlers took ${Date.now() - handlersStartTime}ms`);
 
         if (this.isSchedulingEnabled) {
-            await this.registerAutoRegisterableJobs();
-
-            if (this.debug) {
-                log.bgColorLog("cyan", `Register jobs took ${Date.now() - offsetTime} ms`);
-                offsetTime = Date.now();
+            if (this.leaderElectionConfig) {
+                await this.startLeaderElection();
+            } else {
+                const jobsStartTime = Date.now();
+                await this.registerAutoRegisterableJobs();
+                perfLog.debug(`Register jobs took ${Date.now() - jobsStartTime}ms`);
             }
         }
 
@@ -422,7 +539,7 @@ export class FlinkApp<C extends FlinkContext> {
         });
 
         if (this.disableHttpServer) {
-            log.info("🚧 HTTP server is disabled, but flink app is running");
+            initLog.info("🚧 HTTP server is disabled, but flink app is running");
             this.started = true;
         } else {
             this.expressServer = this.expressApp?.listen(this.port, () => {
@@ -431,16 +548,27 @@ export class FlinkApp<C extends FlinkContext> {
             });
         }
 
+        const totalStartTime = Date.now() - startTime;
+        perfLog.info(`✓ FlinkApp started in ${totalStartTime}ms`);
+
         return this;
     }
 
     async stop() {
         log.info("🛑 Stopping Flink app...");
 
+        if (this.leaderElection) {
+            await this.leaderElection.stop();
+        }
+
         if (this.scheduler) {
             await this.scheduler.stop();
         }
 
+        if (this.allInstanceScheduler) {
+            await this.allInstanceScheduler.stop();
+        }
+
         if (this.expressServer) {
             return new Promise<void>((resolve, reject) => {
                 const int = setTimeout(() => {
@@ -490,6 +618,21 @@ export class FlinkApp<C extends FlinkContext> {
             log.warn(`${methodAndPath} overlaps existing route`);
         }
 
+        // Use direct schemas from routeProps if provided, otherwise fall back to manifest lookup
+        let reqSchema = routeProps.reqSchema;
+        let resSchema = routeProps.resSchema;
+        let queryMetadata: any[] = [];
+        let paramsMetadata: any[] = [];
+
+        if (!reqSchema || !resSchema) {
+            const schemaManifest = this.loadSchemaManifest();
+            const metadata = handler.__file ? schemaManifest.handlers[handler.__file] : undefined;
+            if (!reqSchema) reqSchema = this.resolveSchema(metadata?.reqSchemaName);
+            if (!resSchema) resSchema = this.resolveSchema(metadata?.resSchemaName);
+            queryMetadata = metadata?.queryMetadata || [];
+            paramsMetadata = metadata?.paramsMetadata || [];
+        }
+
         const handlerConfig: HandlerConfigWithMethod = {
             routeProps: {
                 ...routeProps,
@@ -497,21 +640,13 @@ export class FlinkApp<C extends FlinkContext> {
                 path: routeProps.path!,
             },
             schema: {
-                reqSchema: handler.__schemas?.reqSchema,
-                resSchema: handler.__schemas?.resSchema,
+                reqSchema,
+                resSchema,
             },
-            queryMetadata: handler.__query || [],
-            paramsMetadata: handler.__params || [],
+            queryMetadata,
+            paramsMetadata,
         };
 
-        if (handler.__schemas?.reqSchema && !handlerConfig.schema?.reqSchema) {
-            log.warn(`Expected request schema ${handler.__schemas.reqSchema} for handler ${methodAndPath} but no such schema was found`);
-        }
-
-        if (handler.__schemas?.resSchema && !handlerConfig.schema?.resSchema) {
-            log.warn(`Expected response schema ${handler.__schemas.resSchema} for handler ${methodAndPath} but no such schema was found`);
-        }
-
         this.registerHandler(handlerConfig, handler.default);
     }
 
@@ -519,7 +654,7 @@ export class FlinkApp<C extends FlinkContext> {
         this.handlers.push(handlerConfig);
 
         const { routeProps, schema = {} } = handlerConfig;
-        const { method } = routeProps;
+        const { method, streamFormat } = routeProps;
 
         if (!method) {
             log.error(`Route ${routeProps.path} is missing http method`);
@@ -535,17 +670,45 @@ export class FlinkApp<C extends FlinkContext> {
             let validateReq: ValidateFunction<any> | undefined;
             let validateRes: ValidateFunction<any> | undefined;
 
+            // Select AJV instance (use schemaAjv for v2.0 manifests, fallback to global ajv)
+            const ajvInstance = this.schemaAjv || ajv;
+
             // Determine validation mode (default to Validate if not specified)
             const validationMode = routeProps.validation || ValidationMode.Validate;
 
             // Compile request schema if validation mode requires it
             if (schema.reqSchema && validationMode !== ValidationMode.SkipValidation && validationMode !== ValidationMode.ValidateResponse) {
-                validateReq = ajv.compile(schema.reqSchema);
+                // For v2.0 manifests with $id, use getSchema() if available
+                if (schema.reqSchema.$id && this.schemaAjv) {
+                    validateReq = this.schemaAjv.getSchema(schema.reqSchema.$id);
+                    if (!validateReq) {
+                        log.warn(`Schema ${schema.reqSchema.$id} not found in AJV registry, compiling inline`);
+                        validateReq = ajvInstance.compile(schema.reqSchema);
+                    }
+                } else {
+                    validateReq = ajvInstance.compile(schema.reqSchema);
+                }
             }
 
-            // Compile response schema if validation mode requires it
-            if (schema.resSchema && validationMode !== ValidationMode.SkipValidation && validationMode !== ValidationMode.ValidateRequest) {
-                validateRes = ajv.compile(schema.resSchema);
+            // Skip response validation for streaming handlers (responses are stream chunks, not final JSON)
+            // Skip response validation for non-JSON response types (html, csv, etc.)
+            if (
+                !streamFormat &&
+                !routeProps.responseType &&
+                schema.resSchema &&
+                validationMode !== ValidationMode.SkipValidation &&
+                validationMode !== ValidationMode.ValidateRequest
+            ) {
+                // For v2.0 manifests with $id, use getSchema() if available
+                if (schema.resSchema.$id && this.schemaAjv) {
+                    validateRes = this.schemaAjv.getSchema(schema.resSchema.$id);
+                    if (!validateRes) {
+                        log.warn(`Schema ${schema.resSchema.$id} not found in AJV registry, compiling inline`);
+                        validateRes = ajvInstance.compile(schema.resSchema);
+                    }
+                } else {
+                    validateRes = ajvInstance.compile(schema.resSchema);
+                }
             }
 
             this.expressApp![method](routeProps.path, async (req, res) => {
@@ -562,18 +725,23 @@ export class FlinkApp<C extends FlinkContext> {
                         const formattedErrors = formatValidationErrors(validateReq.errors, req.body);
                         log.warn(`[${req.reqId}] ${methodAndRoute}: Bad request\n${formattedErrors}`);
 
-                        return res.status(400).json({
+                        const errorResponse: FlinkResponse<FlinkError> = {
                             status: 400,
                             error: {
                                 id: v4(),
                                 title: "Bad request",
                                 detail: formattedErrors,
                             },
-                        });
+                        };
+
+                        this.invokeOnError(errorResponse, req as FlinkRequest, method!, routeProps.path);
+
+                        return res.status(400).json(errorResponse);
                     }
                 }
 
-                if (routeProps.mockApi && schema.resSchema) {
+                // Skip mock API for streaming handlers
+                if (routeProps.mockApi && schema.resSchema && !streamFormat) {
                     log.warn(`Mock response for ${req.method.toUpperCase()} ${req.path}`);
 
                     const data = generateMockData(schema.resSchema);
@@ -603,16 +771,45 @@ export class FlinkApp<C extends FlinkContext> {
                     req.query = normalizedQuery;
                 }
 
+                // Create stream writer if streaming handler
+                const stream = streamFormat ? StreamWriterFactory.create(res, streamFormat) : undefined;
+
                 let handlerRes: FlinkResponse<any>;
 
                 try {
-                    // 👇 This is where the actual handler gets invoked
-                    handlerRes = await handler({
-                        req: req as FlinkRequest,
-                        ctx: this.ctx,
-                        origin: routeProps.origin,
-                    });
+                    // Wrap handler execution in AsyncLocalStorage context
+                    // Context persists through entire async chain: handler → agent → tools
+                    const flinkReq = req as FlinkRequest;
+                    handlerRes = await requestContext.run(
+                        {
+                            reqId: flinkReq.reqId,
+                            user: flinkReq.user,
+                            userPermissions: flinkReq.userPermissions,
+                            method: method,
+                            path: routeProps.path,
+                            timestamp: Date.now(),
+                            isStreaming: !!streamFormat,
+                        },
+                        async () => {
+                            return await handler({
+                                req: flinkReq,
+                                ctx: this.ctx,
+                                origin: routeProps.origin,
+                                stream,
+                            });
+                        }
+                    );
                 } catch (err: any) {
+                    // Handle errors for streaming handlers
+                    if (streamFormat && stream) {
+                        log.error(`Streaming handler error on ${req.method.toUpperCase()} ${req.path}: ${err.message}`, {
+                            error: err,
+                            path: req.path,
+                            method: req.method,
+                        });
+                        stream.error(err);
+                        return;
+                    }
                     let errorResponse: FlinkResponse<FlinkError>;
 
                     // duck typing to check if it is a FlinkError
@@ -632,50 +829,78 @@ export class FlinkApp<C extends FlinkContext> {
                         errorResponse = internalServerError(err as any);
                     }
 
-                    // Invoke onError callback if provided
-                    if (this.onError) {
-                        try {
-                            const result = this.onError(errorResponse, {
-                                req: req as FlinkRequest,
-                                method: method!,
-                                path: routeProps.path,
-                                reqId: req.reqId,
-                            });
-
-                            // Handle async callbacks - don't wait for them
-                            if (result instanceof Promise) {
-                                result.catch((callbackErr) => {
-                                    log.error(`onError callback rejected with: ${callbackErr}`);
-                                });
-                            }
-                        } catch (callbackErr) {
-                            log.error(`onError callback threw an exception: ${callbackErr}`);
-                        }
-                    }
+                    this.invokeOnError(errorResponse, req as FlinkRequest, method!, routeProps.path);
 
                     return res.status(errorResponse.status || 500).json(errorResponse);
                 }
 
+                // Skip response handling for streaming handlers (stream controls response lifecycle)
+                if (streamFormat) {
+                    return;
+                }
+
+                // Ensure handlerRes is defined for non-streaming handlers
+                if (!handlerRes) {
+                    return res.status(204).send();
+                }
+
                 if (validateRes && !isError(handlerRes)) {
-                    const valid = validateRes(JSON.parse(JSON.stringify(handlerRes.data)));
+                    if (handlerRes.data === undefined) {
+                        if (handlerRes.status !== 204) {
+                            const detail =
+                                "Response schema is defined but handler returned no data";
+                            log.warn(`[${req.reqId}] ${methodAndRoute}: Bad response - ${detail}`);
 
-                    if (!valid) {
-                        const formattedErrors = formatValidationErrors(validateRes.errors, handlerRes.data);
-                        log.warn(`[${req.reqId}] ${methodAndRoute}: Bad response\n${formattedErrors}`);
+                            const errorResponse: FlinkResponse<FlinkError> = {
+                                status: 500,
+                                error: { id: v4(), title: "Bad response", detail },
+                            };
 
-                        return res.status(500).json({
-                            status: 500,
-                            error: {
-                                id: v4(),
-                                title: "Bad response",
-                                detail: formattedErrors,
-                            },
-                        });
+                            this.invokeOnError(errorResponse, req as FlinkRequest, method!, routeProps.path);
+
+                            return res.status(500).json(errorResponse);
+                        }
+                    } else {
+                        const valid = validateRes(JSON.parse(JSON.stringify(handlerRes.data)));
+
+                        if (!valid) {
+                            const formattedErrors = formatValidationErrors(validateRes.errors, handlerRes.data);
+                            log.warn(`[${req.reqId}] ${methodAndRoute}: Bad response\n${formattedErrors}`);
+
+                            const errorResponse: FlinkResponse<FlinkError> = {
+                                status: 500,
+                                error: {
+                                    id: v4(),
+                                    title: "Bad response",
+                                    detail: formattedErrors,
+                                },
+                            };
+
+                            this.invokeOnError(errorResponse, req as FlinkRequest, method!, routeProps.path);
+
+                            return res.status(500).json(errorResponse);
+                        }
                     }
                 }
 
                 res.set(handlerRes.headers);
 
+                if (routeProps.responseType) {
+                    return res
+                        .status(handlerRes.status || 200)
+                        .type(routeProps.responseType)
+                        .send(handlerRes.data);
+                }
+
+                if (handlerRes.error?.meta !== undefined) {
+                    try {
+                        JSON.stringify(handlerRes.error.meta);
+                    } catch (e) {
+                        log.warn(`[${handlerRes.reqId}] error.meta stripped from error ${handlerRes.error.id}: not JSON-serializable (${(e as Error).message})`);
+                        delete handlerRes.error.meta;
+                    }
+                }
+
                 res.status(handlerRes.status || 200).json(handlerRes);
             });
 
@@ -684,11 +909,224 @@ export class FlinkApp<C extends FlinkContext> {
                 return process.exit(1); // TODO: Do we need to exit?
             } else {
                 this.handlerRouteCache.set(methodAndRoute, JSON.stringify(routeProps));
-                log.info(`Registered route ${methodAndRoute}`);
+                initLog.info(`Registered ${streamFormat ? "streaming " : ""}route ${methodAndRoute}${streamFormat ? ` (${streamFormat})` : ""}`);
+            }
+        }
+    }
+
+    /**
+     * Load schema manifest from .flink directory.
+     * Returns empty structure if manifest doesn't exist (dev mode without build).
+     *
+     * The manifest contains:
+     * - definitions: ALL JSON Schema type definitions (supports $ref resolution)
+     * - handlers: Handler metadata with schema names (references to definitions)
+     * - tools: Tool metadata with schema names (references to definitions)
+     */
+    private loadSchemaManifest(): {
+        version?: string;
+        definitions?: Record<string, any>;
+        schemas?: Record<string, any>;
+        handlers: Record<
+            string,
+            {
+                reqSchemaName?: string;
+                resSchemaName?: string;
+                queryMetadata?: any[];
+                paramsMetadata?: any[];
+                assumedMethod?: string;
+            }
+        >;
+        tools: Record<
+            string,
+            {
+                inputSchemaName?: string;
+                outputSchemaName?: string;
+                inputTypeHint?: "void" | "any" | "named";
+                outputTypeHint?: "void" | "any" | "named";
+            }
+        >;
+    } {
+        // Return cached manifest if already loaded
+        if (this.schemaManifest) {
+            return this.schemaManifest;
+        }
+
+        const fs = require("fs");
+        const path = require("path");
+        const manifestPath = path.join(process.cwd(), "dist/.flink/schema-manifest.json");
+
+        if (!fs.existsSync(manifestPath)) {
+            log.warn("Schema manifest not found at dist/.flink/schema-manifest.json - handlers/tools may not have validation schemas");
+            const emptyManifest = { definitions: {}, handlers: {}, tools: {} };
+            this.schemaManifest = emptyManifest;
+            return emptyManifest;
+        }
+
+        try {
+            const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+
+            // Check version for backward compatibility
+            if (manifest.version === "2.0") {
+                // New format with schema universe and AJV global registry
+                this.schemaManifest = manifest;
+
+                // Initialize AJV and register all schemas
+                this.schemaAjv = new Ajv({
+                    strict: false, // Allow additional properties by default
+                    allErrors: true, // Return all validation errors, not just first
+                });
+                addFormats(this.schemaAjv);
+
+                // Register all schemas in the universe
+                const schemas = manifest.schemas || {};
+                for (const schema of Object.values(schemas)) {
+                    if (schema && typeof schema === "object" && (schema as any).$id) {
+                        // Skip schemas with unresolved generic type parameters (T, U, V, etc.)
+                        if (this.hasUnresolvedTypeParams(schema, Object.keys(schemas))) {
+                            log.debug(`Skipping registration of generic schema: ${(schema as any).$id}`);
+                            continue;
+                        }
+                        this.schemaAjv.addSchema(schema);
+                    }
+                }
+
+                log.debug(
+                    `Loaded schema manifest v2.0: ${Object.keys(schemas).length} schemas, ${Object.keys(manifest.handlers).length} handlers, ${
+                        Object.keys(manifest.tools).length
+                    } tools`
+                );
+
+                return manifest;
+            } else {
+                // Old format (v1.0) - still supported for migration
+                log.debug(
+                    `Loaded schema manifest v1.0: ${Object.keys(manifest.definitions || {}).length} definitions, ${Object.keys(manifest.handlers).length} handlers, ${
+                        Object.keys(manifest.tools).length
+                    } tools`
+                );
+                this.schemaManifest = manifest;
+                return manifest;
             }
+        } catch (error) {
+            log.error("Failed to parse schema manifest:", error);
+            const errorManifest = { definitions: {}, handlers: {}, tools: {} };
+            this.schemaManifest = errorManifest;
+            return errorManifest;
         }
     }
 
+    /**
+     * Get the AJV instance for validation (v2.0 manifests).
+     * Returns undefined for v1.0 manifests.
+     */
+    public getSchemaAjv(): Ajv | undefined {
+        return this.schemaAjv;
+    }
+
+    /**
+     * Register plugin schemas into the app's AJV instance.
+     * Schema $id values are prefixed with `pluginId::` to avoid collisions
+     * with the app's own schemas or other plugins.
+     *
+     * @param pluginId Unique identifier for the plugin (used as namespace prefix)
+     * @param schemas Record of schema name to JSON Schema object (already prefixed)
+     */
+    public registerSchemas(pluginId: string, schemas: Record<string, any>): void {
+        // Ensure schema manifest and AJV are initialized
+        this.loadSchemaManifest();
+
+        if (!this.schemaAjv) {
+            // If no v2.0 manifest exists, create a fresh AJV instance
+            this.schemaAjv = new Ajv({
+                strict: false,
+                allErrors: true,
+            });
+            addFormats(this.schemaAjv);
+        }
+
+        for (const schema of Object.values(schemas)) {
+            if (schema && typeof schema === "object" && schema.$id) {
+                // Skip if already registered
+                if (this.schemaAjv.getSchema(schema.$id)) {
+                    continue;
+                }
+                try {
+                    this.schemaAjv.addSchema(schema);
+                } catch (err) {
+                    log.warn(`Failed to register plugin schema ${schema.$id}: ${err}`);
+                }
+            }
+        }
+
+        log.debug(`Registered ${Object.keys(schemas).length} schemas from plugin '${pluginId}'`);
+    }
+
+    /**
+     * Check if a schema has unresolved generic type parameter references.
+     * Generic type parameters like T, U, V are single uppercase letters.
+     *
+     * @param schema JSON schema object
+     * @param registeredSchemaIds List of schema IDs in the manifest
+     * @returns true if schema references generic type params that don't exist
+     */
+    private hasUnresolvedTypeParams(schema: any, registeredSchemaIds: string[]): boolean {
+        const schemaIdSet = new Set(registeredSchemaIds);
+
+        const checkRefs = (obj: any): boolean => {
+            if (!obj || typeof obj !== "object") {
+                return false;
+            }
+
+            if (Array.isArray(obj)) {
+                return obj.some((item) => checkRefs(item));
+            }
+
+            for (const [key, value] of Object.entries(obj)) {
+                if (key === "$ref" && typeof value === "string") {
+                    // Check if ref is a single uppercase letter (generic type param)
+                    // and not in the registered schema IDs
+                    if (/^[A-Z]$/.test(value) && !schemaIdSet.has(value)) {
+                        return true;
+                    }
+                } else if (typeof value === "object") {
+                    if (checkRefs(value)) {
+                        return true;
+                    }
+                }
+            }
+
+            return false;
+        };
+
+        return checkRefs(schema);
+    }
+
+    /**
+     * Resolve schema by name from manifest.
+     * Works with both v1.0 (definitions) and v2.0 (schema universe) formats.
+     *
+     * @param schemaName Schema name or $id
+     * @returns JSON schema object or undefined
+     */
+    private resolveSchema(schemaName: string | undefined): any {
+        if (!schemaName) return undefined;
+
+        const manifest = this.loadSchemaManifest();
+
+        // v2.0 manifest - return schema from universe
+        if (manifest.version === "2.0" && manifest.schemas) {
+            return manifest.schemas[schemaName];
+        }
+
+        // v1.0 manifest - return from definitions
+        if (manifest.definitions) {
+            return manifest.definitions[schemaName];
+        }
+
+        return undefined;
+    }
+
     /**
      * Register handlers found within the `/src/handlers`
      * directory in Flink App.
@@ -696,30 +1134,70 @@ export class FlinkApp<C extends FlinkContext> {
      * Will not register any handlers added programmatically.
      */
     private async registerAutoRegisterableHandlers() {
-        for (const { handler, assumedHttpMethod } of autoRegisteredHandlers.sort((a, b) => (a.handler.Route?.order || 0) - (b.handler.Route?.order || 0))) {
+        // Load schema manifest once
+        const schemaManifest = this.loadSchemaManifest();
+        const schemaCount =
+            schemaManifest.version === "2.0" ? Object.keys(schemaManifest.schemas || {}).length : Object.keys(schemaManifest.definitions || {}).length;
+
+        log.debug(`Registering ${schemaCount} schemas with AJV (manifest version: ${schemaManifest.version || "1.0"})`);
+
+        for (const { handler, assumedHttpMethod, __file } of autoRegisteredHandlers.sort((a, b) => {
+            const orderDiff = (a.handler.Route?.order || 0) - (b.handler.Route?.order || 0);
+            if (orderDiff !== 0) return orderDiff;
+            // Static segments must be registered before parameterized ones to avoid
+            // Express matching e.g. GET /jobs/by-tags with the /jobs/:id route.
+            const aHasParam = a.handler.Route?.path?.includes("/:") ? 1 : 0;
+            const bHasParam = b.handler.Route?.path?.includes("/:") ? 1 : 0;
+            return aHasParam - bHasParam;
+        })) {
             if (!handler.Route) {
-                log.error(`Missing Props in handler ${handler.__file}`);
+                log.error(`Missing Props in handler ${__file}`);
                 continue;
             }
 
             if (!handler.default) {
-                log.error(`Missing exported handler function in handler ${handler.__file}`);
+                log.error(`Missing exported handler function in handler ${__file}`);
                 continue;
             }
 
-            if (!!handler.__params?.length) {
+            // Look up ALL metadata by file path from the schema manifest
+            const metadata = schemaManifest.handlers[__file || ""];
+
+            // Use direct schemas from Route if provided, otherwise resolve from manifest
+            const reqSchema = handler.Route.reqSchema || this.resolveSchema(metadata?.reqSchemaName);
+            const resSchema = handler.Route.resSchema || this.resolveSchema(metadata?.resSchemaName);
+
+            // Validation warnings
+            if (
+                !metadata &&
+                (handler.Route.validation === ValidationMode.Validate ||
+                    handler.Route.validation === ValidationMode.ValidateRequest ||
+                    handler.Route.validation === ValidationMode.ValidateResponse)
+            ) {
+                log.warn(`Handler ${__file} expects validation but no metadata found in manifest`);
+            }
+
+            // Warn if schema name doesn't resolve
+            if (metadata?.reqSchemaName && !reqSchema) {
+                log.warn(`Handler ${__file} references reqSchema "${metadata.reqSchemaName}" but not found in schema universe`);
+            }
+            if (metadata?.resSchemaName && !resSchema) {
+                log.warn(`Handler ${__file} references resSchema "${metadata.resSchemaName}" but not found in schema universe`);
+            }
+
+            if (!!metadata?.paramsMetadata?.length) {
                 const pathParams = getPathParams(handler.Route.path);
 
-                for (const param of handler.__params) {
+                for (const param of metadata.paramsMetadata) {
                     if (!pathParams.includes(param.name)) {
-                        log.error(`Handler ${handler.__file} has param ${param.name} but it is not present in the path '${handler.Route.path}'`);
+                        log.error(`Handler ${__file} has param ${param.name} but it is not present in the path '${handler.Route.path}'`);
                         throw new Error("Invalid/missing handler path param");
                     }
                 }
 
-                if (pathParams.length !== handler.__params.length) {
+                if (pathParams.length !== metadata.paramsMetadata.length) {
                     log.warn(
-                        `Handler ${handler.__file} has ${handler.__params.length} typed params but the path '${handler.Route.path}' has ${pathParams.length} params`
+                        `Handler ${__file} has ${metadata.paramsMetadata.length} typed params but the path '${handler.Route.path}' has ${pathParams.length} params`
                     );
                 }
             }
@@ -728,55 +1206,58 @@ export class FlinkApp<C extends FlinkContext> {
                 {
                     routeProps: {
                         ...handler.Route,
-                        method: handler.Route.method || assumedHttpMethod,
+                        method: handler.Route.method || assumedHttpMethod || metadata?.assumedMethod,
                         origin: this.name,
                     },
                     schema: {
-                        reqSchema: handler.__schemas?.reqSchema,
-                        resSchema: handler.__schemas?.resSchema,
+                        reqSchema,
+                        resSchema,
                     },
-                    queryMetadata: handler.__query || [],
-                    paramsMetadata: handler.__params || [],
+                    queryMetadata: metadata?.queryMetadata || [],
+                    paramsMetadata: metadata?.paramsMetadata || [],
                 },
                 handler.default
             );
         }
     }
 
-    private async registerAutoRegisterableJobs() {
+    private async registerAutoRegisterableJobs(filter?: (jobProps: FlinkJobProps) => boolean) {
         if (!this.scheduler) {
             throw new Error("Scheduler not initialized"); // should never happen
         }
 
         for (const { Job: jobProps, default: jobFn, __file } of autoRegisteredJobs) {
+            if (filter && !filter(jobProps)) {
+                continue;
+            }
             if (jobProps.cron && jobProps.interval) {
-                log.error(`Cannot register job ${jobProps.id} - both cron and interval are set in ${__file}`);
+                schedulerLog.error(`Cannot register job ${jobProps.id} - both cron and interval are set in ${__file}`);
                 continue;
             }
 
             if (jobProps.cron && jobProps.afterDelay) {
-                log.error(`Cannot register job ${jobProps.id} - both cron and afterDelay are set in ${__file}`);
+                schedulerLog.error(`Cannot register job ${jobProps.id} - both cron and afterDelay are set in ${__file}`);
                 continue;
             }
 
             if (jobProps.interval && jobProps.afterDelay) {
-                log.error(`Cannot register job ${jobProps.id} - both interval and afterDelay are set in ${__file}`);
+                schedulerLog.error(`Cannot register job ${jobProps.id} - both interval and afterDelay are set in ${__file}`);
                 continue;
             }
 
             if (this.scheduler.existsById(jobProps.id)) {
-                log.error(`Job with id ${jobProps.id} is already registered, found duplicate in ${__file}`);
+                schedulerLog.error(`Job with id ${jobProps.id} is already registered, found duplicate in ${__file}`);
                 continue;
             }
 
-            log.debug(`Registering job ${jobProps.id}: ${JSON.stringify(jobProps)} from ${__file}`);
+            schedulerLog.debug(`Registering job ${jobProps.id}: ${JSON.stringify(jobProps)} from ${__file}`);
 
             const task = new AsyncTask(
                 jobProps.id,
                 async () => {
                     await jobFn({ ctx: this.ctx });
 
-                    log.debug(`Job ${jobProps.id} completed`);
+                    schedulerLog.debug(`Job ${jobProps.id} completed`);
 
                     if (jobProps.afterDelay) {
                         // afterDelay runs only once, so we remove the job
@@ -784,7 +1265,7 @@ export class FlinkApp<C extends FlinkContext> {
                     }
                 },
                 (err) => {
-                    log.error(`Job ${jobProps.id} threw unhandled exception ${err}`);
+                    schedulerLog.error(`Job ${jobProps.id} threw unhandled exception ${err}`);
                     console.error(err);
                 }
             );
@@ -811,20 +1292,32 @@ export class FlinkApp<C extends FlinkContext> {
 
                 this.scheduler.addSimpleIntervalJob(job);
             } else if (jobProps.afterDelay !== undefined) {
-                const job = new SimpleIntervalJob(
-                    {
-                        milliseconds: ms(jobProps.afterDelay),
-                        runImmediately: false,
-                    },
-                    task,
-                    {
-                        id: jobProps.id,
-                        preventOverrun: jobProps.singleton,
-                    }
-                );
-                this.scheduler.addSimpleIntervalJob(job);
+                const delayMs = ms(jobProps.afterDelay);
+                if (delayMs === 0) {
+                    setImmediate(async () => {
+                        try {
+                            await jobFn({ ctx: this.ctx });
+                        } catch (err) {
+                            schedulerLog.error(`Job ${jobProps.id} threw unhandled exception ${err}`);
+                            console.error(err);
+                        }
+                    });
+                } else {
+                    const job = new SimpleIntervalJob(
+                        {
+                            milliseconds: delayMs,
+                            runImmediately: false,
+                        },
+                        task,
+                        {
+                            id: jobProps.id,
+                            preventOverrun: jobProps.singleton,
+                        }
+                    );
+                    this.scheduler.addSimpleIntervalJob(job);
+                }
             } else {
-                log.error(`Cannot register job ${jobProps.id} - no cron, interval or once set in ${__file}`);
+                schedulerLog.error(`Cannot register job ${jobProps.id} - no cron, interval or once set in ${__file}`);
                 continue;
             }
         }
@@ -836,6 +1329,142 @@ export class FlinkApp<C extends FlinkContext> {
         // repoInstance.ctx = this.ctx;
     }
 
+    private async registerAutoRegisterableTools() {
+        const { ToolExecutor } = require("./ai/ToolExecutor");
+        const { getRepoInstanceName } = require("./utils");
+
+        // Load schema manifest once
+        const schemaManifest = this.loadSchemaManifest();
+
+        for (const toolFile of autoRegisteredTools) {
+            if (!toolFile.Tool) {
+                log.error(`Missing FlinkToolProps export in tool ${toolFile.__file}`);
+                continue;
+            }
+
+            if (!toolFile.default) {
+                log.error(`Missing exported tool function in tool ${toolFile.__file}`);
+                continue;
+            }
+
+            const toolId = toolFile.Tool.id;
+            if (!toolId) {
+                log.error(`Tool ${toolFile.__file} missing 'id' property`);
+                continue;
+            }
+
+            const toolInstanceName = getRepoInstanceName(toolId);
+
+            // Look up schema names from manifest
+            const metadata = schemaManifest.tools[toolFile.__file || ""];
+
+            // Resolve schemas using helper (works with both v1.0 and v2.0)
+            const schemas = metadata
+                ? {
+                      inputSchema: this.resolveSchema(metadata.inputSchemaName),
+                      outputSchema: this.resolveSchema(metadata.outputSchemaName),
+                      inputTypeHint: metadata.inputTypeHint,
+                      outputTypeHint: metadata.outputTypeHint,
+                  }
+                : undefined;
+
+            // Warn if schema name doesn't resolve
+            if (metadata?.inputSchemaName && !schemas?.inputSchema) {
+                log.warn(`Tool ${toolFile.__file} references inputSchema "${metadata.inputSchemaName}" but not found in schema universe`);
+            }
+            if (metadata?.outputSchemaName && !schemas?.outputSchema) {
+                log.warn(`Tool ${toolFile.__file} references outputSchema "${metadata.outputSchemaName}" but not found in schema universe`);
+            }
+
+            // Pass full schema universe so AJV can resolve $ref across schemas
+            const allSchemas = schemaManifest.version === "2.0" ? schemaManifest.schemas : schemaManifest.definitions;
+
+            const toolExecutor = new ToolExecutor(
+                toolFile.Tool,
+                toolFile.default,
+                this.ctx,
+                schemas, // Auto-generated schemas from manifest (resolved from definitions)
+                allSchemas
+            );
+            this.tools[toolInstanceName] = toolExecutor;
+
+            initLog.info(`Registered tool ${toolInstanceName} (${toolId})`);
+        }
+    }
+
+    private async registerAutoRegisterableAgents() {
+        const { getRepoInstanceName, toKebabCase } = require("./utils");
+
+        for (const agentFile of autoRegisteredAgents) {
+            // agentFile now exports a class, not a config object
+            const AgentClass = agentFile.default;
+
+            if (!AgentClass) {
+                log.error(`Missing default export in agent ${agentFile.__file}`);
+                continue;
+            }
+
+            // Instantiate agent (similar to repo instantiation)
+            const agentInstance = new AgentClass();
+
+            // Derive instance name from class name (camelCase)
+            const agentInstanceName = getRepoInstanceName(AgentClass.name);
+
+            // Get agent ID (kebab-case) - either explicit or derived
+            const agentId = agentInstance.id;
+
+            // Check for duplicate instance name
+            if (this.agents[agentInstanceName]) {
+                const existingAgent = this.agents[agentInstanceName];
+                throw new Error(
+                    `Duplicate agent instance name: "${agentInstanceName}". ` +
+                        `Agent class "${AgentClass.name}" conflicts with existing agent "${existingAgent.constructor.name}". ` +
+                        `Instance names are derived by lowercasing the first letter of the class name. ` +
+                        `Rename one of the classes or use a unique explicit 'id' property.`
+                );
+            }
+
+            // Check for duplicate agent ID
+            const existingAgentWithSameId = Object.values(this.agents).find((agent: any) => agent.id === agentId);
+
+            if (existingAgentWithSameId) {
+                throw new Error(
+                    `Duplicate agent ID: "${agentId}". ` +
+                        `Agent class "${AgentClass.name}" conflicts with existing agent "${existingAgentWithSameId.constructor.name}". ` +
+                        `Change the 'id' property on one of them to resolve the conflict.`
+                );
+            }
+
+            // Validate tools exist
+            if (agentInstance.tools) {
+                for (const toolRef of agentInstance.tools) {
+                    // Handle string IDs, tool file references, and tool props
+                    let toolId: string;
+                    if (typeof toolRef === "string") {
+                        toolId = toolRef;
+                    } else if ("Tool" in toolRef) {
+                        // FlinkToolFile - extract ID from Tool property
+                        toolId = toolRef.Tool.id;
+                    } else {
+                        // FlinkToolProps - extract ID directly
+                        toolId = toolRef.id;
+                    }
+
+                    const tool = this.tools[toolId];
+
+                    if (!tool) {
+                        log.error(`Agent ${AgentClass.name} references tool ${toolId} which is not registered`);
+                        throw new Error(`Invalid tool reference in agent ${AgentClass.name}`);
+                    }
+                }
+            }
+
+            // Register agent (duplicate checks already performed above)
+            this.agents[agentInstanceName] = agentInstance;
+            initLog.info(`Registered agent ${agentInstanceName} (${AgentClass.name}) with ID: ${agentId}`);
+        }
+    }
+
     /**
      * Constructs the app context. Will inject context in all components
      * except for handlers which are handled in later stage.
@@ -847,7 +1476,7 @@ export class FlinkApp<C extends FlinkContext> {
 
                 this.repos[repoInstanceName] = repoInstance;
 
-                log.info(`Registered repo ${repoInstanceName}`);
+                initLog.info(`Registered repo ${repoInstanceName}`);
             }
         } else if (autoRegisteredRepos.length > 0) {
             log.warn(`No db configured but found repo(s)`);
@@ -861,15 +1490,47 @@ export class FlinkApp<C extends FlinkContext> {
             return out;
         }, {});
 
+        // Instantiate services (ctx not yet available - constructors must not access it)
+        for (const { serviceInstanceName, Service } of autoRegisteredServices) {
+            const serviceInstance: FlinkService<C> = new Service();
+            this.services[serviceInstanceName] = serviceInstance;
+            initLog.info(`Registered service ${serviceInstanceName}`);
+        }
+
         this._ctx = {
             repos: this.repos,
             plugins: pluginCtx,
             auth: this.auth,
+            agents: this.agents,
+            services: this.services,
         } as C;
 
+        // Inject context into repos
         for (const repo of Object.values(this.repos)) {
             repo.ctx = this.ctx;
         }
+
+        // Inject context into services, then call onInit() in parallel
+        for (const service of Object.values(this.services)) {
+            service.ctx = this.ctx;
+        }
+
+        const servicesWithInit = Object.values(this.services).filter((s) => typeof s.onInit === "function");
+        if (servicesWithInit.length > 0) {
+            await Promise.all(servicesWithInit.map((s) => s.onInit!()));
+        }
+    }
+
+    /**
+     * Initialize agents after they've been registered and context is ready
+     * Must be called after registerAutoRegisterableAgents()
+     */
+    private async initializeAgents() {
+        // Inject context and initialize agents
+        for (const agent of Object.values(this.agents)) {
+            agent.ctx = this.ctx;
+            agent.__init(this.llmAdapters, this.tools, this.agentObserver);
+        }
     }
 
     /**
@@ -911,7 +1572,7 @@ export class FlinkApp<C extends FlinkContext> {
                 }
             } else if (plugin.db.uri) {
                 try {
-                    log.debug(`Connecting to '${plugin.id}' db`);
+                    initLog.debug(`Connecting to '${plugin.id}' db`);
                     const client = await MongoClient.connect(plugin.db.uri, this.getMongoConnectionOptions());
                     return client.db();
                 } catch (err) {
@@ -925,7 +1586,37 @@ export class FlinkApp<C extends FlinkContext> {
         if (!this.auth) {
             throw new Error(`Attempting to authenticate request (${req.method} ${req.path}) but no authPlugin is set`);
         }
-        return await this.auth.authenticateRequest(req as FlinkRequest, permissions);
+        return await this.auth.authenticateRequest(req as FlinkRequest, permissions, this._ctx);
+    }
+
+    /**
+     * Invokes the optional onError callback in a fire-and-forget manner.
+     * Any error thrown or rejected by the callback is caught and logged so
+     * it never affects the error response sent to the client.
+     */
+    private invokeOnError(errorResponse: FlinkResponse<FlinkError>, req: FlinkRequest, method: HttpMethod, path: string) {
+        if (!this.onError) {
+            return;
+        }
+
+        try {
+            const result = this.onError(errorResponse, {
+                req,
+                method,
+                path,
+                reqId: req.reqId,
+                ctx: this.ctx,
+            });
+
+            // Handle async callbacks - don't wait for them
+            if (result instanceof Promise) {
+                result.catch((callbackErr) => {
+                    log.error(`onError callback rejected with: ${callbackErr}`);
+                });
+            }
+        } catch (callbackErr) {
+            log.error(`onError callback threw an exception: ${callbackErr}`);
+        }
     }
 
     public getRegisteredRoutes() {
@@ -936,6 +1627,56 @@ export class FlinkApp<C extends FlinkContext> {
         return this.schedulingOptions?.enabled !== false;
     }
 
+    private get leaderElectionConfig(): LeaderElectionOptions | undefined {
+        const opt = this.schedulingOptions?.leaderElection;
+        if (!opt) return undefined;
+        return opt === true ? {} : opt;
+    }
+
+    private async startLeaderElection() {
+        if (!this.db) {
+            schedulerLog.warn(
+                "Leader election is enabled but no database is configured. " +
+                    "Leader election requires a MongoDB connection to coordinate between instances. " +
+                    "Either add a database connection via the `db` option, or remove `scheduling.leaderElection` from your config. " +
+                    "Jobs will run on ALL instances without leader election."
+            );
+            // Fall back to running jobs on all instances
+            this.scheduler = new ToadScheduler();
+            await this.registerAutoRegisterableJobs();
+            return;
+        }
+
+        // Register runOnAllInstances jobs immediately on a separate scheduler
+        const hasAllInstanceJobs = autoRegisteredJobs.some((j) => j.Job.runOnAllInstances);
+        if (hasAllInstanceJobs) {
+            this.allInstanceScheduler = new ToadScheduler();
+            this.scheduler = this.allInstanceScheduler;
+            await this.registerAutoRegisterableJobs((job) => !!job.runOnAllInstances);
+            this.scheduler = undefined;
+        }
+
+        const opts = this.leaderElectionConfig;
+        this.leaderElection = new LeaderElection(this.db, opts);
+
+        await this.leaderElection.start(
+            // onBecameLeader
+            async () => {
+                schedulerLog.info("This instance is now the leader - starting scheduled jobs");
+                this.scheduler = new ToadScheduler();
+                await this.registerAutoRegisterableJobs((job) => !job.runOnAllInstances);
+            },
+            // onLostLeadership
+            () => {
+                schedulerLog.info("This instance lost leadership - stopping scheduled jobs");
+                if (this.scheduler) {
+                    this.scheduler.stop();
+                    this.scheduler = undefined;
+                }
+            }
+        );
+    }
+
     private getMongoConnectionOptions() {
         if (!this.dbOpts) {
             throw new Error("No db configured");
@@ -944,14 +1685,14 @@ export class FlinkApp<C extends FlinkContext> {
         const { version: driverVersion } = require("mongodb/package.json");
 
         if (driverVersion.startsWith("3")) {
-            log.debug(`Using legacy mongodb connection options as mongo client is version ${driverVersion}`);
+            initLog.debug(`Using legacy mongodb connection options as mongo client is version ${driverVersion}`);
             return {
                 useNewUrlParser: true,
                 useUnifiedTopology: true,
             };
         }
 
-        log.debug(`Using modern MongoDB client options (driver version ${driverVersion})`);
+        initLog.debug(`Using modern MongoDB client options (driver version ${driverVersion})`);
 
         return {
             serverApi: {