mongodash 2.6.0 → 2.7.0

package/dist/types/cronTasks.d.ts

@@ -1,6 +1,19 @@
 import { CronExpressionOptions } from 'cron-parser';
 export interface InitOptions {
     runCronTasks: boolean;
+    /**
+     * Maximum number of cron tasks this instance will execute in parallel.
+     *
+     * The default of `1` preserves the historical behaviour: one task is
+     * processed at a time per instance. Raise it when you have many
+     * independent cron tasks and want to avoid head-of-line blocking (a
+     * long-running task delaying unrelated ones).
+     *
+     * Tasks with the same id are always serialised via the per-task lock
+     * (`lockedTill`), so raising this does not cause a single task to run
+     * twice in parallel.
+     */
+    cronTaskConcurrency: number;
     cronExpressionParserOptions: CronExpressionOptions;
     cronTaskCaller: CronTaskCaller;
     cronTaskFilter: CronTaskFilter;
@@ -64,8 +77,10 @@ export declare function cronTask(taskId: TaskId, interval: Interval, task: TaskF
  */
 export declare function getCronTasksList(query?: CronTaskQuery): Promise<CronPagedResult<CronTaskRecord>>;
 /**
- * Triggers a cron task immediately.
- * Alias for scheduleCronTaskImmediately but returns the new state or confirmation.
+ * @deprecated Alias for {@link scheduleCronTaskImmediately}. Prefer that name for
+ * clarity - it describes exactly what happens (the task is scheduled to run on
+ * the next polling tick, not necessarily this very millisecond). This alias will
+ * be removed in a future major version.
  */
 export declare function triggerCronTask(taskId: TaskId): Promise<void>;
 /**

package/dist/types/index.d.ts

@@ -10,7 +10,7 @@ export { getCollection } from './getCollection';
 export { getMongoClient } from './getMongoClient';
 export { OnError } from './OnError';
 export { processInBatches, ProcessInBatchesOptions, ProcessInBatchesResult } from './processInBatches';
-export { CODE_REACTIVE_TASK_FAILED, CODE_REACTIVE_TASK_FINISHED, CODE_REACTIVE_TASK_LEADER_LOCK_LOST, CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_FINISHED, CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_STARTED, CODE_REACTIVE_TASK_PLANNER_STARTED, CODE_REACTIVE_TASK_PLANNER_STOPPED, CODE_REACTIVE_TASK_PLANNER_STREAM_ERROR, CODE_REACTIVE_TASK_STARTED, countReactiveTasks, getPrometheusMetrics, getReactiveTasks, reactiveTask, ReactiveTask, ReactiveTaskHandler, retryReactiveTasks, startReactiveTasks, stopReactiveTasks, TaskConditionFailedError, _scheduler, } from './reactiveTasks';
+export { CODE_REACTIVE_TASK_CLEANUP, CODE_REACTIVE_TASK_FAILED, CODE_REACTIVE_TASK_FINISHED, CODE_REACTIVE_TASK_INITIALIZED, CODE_REACTIVE_TASK_LEADER_LOCK_LOST, CODE_REACTIVE_TASK_LOCK_LOST, CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_FINISHED, CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_STARTED, CODE_REACTIVE_TASK_PLANNER_STARTED, CODE_REACTIVE_TASK_PLANNER_STOPPED, CODE_REACTIVE_TASK_PLANNER_STREAM_ERROR, CODE_REACTIVE_TASK_STARTED, countReactiveTasks, getPrometheusMetrics, getReactiveTasks, PagedResult, PaginationOptions, reactiveTask, ReactiveTask, ReactiveTaskHandler, ReactiveTaskQuery, ReactiveTaskRecord, ReactiveTaskStatus, retryReactiveTasks, startReactiveTasks, stopReactiveTasks, TaskConditionFailedError, _scheduler, } from './reactiveTasks';
 export { OperationalTaskController, serveDashboard } from './task-management';
 export * from './testing';
 export { isLockAlreadyAcquiredError, LockAlreadyAcquiredError, withLock, WithLockOptions } from './withLock';

package/dist/types/reactiveTasks/LeaderElector.d.ts

@@ -1,6 +1,6 @@
 import { GlobalsCollection } from '../globalsCollection';
-import { OnInfo } from '../OnInfo';
 import { OnError } from '../OnError';
+import { OnInfo } from '../OnInfo';
 export interface LeaderElectorCallbacks {
     onBecomeLeader: () => Promise<void>;
     onLoseLeader: () => Promise<void>;
@@ -35,6 +35,20 @@ export declare class LeaderElector {
     get isLeader(): boolean;
     start(): Promise<void>;
     stop(): Promise<void>;
+    /**
+     * Give up leadership locally. The DB lock is NOT released - the next
+     * heartbeat will likely re-acquire it (unless another instance raced
+     * in). onLoseLeader is fired asynchronously so callers (e.g. the
+     * scheduler wiring this to a flush-failure path) get a clean
+     * planner.stop() before the next heartbeat restarts it, rather than
+     * starting a new planner on top of a live one.
+     *
+     * Note: the follow-up onBecomeLeader that fires after a forced loss
+     * looks identical to a real leader election and will increment
+     * reactive_tasks_leader_elections_total; see the event codes
+     * CODE_REACTIVE_TASK_PLANNER_STREAM_ERROR and the flush-failure
+     * counter to disambiguate "real" flapping from restart-driven ones.
+     */
     forceLoseLeader(): void;
     private runLeaderElectionLoop;
     private tryAcquireLock;

package/dist/types/reactiveTasks/MetricsCollector.d.ts

@@ -31,6 +31,10 @@ export declare class MetricsCollector {
     private globalStatsRegistry?;
     private metricDuration?;
     private metricRetries?;
+    private metricLeaderElections?;
+    private metricLockLost?;
+    private metricStreamErrors?;
+    private metricFlushFailures?;
     private pushInterval?;
     private queueMetricsPromise;
     planner?: ReactiveTaskPlanner;
@@ -42,6 +46,10 @@ export declare class MetricsCollector {
     stop(): void;
     recordTaskExecution(task: string, status: 'success' | 'failed', durationMs: number): void;
     recordRetry(task: string): void;
+    recordLeaderElection(): void;
+    recordLockLost(task: string): void;
+    recordStreamError(): void;
+    recordFlushFailure(): void;
     getPrometheusMetrics(): Promise<Registry | null>;
     /**
      * Returns aggregated metrics from ALL instances.

package/dist/types/reactiveTasks/ReactiveTaskPlanner.d.ts

@@ -6,6 +6,15 @@ import { ReactiveTaskRegistry } from './ReactiveTaskRegistry';
 export interface PlannerCallbacks {
     onStreamError: () => void;
     onTaskPlanned: (tasksCollectionName: string, debounceMs: number) => void;
+    /** Fired when a batch flush fails. Records the metric and should trigger a planner restart. */
+    onFlushFailure?: () => void;
+    /**
+     * Fired when the planner needs to restart due to a flush failure (distinct from a
+     * real change-stream error). Callers should trigger a leader-election cycle here
+     * instead of reacting to `onStreamError`, so flush failures don't pollute the
+     * stream-error metric.
+     */
+    onRequestRestart?: () => void;
 }
 /**
  * Responsible for listening to MongoDB Change Stream events and planning tasks.
@@ -31,6 +40,7 @@ export declare class ReactiveTaskPlanner {
     private batchFlushTimer;
     private batchFirstEventTime;
     private isFlushing;
+    private lastFlushFailed;
     private metaDocId;
     private lastClusterTime;
     private ops;
@@ -59,6 +69,7 @@ export declare class ReactiveTaskPlanner {
     private groupEventsByCollection;
     private processDeletions;
     private executeUpsertOperations;
+    private throwOnAnyRejection;
     private handleStreamError;
     private checkEvolutionStrategies;
     private checkTriggerEvolution;

package/dist/types/reactiveTasks/ReactiveTaskRepository.d.ts

@@ -24,11 +24,20 @@ export declare class ReactiveTaskRepository<T extends Document> {
     findAndLockNextTask(taskDefs: ReactiveTaskInternal<T>[], options: {
         visibilityTimeoutMs: number;
     }): Promise<ReactiveTaskRecord<T> | null>;
+    /**
+     * Finalize a task record (success or failure). Returns `true` when the
+     * update matched the record, `false` when it did not - which in
+     * practice means another worker has since re-claimed the task (its
+     * startedAt no longer matches) and this call was a no-op.
+     *
+     * Callers that care about the distinction (e.g. to suppress success /
+     * failure metrics for a stolen task) should inspect the return value.
+     */
     finalizeTask(taskRecord: ReactiveTaskRecord<T>, strategy: ReactiveTaskRetryStrategy, error?: Error, debounceMs?: number, executionStats?: {
         durationMs: number;
     }, executionHistoryLimit?: number, options?: {
         session?: import('mongodb').ClientSession;
-    }): Promise<void>;
+    }): Promise<boolean>;
     deferTask(taskRecord: ReactiveTaskRecord<T>, delay: number | Date): Promise<void>;
     executeBulkWrite(operations: Parameters<Collection<ReactiveTaskRecord<T>>['bulkWrite']>[0], options?: CompatibleBulkWriteOptions): Promise<void>;
     findTasks(filter: Filter<ReactiveTaskRecord<T>>, options?: {

package/dist/types/reactiveTasks/ReactiveTaskTypes.d.ts

@@ -336,6 +336,7 @@ export interface ReactiveTaskCaller {
 export declare const CODE_REACTIVE_TASK_STARTED = "reactiveTaskStarted";
 export declare const CODE_REACTIVE_TASK_FINISHED = "reactiveTaskFinished";
 export declare const CODE_REACTIVE_TASK_FAILED = "reactiveTaskFailed";
+export declare const CODE_REACTIVE_TASK_LOCK_LOST = "reactiveTaskLockLost";
 export declare const CODE_REACTIVE_TASK_PLANNER_STARTED = "reactiveTaskPlannerStarted";
 export declare const CODE_REACTIVE_TASK_PLANNER_STOPPED = "reactiveTaskPlannerStopped";
 export declare const CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_STARTED = "reactiveTaskPlannerReconciliationStarted";
@@ -345,6 +346,11 @@ export declare const CODE_REACTIVE_TASK_LEADER_LOCK_LOST = "reactiveTaskLeaderLo
 export declare const CODE_REACTIVE_TASK_INITIALIZED = "reactiveTaskInitialized";
 export declare const CODE_REACTIVE_TASK_CLEANUP = "reactiveTaskCleanup";
 export declare const CODE_MANUAL_TRIGGER = "manualTrigger";
+/**
+ * @internal
+ * Document id used by the planner for its meta document. Exposed for the
+ * dashboard and advanced tooling - not part of the public API contract.
+ */
 export declare const REACTIVE_TASK_META_DOC_ID = "_mongodash_planner_meta";
 /**
  * Filter for querying tasks.

package/dist/types/reactiveTasks/index.d.ts

@@ -6,7 +6,13 @@ import { ReactiveTaskManager } from './ReactiveTaskManager';
 import { ReactiveTaskPlanner } from './ReactiveTaskPlanner';
 import { ReactiveTaskRegistry } from './ReactiveTaskRegistry';
 import { PagedResult, PaginationOptions, ReactiveTask, ReactiveTaskQuery, ReactiveTaskRecord, ReactiveTaskSchedulerOptions } from './ReactiveTaskTypes';
-export { CODE_REACTIVE_TASK_CLEANUP, CODE_REACTIVE_TASK_FAILED, CODE_REACTIVE_TASK_FINISHED, CODE_REACTIVE_TASK_INITIALIZED, CODE_REACTIVE_TASK_LEADER_LOCK_LOST, CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_FINISHED, CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_STARTED, CODE_REACTIVE_TASK_PLANNER_STARTED, CODE_REACTIVE_TASK_PLANNER_STOPPED, CODE_REACTIVE_TASK_PLANNER_STREAM_ERROR, CODE_REACTIVE_TASK_STARTED, PagedResult, PaginationOptions, ReactiveTask, ReactiveTaskCaller, ReactiveTaskFilter, ReactiveTaskHandler, ReactiveTaskQuery, ReactiveTaskRecord, ReactiveTaskSchedulerOptions, ReactiveTaskStatus, REACTIVE_TASK_META_DOC_ID, TaskConditionFailedError, } from './ReactiveTaskTypes';
+export { CODE_REACTIVE_TASK_CLEANUP, CODE_REACTIVE_TASK_FAILED, CODE_REACTIVE_TASK_FINISHED, CODE_REACTIVE_TASK_INITIALIZED, CODE_REACTIVE_TASK_LEADER_LOCK_LOST, CODE_REACTIVE_TASK_LOCK_LOST, CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_FINISHED, CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_STARTED, CODE_REACTIVE_TASK_PLANNER_STARTED, CODE_REACTIVE_TASK_PLANNER_STOPPED, CODE_REACTIVE_TASK_PLANNER_STREAM_ERROR, CODE_REACTIVE_TASK_STARTED, PagedResult, PaginationOptions, ReactiveTask, ReactiveTaskCaller, ReactiveTaskFilter, ReactiveTaskHandler, ReactiveTaskQuery, ReactiveTaskRecord, ReactiveTaskSchedulerOptions, ReactiveTaskStatus, REACTIVE_TASK_META_DOC_ID, TaskConditionFailedError, } from './ReactiveTaskTypes';
+/**
+ * @internal
+ * Exported only for the built-in OperationalTaskController / dashboard bridge
+ * and for advanced testing. Not part of the public API contract: fields and
+ * methods on the scheduler instance can change between minor versions.
+ */
 export { scheduler as _scheduler };
 export type InitOptions = {
     globalsCollection: GlobalsCollection;
@@ -54,7 +60,7 @@ export declare class ReactiveTaskScheduler {
         debounce?: number;
     }): void;
     get forceDebounce(): number | string | undefined;
-    addTask(taskDef: ReactiveTask<Document>): Promise<void>;
+    addTask<T extends Document>(taskDef: ReactiveTask<T>): Promise<void>;
     /**
      * Starts the entire system - leader election and workers.
      */

package/dist/types/testing/assertNoReactiveTaskErrors.d.ts

@@ -1,4 +1,5 @@
 import { ReactiveTaskScheduler } from '../reactiveTasks';
+import { WhitelistRule } from './resolveWhitelistFilter';
 export interface AssertNoReactiveTaskErrorsOptions {
     /**
      * Check for errors occurring after this time.
@@ -6,11 +7,10 @@ export interface AssertNoReactiveTaskErrorsOptions {
      */
     since: Date;
     /**
-     * Optional: Check only tasks related to these specific source documents.
-     * Useful when other tests might be generating noise in the background.
-     * Supports generic ID types (ObjectId, string, number).
+     * Optional: Check only tasks related to specific entities.
+     * If provided, errors in collections/tasks not matching the whitelist are ignored.
      */
-    sourceDocIds?: unknown[];
+    whitelist?: WhitelistRule[];
     /**
      * Optional: Whitelist specific errors.
      * If a string is provided, exact match is required.

package/dist/types/testing/index.d.ts

@@ -1,3 +1,5 @@
 export * from './assertNoReactiveTaskErrors';
 export * from './configureForTesting';
+export * from './resolveWhitelistFilter';
+export * from './waitUntil';
 export * from './waitUntilReactiveTasksIdle';

package/dist/types/testing/resolveWhitelistFilter.d.ts

@@ -0,0 +1,35 @@
+import { Collection, Document, Filter } from 'mongodb';
+import { ReactiveTaskRecord } from '../reactiveTasks';
+/**
+ * A single rule used by the testing utilities to scope checks to a set of
+ * source documents.
+ */
+export interface WhitelistRule {
+    collection: string;
+    /**
+     * Filter to find relevant source documents. When omitted every document
+     * in the collection is considered.
+     */
+    filter?: Filter<Document>;
+    /**
+     * Optional: restrict to a specific reactive task name.
+     */
+    task?: string;
+}
+/**
+ * Resolution outcome for a whitelist against one registry entry.
+ *
+ * - `'skip'`: the whitelist has rules, but none apply to this collection or
+ *   the source filters matched zero documents. Callers should skip this
+ *   entry entirely.
+ * - `'matchAll'`: at least one rule for this collection wants the full
+ *   collection. Callers should apply no extra filter.
+ * - An object: the caller should AND this filter with its base query.
+ */
+export type WhitelistResolution = 'skip' | 'matchAll' | Filter<ReactiveTaskRecord>;
+/**
+ * Build the `Filter<ReactiveTaskRecord>` for a single registry entry based on
+ * the provided whitelist rules. Extracted from `waitUntilReactiveTasksIdle` /
+ * `assertNoReactiveTaskErrors` so the two utilities cannot drift.
+ */
+export declare function resolveWhitelistFilter(whitelist: WhitelistRule[], sourceCollection: Pick<Collection<Document>, 'collectionName' | 'find'>): Promise<WhitelistResolution>;

package/dist/types/testing/waitUntilReactiveTasksIdle.d.ts

@@ -1,4 +1,4 @@
-import { Document, Filter } from 'mongodb';
+import { WhitelistRule } from './resolveWhitelistFilter';
 import { WaitUntilOptions } from './waitUntil';
 /**
  * Waits until the reactive task system is idle.
@@ -8,6 +8,11 @@ import { WaitUntilOptions } from './waitUntil';
  * 3. No tasks in the database are in a pending or processing state.
  *
  * This enables robust E2E testing by ensuring that all side effects and cascading tasks have finished.
+ *
+ * @remarks
+ * Pending tasks scheduled far in the future (beyond `timeoutMs + stabilityDurationMs + 100ms`)
+ * are treated as "future work" and ignored. This prevents long-running retries (e.g. exponential backoff
+ * pushing `nextRunAt` hours ahead) from blocking the idle check forever.
  */
 export interface WaitUntilReactiveTasksIdleOptions extends Partial<WaitUntilOptions> {
     /**
@@ -15,17 +20,6 @@ export interface WaitUntilReactiveTasksIdleOptions extends Partial<WaitUntilOpti
      * Global checks (Planner buffer, Active workers) are SKIPPED in this mode to ensure isolation
      * from other running tests.
      */
-    whitelist?: Array<{
-        collection: string;
-        /**
-         * Filter to find relevant documents.
-         * If not provided, ALL documents in the collection are considered (use carefully!).
-         */
-        filter?: Filter<Document>;
-        /**
-         * Optional task name filter.
-         */
-        task?: string;
-    }>;
+    whitelist?: WhitelistRule[];
 }
 export declare function waitUntilReactiveTasksIdle(customOptions?: WaitUntilReactiveTasksIdleOptions): Promise<void>;

package/docs/.vitepress/config.mts

@@ -53,7 +53,15 @@ export default defineConfig({
                 text: 'Utilities',
                 items: [
                     { text: 'Process In Batches', link: '/process-in-batches' },
-                    { text: 'Getters', link: '/getters' }
+                    { text: 'Getters', link: '/getters' },
+                    { text: 'Error Handling', link: '/error-handling' }
+                ]
+            },
+            {
+                text: 'Testing',
+                items: [
+                    { text: 'Overview', link: '/testing' },
+                    { text: 'Testing Reactive Tasks', link: '/reactive-tasks/testing' }
                 ]
             }
         ],

package/docs/cron-tasks.md

@@ -135,10 +135,15 @@ import mongodash from 'mongodash';
 mongodash.init({
     // database connection
     uri: 'mongodb://mongodb0.example.com:27017',
-    
+
     // true by default
     runCronTasks: false,
 
+    // Maximum number of cron tasks this instance executes in parallel.
+    // Default 1 (serial). See the "Parallel execution within one instance"
+    // section earlier on this page.
+    cronTaskConcurrency: 5,
+
     // valid only if CRON expressions used
     // see https://www.npmjs.com/package/cron-parser for valid options
     cronExpressionParserOptions: {
@@ -170,3 +175,127 @@ The system handles concurrency by locking tasks in MongoDB.
 The system maintains a brief execution history in the database:
 - **Limit**: Only the **last 5 runs** are stored in the `runLog` of the task document.
 - Use this to monitor recent successes or failures.
+
+### Parallel execution within one instance
+
+By default each instance runs one cron task at a time. When you have many
+independent cron tasks and a single long-running one would block the
+others, opt in to parallel execution:
+
+```typescript
+await mongodash.init({
+    // ...
+    cronTaskConcurrency: 5, // up to 5 cron tasks in flight on this instance
+});
+```
+
+- A single task can **never** run twice in parallel, regardless of the
+  value. The per-task `lockedTill` lock guarantees that even within one
+  instance — and across instances — only one execution of a given
+  `taskId` is in flight at a time.
+- `cronTaskConcurrency: 1` (the default) keeps the historical single-loop
+  behaviour.
+- Raising the value only affects *different* tasks running at the same
+  time. Use it when you see head-of-line blocking on the cron collection.
+
+## Monitoring
+
+Cron tasks emit structured events through the `onInfo` callback. Each event
+has a stable `code` that you can route to your logging stack without
+parsing strings.
+
+| Code constant | When it fires | Payload |
+| :--- | :--- | :--- |
+| `CODE_CRON_TASK_STARTED` | Handler is about to be invoked. Also fired once during `init` to announce that cron processing has begun. | `{ taskId, code }` |
+| `CODE_CRON_TASK_FINISHED` | Handler returned without throwing. | `{ taskId, code, duration }` |
+| `CODE_CRON_TASK_FAILED` | Handler threw. The same error is also passed to `onError`. | `{ taskId, code, reason, duration }` |
+| `CODE_CRON_TASK_SCHEDULED` | The task has been scheduled for its next run. | `{ taskId, code, nextRunDate }` |
+
+```typescript
+import { CODE_CRON_TASK_FAILED } from 'mongodash';
+
+await mongodash.init({
+    onInfo: (event) => {
+        if (event.code === CODE_CRON_TASK_FAILED) {
+            metrics.increment('cron.failed', { task: event.taskId });
+        }
+    },
+});
+```
+
+See also [**Error Handling**](./error-handling.md) for how `onError` and
+`onInfo` compose.
+
+## Task Management
+
+### getCronTasksList(query?) => Promise<CronPagedResult\<CronTaskRecord\>>
+
+Inspect the state of registered tasks - useful for admin UIs, health
+checks, or integration tests.
+
+```typescript
+import { getCronTasksList } from 'mongodash';
+
+const page = await getCronTasksList({
+    filter: 'daily', // regex match against taskId (case-insensitive)
+    limit: 20,
+    skip: 0,
+    sort: { field: 'nextRunAt', direction: 1 },
+});
+
+for (const task of page.items) {
+    console.log(task._id, task.status, task.lastRun?.error);
+}
+```
+
+`status` can be `'idle'`, `'running'` (lock held), `'scheduled'`
+(manual trigger pending), or `'failed'` (last run errored).
+
+### getRegisteredCronTaskIds() => string[]
+
+Returns the IDs of tasks registered *on this instance* (useful when
+`runCronTasks: false` on some instances).
+
+## Testing
+
+Cron tasks expose three helpers that are primarily useful in tests. They
+live on the main `mongodash` module alongside the rest of the cron API.
+
+### Run a task synchronously
+
+```typescript
+import { runCronTask } from 'mongodash';
+
+it('processes pending invoices', async () => {
+    await runCronTask('invoice-sweep');
+    const processed = await invoices.countDocuments({ status: 'processed' });
+    expect(processed).toBeGreaterThan(0);
+});
+```
+
+`runCronTask(taskId)` enqueues the task and awaits its completion. It
+throws if called from inside another running cron task — use
+`scheduleCronTaskImmediately` / `triggerCronTask` for the "fire and
+forget" case.
+
+### Disable the scheduler in tests
+
+Running cron jobs in the background of unit tests causes non-determinism.
+Two options:
+
+```typescript
+// Option A: never auto-start. Tests trigger everything explicitly.
+await mongodash.init({ ..., runCronTasks: false });
+
+// Option B: stop after init. Useful for tests that register tasks and
+// then inspect state without running them.
+import { stopCronTasks, startCronTasks } from 'mongodash';
+stopCronTasks();
+// ...
+startCronTasks(); // if a test needs it back
+```
+
+Called before the first `cronTask()` registration, `stopCronTasks()`
+also prevents any task from starting later in the process.
+
+See [**Testing overview**](./testing.md) for cross-subsystem test helpers.

package/docs/error-handling.md

@@ -0,0 +1,156 @@
+# Error handling
+
+Mongodash routes all runtime errors and informational events through two
+pluggable callbacks you supply at `init` time: `onError` and `onInfo`. Both
+default to `console.error` / `console.log` respectively, so you can adopt
+the library without any observability plumbing and tighten it later.
+
+## `onError`
+
+Called with an `Error` whenever something went wrong **but the library was
+able to continue running** — a failed cron task, a change-stream hiccup,
+a planner flush that needed to be retried, etc. Unrecoverable errors
+throw from the calling code directly (e.g. `init()` on a bad URI); they
+are never routed through `onError`.
+
+```typescript
+import mongodash, { OnError } from 'mongodash';
+
+const onError: OnError = (err) => {
+    sentry.captureException(err);
+    logger.error({ err }, 'mongodash runtime error');
+};
+
+await mongodash.init({ uri: '...', onError });
+```
+
+### Signature
+
+```typescript
+type OnError = (error: Error) => void;
+```
+
+The callback is wrapped in a secure handler internally — if your
+`onError` itself throws, the wrapper catches and logs it so a faulty
+observability layer cannot crash the library. Prefer to keep the
+callback fast and synchronous; offload heavy work (HTTP to an APM, disk
+IO) to a queue you drain elsewhere.
+
+## `onInfo`
+
+Called with a structured event object whenever the library wants to
+announce something interesting that is **not an error**: task lifecycle
+transitions, reconciliation progress, leader elections, metric pushes.
+
+Each event carries a stable `code` that you can match on without
+parsing the human-readable `message`:
+
+```typescript
+import mongodash, {
+    OnInfo,
+    CODE_CRON_TASK_FAILED,
+    CODE_REACTIVE_TASK_FAILED,
+    CODE_REACTIVE_TASK_LOCK_LOST,
+} from 'mongodash';
+
+const onInfo: OnInfo = (event) => {
+    switch (event.code) {
+        case CODE_CRON_TASK_FAILED:
+        case CODE_REACTIVE_TASK_FAILED:
+            metrics.increment('tasks.failed', { task: event.taskId });
+            break;
+        case CODE_REACTIVE_TASK_LOCK_LOST:
+            metrics.increment('tasks.lock_lost', { task: event.taskId });
+            break;
+    }
+    logger.info(event);
+};
+
+await mongodash.init({ uri: '...', onInfo });
+```
+
+### Signature
+
+```typescript
+type OnInfo = (event: { message: string; code: string; [key: string]: unknown }) => void;
+```
+
+### Event catalog
+
+| Code constant | Subsystem | When it fires |
+| :--- | :--- | :--- |
+| `CODE_CRON_TASK_STARTED` | cron | Handler about to be invoked (also on `init` to announce cron processing). |
+| `CODE_CRON_TASK_FINISHED` | cron | Handler returned successfully. |
+| `CODE_CRON_TASK_FAILED` | cron | Handler threw. The same error is also passed to `onError`. |
+| `CODE_CRON_TASK_SCHEDULED` | cron | Task scheduled for next run. |
+| `CODE_REACTIVE_TASK_STARTED` | reactive | Handler about to be invoked. |
+| `CODE_REACTIVE_TASK_FINISHED` | reactive | Handler succeeded (or skipped via `TaskConditionFailedError`). |
+| `CODE_REACTIVE_TASK_FAILED` | reactive | Handler threw. |
+| `CODE_REACTIVE_TASK_LOCK_LOST` | reactive | A long-running worker's lock was stolen by another; the worker is backing off. |
+| `CODE_REACTIVE_TASK_CLEANUP` | reactive | Orphaned task records were deleted by the cleanup policy. |
+| `CODE_REACTIVE_TASK_INITIALIZED` | reactive | A reactive task was registered (also fires on startup for existing registrations). |
+| `CODE_REACTIVE_TASK_PLANNER_STARTED` | reactive | Planner started (leader elected or restarted after an error). |
+| `CODE_REACTIVE_TASK_PLANNER_STOPPED` | reactive | Planner stopped (leader lost or shutdown). |
+| `CODE_REACTIVE_TASK_PLANNER_STREAM_ERROR` | reactive | Raw change-stream error observed. |
+| `CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_STARTED` | reactive | Full-scan reconciliation began. |
+| `CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_FINISHED` | reactive | Full-scan reconciliation finished. |
+| `CODE_REACTIVE_TASK_LEADER_LOCK_LOST` | reactive | This instance was the leader and the lock expired on it. |
+
+See [**Reactive tasks - Monitoring**](./reactive-tasks/monitoring.md) for
+the matching Prometheus metrics and
+[**Cron tasks - Monitoring**](./cron-tasks.md#monitoring) for the cron
+side.
+
+## Typed errors
+
+A handful of errors can be recognised by reference (they are exported
+classes) and deserve special handling:
+
+### `TaskConditionFailedError`
+
+Thrown from `context.getDocument()` inside a **reactive-task handler**
+when the source document no longer matches the task filter (typically
+because the user deleted or updated it between planning and execution).
+The library treats it as a soft skip — the task record is marked
+finished without raising an error. Operators generally do not need to
+react.
+
+```typescript
+import { reactiveTask, TaskConditionFailedError } from 'mongodash';
+
+await reactiveTask({
+    // ...
+    handler: async (ctx) => {
+        try {
+            const doc = await ctx.getDocument();
+            // ...
+        } catch (err) {
+            if (err instanceof TaskConditionFailedError) {
+                // Expected - the upstream filter no longer matches. Skip silently.
+                return;
+            }
+            throw err;
+        }
+    },
+});
+```
+
+### `LockAlreadyAcquiredError` / `isLockAlreadyAcquiredError`
+
+Thrown from `withLock` when another caller already holds the lock and
+`maxWaitForLock` elapses. Use `isLockAlreadyAcquiredError(err)` when you
+do not want to take a static import dependency on the class.
+
+```typescript
+import { withLock, LockAlreadyAcquiredError, isLockAlreadyAcquiredError } from 'mongodash';
+
+try {
+    await withLock('nightly-rollup', async () => { /* ... */ });
+} catch (err) {
+    if (isLockAlreadyAcquiredError(err)) {
+        // Another instance is already running the rollup - that's fine.
+        return;
+    }
+    throw err;
+}
+```

package/docs/reactive-tasks/guides.md

@@ -301,4 +301,4 @@ Testing asynchronous, event-driven workflows can be challenging. Mongodash provi
 
 Use \`waitUntilReactiveTasksIdle\` to robustly wait for all side-effects (including retries and cascading tasks) to finish before making assertions.
 
-See **[Testing Utilities](../testing.md)** for detailed usage and examples.
+See **[Testing Reactive Tasks](./testing.md)** for detailed usage and examples.

package/docs/reactive-tasks/index.md

@@ -15,7 +15,7 @@ Reactive Tasks allow you to define background jobs that trigger automatically wh
 -   **[Concurrency Control](./configuration.md)**: Limit parallel execution to protect downstream resources.
 -   **[Deduplication](./guides.md#idempotency--re-execution)**: Automatic debouncing ("wait for data to settle") and task merging.
 -   **[Observability](./monitoring.md)**: First-class Prometheus metrics support.
--   **[Testing Support](../testing.md)**: Built-in utilities (`waitUntilReactiveTasksIdle`) to ensure your reactive flows are robust and error-free.
+-   **[Testing Support](./testing.md)**: Built-in utilities (`waitUntilReactiveTasksIdle`) to ensure your reactive flows are robust and error-free.
 -   **[Dashboard](../dashboard.md)**: A visual Dashboard to monitor, retry, and debug tasks.
 -   **Developer Friendly**: Zero-config local development, fully typed with TypeScript.
 

package/docs/reactive-tasks/monitoring.md

@@ -59,6 +59,13 @@ The system exposes the following metrics with standardized labels:
 | `reactive_tasks_global_lag_seconds` | Gauge | `task_name` | Age of the oldest `pending` task, measured from `dueAt`. This ensures deferred tasks still reflect their true waiting time. |
 | `reactive_tasks_change_stream_lag_seconds` | Gauge | *none* | Time difference between now and the last processed Change Stream event. |
 | `reactive_tasks_last_reconciliation_timestamp_seconds` | Gauge | *none* | Timestamp when the last full reconciliation (recovery) finished. |
+| `reactive_tasks_leader_elections_total` | Counter | *none* | Number of times this instance became leader. A high rate indicates leader flapping (clock skew, slow heartbeats, network partitions). |
+| `reactive_tasks_lock_lost_total` | Counter | `task_name` | Number of tasks whose execution lock was stolen by another worker (detected via CAS). A non-zero value means work was duplicated; usually a signal to increase `visibilityTimeoutMs` or investigate slow handlers. |
+| `reactive_tasks_stream_errors_total` | Counter | *none* | Number of change-stream errors observed by this instance (disconnects, oplog lost, etc.). |
+| `reactive_tasks_flush_failures_total` | Counter | *none* | Number of planner batches that failed and required a stream restart. Distinct from stream errors: the DB was reachable but the upsert pipeline rejected a batch. |
+
+> [!NOTE]
+> All new counters are **per-instance** (exported via the instance's local registry). In `cluster` mode they are summed across instances at scrape time; in `local` mode each instance reports its own value.
 
 ## Grafana Dashboard