mongodash 2.6.0 → 2.8.0

package/dist/types/testing/waitUntilReactiveTasksIdle.d.ts

@@ -1,4 +1,4 @@
-import { Document, Filter } from 'mongodb';
+import { WhitelistRule } from './resolveWhitelistFilter';
 import { WaitUntilOptions } from './waitUntil';
 /**
  * Waits until the reactive task system is idle.
@@ -8,6 +8,11 @@ import { WaitUntilOptions } from './waitUntil';
  * 3. No tasks in the database are in a pending or processing state.
  *
  * This enables robust E2E testing by ensuring that all side effects and cascading tasks have finished.
+ *
+ * @remarks
+ * Pending tasks scheduled far in the future (beyond `timeoutMs + stabilityDurationMs + 100ms`)
+ * are treated as "future work" and ignored. This prevents long-running retries (e.g. exponential backoff
+ * pushing `nextRunAt` hours ahead) from blocking the idle check forever.
  */
 export interface WaitUntilReactiveTasksIdleOptions extends Partial<WaitUntilOptions> {
     /**
@@ -15,17 +20,6 @@ export interface WaitUntilReactiveTasksIdleOptions extends Partial<WaitUntilOpti
      * Global checks (Planner buffer, Active workers) are SKIPPED in this mode to ensure isolation
      * from other running tests.
      */
-    whitelist?: Array<{
-        collection: string;
-        /**
-         * Filter to find relevant documents.
-         * If not provided, ALL documents in the collection are considered (use carefully!).
-         */
-        filter?: Filter<Document>;
-        /**
-         * Optional task name filter.
-         */
-        task?: string;
-    }>;
+    whitelist?: WhitelistRule[];
 }
 export declare function waitUntilReactiveTasksIdle(customOptions?: WaitUntilReactiveTasksIdleOptions): Promise<void>;

package/docs/.vitepress/config.mts

@@ -53,7 +53,15 @@ export default defineConfig({
                 text: 'Utilities',
                 items: [
                     { text: 'Process In Batches', link: '/process-in-batches' },
-                    { text: 'Getters', link: '/getters' }
+                    { text: 'Getters', link: '/getters' },
+                    { text: 'Error Handling', link: '/error-handling' }
+                ]
+            },
+            {
+                text: 'Testing',
+                items: [
+                    { text: 'Overview', link: '/testing' },
+                    { text: 'Testing Reactive Tasks', link: '/reactive-tasks/testing' }
                 ]
             }
         ],

package/docs/cron-tasks.md

@@ -135,10 +135,15 @@ import mongodash from 'mongodash';
 mongodash.init({
     // database connection
     uri: 'mongodb://mongodb0.example.com:27017',
-    
+
     // true by default
     runCronTasks: false,
 
+    // Maximum number of cron tasks this instance executes in parallel.
+    // Default 1 (serial). See the "Parallel execution within one instance"
+    // section earlier on this page.
+    cronTaskConcurrency: 5,
+
     // valid only if CRON expressions used
     // see https://www.npmjs.com/package/cron-parser for valid options
     cronExpressionParserOptions: {
@@ -170,3 +175,127 @@ The system handles concurrency by locking tasks in MongoDB.
 The system maintains a brief execution history in the database:
 - **Limit**: Only the **last 5 runs** are stored in the `runLog` of the task document.
 - Use this to monitor recent successes or failures.
+
+### Parallel execution within one instance
+
+By default each instance runs one cron task at a time. When you have many
+independent cron tasks and a single long-running one would block the
+others, opt in to parallel execution:
+
+```typescript
+await mongodash.init({
+    // ...
+    cronTaskConcurrency: 5, // up to 5 cron tasks in flight on this instance
+});
+```
+
+- A single task can **never** run twice in parallel, regardless of the
+  value. The per-task `lockedTill` lock guarantees that even within one
+  instance — and across instances — only one execution of a given
+  `taskId` is in flight at a time.
+- `cronTaskConcurrency: 1` (the default) keeps the historical single-loop
+  behaviour.
+- Raising the value only affects *different* tasks running at the same
+  time. Use it when you see head-of-line blocking on the cron collection.
+
+## Monitoring
+
+Cron tasks emit structured events through the `onInfo` callback. Each event
+has a stable `code` that you can route to your logging stack without
+parsing strings.
+
+| Code constant | When it fires | Payload |
+| :--- | :--- | :--- |
+| `CODE_CRON_TASK_STARTED` | Handler is about to be invoked. Also fired once during `init` to announce that cron processing has begun. | `{ taskId, code }` |
+| `CODE_CRON_TASK_FINISHED` | Handler returned without throwing. | `{ taskId, code, duration }` |
+| `CODE_CRON_TASK_FAILED` | Handler threw. The same error is also passed to `onError`. | `{ taskId, code, reason, duration }` |
+| `CODE_CRON_TASK_SCHEDULED` | The task has been scheduled for its next run. | `{ taskId, code, nextRunDate }` |
+
+```typescript
+import { CODE_CRON_TASK_FAILED } from 'mongodash';
+
+await mongodash.init({
+    onInfo: (event) => {
+        if (event.code === CODE_CRON_TASK_FAILED) {
+            metrics.increment('cron.failed', { task: event.taskId });
+        }
+    },
+});
+```
+
+See also [**Error Handling**](./error-handling.md) for how `onError` and
+`onInfo` compose.
+
+## Task Management
+
+### getCronTasksList(query?) => Promise<CronPagedResult\<CronTaskRecord\>>
+
+Inspect the state of registered tasks - useful for admin UIs, health
+checks, or integration tests.
+
+```typescript
+import { getCronTasksList } from 'mongodash';
+
+const page = await getCronTasksList({
+    filter: 'daily', // regex match against taskId (case-insensitive)
+    limit: 20,
+    skip: 0,
+    sort: { field: 'nextRunAt', direction: 1 },
+});
+
+for (const task of page.items) {
+    console.log(task._id, task.status, task.lastRun?.error);
+}
+```
+
+`status` can be `'idle'`, `'running'` (lock held), `'scheduled'`
+(manual trigger pending), or `'failed'` (last run errored).
+
+### getRegisteredCronTaskIds() => string[]
+
+Returns the IDs of tasks registered *on this instance* (useful when
+`runCronTasks: false` on some instances).
+
+## Testing
+
+Cron tasks expose three helpers that are primarily useful in tests. They
+live on the main `mongodash` module alongside the rest of the cron API.
+
+### Run a task synchronously
+
+```typescript
+import { runCronTask } from 'mongodash';
+
+it('processes pending invoices', async () => {
+    await runCronTask('invoice-sweep');
+    const processed = await invoices.countDocuments({ status: 'processed' });
+    expect(processed).toBeGreaterThan(0);
+});
+```
+
+`runCronTask(taskId)` enqueues the task and awaits its completion. It
+throws if called from inside another running cron task — use
+`scheduleCronTaskImmediately` / `triggerCronTask` for the "fire and
+forget" case.
+
+### Disable the scheduler in tests
+
+Running cron jobs in the background of unit tests causes non-determinism.
+Two options:
+
+```typescript
+// Option A: never auto-start. Tests trigger everything explicitly.
+await mongodash.init({ ..., runCronTasks: false });
+
+// Option B: stop after init. Useful for tests that register tasks and
+// then inspect state without running them.
+import { stopCronTasks, startCronTasks } from 'mongodash';
+stopCronTasks();
+// ...
+startCronTasks(); // if a test needs it back
+```
+
+Called before the first `cronTask()` registration, `stopCronTasks()`
+also prevents any task from starting later in the process.
+
+See [**Testing overview**](./testing.md) for cross-subsystem test helpers.

package/docs/error-handling.md

@@ -0,0 +1,156 @@
+# Error handling
+
+Mongodash routes all runtime errors and informational events through two
+pluggable callbacks you supply at `init` time: `onError` and `onInfo`. Both
+default to `console.error` / `console.log` respectively, so you can adopt
+the library without any observability plumbing and tighten it later.
+
+## `onError`
+
+Called with an `Error` whenever something went wrong **but the library was
+able to continue running** — a failed cron task, a change-stream hiccup,
+a planner flush that needed to be retried, etc. Unrecoverable errors
+throw from the calling code directly (e.g. `init()` on a bad URI); they
+are never routed through `onError`.
+
+```typescript
+import mongodash, { OnError } from 'mongodash';
+
+const onError: OnError = (err) => {
+    sentry.captureException(err);
+    logger.error({ err }, 'mongodash runtime error');
+};
+
+await mongodash.init({ uri: '...', onError });
+```
+
+### Signature
+
+```typescript
+type OnError = (error: Error) => void;
+```
+
+The callback is wrapped in a secure handler internally — if your
+`onError` itself throws, the wrapper catches and logs it so a faulty
+observability layer cannot crash the library. Prefer to keep the
+callback fast and synchronous; offload heavy work (HTTP to an APM, disk
+IO) to a queue you drain elsewhere.
+
+## `onInfo`
+
+Called with a structured event object whenever the library wants to
+announce something interesting that is **not an error**: task lifecycle
+transitions, reconciliation progress, leader elections, metric pushes.
+
+Each event carries a stable `code` that you can match on without
+parsing the human-readable `message`:
+
+```typescript
+import mongodash, {
+    OnInfo,
+    CODE_CRON_TASK_FAILED,
+    CODE_REACTIVE_TASK_FAILED,
+    CODE_REACTIVE_TASK_LOCK_LOST,
+} from 'mongodash';
+
+const onInfo: OnInfo = (event) => {
+    switch (event.code) {
+        case CODE_CRON_TASK_FAILED:
+        case CODE_REACTIVE_TASK_FAILED:
+            metrics.increment('tasks.failed', { task: event.taskId });
+            break;
+        case CODE_REACTIVE_TASK_LOCK_LOST:
+            metrics.increment('tasks.lock_lost', { task: event.taskId });
+            break;
+    }
+    logger.info(event);
+};
+
+await mongodash.init({ uri: '...', onInfo });
+```
+
+### Signature
+
+```typescript
+type OnInfo = (event: { message: string; code: string; [key: string]: unknown }) => void;
+```
+
+### Event catalog
+
+| Code constant | Subsystem | When it fires |
+| :--- | :--- | :--- |
+| `CODE_CRON_TASK_STARTED` | cron | Handler about to be invoked (also on `init` to announce cron processing). |
+| `CODE_CRON_TASK_FINISHED` | cron | Handler returned successfully. |
+| `CODE_CRON_TASK_FAILED` | cron | Handler threw. The same error is also passed to `onError`. |
+| `CODE_CRON_TASK_SCHEDULED` | cron | Task scheduled for next run. |
+| `CODE_REACTIVE_TASK_STARTED` | reactive | Handler about to be invoked. |
+| `CODE_REACTIVE_TASK_FINISHED` | reactive | Handler succeeded (or skipped via `TaskConditionFailedError`). |
+| `CODE_REACTIVE_TASK_FAILED` | reactive | Handler threw. |
+| `CODE_REACTIVE_TASK_LOCK_LOST` | reactive | A long-running worker's lock was stolen by another; the worker is backing off. |
+| `CODE_REACTIVE_TASK_CLEANUP` | reactive | Orphaned task records were deleted by the cleanup policy. |
+| `CODE_REACTIVE_TASK_INITIALIZED` | reactive | A reactive task was registered (also fires on startup for existing registrations). |
+| `CODE_REACTIVE_TASK_PLANNER_STARTED` | reactive | Planner started (leader elected or restarted after an error). |
+| `CODE_REACTIVE_TASK_PLANNER_STOPPED` | reactive | Planner stopped (leader lost or shutdown). |
+| `CODE_REACTIVE_TASK_PLANNER_STREAM_ERROR` | reactive | Raw change-stream error observed. |
+| `CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_STARTED` | reactive | Full-scan reconciliation began. |
+| `CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_FINISHED` | reactive | Full-scan reconciliation finished. |
+| `CODE_REACTIVE_TASK_LEADER_LOCK_LOST` | reactive | This instance was the leader and the lock expired on it. |
+
+See [**Reactive tasks - Monitoring**](./reactive-tasks/monitoring.md) for
+the matching Prometheus metrics and
+[**Cron tasks - Monitoring**](./cron-tasks.md#monitoring) for the cron
+side.
+
+## Typed errors
+
+A handful of errors can be recognised by reference (they are exported
+classes) and deserve special handling:
+
+### `TaskConditionFailedError`
+
+Thrown from `context.getDocument()` inside a **reactive-task handler**
+when the source document no longer matches the task filter (typically
+because the user deleted or updated it between planning and execution).
+The library treats it as a soft skip — the task record is marked
+finished without raising an error. Operators generally do not need to
+react.
+
+```typescript
+import { reactiveTask, TaskConditionFailedError } from 'mongodash';
+
+await reactiveTask({
+    // ...
+    handler: async (ctx) => {
+        try {
+            const doc = await ctx.getDocument();
+            // ...
+        } catch (err) {
+            if (err instanceof TaskConditionFailedError) {
+                // Expected - the upstream filter no longer matches. Skip silently.
+                return;
+            }
+            throw err;
+        }
+    },
+});
+```
+
+### `LockAlreadyAcquiredError` / `isLockAlreadyAcquiredError`
+
+Thrown from `withLock` when another caller already holds the lock and
+`maxWaitForLock` elapses. Use `isLockAlreadyAcquiredError(err)` when you
+do not want to take a static import dependency on the class.
+
+```typescript
+import { withLock, LockAlreadyAcquiredError, isLockAlreadyAcquiredError } from 'mongodash';
+
+try {
+    await withLock('nightly-rollup', async () => { /* ... */ });
+} catch (err) {
+    if (isLockAlreadyAcquiredError(err)) {
+        // Another instance is already running the rollup - that's fine.
+        return;
+    }
+    throw err;
+}
+```

package/docs/reactive-tasks/guides.md

@@ -301,4 +301,4 @@ Testing asynchronous, event-driven workflows can be challenging. Mongodash provi
 
 Use \`waitUntilReactiveTasksIdle\` to robustly wait for all side-effects (including retries and cascading tasks) to finish before making assertions.
 
-See **[Testing Utilities](../testing.md)** for detailed usage and examples.
+See **[Testing Reactive Tasks](./testing.md)** for detailed usage and examples.

package/docs/reactive-tasks/index.md

@@ -15,7 +15,7 @@ Reactive Tasks allow you to define background jobs that trigger automatically wh
 -   **[Concurrency Control](./configuration.md)**: Limit parallel execution to protect downstream resources.
 -   **[Deduplication](./guides.md#idempotency--re-execution)**: Automatic debouncing ("wait for data to settle") and task merging.
 -   **[Observability](./monitoring.md)**: First-class Prometheus metrics support.
--   **[Testing Support](../testing.md)**: Built-in utilities (`waitUntilReactiveTasksIdle`) to ensure your reactive flows are robust and error-free.
+-   **[Testing Support](./testing.md)**: Built-in utilities (`waitUntilReactiveTasksIdle`) to ensure your reactive flows are robust and error-free.
 -   **[Dashboard](../dashboard.md)**: A visual Dashboard to monitor, retry, and debug tasks.
 -   **Developer Friendly**: Zero-config local development, fully typed with TypeScript.
 
@@ -28,7 +28,7 @@ It is important to distinguish between Reactive Tasks and standard schedulers (l
 
 Reactive Tasks support time-based operations via `debounce` (e.g., "Wait 1m after data change to settle") and `deferCurrent` (e.g., "Retry in 5m"), but they are fundamentally event-driven. If you need purely time-based jobs (e.g., "Daily Report" without any data change trigger), you can trigger them via a [Cron job](../cron-tasks.md), although you can model them as "Run on insert to 'daily_reports' collection".
 
-## Advantages over Standard Messaging
+## Advantages over Standard Queue systems
 
 Using Reactive Tasks instead of a traditional message broker (RabbitMQ, Kafka) provides distinct architectural benefits:
 

package/docs/reactive-tasks/monitoring.md

@@ -59,6 +59,13 @@ The system exposes the following metrics with standardized labels:
 | `reactive_tasks_global_lag_seconds` | Gauge | `task_name` | Age of the oldest `pending` task, measured from `dueAt`. This ensures deferred tasks still reflect their true waiting time. |
 | `reactive_tasks_change_stream_lag_seconds` | Gauge | *none* | Time difference between now and the last processed Change Stream event. |
 | `reactive_tasks_last_reconciliation_timestamp_seconds` | Gauge | *none* | Timestamp when the last full reconciliation (recovery) finished. |
+| `reactive_tasks_leader_elections_total` | Counter | *none* | Number of times this instance became leader. A high rate indicates leader flapping (clock skew, slow heartbeats, network partitions). |
+| `reactive_tasks_lock_lost_total` | Counter | `task_name` | Number of tasks whose execution lock was stolen by another worker (detected via CAS). A non-zero value means work was duplicated; usually a signal to increase `visibilityTimeoutMs` or investigate slow handlers. |
+| `reactive_tasks_stream_errors_total` | Counter | *none* | Number of change-stream errors observed by this instance (disconnects, oplog lost, etc.). |
+| `reactive_tasks_flush_failures_total` | Counter | *none* | Number of planner batches that failed and required a stream restart. Distinct from stream errors: the DB was reachable but the upsert pipeline rejected a batch. |
+
+> [!NOTE]
+> All new counters are **per-instance** (exported via the instance's local registry). In `cluster` mode they are summed across instances at scrape time; in `local` mode each instance reports its own value.
 
 ## Grafana Dashboard
 

package/docs/reactive-tasks/testing.md

@@ -0,0 +1,187 @@
+# Testing Reactive Tasks
+
+Testing asynchronous, event-driven workflows is notoriously hard: the "done"
+state is not a return value from the code you called but a quiescent state of
+a background system. Mongodash ships with helpers that address this directly.
+
+> [!TIP]
+> For the generic polling helper (not tied to reactive tasks), see
+> [**`waitUntil`**](../testing.md).
+
+## `waitUntilReactiveTasksIdle`
+
+Blocks until the reactive-task subsystem is completely quiesced. This is
+essential for end-to-end tests where you want to assert the final state after
+a chain of cascading tasks has settled.
+
+### What it waits for
+
+It resolves only when **all** of the following validation checks pass
+simultaneously (and remain true for `stabilityDurationMs`):
+
+1.  **Planner empty** — the internal `ReactiveTaskPlanner` has no buffered
+    change-stream events waiting to be flushed.
+2.  **Workers idle** — no `ReactiveTaskWorker` is currently processing a task
+    (active count is 0).
+3.  **Database settled** — no tasks in any registered task collection are in
+    `pending`, `processing`, or `processing_dirty` state.
+    -   **Exception**: pending tasks scheduled for the _distant future_
+        (beyond the current `timeoutMs + stabilityDurationMs + 100ms`) are
+        ignored, so long retries (e.g. "retry in 1 hour") do not block your
+        test forever.
+
+### Usage
+
+```typescript
+import { waitUntilReactiveTasksIdle } from 'mongodash/testing';
+
+it('should process user registration workflow', async () => {
+    // 1. Trigger the workflow
+    await users.insertOne({ email: 'test@example.com', status: 'new' });
+
+    // 2. Wait for the reactive task to process the insert AND any cascading
+    //    tasks it may have triggered (welcome email, provisioning, etc.)
+    await waitUntilReactiveTasksIdle();
+
+    // 3. Assert the final state
+    const emailTask = await emailTasks.findOne({ email: 'test@example.com' });
+    expect(emailTask).toBeDefined();
+    expect(emailTask.status).toBe('sent');
+});
+```
+
+### Configuration
+
+Defaults are tuned for general use; override as needed:
+
+```typescript
+await waitUntilReactiveTasksIdle({
+    timeoutMs: 30000,
+    stabilityDurationMs: 200, // 200ms of "silence" to catch in-flight cascading tasks
+});
+```
+
+### Isolation with `whitelist`
+
+When running tests in parallel against a shared database, use `whitelist` to
+wait only for tasks that belong to this test. In whitelist mode the **active
+workers** check is skipped so another test's worker pool does not block you;
+the planner buffer check is still applied (otherwise we could return idle
+before a change event we care about has even been turned into a task row).
+
+```typescript
+await waitUntilReactiveTasksIdle({
+    whitelist: [
+        { collection: 'users', filter: { _id: userId } }, // specific user
+        { collection: 'orders', task: 'processOrder' },   // specific task on collection
+    ],
+});
+```
+
+> [!NOTE]
+> The idle check queries task records by `sourceDocId`. If your trigger is
+> very fast (e.g. a write immediately followed by `waitUntilReactiveTasksIdle`)
+> and the default `stabilityDurationMs` is low, you may need to raise
+> `stabilityDurationMs` to ensure the planner has had a chance to flush the
+> event into a task record. Alternatively, poll with `waitUntil` for the
+> task record to exist first, then call `waitUntilReactiveTasksIdle`.
+
+## `assertNoReactiveTaskErrors`
+
+Catches "silent failures" — tasks that threw and were logged but never
+propagated to the test as an assertion error.
+
+### Features
+
+-   **Time filtering** — only checks for errors that occurred after a given
+    `since` timestamp (typically test start).
+-   **Scope** — check globally or limit to specific source documents / task
+    names via `whitelist`.
+-   **Exclusions** — allow known errors (string or regex) without failing the
+    test.
+
+### Usage
+
+```typescript
+import { assertNoReactiveTaskErrors } from 'mongodash/testing';
+
+it('should process successfully', async () => {
+    const startTime = new Date();
+
+    // ... run test steps ...
+    await waitUntilReactiveTasksIdle();
+
+    await assertNoReactiveTaskErrors({ since: startTime });
+});
+```
+
+### Options
+
+```typescript
+await assertNoReactiveTaskErrors({
+    since: startTime, // required
+    whitelist: [{
+        collection: 'users',
+        filter: { _id: userId },
+    }],
+    excludeErrors: [
+        'Expected Failure',
+        /Authorization Error/,
+    ],
+});
+```
+
+## `configureForTesting`
+
+Production defaults (`debounce: 1000ms`, `minPollMs: 200ms`, etc.) are tuned
+for real workloads but make tests unnecessarily slow. `configureForTesting`
+overrides these globally with minimal values (typically `10ms`).
+
+Call it **once** in your test setup (`jest.setup.js`, `beforeAll`, etc.). It
+works whether called before or after task registration.
+
+```typescript
+import { configureForTesting } from 'mongodash/testing';
+
+beforeAll(() => {
+    configureForTesting();
+});
+```
+
+### Options
+
+```typescript
+configureForTesting({
+    debounce: 0,        // 0ms debounce (immediate planning)
+    minPollMs: 50,      // polling interval
+    minBatchIntervalMs: 50,
+});
+```
+
+## Whitelist rules
+
+`waitUntilReactiveTasksIdle` and `assertNoReactiveTaskErrors` share a
+`whitelist` option of type `WhitelistRule[]`. The type is exported so test
+suites can share scope definitions:
+
+```typescript
+import type { WhitelistRule } from 'mongodash/testing';
+
+const scopeToUser = (userId: string): WhitelistRule[] => [
+    { collection: 'users', filter: { _id: userId } },
+    { collection: 'notifications', filter: { userId } },
+];
+
+await waitUntilReactiveTasksIdle({ whitelist: scopeToUser(id) });
+await assertNoReactiveTaskErrors({ since, whitelist: scopeToUser(id) });
+```
+
+Rule semantics:
+
+-   **No `filter` and no `task`** — matches every document and every task in
+    that collection ("match-all").
+-   **`filter` that matches zero documents** — the rule is treated as
+    "skip this collection", useful when the rule is built from a variable
+    that may be empty.
+-   **Multiple rules for the same collection** — OR-merged into a single
+    filter. A `match-all` rule wins over others.