mongodash 2.0.0 → 2.1.0

package/dist/types/src/reactiveTasks/LeaderElector.d.ts

@@ -0,0 +1,42 @@
+import { GlobalsCollection } from '../globalsCollection';
+import { OnInfo } from '../OnInfo';
+import { OnError } from '../OnError';
+export interface LeaderElectorCallbacks {
+    onBecomeLeader: () => Promise<void>;
+    onLoseLeader: () => Promise<void>;
+    onHeartbeat: () => Promise<void>;
+}
+export interface LeaderElectorOptions {
+    lockTtlMs: number;
+    lockHeartbeatMs: number;
+    metaDocId?: string;
+}
+/**
+ * Manages leader election among multiple scheduler instances.
+ *
+ * Responsibilities:
+ * - Attempts to acquire a distributed lock in the globals collection.
+ * - Maintains the lock by periodically renewing it (heartbeat).
+ * - Notifies callbacks when the instance becomes leader or loses leadership.
+ * - Ensures only one instance (the leader) runs the `ReactiveTaskPlanner` at a time.
+ */
+export declare class LeaderElector {
+    private globalsCollection;
+    private instanceId;
+    private options;
+    private callbacks;
+    private onInfo;
+    private onError;
+    private isRunning;
+    private _isLeader;
+    private leaderTimer;
+    private metaDocId;
+    constructor(globalsCollection: GlobalsCollection, instanceId: string, options: LeaderElectorOptions, callbacks: LeaderElectorCallbacks, onInfo?: OnInfo, onError?: OnError);
+    get isLeader(): boolean;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    forceLoseLeader(): void;
+    private runLeaderElectionLoop;
+    private tryAcquireLock;
+    private releaseLock;
+}

package/dist/types/src/reactiveTasks/MetricsCollector.d.ts

@@ -0,0 +1,73 @@
+import type { Registry } from 'prom-client';
+import { GlobalsCollection } from '../globalsCollection';
+import { OnError } from '../OnError';
+import { OnInfo } from '../OnInfo';
+import { LeaderElector } from './LeaderElector';
+import { ReactiveTaskPlanner } from './ReactiveTaskPlanner';
+import { ReactiveTaskRegistry } from './ReactiveTaskRegistry';
+import { ReactiveTaskSchedulerOptions } from './ReactiveTaskTypes';
+/**
+ * Collects and aggregates metrics for Reactive Tasks.
+ *
+ * Supports two scrape modes:
+ * - **cluster**: Returns aggregated metrics from ALL instances (via DB registry).
+ *   Use when Prometheus scrapes a single endpoint (e.g., behind a load balancer).
+ * - **local**: Returns metrics from THIS instance only.
+ *   Use when Prometheus scrapes each pod individually (K8s Pod Monitors).
+ *
+ * Global stats (queue depth, lag) are computed on-the-fly by the **Leader only**.
+ */
+export declare class MetricsCollector {
+    private enabled;
+    private readonly options;
+    private readonly instanceId;
+    private readonly registry;
+    private readonly globalsCollection;
+    private readonly leaderElector;
+    private readonly onInfo;
+    private readonly onError;
+    private promClientModule;
+    private localPromRegistry?;
+    private globalStatsRegistry?;
+    private metricDuration?;
+    private metricRetries?;
+    private pushInterval?;
+    private queueMetricsPromise;
+    planner?: ReactiveTaskPlanner;
+    constructor(instanceId: string, registry: ReactiveTaskRegistry, globalsCollection: GlobalsCollection, leaderElector: LeaderElector, options: ReactiveTaskSchedulerOptions['monitoring'], onInfo?: OnInfo, onError?: OnError);
+    private initPrometheus;
+    private initLocalMetrics;
+    private initGlobalStatsMetrics;
+    start(): void;
+    stop(): void;
+    recordTaskExecution(task: string, status: 'success' | 'failed', durationMs: number): void;
+    recordRetry(task: string): void;
+    getPrometheusMetrics(): Promise<Registry | null>;
+    /**
+     * Returns aggregated metrics from ALL instances.
+     * Fetches other instances' metrics from DB and merges with fresh local metrics.
+     * Leader also includes global stats.
+     */
+    private getClusterMetrics;
+    /**
+     * Returns metrics from THIS instance only.
+     * Leader also includes global stats.
+     */
+    private getLocalMetrics;
+    private fetchOtherInstancesMetrics;
+    private getLocalMetricsAsJson;
+    private getGlobalStatsAsJson;
+    private triggerGlobalStatsCollection;
+    private pushLocalMetrics;
+    private publishMetricsToGlobalRegistry;
+    private collectIfLeader;
+    /**
+     * Triggers queue metrics collection with deduplication.
+     * Both QUEUE_DEPTH and GLOBAL_LAG share the same aggregation query.
+     */
+    private triggerQueueMetricsCollection;
+    private collectQueueMetrics;
+    private collectReconciliationMetrics;
+    private collectChangeStreamLag;
+    private getOrCreateMetric;
+}

package/dist/types/src/reactiveTasks/ReactiveTaskManager.d.ts

@@ -0,0 +1,18 @@
+import { Document } from 'mongodb';
+import { ReactiveTaskRegistry } from './ReactiveTaskRegistry';
+import { PagedResult, PaginationOptions, ReactiveTaskQuery, ReactiveTaskRecord, ReactiveTaskStatsResult } from './ReactiveTaskTypes';
+export declare class ReactiveTaskManager {
+    private registry;
+    constructor(registry: ReactiveTaskRegistry);
+    getTasks(query: ReactiveTaskQuery<Document>, pagination?: PaginationOptions): Promise<PagedResult<ReactiveTaskRecord<Document>>>;
+    countTasks(query: ReactiveTaskQuery<Document>): Promise<number>;
+    getAllTaskStats(): Promise<Record<string, ReactiveTaskStatsResult>>;
+    getTaskStats(query: ReactiveTaskQuery<Document>): Promise<ReactiveTaskStatsResult>;
+    retryTasks(query: ReactiveTaskQuery<Document>): Promise<{
+        modifiedCount: number;
+    }>;
+    private resolveTaskNames;
+    private resolveCollectionsAndQueries;
+    private buildDirectQuery;
+    private isComplexSourceFilter;
+}

package/dist/types/src/reactiveTasks/ReactiveTaskOps.d.ts

@@ -0,0 +1,17 @@
+import { ReactiveTaskRegistry } from './ReactiveTaskRegistry';
+/**
+ * Helper class for generating and executing reactive task operations.
+ *
+ * Responsibilities:
+ * - Generates upsert operations for tasks based on source documents and task definitions.
+ * - Executes bulk write operations to the task collections.
+ * - Handles duplicate key errors gracefully (which can occur during reconciliation).
+ * - Manages debouncing logic by tracking the minimum debounce time for planned tasks.
+ */
+export declare class ReactiveTaskOps {
+    private registry;
+    private onTaskPlanned;
+    constructor(registry: ReactiveTaskRegistry, onTaskPlanned: (tasksCollectionName: string, debounceMs: number) => void);
+    executePlanningPipeline(collectionName: string, sourceDocIds: unknown[], allowedTaskNames?: Set<string>): Promise<void>;
+    private generatePlanningPipeline;
+}

package/dist/types/src/reactiveTasks/ReactiveTaskPlanner.d.ts

@@ -0,0 +1,62 @@
+import { ResumeToken } from 'mongodb';
+import { GlobalsCollection } from '../globalsCollection';
+import { ReactiveTaskRegistry } from './ReactiveTaskRegistry';
+import { OnInfo } from '../OnInfo';
+import { OnError } from '../OnError';
+export interface PlannerCallbacks {
+    onStreamError: () => void;
+    onTaskPlanned: (tasksCollectionName: string, debounceMs: number) => void;
+}
+/**
+ * Responsible for listening to MongoDB Change Stream events and planning tasks.
+ *
+ * Responsibilities:
+ * - Manages the lifecycle of the Change Stream (start, stop, error handling).
+ * - Batches Change Stream events to reduce database load.
+ * - Coordinates with `ReactiveTaskOps` to generate and execute task operations.
+ * - Coordinates with `ReactiveTaskReconciler` to handle reconciliation when the stream is interrupted or history is lost.
+ * - Handles critical errors like `ChangeStreamHistoryLost` (code 280) by triggering reconciliation.
+ */
+export declare class ReactiveTaskPlanner {
+    private globalsCollection;
+    private instanceId;
+    private registry;
+    private callbacks;
+    private internalOptions;
+    private onInfo;
+    private onError;
+    private changeStream;
+    private taskBatch;
+    private taskBatchLastResumeToken;
+    private batchFlushTimer;
+    private isFlushing;
+    private metaDocId;
+    private lastClusterTime;
+    private ops;
+    private reconciler;
+    private get isStoppedTester();
+    constructor(globalsCollection: GlobalsCollection, instanceId: string, registry: ReactiveTaskRegistry, callbacks: PlannerCallbacks, internalOptions: {
+        batchSize: number;
+        batchIntervalMs: number;
+        getNextCleanupDate: (date?: Date) => Date;
+    }, onInfo?: OnInfo, onError?: OnError);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    saveResumeToken(token: ResumeToken, lastClusterTime?: Date): Promise<void>;
+    get isEmpty(): boolean;
+    onHeartbeat(): Promise<void>;
+    private isStopping;
+    private startChangeStream;
+    private stopChangeStream;
+    private getChangeStreamPipeline;
+    private getChangeStreamResumeToken;
+    private enqueueTaskChange;
+    private flushTaskBatch;
+    private groupEventsByCollection;
+    private processDeletions;
+    private executeUpsertOperations;
+    private handleStreamError;
+    private checkEvolutionStrategies;
+    private checkTriggerEvolution;
+    private checkLogicEvolution;
+}

package/dist/types/src/reactiveTasks/ReactiveTaskReconciler.d.ts

@@ -0,0 +1,29 @@
+import { GlobalsCollection } from '../globalsCollection';
+import { OnInfo } from '../OnInfo';
+import { ReactiveTaskOps } from './ReactiveTaskOps';
+import { ReactiveTaskRegistry } from './ReactiveTaskRegistry';
+/**
+ * Responsible for reconciling reactive tasks when the Change Stream history is lost or on startup.
+ *
+ * Responsibilities:
+ * - Scans source collections to identify tasks that should exist.
+ * - Uses `ReactiveTaskOps` to generate and execute task operations.
+ * - Tracks reconciliation status in the meta document.
+ */
+export declare class ReactiveTaskReconciler {
+    private instanceId;
+    private globalsCollection;
+    private registry;
+    private ops;
+    private onInfo;
+    private internalOptions;
+    constructor(instanceId: string, globalsCollection: GlobalsCollection, registry: ReactiveTaskRegistry, ops: ReactiveTaskOps, onInfo: OnInfo, internalOptions: {
+        batchSize: number;
+        batchIntervalMs: number;
+        getNextCleanupDate: (date?: Date) => Date;
+    });
+    private nextCleanupTime;
+    reconcile(shouldStop: () => boolean): Promise<void>;
+    markAsUnreconciled(taskNames: string[]): Promise<void>;
+    performPeriodicCleanup(shouldStop: () => boolean): Promise<void>;
+}

package/dist/types/src/reactiveTasks/ReactiveTaskRegistry.d.ts

@@ -0,0 +1,34 @@
+import { Collection, Document } from 'mongodb';
+import { OnError } from '../OnError';
+import { OnInfo } from '../OnInfo';
+import { ReactiveTaskRepository } from './ReactiveTaskRepository';
+import { ReactiveTask, ReactiveTaskInternal, ReactiveTaskRecord } from './ReactiveTaskTypes';
+export interface TaskMapEntry {
+    sourceCollection: Collection<Document>;
+    tasks: Map<string, ReactiveTaskInternal<Document>>;
+    tasksCollection: Collection<ReactiveTaskRecord<Document>>;
+    repository: ReactiveTaskRepository<Document>;
+}
+/**
+ * Registry for managing reactive task definitions and their associated collections.
+ *
+ * Responsibilities:
+ * - Stores task definitions and maps them to source and task collections.
+ * - Initializes the `ReactiveTaskRepository` for each task collection.
+ * - Provides lookup methods to retrieve tasks by name or collection.
+ * - Ensures task uniqueness and validates configuration.
+ */
+export declare class ReactiveTaskRegistry {
+    private map;
+    private tasksByIdentifiers;
+    private onInfo;
+    private onError;
+    constructor(onInfo?: OnInfo, onError?: OnError);
+    setCallbacks(onInfo: OnInfo, onError: OnError): void;
+    addTask(taskDef: ReactiveTask<Document>): Promise<void>;
+    getTask(taskName: string): ReactiveTaskInternal<Document> | undefined;
+    getAllTasks(): ReactiveTaskInternal<Document>[];
+    getEntry(collectionName: string): TaskMapEntry;
+    getAllEntries(): TaskMapEntry[];
+    private validateEvolutionConfig;
+}

package/dist/types/src/reactiveTasks/ReactiveTaskRepository.d.ts

@@ -0,0 +1,59 @@
+import { Collection, Document, Filter, UpdateFilter } from 'mongodb';
+import { CompatibleBulkWriteOptions } from '../mongoCompatibility';
+import { OnError } from '../OnError';
+import { OnInfo } from '../OnInfo';
+import { ReactiveTaskRetryStrategy } from './ReactiveTaskRetryStrategy';
+import { CleanupDeleteWhen, ReactiveTaskInternal, ReactiveTaskRecord, ReactiveTaskStatsOptions, ReactiveTaskStatsResult } from './ReactiveTaskTypes';
+/**
+ * Handles all database interactions for reactive tasks.
+ *
+ * Responsibilities:
+ * - Generates MongoDB operations for creating, updating, and deleting tasks.
+ * - Manages task state transitions (pending -> processing -> completed/failed).
+ * - Implements the locking mechanism for task execution.
+ * - Handles task finalization (retries, completion, failure).
+ * - Manages database indexes for performance.
+ * - Cleans up orphaned tasks.
+ */
+export declare class ReactiveTaskRepository<T extends Document> {
+    private tasksCollection;
+    private onInfo;
+    private onError;
+    readonly initPromise: Promise<void>;
+    constructor(tasksCollection: Collection<ReactiveTaskRecord<T>>, onInfo?: OnInfo, onError?: OnError);
+    findAndLockNextTask(taskDefs: ReactiveTaskInternal<T>[], options: {
+        visibilityTimeoutMs: number;
+    }): Promise<ReactiveTaskRecord<T> | null>;
+    finalizeTask(taskRecord: ReactiveTaskRecord<T>, strategy: ReactiveTaskRetryStrategy, error?: Error, debounceMs?: number, executionStats?: {
+        durationMs: number;
+    }, executionHistoryLimit?: number): Promise<void>;
+    deferTask(taskRecord: ReactiveTaskRecord<T>, delay: number | Date): Promise<void>;
+    executeBulkWrite(operations: Parameters<Collection<ReactiveTaskRecord<T>>['bulkWrite']>[0], options?: CompatibleBulkWriteOptions): Promise<void>;
+    findTasks(filter: Filter<ReactiveTaskRecord<T>>, options?: {
+        limit?: number;
+        skip?: number;
+        sort?: Record<string, 1 | -1>;
+    }): Promise<ReactiveTaskRecord<T>[]>;
+    countTasks(filter: Filter<ReactiveTaskRecord<T>>): Promise<number>;
+    updateTasks(filter: Filter<ReactiveTaskRecord<T>>, update: UpdateFilter<ReactiveTaskRecord<T>> | Document[]): Promise<{
+        matchedCount: number;
+        modifiedCount: number;
+    }>;
+    resetTasks(filter: Filter<ReactiveTaskRecord<T>>): Promise<{
+        matchedCount: number;
+        modifiedCount: number;
+    }>;
+    resetTasksForUpgrade(taskName: string, mode: 'failed' | 'all'): Promise<{
+        modifiedCount: number;
+    }>;
+    private ensureIndexes;
+    /**
+     * Periodically cleans up orphaned tasks that match the cleanupPolicy.
+     * This runs on a schedule (e.g. hourly) in the Leader instance.
+     */
+    deleteOrphanedTasks(taskName: string, sourceCollectionName: string, taskFilter: Filter<Document>, cleanupPolicy: {
+        deleteWhen: CleanupDeleteWhen;
+        keepForMs: number;
+    }, shouldStop: () => boolean, limitToSourceIds?: unknown[]): Promise<void>;
+    getStatistics(filter: Filter<ReactiveTaskRecord<T>>, options: ReactiveTaskStatsOptions): Promise<ReactiveTaskStatsResult>;
+}

package/dist/types/src/reactiveTasks/ReactiveTaskRetryStrategy.d.ts

@@ -0,0 +1,21 @@
+import { RetryPolicy } from './ReactiveTaskTypes';
+export declare class ReactiveTaskRetryStrategy {
+    readonly policy: RetryPolicy;
+    constructor(policy: RetryPolicy);
+    private validatePolicy;
+    private validateDuration;
+    /**
+     * Calculates the timestamp for the next retry attempt.
+     * @param attempts Current number of attempts (will be incremented for next check) called AFTER failure
+     * @param lastError Timestamp of the last error or last attempt? No, policies are usually "after X time".
+     *                  For linear/exp/fixed, we add delay to NOW.
+     *                  For Cron, we find next date after NOW.
+     */
+    calculateNextRetry(attempts: number): Date;
+    /**
+     * Determines if the task should be marked as permanently failed.
+     * @param attempts Number of attempts consumed so far (including the one that just failed)
+     * @param firstErrorAt When the first error in this sequence occurred
+     */
+    shouldFail(attempts: number, firstErrorAt?: Date | null): boolean;
+}