mongodash 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -22
- package/dist/dashboard/index.html +40 -0
- package/dist/lib/playground/server.js +131 -0
- package/dist/lib/playground/server.js.map +1 -0
- package/dist/lib/src/ConcurrentRunner.js +148 -0
- package/dist/lib/src/ConcurrentRunner.js.map +1 -0
- package/dist/lib/{OnError.js → src/OnError.js} +3 -3
- package/dist/lib/src/OnError.js.map +1 -0
- package/dist/lib/{OnInfo.js → src/OnInfo.js} +6 -3
- package/dist/lib/src/OnInfo.js.map +1 -0
- package/dist/lib/{createContinuousLock.js → src/createContinuousLock.js} +5 -3
- package/dist/lib/src/createContinuousLock.js.map +1 -0
- package/dist/lib/{cronTasks.js → src/cronTasks.js} +129 -73
- package/dist/lib/src/cronTasks.js.map +1 -0
- package/dist/lib/{getCollection.js → src/getCollection.js} +2 -2
- package/dist/lib/src/getCollection.js.map +1 -0
- package/dist/lib/{getMongoClient.js → src/getMongoClient.js} +2 -2
- package/dist/lib/src/getMongoClient.js.map +1 -0
- package/dist/lib/src/globalsCollection.js +10 -0
- package/dist/lib/src/globalsCollection.js.map +1 -0
- package/dist/lib/src/index.js +101 -0
- package/dist/lib/src/index.js.map +1 -0
- package/dist/lib/{initPromise.js → src/initPromise.js} +2 -3
- package/dist/lib/src/initPromise.js.map +1 -0
- package/dist/lib/src/mongoCompatibility.js +10 -0
- package/dist/lib/src/mongoCompatibility.js.map +1 -0
- package/dist/lib/src/parseInterval.js +60 -0
- package/dist/lib/src/parseInterval.js.map +1 -0
- package/dist/lib/src/prefixFilterKeys.js +69 -0
- package/dist/lib/src/prefixFilterKeys.js.map +1 -0
- package/dist/lib/src/processInBatches.js +46 -0
- package/dist/lib/src/processInBatches.js.map +1 -0
- package/dist/lib/src/reactiveTasks/LeaderElector.js +155 -0
- package/dist/lib/src/reactiveTasks/LeaderElector.js.map +1 -0
- package/dist/lib/src/reactiveTasks/MetricsCollector.js +410 -0
- package/dist/lib/src/reactiveTasks/MetricsCollector.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskManager.js +288 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskManager.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskOps.js +185 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskOps.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskPlanner.js +443 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskPlanner.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskReconciler.js +218 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskReconciler.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRegistry.js +184 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRegistry.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRepository.js +355 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRepository.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRetryStrategy.js +153 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRetryStrategy.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskTypes.js +34 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskTypes.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskWorker.js +186 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskWorker.js.map +1 -0
- package/dist/lib/src/reactiveTasks/compileWatchProjection.js +65 -0
- package/dist/lib/src/reactiveTasks/compileWatchProjection.js.map +1 -0
- package/dist/lib/src/reactiveTasks/index.js +298 -0
- package/dist/lib/src/reactiveTasks/index.js.map +1 -0
- package/dist/lib/src/reactiveTasks/queryToExpression.js +160 -0
- package/dist/lib/src/reactiveTasks/queryToExpression.js.map +1 -0
- package/dist/lib/src/reactiveTasks/validateTaskFilter.js +88 -0
- package/dist/lib/src/reactiveTasks/validateTaskFilter.js.map +1 -0
- package/dist/lib/src/task-management/OperationalTaskController.js +162 -0
- package/dist/lib/src/task-management/OperationalTaskController.js.map +1 -0
- package/dist/lib/src/task-management/index.js +27 -0
- package/dist/lib/src/task-management/index.js.map +1 -0
- package/dist/lib/src/task-management/serveDashboard.js +149 -0
- package/dist/lib/src/task-management/serveDashboard.js.map +1 -0
- package/dist/lib/src/task-management/types.js +10 -0
- package/dist/lib/src/task-management/types.js.map +1 -0
- package/dist/lib/{withLock.js → src/withLock.js} +3 -4
- package/dist/lib/src/withLock.js.map +1 -0
- package/dist/lib/{withTransaction.js → src/withTransaction.js} +4 -4
- package/dist/lib/src/withTransaction.js.map +1 -0
- package/dist/lib/tools/check-db-connection.js +28 -0
- package/dist/lib/tools/check-db-connection.js.map +1 -0
- package/dist/lib/tools/clean-testing-databases.js +12 -0
- package/dist/lib/tools/clean-testing-databases.js.map +1 -0
- package/dist/lib/tools/prepare-republish.js +27 -0
- package/dist/lib/tools/prepare-republish.js.map +1 -0
- package/dist/lib/tools/test-matrix-local.js +212 -0
- package/dist/lib/tools/test-matrix-local.js.map +1 -0
- package/dist/lib/tools/testingDatabase.js +55 -0
- package/dist/lib/tools/testingDatabase.js.map +1 -0
- package/dist/types/playground/server.d.ts +1 -0
- package/dist/types/src/ConcurrentRunner.d.ts +30 -0
- package/dist/types/{OnInfo.d.ts → src/OnInfo.d.ts} +1 -1
- package/dist/types/{cronTasks.d.ts → src/cronTasks.d.ts} +44 -1
- package/dist/types/src/globalsCollection.d.ts +4 -0
- package/dist/types/src/index.d.ts +28 -0
- package/dist/types/src/mongoCompatibility.d.ts +29 -0
- package/dist/types/src/parseInterval.d.ts +12 -0
- package/dist/types/src/prefixFilterKeys.d.ts +11 -0
- package/dist/types/src/processInBatches.d.ts +10 -0
- package/dist/types/src/reactiveTasks/LeaderElector.d.ts +42 -0
- package/dist/types/src/reactiveTasks/MetricsCollector.d.ts +73 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskManager.d.ts +18 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskOps.d.ts +17 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskPlanner.d.ts +62 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskReconciler.d.ts +29 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskRegistry.d.ts +34 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskRepository.d.ts +59 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskRetryStrategy.d.ts +21 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskTypes.d.ts +389 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskWorker.d.ts +36 -0
- package/dist/types/src/reactiveTasks/compileWatchProjection.d.ts +12 -0
- package/dist/types/src/reactiveTasks/index.d.ts +82 -0
- package/dist/types/src/reactiveTasks/queryToExpression.d.ts +13 -0
- package/dist/types/src/reactiveTasks/validateTaskFilter.d.ts +10 -0
- package/dist/types/src/task-management/OperationalTaskController.d.ts +59 -0
- package/dist/types/src/task-management/index.d.ts +3 -0
- package/dist/types/src/task-management/serveDashboard.d.ts +12 -0
- package/dist/types/src/task-management/types.d.ts +95 -0
- package/dist/types/tools/check-db-connection.d.ts +2 -0
- package/dist/types/tools/clean-testing-databases.d.ts +1 -0
- package/dist/types/tools/prepare-republish.d.ts +2 -0
- package/dist/types/tools/test-matrix-local.d.ts +1 -0
- package/dist/types/tools/testingDatabase.d.ts +2 -0
- package/docs/.vitepress/cache/deps/_metadata.json +31 -0
- package/docs/.vitepress/cache/deps/chunk-LE5NDSFD.js +12824 -0
- package/docs/.vitepress/cache/deps/chunk-LE5NDSFD.js.map +7 -0
- package/docs/.vitepress/cache/deps/package.json +3 -0
- package/docs/.vitepress/cache/deps/vitepress___@vue_devtools-api.js +4505 -0
- package/docs/.vitepress/cache/deps/vitepress___@vue_devtools-api.js.map +7 -0
- package/docs/.vitepress/cache/deps/vitepress___@vueuse_core.js +9731 -0
- package/docs/.vitepress/cache/deps/vitepress___@vueuse_core.js.map +7 -0
- package/docs/.vitepress/cache/deps/vue.js +347 -0
- package/docs/.vitepress/cache/deps/vue.js.map +7 -0
- package/docs/.vitepress/config.mts +48 -0
- package/docs/.vitepress/theme/index.ts +4 -0
- package/docs/.vitepress/theme/style.css +16 -0
- package/docs/assets/dashboard.png +0 -0
- package/docs/cron-tasks.md +172 -0
- package/docs/dashboard.md +117 -0
- package/docs/getters.md +31 -0
- package/docs/getting-started.md +120 -0
- package/docs/index.md +29 -0
- package/docs/initialization.md +59 -0
- package/docs/process-in-batches.md +73 -0
- package/docs/reactive-tasks.md +914 -0
- package/docs/with-lock.md +45 -0
- package/docs/with-transaction.md +65 -0
- package/grafana/reactive_tasks.json +765 -0
- package/package.json +127 -116
- package/dist/lib/OnError.js.map +0 -1
- package/dist/lib/OnInfo.js.map +0 -1
- package/dist/lib/createContinuousLock.js.map +0 -1
- package/dist/lib/cronTasks.js.map +0 -1
- package/dist/lib/getCollection.js.map +0 -1
- package/dist/lib/getMongoClient.js.map +0 -1
- package/dist/lib/index.js +0 -64
- package/dist/lib/index.js.map +0 -1
- package/dist/lib/initPromise.js.map +0 -1
- package/dist/lib/withLock.js.map +0 -1
- package/dist/lib/withTransaction.js.map +0 -1
- package/dist/types/index.d.ts +0 -17
- /package/dist/types/{OnError.d.ts → src/OnError.d.ts} +0 -0
- /package/dist/types/{createContinuousLock.d.ts → src/createContinuousLock.d.ts} +0 -0
- /package/dist/types/{getCollection.d.ts → src/getCollection.d.ts} +0 -0
- /package/dist/types/{getMongoClient.d.ts → src/getMongoClient.d.ts} +0 -0
- /package/dist/types/{initPromise.d.ts → src/initPromise.d.ts} +0 -0
- /package/dist/types/{withLock.d.ts → src/withLock.d.ts} +0 -0
- /package/dist/types/{withTransaction.d.ts → src/withTransaction.d.ts} +0 -0
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
import { Collection, Document, ObjectId, ResumeToken, WithId, Filter, FindOptions } from 'mongodb';
|
|
2
|
+
import type { ReactiveTaskRetryStrategy } from './ReactiveTaskRetryStrategy';
|
|
3
|
+
/**
|
|
4
|
+
* Represents the status of a task in the database.
|
|
5
|
+
*/
|
|
6
|
+
export type ReactiveTaskStatus = 'pending' | 'processing' | 'processing_dirty' | 'completed' | 'failed';
|
|
7
|
+
/**
|
|
8
|
+
* Defines when orphaned task records should be deleted.
|
|
9
|
+
*/
|
|
10
|
+
export type CleanupDeleteWhen = 'sourceDocumentDeleted' | 'sourceDocumentDeletedOrNoLongerMatching' | 'never';
|
|
11
|
+
/**
|
|
12
|
+
* Configuration for automatic cleanup of orphaned task records.
|
|
13
|
+
*/
|
|
14
|
+
export interface CleanupPolicy {
|
|
15
|
+
/**
|
|
16
|
+
* When to delete orphaned task records.
|
|
17
|
+
* Default: 'sourceDocumentDeleted'
|
|
18
|
+
*/
|
|
19
|
+
deleteWhen?: CleanupDeleteWhen;
|
|
20
|
+
/**
|
|
21
|
+
* How long to keep task records after the deletion condition is met.
|
|
22
|
+
* Accepts duration strings ('24h', '7d') or milliseconds.
|
|
23
|
+
* Default: '24h'
|
|
24
|
+
*/
|
|
25
|
+
keepFor?: string | number;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Structure of the task document stored in the `_tasks` collection.
|
|
29
|
+
*/
|
|
30
|
+
export interface ReactiveTaskRecord<T = Document> {
|
|
31
|
+
_id: string | ObjectId;
|
|
32
|
+
task: string;
|
|
33
|
+
sourceDocId: WithId<T>['_id'];
|
|
34
|
+
status: ReactiveTaskStatus;
|
|
35
|
+
attempts: number;
|
|
36
|
+
scheduledAt: Date;
|
|
37
|
+
createdAt: Date;
|
|
38
|
+
updatedAt: Date;
|
|
39
|
+
startedAt?: Date | null;
|
|
40
|
+
completedAt?: Date | null;
|
|
41
|
+
lockExpiresAt?: Date | null;
|
|
42
|
+
firstErrorAt?: Date | null;
|
|
43
|
+
lastError?: string | null;
|
|
44
|
+
lastFinalizedAt?: Date | null;
|
|
45
|
+
initialScheduledAt?: Date | null;
|
|
46
|
+
lastObservedValues?: Record<string, unknown> | null;
|
|
47
|
+
lastSuccess?: {
|
|
48
|
+
at: Date;
|
|
49
|
+
durationMs: number;
|
|
50
|
+
} | null;
|
|
51
|
+
executionHistory?: Array<{
|
|
52
|
+
at: Date;
|
|
53
|
+
status: 'completed' | 'failed';
|
|
54
|
+
durationMs: number;
|
|
55
|
+
error?: string;
|
|
56
|
+
}>;
|
|
57
|
+
}
|
|
58
|
+
export interface MetaDocument {
|
|
59
|
+
_id: string;
|
|
60
|
+
lock?: {
|
|
61
|
+
expiresAt: Date;
|
|
62
|
+
instanceId: string;
|
|
63
|
+
};
|
|
64
|
+
streamState?: {
|
|
65
|
+
resumeToken: ResumeToken;
|
|
66
|
+
lastClusterTime?: Date;
|
|
67
|
+
};
|
|
68
|
+
reconciliation?: {
|
|
69
|
+
[taskName: string]: boolean;
|
|
70
|
+
};
|
|
71
|
+
reconciliationState?: {
|
|
72
|
+
[collectionName: string]: {
|
|
73
|
+
lastId: unknown;
|
|
74
|
+
taskNames: string[];
|
|
75
|
+
updatedAt: Date;
|
|
76
|
+
};
|
|
77
|
+
};
|
|
78
|
+
tasks?: {
|
|
79
|
+
[taskName: string]: {
|
|
80
|
+
triggerConfig: {
|
|
81
|
+
filter: Document;
|
|
82
|
+
watchProjection: Document;
|
|
83
|
+
};
|
|
84
|
+
handlerVersion: number;
|
|
85
|
+
};
|
|
86
|
+
};
|
|
87
|
+
lastReconciledAt?: Date;
|
|
88
|
+
lastCleanupAt?: Date;
|
|
89
|
+
}
|
|
90
|
+
export interface RegistryDocument {
|
|
91
|
+
_id: string;
|
|
92
|
+
instances: Array<{
|
|
93
|
+
id: string;
|
|
94
|
+
lastSeen: Date | string;
|
|
95
|
+
metrics: unknown;
|
|
96
|
+
}>;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Error thrown when `getDocument` fails because the document no longer matches the filter
|
|
100
|
+
* or has been deleted. The worker will catch this and mark the task as skipped (success).
|
|
101
|
+
*/
|
|
102
|
+
export declare class TaskConditionFailedError extends Error {
|
|
103
|
+
constructor(message?: string);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Context passed to the task handler.
|
|
107
|
+
* It provides access to the source document ID, snapshot data, fetching logic, and flow control.
|
|
108
|
+
*/
|
|
109
|
+
export interface ReactiveTaskContext<T = Document> {
|
|
110
|
+
/** The ID of the source document to process. */
|
|
111
|
+
docId: WithId<T>['_id'];
|
|
112
|
+
/**
|
|
113
|
+
* The snapshot of fields that were observed when the task was triggered.
|
|
114
|
+
* Useful for debugging or decision logic without fetching the full document.
|
|
115
|
+
*/
|
|
116
|
+
watchedValues: Record<string, unknown> | null;
|
|
117
|
+
/**
|
|
118
|
+
* Atomically fetches the source document.
|
|
119
|
+
* Applies the task's `filter` logic to ensure the document still matches.
|
|
120
|
+
*
|
|
121
|
+
* @example
|
|
122
|
+
* ```ts
|
|
123
|
+
* const doc = await context.getDocument({ session });
|
|
124
|
+
* ```
|
|
125
|
+
*
|
|
126
|
+
* @param options MongoDB FindOptions (allows passing `session` for transactions, projection, etc.).
|
|
127
|
+
* @throws TaskConditionFailedError if document is not found or mismatch.
|
|
128
|
+
*/
|
|
129
|
+
getDocument(options?: FindOptions): Promise<T>;
|
|
130
|
+
/**
|
|
131
|
+
* Defers the task execution to a later time.
|
|
132
|
+
* The task will remain 'pending' (or effectively so) until the specified time.
|
|
133
|
+
* Use this for temporary external failures (e.g. rate limits) or to postpone processing.
|
|
134
|
+
* This preserves the original `scheduledAt` for global lag calculation.
|
|
135
|
+
*/
|
|
136
|
+
deferCurrent(delay: number | Date): void;
|
|
137
|
+
/**
|
|
138
|
+
* Pauses ALL tasks of this type (on this instance only) until the specified time.
|
|
139
|
+
* Useful for instance-local backoff (e.g. rate limits), but note that other instances will continue processing.
|
|
140
|
+
*
|
|
141
|
+
* IMPORTANT: This does NOT automatically defer the *current* task.
|
|
142
|
+
* The current task will be marked as successful unless you also call `deferCurrent()` or throw an error.
|
|
143
|
+
*/
|
|
144
|
+
throttleAll(until: number | Date): void;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Worker function provided by the user.
|
|
148
|
+
*/
|
|
149
|
+
export type ReactiveTaskHandler<T = Document> = (context: ReactiveTaskContext<T>) => Promise<void>;
|
|
150
|
+
export type RetryPolicy = {
|
|
151
|
+
/**
|
|
152
|
+
* Maximum number of attempts before marking as 'failed'.
|
|
153
|
+
* Includes the first run (e.g. 1 means "try once, do not retry").
|
|
154
|
+
* Set to -1 for infinite attempts.
|
|
155
|
+
* Default: 5 (if maxDuration is not set)
|
|
156
|
+
*/
|
|
157
|
+
maxAttempts?: number;
|
|
158
|
+
/**
|
|
159
|
+
* Maximum duration since the *first* failure in the current sequence.
|
|
160
|
+
* If the task is still failing after this time window, it is marked as 'failed'.
|
|
161
|
+
* Example: '24h', '30m'
|
|
162
|
+
*/
|
|
163
|
+
maxDuration?: string;
|
|
164
|
+
/**
|
|
165
|
+
* If true, resets the retry history (including `firstErrorAt`) when the task data changes.
|
|
166
|
+
* Default: true
|
|
167
|
+
*/
|
|
168
|
+
resetRetriesOnDataChange?: boolean;
|
|
169
|
+
} & ({
|
|
170
|
+
type: 'exponential';
|
|
171
|
+
min: number | string;
|
|
172
|
+
max: number | string;
|
|
173
|
+
factor?: number;
|
|
174
|
+
} | {
|
|
175
|
+
type: 'linear';
|
|
176
|
+
interval: string;
|
|
177
|
+
} | {
|
|
178
|
+
type: 'fixed';
|
|
179
|
+
interval: string;
|
|
180
|
+
} | {
|
|
181
|
+
type: 'series';
|
|
182
|
+
intervals: string[];
|
|
183
|
+
} | {
|
|
184
|
+
type: 'cron';
|
|
185
|
+
expression: string;
|
|
186
|
+
});
|
|
187
|
+
export interface EvolutionConfig {
|
|
188
|
+
/**
|
|
189
|
+
* Version of the task handler logic.
|
|
190
|
+
* Default: 1
|
|
191
|
+
*/
|
|
192
|
+
handlerVersion?: number;
|
|
193
|
+
/**
|
|
194
|
+
* Policy to apply when the handler version increases.
|
|
195
|
+
* - 'none': Do nothing.
|
|
196
|
+
* - 'reprocess_failed': Reset 'failed' tasks to 'pending'.
|
|
197
|
+
* - 'reprocess_all': Reset ALL tasks (completed & failed) to 'pending'.
|
|
198
|
+
* Default: 'none'
|
|
199
|
+
*/
|
|
200
|
+
onHandlerVersionChange?: 'none' | 'reprocess_failed' | 'reprocess_all';
|
|
201
|
+
/**
|
|
202
|
+
* Whether to trigger reconciliation when the trigger configuration (filter/watchProjection) changes.
|
|
203
|
+
* Default: true
|
|
204
|
+
*/
|
|
205
|
+
reconcileOnTriggerChange?: boolean;
|
|
206
|
+
}
|
|
207
|
+
export interface ReactiveTask<T extends Document> {
|
|
208
|
+
collection: string | Collection<T>;
|
|
209
|
+
task: string;
|
|
210
|
+
watchProjection?: NonNullable<Parameters<Collection<T>['find']>[1]>['projection'];
|
|
211
|
+
/**
|
|
212
|
+
* Aggregation Expression (e.g. { $eq: ['$status', 'active'] }) OR Standard Query (e.g. { status: 'active' }).
|
|
213
|
+
* Standard queries are automatically converted to Aggregation Expressions.
|
|
214
|
+
*/
|
|
215
|
+
filter?: Filter<T>;
|
|
216
|
+
handler: ReactiveTaskHandler<T>;
|
|
217
|
+
/** Time (ms) or interval description (e.g. "500ms", "1s") to postpone `scheduledAt` on change (Debouncing). Default: 1000ms */
|
|
218
|
+
debounce?: number | string;
|
|
219
|
+
/** Retry Policy configuration */
|
|
220
|
+
retryPolicy?: RetryPolicy;
|
|
221
|
+
/**
|
|
222
|
+
* Number of execution history entries to keep.
|
|
223
|
+
* Default: 5
|
|
224
|
+
*/
|
|
225
|
+
executionHistoryLimit?: number;
|
|
226
|
+
/**
|
|
227
|
+
* Evolution configuration for handling changes to task definitions across deployments.
|
|
228
|
+
*/
|
|
229
|
+
evolution?: EvolutionConfig;
|
|
230
|
+
/**
|
|
231
|
+
* Cleanup policy for orphaned task records.
|
|
232
|
+
* Controls when and how task records are automatically deleted.
|
|
233
|
+
*/
|
|
234
|
+
cleanupPolicy?: CleanupPolicy;
|
|
235
|
+
}
|
|
236
|
+
export type ReactiveTaskInternal<T extends Document> = Omit<ReactiveTask<T>, 'collection'> & {
|
|
237
|
+
sourceCollection: Collection<T>;
|
|
238
|
+
tasksCollection: Collection<ReactiveTaskRecord<T>>;
|
|
239
|
+
initPromise: Promise<void>;
|
|
240
|
+
debounceMs: number;
|
|
241
|
+
retryStrategy: ReactiveTaskRetryStrategy;
|
|
242
|
+
executionHistoryLimit: number;
|
|
243
|
+
cleanupPolicyParsed: {
|
|
244
|
+
deleteWhen: CleanupDeleteWhen;
|
|
245
|
+
keepForMs: number;
|
|
246
|
+
};
|
|
247
|
+
repository: import('./ReactiveTaskRepository').ReactiveTaskRepository<T>;
|
|
248
|
+
};
|
|
249
|
+
/**
|
|
250
|
+
* Configuration options for the scheduler.
|
|
251
|
+
*/
|
|
252
|
+
export interface ReactiveTaskSchedulerOptions {
|
|
253
|
+
/**
|
|
254
|
+
* Optional unique identifier for this scheduler instance.
|
|
255
|
+
* Used for leader election, metrics aggregation, and debugging.
|
|
256
|
+
* If not provided, a random ObjectId hex string will be generated.
|
|
257
|
+
*/
|
|
258
|
+
instanceId?: string;
|
|
259
|
+
/** Number of concurrent workers polling the DB. */
|
|
260
|
+
reactiveTaskConcurrency: number;
|
|
261
|
+
/** Optional caller to wrap task execution (e.g. for context propagation). */
|
|
262
|
+
reactiveTaskCaller?: ReactiveTaskCaller;
|
|
263
|
+
/** Optional filter to restrict which tasks this worker processes. */
|
|
264
|
+
reactiveTaskFilter?: ReactiveTaskFilter;
|
|
265
|
+
/**
|
|
266
|
+
* Monitoring configuration options.
|
|
267
|
+
*/
|
|
268
|
+
monitoring?: {
|
|
269
|
+
/**
|
|
270
|
+
* Enable/disable metrics collection.
|
|
271
|
+
* Default: true
|
|
272
|
+
*/
|
|
273
|
+
enabled?: boolean;
|
|
274
|
+
/**
|
|
275
|
+
* How often workers push local metrics to the global registry (ms).
|
|
276
|
+
* Default: 60000 (1m)
|
|
277
|
+
*/
|
|
278
|
+
pushIntervalMs?: number;
|
|
279
|
+
/**
|
|
280
|
+
* Controls which set of metrics to return.
|
|
281
|
+
* Default: 'cluster'
|
|
282
|
+
*/
|
|
283
|
+
scrapeMode?: 'local' | 'cluster';
|
|
284
|
+
/**
|
|
285
|
+
* Read preference for fetching the global registry and global stats.
|
|
286
|
+
* Default: 'secondaryPreferred'
|
|
287
|
+
*/
|
|
288
|
+
readPreference?: 'primary' | 'primaryPreferred' | 'secondary' | 'secondaryPreferred' | 'nearest';
|
|
289
|
+
};
|
|
290
|
+
/**
|
|
291
|
+
* How often to run periodic cleanup of orphaned task records.
|
|
292
|
+
* Accepts duration strings ('24h'), milliseconds, or cron expressions ('0 3 * * *').
|
|
293
|
+
* Default: '24h'
|
|
294
|
+
*/
|
|
295
|
+
reactiveTaskCleanupInterval?: number | string;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Filter function to restrict which tasks are processed by the worker.
|
|
299
|
+
*/
|
|
300
|
+
export interface ReactiveTaskFilter {
|
|
301
|
+
({ task }: {
|
|
302
|
+
task: string;
|
|
303
|
+
}): boolean;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Caller function to wrap task execution.
|
|
307
|
+
*/
|
|
308
|
+
export interface ReactiveTaskCaller {
|
|
309
|
+
<T>(task: () => Promise<T>): Promise<T> | T;
|
|
310
|
+
}
|
|
311
|
+
export declare const CODE_REACTIVE_TASK_STARTED = "reactiveTaskStarted";
|
|
312
|
+
export declare const CODE_REACTIVE_TASK_FINISHED = "reactiveTaskFinished";
|
|
313
|
+
export declare const CODE_REACTIVE_TASK_FAILED = "reactiveTaskFailed";
|
|
314
|
+
export declare const CODE_REACTIVE_TASK_PLANNER_STARTED = "reactiveTaskPlannerStarted";
|
|
315
|
+
export declare const CODE_REACTIVE_TASK_PLANNER_STOPPED = "reactiveTaskPlannerStopped";
|
|
316
|
+
export declare const CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_STARTED = "reactiveTaskPlannerReconciliationStarted";
|
|
317
|
+
export declare const CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_FINISHED = "reactiveTaskPlannerReconciliationFinished";
|
|
318
|
+
export declare const CODE_REACTIVE_TASK_PLANNER_STREAM_ERROR = "reactiveTaskPlannerStreamError";
|
|
319
|
+
export declare const CODE_REACTIVE_TASK_LEADER_LOCK_LOST = "reactiveTaskLeaderLockLost";
|
|
320
|
+
export declare const CODE_REACTIVE_TASK_INITIALIZED = "reactiveTaskInitialized";
|
|
321
|
+
export declare const CODE_REACTIVE_TASK_CLEANUP = "reactiveTaskCleanup";
|
|
322
|
+
export declare const REACTIVE_TASK_META_DOC_ID = "_mongodash_planner_meta";
|
|
323
|
+
/**
|
|
324
|
+
* Filter for querying tasks.
|
|
325
|
+
* All fields are optional and operate as AND conditions.
|
|
326
|
+
*/
|
|
327
|
+
export interface ReactiveTaskQuery<T = unknown> {
|
|
328
|
+
/** Filter by specific task name(s) */
|
|
329
|
+
task?: string | string[];
|
|
330
|
+
/** Filter by task record ID */
|
|
331
|
+
_id?: string | ObjectId | string[] | ObjectId[];
|
|
332
|
+
/** Filter by task status */
|
|
333
|
+
status?: ReactiveTaskStatus | ReactiveTaskStatus[];
|
|
334
|
+
/**
|
|
335
|
+
* Filter by Source Document fields.
|
|
336
|
+
* This allows finding tasks based on the state of the original document.
|
|
337
|
+
*
|
|
338
|
+
* Examples:
|
|
339
|
+
* - `{ _id: 'user-123' }` -> Find tasks for specific doc.
|
|
340
|
+
* - `{ region: 'EU', active: true }` -> Find tasks for all active EU docs.
|
|
341
|
+
*/
|
|
342
|
+
sourceDocFilter?: Filter<T>;
|
|
343
|
+
/** Search in lastError message (regex or text search) */
|
|
344
|
+
errorMessage?: string | RegExp;
|
|
345
|
+
/** Filter to tasks that have a lastError (true) or don't (false) */
|
|
346
|
+
hasError?: boolean;
|
|
347
|
+
}
|
|
348
|
+
export interface PaginationOptions {
|
|
349
|
+
limit?: number;
|
|
350
|
+
skip?: number;
|
|
351
|
+
sort?: {
|
|
352
|
+
field: keyof ReactiveTaskRecord;
|
|
353
|
+
direction: 1 | -1;
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
export interface PagedResult<T> {
|
|
357
|
+
items: T[];
|
|
358
|
+
total: number;
|
|
359
|
+
limit: number;
|
|
360
|
+
offset: number;
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Statistical summary of tasks matching a filter.
|
|
364
|
+
*/
|
|
365
|
+
export interface ReactiveTaskStatsOptions {
|
|
366
|
+
readPreference?: import('mongodb').ReadPreferenceLike;
|
|
367
|
+
groupByTask?: boolean;
|
|
368
|
+
includeStatusCounts?: boolean;
|
|
369
|
+
includeErrorCount?: boolean;
|
|
370
|
+
includeGlobalLag?: boolean;
|
|
371
|
+
}
|
|
372
|
+
export interface ReactiveTaskStatsResult {
|
|
373
|
+
statuses: {
|
|
374
|
+
_id: string | {
|
|
375
|
+
task: string;
|
|
376
|
+
status: string;
|
|
377
|
+
};
|
|
378
|
+
count: number;
|
|
379
|
+
}[];
|
|
380
|
+
errorCount?: number;
|
|
381
|
+
errorCounts?: {
|
|
382
|
+
_id: string;
|
|
383
|
+
count: number;
|
|
384
|
+
}[];
|
|
385
|
+
globalLag?: {
|
|
386
|
+
_id: string;
|
|
387
|
+
minScheduledAt: Date | null;
|
|
388
|
+
}[];
|
|
389
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { ReactiveTaskRegistry } from './ReactiveTaskRegistry';
|
|
2
|
+
import { ReactiveTaskCaller, ReactiveTaskFilter } from './ReactiveTaskTypes';
|
|
3
|
+
import { OnInfo } from '../OnInfo';
|
|
4
|
+
import { OnError } from '../OnError';
|
|
5
|
+
import { MetricsCollector } from './MetricsCollector';
|
|
6
|
+
export interface WorkerCallbacks {
|
|
7
|
+
onTaskFound: (collectionName: string) => void;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Responsible for executing reactive tasks.
|
|
11
|
+
*
|
|
12
|
+
* Responsibilities:
|
|
13
|
+
* - Polls for pending tasks from the database.
|
|
14
|
+
* - Applies filtering to restrict which tasks this worker processes.
|
|
15
|
+
* - Locks tasks during execution to prevent concurrent processing.
|
|
16
|
+
* - Fetches the source document and executes the user-defined handler.
|
|
17
|
+
* - Handles task completion, failure, retries, and dead-letter queueing.
|
|
18
|
+
* - Manages the visibility timeout lock extension.
|
|
19
|
+
*/
|
|
20
|
+
export declare class ReactiveTaskWorker {
|
|
21
|
+
private instanceId;
|
|
22
|
+
private registry;
|
|
23
|
+
private callbacks;
|
|
24
|
+
private internalOptions;
|
|
25
|
+
private taskFilter?;
|
|
26
|
+
private onInfo;
|
|
27
|
+
private onError;
|
|
28
|
+
private metricsCollector?;
|
|
29
|
+
private taskCaller;
|
|
30
|
+
private throttledUntil;
|
|
31
|
+
constructor(instanceId: string, registry: ReactiveTaskRegistry, callbacks: WorkerCallbacks, internalOptions?: {
|
|
32
|
+
visibilityTimeoutMs: number;
|
|
33
|
+
}, taskCaller?: ReactiveTaskCaller, taskFilter?: ReactiveTaskFilter | undefined, onInfo?: OnInfo, onError?: OnError, metricsCollector?: MetricsCollector | undefined);
|
|
34
|
+
tryRunATask(collectionName: string): Promise<void>;
|
|
35
|
+
private processTask;
|
|
36
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Document } from 'mongodb';
|
|
2
|
+
/**
|
|
3
|
+
* Compiles a user-friendly watchProjection into a strict Aggregation Expression.
|
|
4
|
+
* This handles:
|
|
5
|
+
* 1. Unflattening dotted keys (e.g. {'a.b': 1} -> {a: {b: '$a.b'}})
|
|
6
|
+
* 2. Converting shorthand inclusion (1/true) to field paths ('$key')
|
|
7
|
+
* 3. Validating that no unsupported features (like exclusion) are used.
|
|
8
|
+
*
|
|
9
|
+
* @param projection The user-provided projection.
|
|
10
|
+
* @returns An aggregation expression or string '$$ROOT'.
|
|
11
|
+
*/
|
|
12
|
+
export declare function compileWatchProjection(projection?: Document): Document | string;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { Document } from 'mongodb';
|
|
2
|
+
import type { Registry } from 'prom-client';
|
|
3
|
+
import { GlobalsCollection } from '../globalsCollection';
|
|
4
|
+
import { OnError } from '../OnError';
|
|
5
|
+
import { OnInfo } from '../OnInfo';
|
|
6
|
+
import { ReactiveTaskManager } from './ReactiveTaskManager';
|
|
7
|
+
import { ReactiveTaskRegistry } from './ReactiveTaskRegistry';
|
|
8
|
+
import { PagedResult, PaginationOptions, ReactiveTask, ReactiveTaskQuery, ReactiveTaskRecord, ReactiveTaskSchedulerOptions } from './ReactiveTaskTypes';
|
|
9
|
+
export { CODE_REACTIVE_TASK_CLEANUP, CODE_REACTIVE_TASK_FAILED, CODE_REACTIVE_TASK_FINISHED, CODE_REACTIVE_TASK_INITIALIZED, CODE_REACTIVE_TASK_LEADER_LOCK_LOST, CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_FINISHED, CODE_REACTIVE_TASK_PLANNER_RECONCILIATION_STARTED, CODE_REACTIVE_TASK_PLANNER_STARTED, CODE_REACTIVE_TASK_PLANNER_STOPPED, CODE_REACTIVE_TASK_PLANNER_STREAM_ERROR, CODE_REACTIVE_TASK_STARTED, PagedResult, PaginationOptions, ReactiveTask, ReactiveTaskCaller, ReactiveTaskFilter, ReactiveTaskHandler, ReactiveTaskQuery, ReactiveTaskRecord, ReactiveTaskSchedulerOptions, ReactiveTaskStatus, REACTIVE_TASK_META_DOC_ID, TaskConditionFailedError, } from './ReactiveTaskTypes';
|
|
10
|
+
export { scheduler as _scheduler };
|
|
11
|
+
export type InitOptions = {
|
|
12
|
+
onError: OnError;
|
|
13
|
+
onInfo: OnInfo;
|
|
14
|
+
globalsCollection: GlobalsCollection;
|
|
15
|
+
} & Partial<ReactiveTaskSchedulerOptions>;
|
|
16
|
+
/**
|
|
17
|
+
* Main scheduler class implementing the described logic.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Main entry point for the Reactive Tasks system.
|
|
21
|
+
*
|
|
22
|
+
* Responsibilities:
|
|
23
|
+
* - Orchestrates the initialization and configuration of the system.
|
|
24
|
+
* - Manages the lifecycle of `LeaderElector`, `ReactiveTaskPlanner`, and `ReactiveTaskWorker`.
|
|
25
|
+
* - Provides the public API for registering tasks and starting/stopping the system.
|
|
26
|
+
* - Configures the `ConcurrentRunner` for worker execution.
|
|
27
|
+
*/
|
|
28
|
+
export declare class ReactiveTaskScheduler {
|
|
29
|
+
private options;
|
|
30
|
+
private registry;
|
|
31
|
+
private _instanceId?;
|
|
32
|
+
private isRunning;
|
|
33
|
+
private concurrentRunner;
|
|
34
|
+
private leaderElector;
|
|
35
|
+
private taskPlanner;
|
|
36
|
+
private worker;
|
|
37
|
+
private metricsCollector;
|
|
38
|
+
private taskManager;
|
|
39
|
+
private internalOptions;
|
|
40
|
+
constructor();
|
|
41
|
+
/**
|
|
42
|
+
* Get the instanceId, generating one if not configured.
|
|
43
|
+
*/
|
|
44
|
+
private get instanceId();
|
|
45
|
+
configure(options: Partial<ReactiveTaskSchedulerOptions>): void;
|
|
46
|
+
addTask(taskDef: ReactiveTask<Document>): Promise<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Starts the entire system - leader election and workers.
|
|
49
|
+
*/
|
|
50
|
+
start(): Promise<void>;
|
|
51
|
+
/**
|
|
52
|
+
* Stops the entire system.
|
|
53
|
+
*/
|
|
54
|
+
stop(): Promise<void>;
|
|
55
|
+
getPrometheusMetrics(): Promise<Registry | null>;
|
|
56
|
+
getTaskManager(): ReactiveTaskManager;
|
|
57
|
+
getRegistry(): ReactiveTaskRegistry;
|
|
58
|
+
}
|
|
59
|
+
declare const scheduler: ReactiveTaskScheduler;
|
|
60
|
+
export declare function init(initOptions: InitOptions): void;
|
|
61
|
+
export declare function reactiveTask<T extends Document = Document>(taskDef: ReactiveTask<T>): Promise<void>;
|
|
62
|
+
export declare function stopReactiveTasks(): Promise<void>;
|
|
63
|
+
export declare function startReactiveTasks(): Promise<void>;
|
|
64
|
+
/**
|
|
65
|
+
* Returns the Prometheus metrics for the Reactive Tasks system.
|
|
66
|
+
* Respects the 'scrapeMode' configuration.
|
|
67
|
+
*/
|
|
68
|
+
export declare function getPrometheusMetrics(): Promise<Registry | null>;
|
|
69
|
+
/**
|
|
70
|
+
* List tasks matching the criteria.
|
|
71
|
+
*/
|
|
72
|
+
export declare function getReactiveTasks(query: ReactiveTaskQuery, pagination?: PaginationOptions): Promise<PagedResult<ReactiveTaskRecord>>;
|
|
73
|
+
/**
|
|
74
|
+
* Count tasks matching criteria.
|
|
75
|
+
*/
|
|
76
|
+
export declare function countReactiveTasks(query: ReactiveTaskQuery): Promise<number>;
|
|
77
|
+
/**
|
|
78
|
+
* Retries/Retriggers tasks matching the query.
|
|
79
|
+
*/
|
|
80
|
+
export declare function retryReactiveTasks(query: ReactiveTaskQuery): Promise<{
|
|
81
|
+
modifiedCount: number;
|
|
82
|
+
}>;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Document } from 'mongodb';
|
|
2
|
+
/**
|
|
3
|
+
* Converts a standard MongoDB query object into an Aggregation Expression.
|
|
4
|
+
* strict support for:
|
|
5
|
+
* - Implicit Equality: { a: 1 }
|
|
6
|
+
* - Comparisons: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin
|
|
7
|
+
* - Logical: $and, $or, $nor, $not
|
|
8
|
+
* - Regex: /pattern/, $regex
|
|
9
|
+
* - Existence: $exists
|
|
10
|
+
*
|
|
11
|
+
* THROWS error on unsupported operators (like $elemMatch, $all, $xyz) to avoid silent failures.
|
|
12
|
+
*/
|
|
13
|
+
export declare function queryToExpression(query: Document): Document;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Document } from 'mongodb';
|
|
2
|
+
/**
|
|
3
|
+
* Normalizes the task filter into a valid Aggregation Expression.
|
|
4
|
+
* If the filter is a standard query (keys not starting with $), it is converted.
|
|
5
|
+
* If the filter is already an expression (keys starting with $), it is kept.
|
|
6
|
+
*
|
|
7
|
+
* @param filter The filter object to normalize.
|
|
8
|
+
* @param taskName The name of the task (for error context).
|
|
9
|
+
*/
|
|
10
|
+
export declare function normalizeTaskFilter(filter: Document | undefined, taskName: string): Document | undefined;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { ReactiveTaskScheduler } from '../reactiveTasks/index';
|
|
2
|
+
import { CronTaskQuery } from '../cronTasks';
|
|
3
|
+
import { Document } from 'mongodb';
|
|
4
|
+
export declare class OperationalTaskController {
|
|
5
|
+
private scheduler;
|
|
6
|
+
constructor(scheduler: ReactiveTaskScheduler);
|
|
7
|
+
getReactiveTasks(params: {
|
|
8
|
+
limit?: number;
|
|
9
|
+
skip?: number;
|
|
10
|
+
task?: string;
|
|
11
|
+
collection?: string;
|
|
12
|
+
status?: string;
|
|
13
|
+
errorMessage?: string;
|
|
14
|
+
hasError?: string;
|
|
15
|
+
sourceDocId?: string;
|
|
16
|
+
}): Promise<{
|
|
17
|
+
stats: import("../reactiveTasks/ReactiveTaskTypes").ReactiveTaskStatsResult;
|
|
18
|
+
items: import("../reactiveTasks/ReactiveTaskTypes").ReactiveTaskRecord<Document>[];
|
|
19
|
+
total: number;
|
|
20
|
+
limit: number;
|
|
21
|
+
offset: number;
|
|
22
|
+
}>;
|
|
23
|
+
retryReactiveTasks(body: {
|
|
24
|
+
task?: string;
|
|
25
|
+
status?: string;
|
|
26
|
+
errorMessage?: string;
|
|
27
|
+
_id?: string;
|
|
28
|
+
sourceDocId?: string;
|
|
29
|
+
}): Promise<{
|
|
30
|
+
modifiedCount: number;
|
|
31
|
+
}>;
|
|
32
|
+
private createSmartIdFilter;
|
|
33
|
+
getCronTasks(params: CronTaskQuery): Promise<import("../cronTasks").CronPagedResult<import("../cronTasks").CronTaskRecord>>;
|
|
34
|
+
triggerCronTask(body: {
|
|
35
|
+
taskId: string;
|
|
36
|
+
}): Promise<{
|
|
37
|
+
success: boolean;
|
|
38
|
+
}>;
|
|
39
|
+
getInfo(): Promise<{
|
|
40
|
+
databaseName: string;
|
|
41
|
+
reactiveTasks: {
|
|
42
|
+
name: string;
|
|
43
|
+
collection: string;
|
|
44
|
+
stats: {
|
|
45
|
+
success: number;
|
|
46
|
+
failed: number;
|
|
47
|
+
processing: number;
|
|
48
|
+
pending: number;
|
|
49
|
+
error: number;
|
|
50
|
+
};
|
|
51
|
+
}[];
|
|
52
|
+
cronTasks: {
|
|
53
|
+
id: string;
|
|
54
|
+
status: import("../cronTasks").CronTaskStatus;
|
|
55
|
+
lastRunError: string | null | undefined;
|
|
56
|
+
nextRunAt: Date;
|
|
57
|
+
}[];
|
|
58
|
+
}>;
|
|
59
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { IncomingMessage, ServerResponse } from 'http';
|
|
2
|
+
import { ReactiveTaskScheduler } from '../reactiveTasks/index';
|
|
3
|
+
export interface ServeDashboardOptions {
|
|
4
|
+
dashboardPath?: string;
|
|
5
|
+
scheduler?: ReactiveTaskScheduler;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Serve the mongodash dashboard.
|
|
9
|
+
* Framework-agnostic: Works with Express, Koa, or native Node.js http.
|
|
10
|
+
* Returns true if the request was handled.
|
|
11
|
+
*/
|
|
12
|
+
export declare function serveDashboard(req: IncomingMessage, res: ServerResponse, options?: ServeDashboardOptions): Promise<boolean>;
|