mongodash 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -22
- package/dist/dashboard/index.html +40 -0
- package/dist/lib/playground/server.js +131 -0
- package/dist/lib/playground/server.js.map +1 -0
- package/dist/lib/src/ConcurrentRunner.js +148 -0
- package/dist/lib/src/ConcurrentRunner.js.map +1 -0
- package/dist/lib/{OnError.js → src/OnError.js} +3 -3
- package/dist/lib/src/OnError.js.map +1 -0
- package/dist/lib/{OnInfo.js → src/OnInfo.js} +6 -3
- package/dist/lib/src/OnInfo.js.map +1 -0
- package/dist/lib/{createContinuousLock.js → src/createContinuousLock.js} +5 -3
- package/dist/lib/src/createContinuousLock.js.map +1 -0
- package/dist/lib/{cronTasks.js → src/cronTasks.js} +129 -73
- package/dist/lib/src/cronTasks.js.map +1 -0
- package/dist/lib/{getCollection.js → src/getCollection.js} +2 -2
- package/dist/lib/src/getCollection.js.map +1 -0
- package/dist/lib/{getMongoClient.js → src/getMongoClient.js} +2 -2
- package/dist/lib/src/getMongoClient.js.map +1 -0
- package/dist/lib/src/globalsCollection.js +10 -0
- package/dist/lib/src/globalsCollection.js.map +1 -0
- package/dist/lib/src/index.js +101 -0
- package/dist/lib/src/index.js.map +1 -0
- package/dist/lib/{initPromise.js → src/initPromise.js} +2 -3
- package/dist/lib/src/initPromise.js.map +1 -0
- package/dist/lib/src/mongoCompatibility.js +10 -0
- package/dist/lib/src/mongoCompatibility.js.map +1 -0
- package/dist/lib/src/parseInterval.js +60 -0
- package/dist/lib/src/parseInterval.js.map +1 -0
- package/dist/lib/src/prefixFilterKeys.js +69 -0
- package/dist/lib/src/prefixFilterKeys.js.map +1 -0
- package/dist/lib/src/processInBatches.js +46 -0
- package/dist/lib/src/processInBatches.js.map +1 -0
- package/dist/lib/src/reactiveTasks/LeaderElector.js +155 -0
- package/dist/lib/src/reactiveTasks/LeaderElector.js.map +1 -0
- package/dist/lib/src/reactiveTasks/MetricsCollector.js +410 -0
- package/dist/lib/src/reactiveTasks/MetricsCollector.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskManager.js +288 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskManager.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskOps.js +185 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskOps.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskPlanner.js +443 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskPlanner.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskReconciler.js +218 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskReconciler.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRegistry.js +184 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRegistry.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRepository.js +355 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRepository.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRetryStrategy.js +153 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskRetryStrategy.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskTypes.js +34 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskTypes.js.map +1 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskWorker.js +186 -0
- package/dist/lib/src/reactiveTasks/ReactiveTaskWorker.js.map +1 -0
- package/dist/lib/src/reactiveTasks/compileWatchProjection.js +65 -0
- package/dist/lib/src/reactiveTasks/compileWatchProjection.js.map +1 -0
- package/dist/lib/src/reactiveTasks/index.js +298 -0
- package/dist/lib/src/reactiveTasks/index.js.map +1 -0
- package/dist/lib/src/reactiveTasks/queryToExpression.js +160 -0
- package/dist/lib/src/reactiveTasks/queryToExpression.js.map +1 -0
- package/dist/lib/src/reactiveTasks/validateTaskFilter.js +88 -0
- package/dist/lib/src/reactiveTasks/validateTaskFilter.js.map +1 -0
- package/dist/lib/src/task-management/OperationalTaskController.js +162 -0
- package/dist/lib/src/task-management/OperationalTaskController.js.map +1 -0
- package/dist/lib/src/task-management/index.js +27 -0
- package/dist/lib/src/task-management/index.js.map +1 -0
- package/dist/lib/src/task-management/serveDashboard.js +149 -0
- package/dist/lib/src/task-management/serveDashboard.js.map +1 -0
- package/dist/lib/src/task-management/types.js +10 -0
- package/dist/lib/src/task-management/types.js.map +1 -0
- package/dist/lib/{withLock.js → src/withLock.js} +3 -4
- package/dist/lib/src/withLock.js.map +1 -0
- package/dist/lib/{withTransaction.js → src/withTransaction.js} +4 -4
- package/dist/lib/src/withTransaction.js.map +1 -0
- package/dist/lib/tools/check-db-connection.js +28 -0
- package/dist/lib/tools/check-db-connection.js.map +1 -0
- package/dist/lib/tools/clean-testing-databases.js +12 -0
- package/dist/lib/tools/clean-testing-databases.js.map +1 -0
- package/dist/lib/tools/prepare-republish.js +27 -0
- package/dist/lib/tools/prepare-republish.js.map +1 -0
- package/dist/lib/tools/test-matrix-local.js +212 -0
- package/dist/lib/tools/test-matrix-local.js.map +1 -0
- package/dist/lib/tools/testingDatabase.js +55 -0
- package/dist/lib/tools/testingDatabase.js.map +1 -0
- package/dist/types/playground/server.d.ts +1 -0
- package/dist/types/src/ConcurrentRunner.d.ts +30 -0
- package/dist/types/{OnInfo.d.ts → src/OnInfo.d.ts} +1 -1
- package/dist/types/{cronTasks.d.ts → src/cronTasks.d.ts} +44 -1
- package/dist/types/src/globalsCollection.d.ts +4 -0
- package/dist/types/src/index.d.ts +28 -0
- package/dist/types/src/mongoCompatibility.d.ts +29 -0
- package/dist/types/src/parseInterval.d.ts +12 -0
- package/dist/types/src/prefixFilterKeys.d.ts +11 -0
- package/dist/types/src/processInBatches.d.ts +10 -0
- package/dist/types/src/reactiveTasks/LeaderElector.d.ts +42 -0
- package/dist/types/src/reactiveTasks/MetricsCollector.d.ts +73 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskManager.d.ts +18 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskOps.d.ts +17 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskPlanner.d.ts +62 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskReconciler.d.ts +29 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskRegistry.d.ts +34 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskRepository.d.ts +59 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskRetryStrategy.d.ts +21 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskTypes.d.ts +389 -0
- package/dist/types/src/reactiveTasks/ReactiveTaskWorker.d.ts +36 -0
- package/dist/types/src/reactiveTasks/compileWatchProjection.d.ts +12 -0
- package/dist/types/src/reactiveTasks/index.d.ts +82 -0
- package/dist/types/src/reactiveTasks/queryToExpression.d.ts +13 -0
- package/dist/types/src/reactiveTasks/validateTaskFilter.d.ts +10 -0
- package/dist/types/src/task-management/OperationalTaskController.d.ts +59 -0
- package/dist/types/src/task-management/index.d.ts +3 -0
- package/dist/types/src/task-management/serveDashboard.d.ts +12 -0
- package/dist/types/src/task-management/types.d.ts +95 -0
- package/dist/types/tools/check-db-connection.d.ts +2 -0
- package/dist/types/tools/clean-testing-databases.d.ts +1 -0
- package/dist/types/tools/prepare-republish.d.ts +2 -0
- package/dist/types/tools/test-matrix-local.d.ts +1 -0
- package/dist/types/tools/testingDatabase.d.ts +2 -0
- package/docs/.vitepress/cache/deps/_metadata.json +31 -0
- package/docs/.vitepress/cache/deps/chunk-LE5NDSFD.js +12824 -0
- package/docs/.vitepress/cache/deps/chunk-LE5NDSFD.js.map +7 -0
- package/docs/.vitepress/cache/deps/package.json +3 -0
- package/docs/.vitepress/cache/deps/vitepress___@vue_devtools-api.js +4505 -0
- package/docs/.vitepress/cache/deps/vitepress___@vue_devtools-api.js.map +7 -0
- package/docs/.vitepress/cache/deps/vitepress___@vueuse_core.js +9731 -0
- package/docs/.vitepress/cache/deps/vitepress___@vueuse_core.js.map +7 -0
- package/docs/.vitepress/cache/deps/vue.js +347 -0
- package/docs/.vitepress/cache/deps/vue.js.map +7 -0
- package/docs/.vitepress/config.mts +48 -0
- package/docs/.vitepress/theme/index.ts +4 -0
- package/docs/.vitepress/theme/style.css +16 -0
- package/docs/assets/dashboard.png +0 -0
- package/docs/cron-tasks.md +172 -0
- package/docs/dashboard.md +117 -0
- package/docs/getters.md +31 -0
- package/docs/getting-started.md +120 -0
- package/docs/index.md +29 -0
- package/docs/initialization.md +59 -0
- package/docs/process-in-batches.md +73 -0
- package/docs/reactive-tasks.md +914 -0
- package/docs/with-lock.md +45 -0
- package/docs/with-transaction.md +65 -0
- package/grafana/reactive_tasks.json +765 -0
- package/package.json +127 -116
- package/dist/lib/OnError.js.map +0 -1
- package/dist/lib/OnInfo.js.map +0 -1
- package/dist/lib/createContinuousLock.js.map +0 -1
- package/dist/lib/cronTasks.js.map +0 -1
- package/dist/lib/getCollection.js.map +0 -1
- package/dist/lib/getMongoClient.js.map +0 -1
- package/dist/lib/index.js +0 -64
- package/dist/lib/index.js.map +0 -1
- package/dist/lib/initPromise.js.map +0 -1
- package/dist/lib/withLock.js.map +0 -1
- package/dist/lib/withTransaction.js.map +0 -1
- package/dist/types/index.d.ts +0 -17
- /package/dist/types/{OnError.d.ts → src/OnError.d.ts} +0 -0
- /package/dist/types/{createContinuousLock.d.ts → src/createContinuousLock.d.ts} +0 -0
- /package/dist/types/{getCollection.d.ts → src/getCollection.d.ts} +0 -0
- /package/dist/types/{getMongoClient.d.ts → src/getMongoClient.d.ts} +0 -0
- /package/dist/types/{initPromise.d.ts → src/initPromise.d.ts} +0 -0
- /package/dist/types/{withLock.d.ts → src/withLock.d.ts} +0 -0
- /package/dist/types/{withTransaction.d.ts → src/withTransaction.d.ts} +0 -0
package/docs/getters.md
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Collection and Client getters
|
|
2
|
+
|
|
3
|
+
Utility functions for global access to Collections and MongoClient.
|
|
4
|
+
|
|
5
|
+
## getCollection(name [, options]) => Collection
|
|
6
|
+
|
|
7
|
+
```typescript
|
|
8
|
+
getCollection(name [, options]) => Collection
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Get a new instance of a [collection](https://mongodb.github.io/node-mongodb-native/4.0/classes/collection.html). Uses default database determined from the connection string. See [official driver documentation](http://mongodb.github.io/node-mongodb-native/3.6/api/Db.html#collection) for valid options.
|
|
12
|
+
|
|
13
|
+
```typescript
|
|
14
|
+
import { getCollection } from 'mongodash';
|
|
15
|
+
const myCollection = getCollection('myCollectionName');
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
The method just shortens `getMongoClient().db().collection('myCollectionName')`.
|
|
19
|
+
|
|
20
|
+
## getMongoClient() => MongoClient
|
|
21
|
+
|
|
22
|
+
```typescript
|
|
23
|
+
getMongoClient() => MongoClient
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Returns the global instance of MongoClient. See [official driver documentation](https://mongodb.github.io/node-mongodb-native/4.0/classes/mongoclient.html) for documentation of MongoClient.
|
|
27
|
+
|
|
28
|
+
```typescript
|
|
29
|
+
import { getMongoClient } from 'mongodash';
|
|
30
|
+
const mongoClient = getMongoClient();
|
|
31
|
+
```
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Getting Started
|
|
3
|
+
---
|
|
4
|
+
|
|
5
|
+
<!--
|
|
6
|
+
⚠️ WARNING: DO NOT EDIT THIS FILE MANUALLY ⚠️
|
|
7
|
+
|
|
8
|
+
This file is strictly auto-generated from README.md.
|
|
9
|
+
Any changes made here will be overwritten by the next build.
|
|
10
|
+
To make changes, edit the README.md file in the root directory.
|
|
11
|
+
-->
|
|
12
|
+
|
|
13
|
+
# Getting Started
|
|
14
|
+
|
|
15
|
+
Installation:
|
|
16
|
+
```bash
|
|
17
|
+
npm install mongodash
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Initialization
|
|
21
|
+
```typescript
|
|
22
|
+
import mongodash from 'mongodash';
|
|
23
|
+
|
|
24
|
+
await mongodash.init({
|
|
25
|
+
uri: 'mongodb://mongodb0.example.com:27017/myDatabase'
|
|
26
|
+
});
|
|
27
|
+
```
|
|
28
|
+
See more initialization options [here](https://vaclavobornik.github.io/mongodash/initialization).
|
|
29
|
+
|
|
30
|
+
<br>
|
|
31
|
+
|
|
32
|
+
## Reactive Tasks
|
|
33
|
+
|
|
34
|
+
```typescript
|
|
35
|
+
import { reactiveTask } from 'mongodash';
|
|
36
|
+
|
|
37
|
+
// Trigger a task when a user is updated
|
|
38
|
+
await reactiveTask({
|
|
39
|
+
task: 'on-user-update',
|
|
40
|
+
collection: 'users',
|
|
41
|
+
handler: async (doc) => {
|
|
42
|
+
console.log('User changed:', doc._id);
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
```
|
|
46
|
+
See detailed description [here](https://vaclavobornik.github.io/mongodash/reactive-tasks).
|
|
47
|
+
|
|
48
|
+
<br>
|
|
49
|
+
|
|
50
|
+
## cronTask
|
|
51
|
+
```typescript
|
|
52
|
+
import { cronTask } from 'mongodash';
|
|
53
|
+
|
|
54
|
+
await cronTask('my-task-id', '5m 20s', async () => {
|
|
55
|
+
|
|
56
|
+
console.log('Hurray the task is running!');
|
|
57
|
+
|
|
58
|
+
});
|
|
59
|
+
```
|
|
60
|
+
See detailed description and more cron tasks methods [here](https://vaclavobornik.github.io/mongodash/cron-tasks).
|
|
61
|
+
|
|
62
|
+
<br>
|
|
63
|
+
|
|
64
|
+
## withLock
|
|
65
|
+
|
|
66
|
+
```typescript
|
|
67
|
+
import { withLock } from 'mongodash';
|
|
68
|
+
|
|
69
|
+
await withLock('my-lock-id', async () => {
|
|
70
|
+
|
|
71
|
+
// it is quaranteed this callback will never run in parallel,
|
|
72
|
+
// so all race-conditions are solved
|
|
73
|
+
const data = await loadFromDatabase();
|
|
74
|
+
data.counter += 1;
|
|
75
|
+
await saveToDatabase(data);
|
|
76
|
+
|
|
77
|
+
});
|
|
78
|
+
```
|
|
79
|
+
See detailed description [here](https://vaclavobornik.github.io/mongodash/with-lock).
|
|
80
|
+
|
|
81
|
+
<br>
|
|
82
|
+
|
|
83
|
+
## withTransaction
|
|
84
|
+
```typescript
|
|
85
|
+
import { withTransaction, getCollection } from 'mongodash';
|
|
86
|
+
|
|
87
|
+
const createdDocuments = await withTransaction(async (session) => {
|
|
88
|
+
|
|
89
|
+
const myDocument1 = { value: 1 };
|
|
90
|
+
const myDocument2 = { value: 2 };
|
|
91
|
+
|
|
92
|
+
const collection = getCollection('myCollection');
|
|
93
|
+
await collection.insertOne(myDocument1, { session });
|
|
94
|
+
await collection.insertOne(myDocument2, { session });
|
|
95
|
+
|
|
96
|
+
return [myDocument1, myDocument2];
|
|
97
|
+
});
|
|
98
|
+
```
|
|
99
|
+
See detailed description [here](https://vaclavobornik.github.io/mongodash/with-transaction).
|
|
100
|
+
|
|
101
|
+
<br>
|
|
102
|
+
|
|
103
|
+
## getCollection
|
|
104
|
+
```typescript
|
|
105
|
+
import { getCollection } from 'mongodash';
|
|
106
|
+
|
|
107
|
+
const myCollection = getCollection('myCollectionName');
|
|
108
|
+
```
|
|
109
|
+
See detailed description [here](https://vaclavobornik.github.io/mongodash/getters).
|
|
110
|
+
|
|
111
|
+
<br>
|
|
112
|
+
|
|
113
|
+
## getMongoClient
|
|
114
|
+
```typescript
|
|
115
|
+
import { getMongoClient } from 'mongodash';
|
|
116
|
+
|
|
117
|
+
const mongoClient = getMongoClient();
|
|
118
|
+
```
|
|
119
|
+
See detailed description [here](https://vaclavobornik.github.io/mongodash/getters).
|
|
120
|
+
|
package/docs/index.md
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
---
|
|
2
|
+
layout: home
|
|
3
|
+
|
|
4
|
+
hero:
|
|
5
|
+
name: "Mongodash"
|
|
6
|
+
text: "Modern MongoDB Utility Library"
|
|
7
|
+
tagline: "Cron, Locks, Transactions, and more made simple."
|
|
8
|
+
actions:
|
|
9
|
+
- theme: brand
|
|
10
|
+
text: Getting Started
|
|
11
|
+
link: /getting-started
|
|
12
|
+
- theme: alt
|
|
13
|
+
text: View on GitHub
|
|
14
|
+
link: https://github.com/VaclavObornik/mongodash
|
|
15
|
+
|
|
16
|
+
features:
|
|
17
|
+
- title: Reactive Tasks
|
|
18
|
+
details: Instant background jobs triggered by MongoDB Change Streams.
|
|
19
|
+
link: /reactive-tasks
|
|
20
|
+
- title: Cron Tasks
|
|
21
|
+
details: Reliable distributed cron tasks stored in MongoDB.
|
|
22
|
+
link: /cron-tasks
|
|
23
|
+
- title: Concurrency Control
|
|
24
|
+
details: Distributed locks to manage critical sections across instances.
|
|
25
|
+
link: /with-lock
|
|
26
|
+
- title: Transactions
|
|
27
|
+
details: Simplified transaction management with automatic session handling.
|
|
28
|
+
link: /with-transaction
|
|
29
|
+
---
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Initialization
|
|
2
|
+
|
|
3
|
+
## Installation
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
npm install mongodash
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## mongodash.init(options) => Promise
|
|
10
|
+
|
|
11
|
+
Initializes the `mongodash` library. This method must be called before using any other features.
|
|
12
|
+
|
|
13
|
+
```typescript
|
|
14
|
+
import mongodash from 'mongodash';
|
|
15
|
+
|
|
16
|
+
// Initialize with connection string
|
|
17
|
+
await mongodash.init({
|
|
18
|
+
uri: 'mongodb://localhost:27017/my-app',
|
|
19
|
+
// ...other options
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
// OR initialize with existing MongoClient
|
|
23
|
+
await mongodash.init({
|
|
24
|
+
mongoClient: myExistingClient,
|
|
25
|
+
// ...other options
|
|
26
|
+
});
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
### Options Reference
|
|
30
|
+
|
|
31
|
+
#### Connection Options
|
|
32
|
+
|
|
33
|
+
| Option | Type | Description |
|
|
34
|
+
| :--- | :--- | :--- |
|
|
35
|
+
| `uri` | `string` | **Required** (if `mongoClient` is not provided). The MongoDB connection string. `mongodash` will connect automatically. |
|
|
36
|
+
| `mongoClient` | `MongoClient` | **Required** (if `uri` is not provided). An existing, connected `MongoClient` instance. |
|
|
37
|
+
| `clientOptions` | `MongoClientOptions` | Optional. Used only when `uri` is provided. Options passed to the MongoDB driver. |
|
|
38
|
+
|
|
39
|
+
#### General Options
|
|
40
|
+
|
|
41
|
+
| Option | Type | Description |
|
|
42
|
+
| :--- | :--- | :--- |
|
|
43
|
+
| `globalsCollection` | `string` \| `Collection` | Name of the collection used for distributed locking and coordination. Mongodash uses a fixed set of documents identified by unique derived `_id`s, so it has a minimal storage footprint and doesn't pollute the collection. Default: `_mongodash_globals`. |
|
|
44
|
+
| `onError` | `(err: Error) => void` | Global error handler. Default: `console.error`. |
|
|
45
|
+
| `onInfo` | `(info: Info) => void` | Global info/debug handler. Default: `console.info`. |
|
|
46
|
+
|
|
47
|
+
#### Cron Tasks Options
|
|
48
|
+
|
|
49
|
+
See [Cron Tasks](./cron-tasks.md#initialization-options-optional) for feature-specific options (e.g. `runCronTasks`, `cronExpressionParserOptions`, etc).
|
|
50
|
+
|
|
51
|
+
#### Reactive Tasks Options
|
|
52
|
+
|
|
53
|
+
See [Reactive Tasks](./reactive-tasks.md#5-advanced-initialization) for feature-specific options (e.g. `concurrency`, `monitoring`, etc).
|
|
54
|
+
|
|
55
|
+
#### Testing Options
|
|
56
|
+
|
|
57
|
+
| Option | Type | Description |
|
|
58
|
+
| :--- | :--- | :--- |
|
|
59
|
+
| `collectionFactory` | `Function` | **Internal**. Used for injecting mock collections during testing. |
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Process In Batches
|
|
2
|
+
|
|
3
|
+
Efficiently process large datasets by iterating through a collection in batches.
|
|
4
|
+
|
|
5
|
+
This utility handles cursor management, batch accumulation, and memory efficiency for you.
|
|
6
|
+
|
|
7
|
+
## processInBatches
|
|
8
|
+
|
|
9
|
+
```typescript
|
|
10
|
+
import { processInBatches } from 'mongodash';
|
|
11
|
+
|
|
12
|
+
// 1. Fetch users
|
|
13
|
+
// 2. Transform them (e.g. enrich data)
|
|
14
|
+
// 3. Save them (e.g. bulk write)
|
|
15
|
+
await processInBatches(
|
|
16
|
+
// 1. Collection
|
|
17
|
+
db.collection('users'),
|
|
18
|
+
|
|
19
|
+
// 2. Filter or Pipeline
|
|
20
|
+
{ status: 'active' },
|
|
21
|
+
// OR: [{ $match: { status: 'active' } }]
|
|
22
|
+
|
|
23
|
+
// 3. Transform: Return an operation, or null to skip
|
|
24
|
+
async (user) => {
|
|
25
|
+
const enrichment = await fetchEnrichment(user._id);
|
|
26
|
+
return {
|
|
27
|
+
updateOne: {
|
|
28
|
+
filter: { _id: user._id },
|
|
29
|
+
update: { $set: { enrichment } }
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
},
|
|
33
|
+
|
|
34
|
+
// 4. Execute Batch: Receives array of operations
|
|
35
|
+
async (batchOps) => {
|
|
36
|
+
if (batchOps.length > 0) {
|
|
37
|
+
await db.collection('users').bulkWrite(batchOps);
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
|
|
41
|
+
// 5. Options
|
|
42
|
+
{ batchSize: 500 }
|
|
43
|
+
);
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
### Signature
|
|
47
|
+
|
|
48
|
+
```typescript
|
|
49
|
+
function processInBatches<TDoc, TOp>(
|
|
50
|
+
collection: Collection<TDoc>,
|
|
51
|
+
queryOrPipeline: Filter<TDoc> | Document[],
|
|
52
|
+
transform: (doc: TDoc) => TOp | TOp[] | null | undefined | Promise<...>,
|
|
53
|
+
executeBatch: (ops: TOp[]) => Promise<void>,
|
|
54
|
+
options?: { batchSize?: number; shouldStop?: () => boolean }
|
|
55
|
+
): Promise<{ processedDocuments: number, operationsPerformed: number }>
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### Parameters
|
|
59
|
+
|
|
60
|
+
| Parameter | Type | Description |
|
|
61
|
+
| :--- | :--- | :--- |
|
|
62
|
+
| `collection` | `Collection` | The source MongoDB collection. |
|
|
63
|
+
| `queryOrPipeline` | `Filter` \| `Document[]` | Standard query object OR Aggregation Pipeline. |
|
|
64
|
+
| `transform` | `Function` | Function called for **each document**. Returns an operation (to be batched), an array of operations, or `null`/`undefined` to skip. |
|
|
65
|
+
| `executeBatch` | `Function` | Function called when `batchSize` is reached. Receives the array of accumulated operations. |
|
|
66
|
+
| `options` | `Object` | Optional configuration. |
|
|
67
|
+
|
|
68
|
+
### Options
|
|
69
|
+
|
|
70
|
+
| Option | Type | Default | Description |
|
|
71
|
+
| :--- | :--- | :--- | :--- |
|
|
72
|
+
| `batchSize` | `number` | `1000` | Size of the batch passed to `executeBatch`. |
|
|
73
|
+
| `shouldStop` | `() => boolean` | `undefined` | Callback checked before processing each document. If returns `true`, processing stops gracefully. |
|