@unchainedshop/core-worker 3.0.0-alpha2 → 3.0.0-alpha3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/db/WorkQueueCollection.d.ts +1 -1
- package/lib/db/WorkQueueCollection.d.ts.map +1 -1
- package/lib/director/WorkerAdapter.d.ts +13 -1
- package/lib/director/WorkerAdapter.d.ts.map +1 -1
- package/lib/director/WorkerAdapter.js.map +1 -1
- package/lib/director/WorkerDirector.d.ts +24 -1
- package/lib/director/WorkerDirector.d.ts.map +1 -1
- package/lib/director/WorkerDirector.js +2 -2
- package/lib/director/WorkerDirector.js.map +1 -1
- package/lib/module/configureWorkerModule.d.ts +78 -3
- package/lib/module/configureWorkerModule.d.ts.map +1 -1
- package/lib/module/configureWorkerModule.js +160 -42
- package/lib/module/configureWorkerModule.js.map +1 -1
- package/lib/module/migrations/addMigrations.d.ts +3 -0
- package/lib/module/migrations/addMigrations.d.ts.map +1 -0
- package/lib/module/migrations/addMigrations.js +16 -0
- package/lib/module/migrations/addMigrations.js.map +1 -0
- package/lib/schedulers/FailedRescheduler.d.ts +11 -1
- package/lib/schedulers/FailedRescheduler.d.ts.map +1 -1
- package/lib/schedulers/FailedRescheduler.js.map +1 -1
- package/lib/types.d.ts +20 -0
- package/lib/types.d.ts.map +1 -0
- package/lib/types.js +2 -0
- package/lib/types.js.map +1 -0
- package/lib/worker-index.d.ts +8 -7
- package/lib/worker-index.d.ts.map +1 -1
- package/lib/worker-index.js +8 -7
- package/lib/worker-index.js.map +1 -1
- package/lib/workers/BaseWorker.d.ts +27 -1
- package/lib/workers/BaseWorker.d.ts.map +1 -1
- package/lib/workers/BaseWorker.js +3 -2
- package/lib/workers/BaseWorker.js.map +1 -1
- package/lib/workers/EventListenerWorker.d.ts +1 -1
- package/lib/workers/EventListenerWorker.d.ts.map +1 -1
- package/lib/workers/EventListenerWorker.js.map +1 -1
- package/lib/workers/IntervalWorker.d.ts +2 -1
- package/lib/workers/IntervalWorker.d.ts.map +1 -1
- package/lib/workers/IntervalWorker.js.map +1 -1
- package/package.json +6 -8
- package/src/db/WorkQueueCollection.ts +1 -1
- package/src/director/WorkerAdapter.ts +15 -1
- package/src/director/WorkerDirector.ts +31 -8
- package/src/module/configureWorkerModule.ts +285 -57
- package/src/module/migrations/addMigrations.ts +18 -0
- package/src/schedulers/FailedRescheduler.ts +17 -1
- package/src/types.ts +20 -0
- package/src/worker-index.ts +8 -8
- package/src/workers/BaseWorker.ts +31 -3
- package/src/workers/EventListenerWorker.ts +1 -2
- package/src/workers/IntervalWorker.ts +2 -2
- package/lib/db/WorkQueueSchema.d.ts +0 -3
- package/lib/db/WorkQueueSchema.d.ts.map +0 -1
- package/lib/db/WorkQueueSchema.js +0 -20
- package/lib/db/WorkQueueSchema.js.map +0 -1
- package/src/db/WorkQueueSchema.ts +0 -23
|
@@ -1,26 +1,122 @@
|
|
|
1
|
+
import type { WorkData, WorkResult } from '../worker-index.js';
|
|
2
|
+
import type { ModuleInput, UnchainedCore } from '@unchainedshop/core';
|
|
3
|
+
|
|
1
4
|
import os from 'os';
|
|
2
|
-
import { ModuleInput, ModuleMutations } from '@unchainedshop/types/core.js';
|
|
3
|
-
import { Work, WorkerModule, WorkerSettingsOptions } from '@unchainedshop/types/worker.js';
|
|
4
5
|
import { createLogger } from '@unchainedshop/logger';
|
|
5
6
|
import {
|
|
6
7
|
generateDbFilterById,
|
|
7
|
-
generateDbMutations,
|
|
8
8
|
buildSortOptions,
|
|
9
9
|
mongodb,
|
|
10
|
+
generateDbObjectId,
|
|
10
11
|
} from '@unchainedshop/mongodb';
|
|
11
|
-
import { SortDirection } from '@unchainedshop/types/api.js';
|
|
12
12
|
import { emit, registerEvents } from '@unchainedshop/events';
|
|
13
|
-
import { buildObfuscatedFieldsFilter } from '@unchainedshop/utils';
|
|
13
|
+
import { buildObfuscatedFieldsFilter, SortDirection, SortOption } from '@unchainedshop/utils';
|
|
14
14
|
import { WorkQueueCollection } from '../db/WorkQueueCollection.js';
|
|
15
|
-
import { WorkQueueSchema } from '../db/WorkQueueSchema.js';
|
|
16
15
|
import { DIRECTOR_MARKED_FAILED_ERROR, WorkerDirector } from '../director/WorkerDirector.js';
|
|
17
16
|
import { WorkerEventTypes } from '../director/WorkerEventTypes.js';
|
|
18
17
|
import { WorkStatus } from '../director/WorkStatus.js';
|
|
18
|
+
import { Work } from '../types.js';
|
|
19
|
+
import addMigrations from './migrations/addMigrations.js';
|
|
19
20
|
|
|
20
21
|
const { UNCHAINED_WORKER_ID = os.hostname() } = process.env;
|
|
21
22
|
|
|
22
23
|
const logger = createLogger('unchained:core-worker');
|
|
23
24
|
|
|
25
|
+
export interface WorkerSettingsOptions {
|
|
26
|
+
blacklistedVariables?: string[];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export type WorkerReport = {
|
|
30
|
+
type: string;
|
|
31
|
+
newCount: number;
|
|
32
|
+
errorCount: number;
|
|
33
|
+
successCount: number;
|
|
34
|
+
startCount: number;
|
|
35
|
+
deleteCount: number;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export type WorkQueueQuery = {
|
|
39
|
+
created?: { end?: Date; start?: Date };
|
|
40
|
+
types?: Array<string>;
|
|
41
|
+
status: Array<WorkStatus>;
|
|
42
|
+
queryString?: string;
|
|
43
|
+
scheduled?: { end?: Date; start?: Date };
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
export type WorkerModule = {
|
|
47
|
+
activeWorkTypes: () => Promise<Array<string>>;
|
|
48
|
+
findWork: (query: { workId?: string; originalWorkId?: string }) => Promise<Work>;
|
|
49
|
+
findWorkQueue: (
|
|
50
|
+
query: WorkQueueQuery & {
|
|
51
|
+
sort?: Array<SortOption>;
|
|
52
|
+
limit?: number;
|
|
53
|
+
skip?: number;
|
|
54
|
+
},
|
|
55
|
+
) => Promise<Array<Work>>;
|
|
56
|
+
count: (query: WorkQueueQuery) => Promise<number>;
|
|
57
|
+
workExists: (query: { workId?: string; originalWorkId?: string }) => Promise<boolean>;
|
|
58
|
+
|
|
59
|
+
// Transformations
|
|
60
|
+
status: (work: Work) => WorkStatus;
|
|
61
|
+
|
|
62
|
+
type: (work: Work) => string;
|
|
63
|
+
|
|
64
|
+
// Mutations
|
|
65
|
+
addWork: (data: WorkData) => Promise<Work>;
|
|
66
|
+
|
|
67
|
+
allocateWork: (doc: { types: Array<string>; worker: string }) => Promise<Work>;
|
|
68
|
+
|
|
69
|
+
processNextWork: (unchainedAPI: UnchainedCore, workerId?: string) => Promise<Work>;
|
|
70
|
+
|
|
71
|
+
rescheduleWork: (work: Work, scheduled: Date, unchainedAPI: UnchainedCore) => Promise<Work>;
|
|
72
|
+
|
|
73
|
+
ensureOneWork: (work: WorkData) => Promise<Work>;
|
|
74
|
+
|
|
75
|
+
ensureNoWork: (work: { priority: number; type: string; scheduleId: string }) => Promise<void>;
|
|
76
|
+
|
|
77
|
+
finishWork: (
|
|
78
|
+
_id: string,
|
|
79
|
+
data: WorkResult<any> & {
|
|
80
|
+
finished?: Date;
|
|
81
|
+
started?: Date;
|
|
82
|
+
worker?: string;
|
|
83
|
+
},
|
|
84
|
+
) => Promise<Work | null>;
|
|
85
|
+
|
|
86
|
+
deleteWork: (_id: string) => Promise<Work | null>;
|
|
87
|
+
|
|
88
|
+
markOldWorkAsFailed: (params: {
|
|
89
|
+
types: Array<string>;
|
|
90
|
+
worker: string;
|
|
91
|
+
referenceDate: Date;
|
|
92
|
+
}) => Promise<Array<Work>>;
|
|
93
|
+
|
|
94
|
+
getReport: (params?: { types?: string[]; from?: Date; to?: Date }) => Promise<WorkerReport[]>;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const WORK_STATUS_FILTER_MAP = {
|
|
98
|
+
[WorkStatus.DELETED]: { deleted: { $exists: true } },
|
|
99
|
+
[WorkStatus.NEW]: {
|
|
100
|
+
started: { $exists: false },
|
|
101
|
+
deleted: { $exists: false },
|
|
102
|
+
},
|
|
103
|
+
[WorkStatus.ALLOCATED]: {
|
|
104
|
+
started: { $exists: true },
|
|
105
|
+
finished: { $exists: false },
|
|
106
|
+
deleted: { $exists: false },
|
|
107
|
+
},
|
|
108
|
+
[WorkStatus.SUCCESS]: {
|
|
109
|
+
finished: { $exists: true },
|
|
110
|
+
success: true,
|
|
111
|
+
deleted: { $exists: false },
|
|
112
|
+
},
|
|
113
|
+
[WorkStatus.FAILED]: {
|
|
114
|
+
finished: { $exists: true },
|
|
115
|
+
success: false,
|
|
116
|
+
deleted: { $exists: false },
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
|
|
24
120
|
export const buildQuerySelector = ({
|
|
25
121
|
created,
|
|
26
122
|
scheduled,
|
|
@@ -37,30 +133,8 @@ export const buildQuerySelector = ({
|
|
|
37
133
|
queryString?: string;
|
|
38
134
|
types?: Array<string>;
|
|
39
135
|
}) => {
|
|
40
|
-
const filterMap = {
|
|
41
|
-
[WorkStatus.DELETED]: { deleted: { $exists: true } },
|
|
42
|
-
[WorkStatus.NEW]: {
|
|
43
|
-
started: { $exists: false },
|
|
44
|
-
deleted: { $exists: false },
|
|
45
|
-
},
|
|
46
|
-
[WorkStatus.ALLOCATED]: {
|
|
47
|
-
started: { $exists: true },
|
|
48
|
-
finished: { $exists: false },
|
|
49
|
-
deleted: { $exists: false },
|
|
50
|
-
},
|
|
51
|
-
[WorkStatus.SUCCESS]: {
|
|
52
|
-
finished: { $exists: true },
|
|
53
|
-
success: true,
|
|
54
|
-
deleted: { $exists: false },
|
|
55
|
-
},
|
|
56
|
-
[WorkStatus.FAILED]: {
|
|
57
|
-
finished: { $exists: true },
|
|
58
|
-
success: false,
|
|
59
|
-
deleted: { $exists: false },
|
|
60
|
-
},
|
|
61
|
-
};
|
|
62
136
|
const statusQuery = {
|
|
63
|
-
$or: Object.entries(
|
|
137
|
+
$or: Object.entries(WORK_STATUS_FILTER_MAP).reduce(
|
|
64
138
|
(acc, [key, filter]) => (status?.includes(key as WorkStatus) ? [...acc, filter] : acc),
|
|
65
139
|
[],
|
|
66
140
|
),
|
|
@@ -91,6 +165,24 @@ export const buildQuerySelector = ({
|
|
|
91
165
|
return { ...query, ...rest };
|
|
92
166
|
};
|
|
93
167
|
|
|
168
|
+
const convertFilterMapToPipelineBranches = () => {
|
|
169
|
+
return Object.entries(WORK_STATUS_FILTER_MAP).map(([status, conditions]) => {
|
|
170
|
+
const caseConditions = Object.entries(conditions).map(([field, condition]) => {
|
|
171
|
+
if (typeof condition === 'object' && '$exists' in condition) {
|
|
172
|
+
return condition.$exists
|
|
173
|
+
? { $ne: [{ $type: `$${field}` }, 'missing'] }
|
|
174
|
+
: { $eq: [{ $type: `$${field}` }, 'missing'] };
|
|
175
|
+
}
|
|
176
|
+
return { $eq: [`$${field}`, condition] };
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
return {
|
|
180
|
+
case: caseConditions.length > 1 ? { $and: caseConditions } : caseConditions[0],
|
|
181
|
+
then: status,
|
|
182
|
+
};
|
|
183
|
+
});
|
|
184
|
+
};
|
|
185
|
+
|
|
94
186
|
const defaultSort: Array<{ key: string; value: SortDirection }> = [
|
|
95
187
|
{ key: 'started', value: SortDirection.DESC },
|
|
96
188
|
{ key: 'priority', value: SortDirection.DESC },
|
|
@@ -98,18 +190,48 @@ const defaultSort: Array<{ key: string; value: SortDirection }> = [
|
|
|
98
190
|
{ key: 'created', value: SortDirection.ASC },
|
|
99
191
|
];
|
|
100
192
|
|
|
193
|
+
const normalizeWorkQueueAggregateResult = (data = []): WorkerReport[] => {
|
|
194
|
+
const statusToFieldMap = {
|
|
195
|
+
NEW: 'newCount',
|
|
196
|
+
ALLOCATED: 'startCount',
|
|
197
|
+
FAILED: 'errorCount',
|
|
198
|
+
SUCCESS: 'successCount',
|
|
199
|
+
DELETED: 'deleteCount',
|
|
200
|
+
};
|
|
201
|
+
|
|
202
|
+
return data.map((item) => {
|
|
203
|
+
const workStatistics: WorkerReport = {
|
|
204
|
+
type: item.type,
|
|
205
|
+
...(Object.values(statusToFieldMap).reduce((acc, key) => ({ ...acc, [key]: 0 }), {}) as any),
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
item.statuses.forEach(({ status, count }) => {
|
|
209
|
+
if (statusToFieldMap[status]) {
|
|
210
|
+
workStatistics[statusToFieldMap[status]] = count;
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
const { errorCount, successCount, deleteCount } = workStatistics;
|
|
214
|
+
|
|
215
|
+
workStatistics.startCount += errorCount + successCount + deleteCount;
|
|
216
|
+
workStatistics.newCount += workStatistics.startCount;
|
|
217
|
+
|
|
218
|
+
return workStatistics;
|
|
219
|
+
}) as unknown as WorkerReport[];
|
|
220
|
+
};
|
|
221
|
+
|
|
101
222
|
export const configureWorkerModule = async ({
|
|
102
223
|
db,
|
|
224
|
+
migrationRepository,
|
|
103
225
|
options,
|
|
104
226
|
}: ModuleInput<WorkerSettingsOptions>): Promise<WorkerModule> => {
|
|
227
|
+
addMigrations(migrationRepository);
|
|
228
|
+
|
|
105
229
|
registerEvents(Object.values(WorkerEventTypes));
|
|
106
230
|
|
|
107
231
|
const WorkQueue = await WorkQueueCollection(db);
|
|
108
232
|
|
|
109
233
|
const removePrivateFields = buildObfuscatedFieldsFilter(options?.blacklistedVariables);
|
|
110
234
|
|
|
111
|
-
const mutations = generateDbMutations<Work>(WorkQueue, WorkQueueSchema) as ModuleMutations<Work>;
|
|
112
|
-
|
|
113
235
|
const allocateWork: WorkerModule['allocateWork'] = async ({ types, worker = UNCHAINED_WORKER_ID }) => {
|
|
114
236
|
// Find a work item that is scheduled for now and is not started.
|
|
115
237
|
// Also:
|
|
@@ -153,18 +275,21 @@ export const configureWorkerModule = async ({
|
|
|
153
275
|
|
|
154
276
|
if (!workBeforeUpdate) return null;
|
|
155
277
|
|
|
156
|
-
await
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
278
|
+
const work = await WorkQueue.findOneAndUpdate(
|
|
279
|
+
generateDbFilterById(workId),
|
|
280
|
+
{
|
|
281
|
+
$set: {
|
|
282
|
+
updated: new Date(),
|
|
283
|
+
finished,
|
|
284
|
+
success,
|
|
285
|
+
error,
|
|
286
|
+
result,
|
|
287
|
+
...(!workBeforeUpdate.started ? { started } : {}),
|
|
288
|
+
worker,
|
|
289
|
+
},
|
|
164
290
|
},
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
const work = await WorkQueue.findOne(generateDbFilterById(workId), {});
|
|
291
|
+
{ returnDocument: 'after' },
|
|
292
|
+
);
|
|
168
293
|
|
|
169
294
|
const duration = new Date(work.finished).getTime() - new Date(work.started).getTime();
|
|
170
295
|
if (work.success) {
|
|
@@ -332,14 +457,15 @@ export const configureWorkerModule = async ({
|
|
|
332
457
|
}
|
|
333
458
|
|
|
334
459
|
const created = new Date();
|
|
335
|
-
const workId = await
|
|
460
|
+
const { insertedId: workId } = await WorkQueue.insertOne({
|
|
461
|
+
_id: generateDbObjectId(),
|
|
462
|
+
created,
|
|
336
463
|
type,
|
|
337
464
|
input,
|
|
338
465
|
priority,
|
|
339
466
|
scheduled: scheduled || created,
|
|
340
467
|
originalWorkId,
|
|
341
468
|
retries,
|
|
342
|
-
created,
|
|
343
469
|
worker,
|
|
344
470
|
});
|
|
345
471
|
|
|
@@ -354,13 +480,17 @@ export const configureWorkerModule = async ({
|
|
|
354
480
|
},
|
|
355
481
|
|
|
356
482
|
rescheduleWork: async (currentWork, scheduled) => {
|
|
357
|
-
await
|
|
358
|
-
|
|
359
|
-
|
|
483
|
+
const work = await WorkQueue.findOneAndUpdate(
|
|
484
|
+
generateDbFilterById(currentWork._id),
|
|
485
|
+
{
|
|
486
|
+
$set: {
|
|
487
|
+
updated: new Date(),
|
|
488
|
+
scheduled,
|
|
489
|
+
},
|
|
360
490
|
},
|
|
361
|
-
|
|
491
|
+
{ returnDocument: 'after' },
|
|
492
|
+
);
|
|
362
493
|
|
|
363
|
-
const work = await WorkQueue.findOne(generateDbFilterById(currentWork._id), {});
|
|
364
494
|
emit(WorkerEventTypes.RESCHEDULED, {
|
|
365
495
|
work: removePrivateFields(work),
|
|
366
496
|
oldScheduled: currentWork.scheduled,
|
|
@@ -371,12 +501,13 @@ export const configureWorkerModule = async ({
|
|
|
371
501
|
|
|
372
502
|
allocateWork,
|
|
373
503
|
|
|
374
|
-
ensureNoWork: async ({ type, priority = 0 }) => {
|
|
504
|
+
ensureNoWork: async ({ type, priority = 0, scheduleId }) => {
|
|
375
505
|
const query = buildQuerySelector({
|
|
376
506
|
type,
|
|
377
507
|
status: [WorkStatus.NEW],
|
|
378
508
|
priority,
|
|
379
509
|
autoscheduled: true,
|
|
510
|
+
scheduleId,
|
|
380
511
|
});
|
|
381
512
|
|
|
382
513
|
await WorkQueue.updateMany(query, {
|
|
@@ -386,16 +517,25 @@ export const configureWorkerModule = async ({
|
|
|
386
517
|
});
|
|
387
518
|
},
|
|
388
519
|
|
|
389
|
-
ensureOneWork: async (
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
520
|
+
ensureOneWork: async ({
|
|
521
|
+
type,
|
|
522
|
+
input,
|
|
523
|
+
priority = 0,
|
|
524
|
+
scheduled,
|
|
525
|
+
timeout,
|
|
526
|
+
originalWorkId,
|
|
527
|
+
retries = 20,
|
|
528
|
+
scheduleId,
|
|
529
|
+
}) => {
|
|
530
|
+
const workId = `${scheduleId}:${scheduled.getTime()}`;
|
|
531
|
+
|
|
393
532
|
const created = new Date();
|
|
394
533
|
const query = buildQuerySelector({
|
|
395
534
|
type,
|
|
396
535
|
status: [WorkStatus.NEW],
|
|
397
536
|
priority,
|
|
398
537
|
autoscheduled: true,
|
|
538
|
+
scheduleId,
|
|
399
539
|
});
|
|
400
540
|
try {
|
|
401
541
|
const result = await WorkQueue.findOneAndUpdate(
|
|
@@ -414,6 +554,7 @@ export const configureWorkerModule = async ({
|
|
|
414
554
|
},
|
|
415
555
|
$setOnInsert: {
|
|
416
556
|
_id: workId,
|
|
557
|
+
scheduleId,
|
|
417
558
|
scheduled,
|
|
418
559
|
type,
|
|
419
560
|
created,
|
|
@@ -428,6 +569,7 @@ export const configureWorkerModule = async ({
|
|
|
428
569
|
upsert: true,
|
|
429
570
|
},
|
|
430
571
|
);
|
|
572
|
+
|
|
431
573
|
if (!result.lastErrorObject.updatedExisting) {
|
|
432
574
|
logger.info(`${type} auto-scheduled @ ${new Date(scheduled).toISOString()}`, {
|
|
433
575
|
workId,
|
|
@@ -435,7 +577,7 @@ export const configureWorkerModule = async ({
|
|
|
435
577
|
emit(WorkerEventTypes.ADDED, removePrivateFields(result.value));
|
|
436
578
|
}
|
|
437
579
|
return result.value;
|
|
438
|
-
} catch
|
|
580
|
+
} catch {
|
|
439
581
|
/* TODO:
|
|
440
582
|
If the findOneAndUpdate call failed because of _id conflict with a DELETED work,
|
|
441
583
|
we should permanently remove the conflicting deleted work
|
|
@@ -458,9 +600,16 @@ export const configureWorkerModule = async ({
|
|
|
458
600
|
);
|
|
459
601
|
if (!workBeforeRemoval) return null;
|
|
460
602
|
|
|
461
|
-
await
|
|
603
|
+
const work = await WorkQueue.findOneAndUpdate(
|
|
604
|
+
generateDbFilterById(workId),
|
|
605
|
+
{
|
|
606
|
+
$set: {
|
|
607
|
+
deleted: new Date(),
|
|
608
|
+
},
|
|
609
|
+
},
|
|
610
|
+
{ returnDocument: 'after' },
|
|
611
|
+
);
|
|
462
612
|
|
|
463
|
-
const work = await WorkQueue.findOne(generateDbFilterById(workId), {});
|
|
464
613
|
emit(WorkerEventTypes.DELETED, removePrivateFields(work));
|
|
465
614
|
|
|
466
615
|
return work;
|
|
@@ -493,5 +642,84 @@ export const configureWorkerModule = async ({
|
|
|
493
642
|
),
|
|
494
643
|
);
|
|
495
644
|
},
|
|
645
|
+
|
|
646
|
+
getReport: async ({ types, from, to } = { types: null, from: null, to: null }) => {
|
|
647
|
+
const pipeline = [];
|
|
648
|
+
const matchConditions = [];
|
|
649
|
+
// build date filter based on provided values it can be a range if both to and from is supplied
|
|
650
|
+
// a upper or lowe limit if either from or to is provided
|
|
651
|
+
// or all if none is provided
|
|
652
|
+
if (from || to) {
|
|
653
|
+
const dateConditions = [];
|
|
654
|
+
if (from) {
|
|
655
|
+
const fromDate = new Date(from);
|
|
656
|
+
dateConditions.push({
|
|
657
|
+
$or: [{ created: { $gte: fromDate } }, { updated: { $gte: fromDate } }],
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
if (to) {
|
|
661
|
+
const toDate = new Date(to);
|
|
662
|
+
dateConditions.push({
|
|
663
|
+
$or: [{ created: { $lte: toDate } }, { updated: { $lte: toDate } }],
|
|
664
|
+
});
|
|
665
|
+
}
|
|
666
|
+
if (dateConditions.length > 0) {
|
|
667
|
+
matchConditions.push({ $and: dateConditions });
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
// build types filter if type is provided or ignore types if it is not provided
|
|
671
|
+
if (types && Array.isArray(types) && types.length) {
|
|
672
|
+
matchConditions.push({ type: { $in: types } });
|
|
673
|
+
}
|
|
674
|
+
if (matchConditions.length > 0) {
|
|
675
|
+
pipeline.push({
|
|
676
|
+
$match: {
|
|
677
|
+
$and: matchConditions,
|
|
678
|
+
},
|
|
679
|
+
});
|
|
680
|
+
}
|
|
681
|
+
pipeline.push(
|
|
682
|
+
{
|
|
683
|
+
$addFields: {
|
|
684
|
+
status: {
|
|
685
|
+
$switch: {
|
|
686
|
+
branches: convertFilterMapToPipelineBranches(),
|
|
687
|
+
default: 'UNKNOWN',
|
|
688
|
+
},
|
|
689
|
+
},
|
|
690
|
+
},
|
|
691
|
+
},
|
|
692
|
+
{
|
|
693
|
+
$group: {
|
|
694
|
+
_id: {
|
|
695
|
+
type: '$type',
|
|
696
|
+
status: '$status',
|
|
697
|
+
},
|
|
698
|
+
count: { $sum: 1 },
|
|
699
|
+
},
|
|
700
|
+
},
|
|
701
|
+
{
|
|
702
|
+
$group: {
|
|
703
|
+
_id: '$_id.type',
|
|
704
|
+
statuses: {
|
|
705
|
+
$push: {
|
|
706
|
+
status: '$_id.status',
|
|
707
|
+
count: '$count',
|
|
708
|
+
},
|
|
709
|
+
},
|
|
710
|
+
},
|
|
711
|
+
},
|
|
712
|
+
|
|
713
|
+
{
|
|
714
|
+
$project: {
|
|
715
|
+
_id: 0,
|
|
716
|
+
type: '$_id',
|
|
717
|
+
statuses: 1,
|
|
718
|
+
},
|
|
719
|
+
},
|
|
720
|
+
);
|
|
721
|
+
|
|
722
|
+
return normalizeWorkQueueAggregateResult(await WorkQueue.aggregate(pipeline).toArray()) as any;
|
|
723
|
+
},
|
|
496
724
|
};
|
|
497
725
|
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { Migration, MigrationRepository } from '@unchainedshop/core';
|
|
2
|
+
import { WorkQueueCollection } from '../../db/WorkQueueCollection.js';
|
|
3
|
+
|
|
4
|
+
export default function addMigrations(repository: MigrationRepository<Migration>) {
|
|
5
|
+
repository?.register({
|
|
6
|
+
id: 20240905161400,
|
|
7
|
+
name: 'Remove all autoscheduled work items so they get scheduled freshly',
|
|
8
|
+
up: async () => {
|
|
9
|
+
const WorkQueue = await WorkQueueCollection(repository.db);
|
|
10
|
+
|
|
11
|
+
await WorkQueue.deleteMany({
|
|
12
|
+
autoscheduled: true,
|
|
13
|
+
started: null,
|
|
14
|
+
deleted: null,
|
|
15
|
+
});
|
|
16
|
+
},
|
|
17
|
+
});
|
|
18
|
+
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import { IScheduler, Work, WorkData } from '@unchainedshop/types/worker.js';
|
|
2
1
|
import { log } from '@unchainedshop/logger';
|
|
3
2
|
import { subscribe } from '@unchainedshop/events';
|
|
4
3
|
import { WorkerEventTypes } from '../director/WorkerEventTypes.js';
|
|
4
|
+
import { UnchainedCore } from '@unchainedshop/core';
|
|
5
|
+
import { WorkData } from '../worker-index.js';
|
|
6
|
+
import { Work } from '../types.js';
|
|
5
7
|
|
|
6
8
|
export interface FailedReschedulerParams {
|
|
7
9
|
retryInput?: (
|
|
@@ -10,6 +12,20 @@ export interface FailedReschedulerParams {
|
|
|
10
12
|
) => Promise<Record<string, any> | null>;
|
|
11
13
|
}
|
|
12
14
|
|
|
15
|
+
export type IScheduler<P> = {
|
|
16
|
+
key: string;
|
|
17
|
+
label: string;
|
|
18
|
+
version: string;
|
|
19
|
+
|
|
20
|
+
actions: (
|
|
21
|
+
params: P,
|
|
22
|
+
unchainedAPI: UnchainedCore,
|
|
23
|
+
) => {
|
|
24
|
+
start: () => void;
|
|
25
|
+
stop: () => void;
|
|
26
|
+
};
|
|
27
|
+
};
|
|
28
|
+
|
|
13
29
|
export const FailedRescheduler: IScheduler<FailedReschedulerParams> = {
|
|
14
30
|
key: 'shop.unchained.scheduler.failed',
|
|
15
31
|
label: 'Reschedule failed works',
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { TimestampFields } from '@unchainedshop/mongodb';
|
|
2
|
+
|
|
3
|
+
export type Work = {
|
|
4
|
+
_id?: string;
|
|
5
|
+
priority: number;
|
|
6
|
+
retries: number;
|
|
7
|
+
scheduled: Date;
|
|
8
|
+
type: string;
|
|
9
|
+
input: Record<string, any>;
|
|
10
|
+
error?: any;
|
|
11
|
+
finished?: Date;
|
|
12
|
+
originalWorkId?: string;
|
|
13
|
+
result?: any;
|
|
14
|
+
started?: Date;
|
|
15
|
+
success?: boolean;
|
|
16
|
+
timeout?: number;
|
|
17
|
+
worker?: string;
|
|
18
|
+
autoscheduled?: boolean;
|
|
19
|
+
scheduleId?: string;
|
|
20
|
+
} & TimestampFields;
|
package/src/worker-index.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
export
|
|
1
|
+
export * from './types.js';
|
|
2
2
|
|
|
3
|
-
export
|
|
4
|
-
export
|
|
3
|
+
export * from './module/configureWorkerModule.js';
|
|
4
|
+
export * from './director/WorkerDirector.js';
|
|
5
|
+
export * from './director/WorkerAdapter.js';
|
|
5
6
|
export { WorkStatus } from './director/WorkStatus.js';
|
|
6
7
|
export { WorkerEventTypes } from './director/WorkerEventTypes.js';
|
|
7
8
|
|
|
8
|
-
export
|
|
9
|
-
|
|
10
|
-
export
|
|
11
|
-
export
|
|
12
|
-
export { IntervalWorker } from './workers/IntervalWorker.js';
|
|
9
|
+
export * from './schedulers/FailedRescheduler.js';
|
|
10
|
+
export * from './workers/BaseWorker.js';
|
|
11
|
+
export * from './workers/EventListenerWorker.js';
|
|
12
|
+
export * from './workers/IntervalWorker.js';
|
|
@@ -1,7 +1,34 @@
|
|
|
1
|
-
import { IWorker, WorkData } from '@unchainedshop/types/worker.js';
|
|
2
1
|
import later from '@breejs/later';
|
|
3
2
|
import { log } from '@unchainedshop/logger';
|
|
4
3
|
import { WorkerDirector } from '../director/WorkerDirector.js';
|
|
4
|
+
import { UnchainedCore } from '@unchainedshop/core';
|
|
5
|
+
import { Work } from '../types.js';
|
|
6
|
+
|
|
7
|
+
export type WorkData = Pick<
|
|
8
|
+
Partial<Work>,
|
|
9
|
+
'input' | 'originalWorkId' | 'priority' | 'retries' | 'timeout' | 'scheduled' | 'worker' | 'scheduleId'
|
|
10
|
+
> & { type: string };
|
|
11
|
+
|
|
12
|
+
export type IWorker<P extends { workerId?: string }> = {
|
|
13
|
+
key: string;
|
|
14
|
+
label: string;
|
|
15
|
+
version: string;
|
|
16
|
+
type: string;
|
|
17
|
+
external: boolean;
|
|
18
|
+
|
|
19
|
+
getFloorDate: (date?: Date) => Date;
|
|
20
|
+
|
|
21
|
+
actions: (
|
|
22
|
+
params: P,
|
|
23
|
+
unchainedAPI: UnchainedCore,
|
|
24
|
+
) => {
|
|
25
|
+
autorescheduleTypes: (options: { referenceDate: Date }) => Promise<Array<Work | null>>;
|
|
26
|
+
process: (options: { maxWorkItemCount?: number; referenceDate?: Date }) => Promise<void>;
|
|
27
|
+
reset: () => Promise<void>;
|
|
28
|
+
start: () => void;
|
|
29
|
+
stop: () => void;
|
|
30
|
+
};
|
|
31
|
+
};
|
|
5
32
|
|
|
6
33
|
interface WorkerParams {
|
|
7
34
|
workerId?: string;
|
|
@@ -47,6 +74,7 @@ export const BaseWorker: IWorker<WorkerParams> = {
|
|
|
47
74
|
await unchainedAPI.modules.worker.ensureNoWork({
|
|
48
75
|
type: workConfig.type,
|
|
49
76
|
priority: workConfig.priority || 0,
|
|
77
|
+
scheduleId,
|
|
50
78
|
});
|
|
51
79
|
return null;
|
|
52
80
|
}
|
|
@@ -57,6 +85,7 @@ export const BaseWorker: IWorker<WorkerParams> = {
|
|
|
57
85
|
nextDate.setMilliseconds(0);
|
|
58
86
|
const workData: WorkData = {
|
|
59
87
|
type: workConfig.type,
|
|
88
|
+
scheduleId,
|
|
60
89
|
scheduled: nextDate,
|
|
61
90
|
timeout: workConfig.timeout,
|
|
62
91
|
priority: workConfig.priority || 0,
|
|
@@ -68,8 +97,7 @@ export const BaseWorker: IWorker<WorkerParams> = {
|
|
|
68
97
|
// when it explicitly returns a falsish input instead of a dictionary
|
|
69
98
|
if (!workData.input) return null;
|
|
70
99
|
}
|
|
71
|
-
|
|
72
|
-
return unchainedAPI.modules.worker.ensureOneWork(workData, workId);
|
|
100
|
+
return unchainedAPI.modules.worker.ensureOneWork(workData);
|
|
73
101
|
}),
|
|
74
102
|
);
|
|
75
103
|
},
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { IWorker } from '@unchainedshop/types/worker.js';
|
|
2
1
|
import { subscribe } from '@unchainedshop/events';
|
|
3
2
|
import { WorkerEventTypes } from '../director/WorkerEventTypes.js';
|
|
4
|
-
import { BaseWorker } from './BaseWorker.js';
|
|
3
|
+
import { BaseWorker, IWorker } from './BaseWorker.js';
|
|
5
4
|
|
|
6
5
|
function debounce<T extends (...args: any) => any>(func: T, wait) {
|
|
7
6
|
let timeout;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { IWorker, WorkerSchedule } from '@unchainedshop/types/worker.js';
|
|
2
1
|
import later from '@breejs/later';
|
|
3
|
-
import { BaseWorker } from './BaseWorker.js';
|
|
2
|
+
import { BaseWorker, IWorker } from './BaseWorker.js';
|
|
3
|
+
import { WorkerSchedule } from '../worker-index.js';
|
|
4
4
|
|
|
5
5
|
const { NODE_ENV } = process.env;
|
|
6
6
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"WorkQueueSchema.d.ts","sourceRoot":"","sources":["../../src/db/WorkQueueSchema.ts"],"names":[],"mappings":"AACA,OAAO,YAAY,MAAM,cAAc,CAAC;AAExC,eAAO,MAAM,eAAe,cAmB3B,CAAC"}
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import { Schemas } from '@unchainedshop/utils';
|
|
2
|
-
import SimpleSchema from 'simpl-schema';
|
|
3
|
-
export const WorkQueueSchema = new SimpleSchema({
|
|
4
|
-
finished: Date,
|
|
5
|
-
originalWorkId: { type: String },
|
|
6
|
-
priority: { type: Number, required: true },
|
|
7
|
-
retries: { type: Number, required: true },
|
|
8
|
-
scheduled: { type: Date, required: true },
|
|
9
|
-
started: Date,
|
|
10
|
-
success: Boolean,
|
|
11
|
-
timeout: Number,
|
|
12
|
-
type: { type: String, required: true },
|
|
13
|
-
worker: String,
|
|
14
|
-
// Worker data
|
|
15
|
-
input: { type: Object, blackbox: true },
|
|
16
|
-
result: { type: Object, blackbox: true },
|
|
17
|
-
error: { type: Object, blackbox: true },
|
|
18
|
-
...Schemas.timestampFields,
|
|
19
|
-
}, { requiredByDefault: false });
|
|
20
|
-
//# sourceMappingURL=WorkQueueSchema.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"WorkQueueSchema.js","sourceRoot":"","sources":["../../src/db/WorkQueueSchema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,YAAY,MAAM,cAAc,CAAC;AAExC,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,YAAY,CAC7C;IACE,QAAQ,EAAE,IAAI;IACd,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;IAChC,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC1C,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IACzC,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE;IACzC,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,OAAO;IAChB,OAAO,EAAE,MAAM;IACf,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IACtC,MAAM,EAAE,MAAM;IACd,cAAc;IACd,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IACvC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IACxC,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IACvC,GAAG,OAAO,CAAC,eAAe;CAC3B,EACD,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAC7B,CAAC"}
|