@unchainedshop/core-worker 3.0.0-alpha2 → 3.0.0-alpha4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/db/WorkQueueCollection.d.ts +1 -1
- package/lib/db/WorkQueueCollection.d.ts.map +1 -1
- package/lib/director/WorkerAdapter.d.ts +13 -1
- package/lib/director/WorkerAdapter.d.ts.map +1 -1
- package/lib/director/WorkerAdapter.js.map +1 -1
- package/lib/director/WorkerDirector.d.ts +24 -1
- package/lib/director/WorkerDirector.d.ts.map +1 -1
- package/lib/director/WorkerDirector.js +2 -2
- package/lib/director/WorkerDirector.js.map +1 -1
- package/lib/module/configureWorkerModule.d.ts +79 -4
- package/lib/module/configureWorkerModule.d.ts.map +1 -1
- package/lib/module/configureWorkerModule.js +160 -42
- package/lib/module/configureWorkerModule.js.map +1 -1
- package/lib/module/migrations/addMigrations.d.ts +3 -0
- package/lib/module/migrations/addMigrations.d.ts.map +1 -0
- package/lib/module/migrations/addMigrations.js +16 -0
- package/lib/module/migrations/addMigrations.js.map +1 -0
- package/lib/schedulers/FailedRescheduler.d.ts +11 -1
- package/lib/schedulers/FailedRescheduler.d.ts.map +1 -1
- package/lib/schedulers/FailedRescheduler.js.map +1 -1
- package/lib/types.d.ts +20 -0
- package/lib/types.d.ts.map +1 -0
- package/lib/types.js +2 -0
- package/lib/types.js.map +1 -0
- package/lib/worker-index.d.ts +8 -7
- package/lib/worker-index.d.ts.map +1 -1
- package/lib/worker-index.js +8 -7
- package/lib/worker-index.js.map +1 -1
- package/lib/workers/BaseWorker.d.ts +27 -1
- package/lib/workers/BaseWorker.d.ts.map +1 -1
- package/lib/workers/BaseWorker.js +3 -2
- package/lib/workers/BaseWorker.js.map +1 -1
- package/lib/workers/EventListenerWorker.d.ts +1 -1
- package/lib/workers/EventListenerWorker.d.ts.map +1 -1
- package/lib/workers/EventListenerWorker.js.map +1 -1
- package/lib/workers/IntervalWorker.d.ts +2 -1
- package/lib/workers/IntervalWorker.d.ts.map +1 -1
- package/lib/workers/IntervalWorker.js.map +1 -1
- package/package.json +6 -8
- package/src/db/WorkQueueCollection.ts +1 -1
- package/src/director/WorkerAdapter.ts +15 -1
- package/src/director/WorkerDirector.ts +31 -8
- package/src/module/buildQuerySelector.test.ts +144 -124
- package/src/module/configureWorkerModule.ts +286 -57
- package/src/module/migrations/addMigrations.ts +18 -0
- package/src/schedulers/FailedRescheduler.ts +17 -1
- package/src/types.ts +20 -0
- package/src/worker-index.ts +8 -8
- package/src/workers/BaseWorker.ts +31 -3
- package/src/workers/EventListenerWorker.ts +1 -2
- package/src/workers/IntervalWorker.ts +2 -2
- package/lib/db/WorkQueueSchema.d.ts +0 -3
- package/lib/db/WorkQueueSchema.d.ts.map +0 -1
- package/lib/db/WorkQueueSchema.js +0 -20
- package/lib/db/WorkQueueSchema.js.map +0 -1
- package/src/db/WorkQueueSchema.ts +0 -23
|
@@ -1,26 +1,123 @@
|
|
|
1
|
+
import type { WorkData, WorkResult } from '../worker-index.js';
|
|
2
|
+
import type { UnchainedCore } from '@unchainedshop/core';
|
|
3
|
+
|
|
1
4
|
import os from 'os';
|
|
2
|
-
import { ModuleInput, ModuleMutations } from '@unchainedshop/types/core.js';
|
|
3
|
-
import { Work, WorkerModule, WorkerSettingsOptions } from '@unchainedshop/types/worker.js';
|
|
4
5
|
import { createLogger } from '@unchainedshop/logger';
|
|
5
6
|
import {
|
|
6
7
|
generateDbFilterById,
|
|
7
|
-
generateDbMutations,
|
|
8
8
|
buildSortOptions,
|
|
9
9
|
mongodb,
|
|
10
|
+
generateDbObjectId,
|
|
11
|
+
ModuleInput,
|
|
10
12
|
} from '@unchainedshop/mongodb';
|
|
11
|
-
import { SortDirection } from '@unchainedshop/types/api.js';
|
|
12
13
|
import { emit, registerEvents } from '@unchainedshop/events';
|
|
13
|
-
import { buildObfuscatedFieldsFilter } from '@unchainedshop/utils';
|
|
14
|
+
import { buildObfuscatedFieldsFilter, SortDirection, SortOption } from '@unchainedshop/utils';
|
|
14
15
|
import { WorkQueueCollection } from '../db/WorkQueueCollection.js';
|
|
15
|
-
import { WorkQueueSchema } from '../db/WorkQueueSchema.js';
|
|
16
16
|
import { DIRECTOR_MARKED_FAILED_ERROR, WorkerDirector } from '../director/WorkerDirector.js';
|
|
17
17
|
import { WorkerEventTypes } from '../director/WorkerEventTypes.js';
|
|
18
18
|
import { WorkStatus } from '../director/WorkStatus.js';
|
|
19
|
+
import { Work } from '../types.js';
|
|
20
|
+
import addMigrations from './migrations/addMigrations.js';
|
|
19
21
|
|
|
20
22
|
const { UNCHAINED_WORKER_ID = os.hostname() } = process.env;
|
|
21
23
|
|
|
22
24
|
const logger = createLogger('unchained:core-worker');
|
|
23
25
|
|
|
26
|
+
export interface WorkerSettingsOptions {
|
|
27
|
+
blacklistedVariables?: string[];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export type WorkerReport = {
|
|
31
|
+
type: string;
|
|
32
|
+
newCount: number;
|
|
33
|
+
errorCount: number;
|
|
34
|
+
successCount: number;
|
|
35
|
+
startCount: number;
|
|
36
|
+
deleteCount: number;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export type WorkQueueQuery = {
|
|
40
|
+
created?: { end?: Date; start?: Date };
|
|
41
|
+
types?: Array<string>;
|
|
42
|
+
status: Array<WorkStatus>;
|
|
43
|
+
queryString?: string;
|
|
44
|
+
scheduled?: { end?: Date; start?: Date };
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export type WorkerModule = {
|
|
48
|
+
activeWorkTypes: () => Promise<Array<string>>;
|
|
49
|
+
findWork: (query: { workId?: string; originalWorkId?: string }) => Promise<Work>;
|
|
50
|
+
findWorkQueue: (
|
|
51
|
+
query: WorkQueueQuery & {
|
|
52
|
+
sort?: Array<SortOption>;
|
|
53
|
+
limit?: number;
|
|
54
|
+
skip?: number;
|
|
55
|
+
},
|
|
56
|
+
) => Promise<Array<Work>>;
|
|
57
|
+
count: (query: WorkQueueQuery) => Promise<number>;
|
|
58
|
+
workExists: (query: { workId?: string; originalWorkId?: string }) => Promise<boolean>;
|
|
59
|
+
|
|
60
|
+
// Transformations
|
|
61
|
+
status: (work: Work) => WorkStatus;
|
|
62
|
+
|
|
63
|
+
type: (work: Work) => string;
|
|
64
|
+
|
|
65
|
+
// Mutations
|
|
66
|
+
addWork: (data: WorkData) => Promise<Work>;
|
|
67
|
+
|
|
68
|
+
allocateWork: (doc: { types: Array<string>; worker: string }) => Promise<Work>;
|
|
69
|
+
|
|
70
|
+
processNextWork: (unchainedAPI: UnchainedCore, workerId?: string) => Promise<Work>;
|
|
71
|
+
|
|
72
|
+
rescheduleWork: (work: Work, scheduled: Date, unchainedAPI: UnchainedCore) => Promise<Work>;
|
|
73
|
+
|
|
74
|
+
ensureOneWork: (work: WorkData) => Promise<Work>;
|
|
75
|
+
|
|
76
|
+
ensureNoWork: (work: { priority: number; type: string; scheduleId: string }) => Promise<void>;
|
|
77
|
+
|
|
78
|
+
finishWork: (
|
|
79
|
+
_id: string,
|
|
80
|
+
data: WorkResult<any> & {
|
|
81
|
+
finished?: Date;
|
|
82
|
+
started?: Date;
|
|
83
|
+
worker?: string;
|
|
84
|
+
},
|
|
85
|
+
) => Promise<Work | null>;
|
|
86
|
+
|
|
87
|
+
deleteWork: (_id: string) => Promise<Work | null>;
|
|
88
|
+
|
|
89
|
+
markOldWorkAsFailed: (params: {
|
|
90
|
+
types: Array<string>;
|
|
91
|
+
worker: string;
|
|
92
|
+
referenceDate: Date;
|
|
93
|
+
}) => Promise<Array<Work>>;
|
|
94
|
+
|
|
95
|
+
getReport: (params?: { types?: string[]; from?: Date; to?: Date }) => Promise<WorkerReport[]>;
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const WORK_STATUS_FILTER_MAP = {
|
|
99
|
+
[WorkStatus.DELETED]: { deleted: { $exists: true } },
|
|
100
|
+
[WorkStatus.NEW]: {
|
|
101
|
+
started: { $exists: false },
|
|
102
|
+
deleted: { $exists: false },
|
|
103
|
+
},
|
|
104
|
+
[WorkStatus.ALLOCATED]: {
|
|
105
|
+
started: { $exists: true },
|
|
106
|
+
finished: { $exists: false },
|
|
107
|
+
deleted: { $exists: false },
|
|
108
|
+
},
|
|
109
|
+
[WorkStatus.SUCCESS]: {
|
|
110
|
+
finished: { $exists: true },
|
|
111
|
+
success: true,
|
|
112
|
+
deleted: { $exists: false },
|
|
113
|
+
},
|
|
114
|
+
[WorkStatus.FAILED]: {
|
|
115
|
+
finished: { $exists: true },
|
|
116
|
+
success: false,
|
|
117
|
+
deleted: { $exists: false },
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
|
|
24
121
|
export const buildQuerySelector = ({
|
|
25
122
|
created,
|
|
26
123
|
scheduled,
|
|
@@ -37,30 +134,8 @@ export const buildQuerySelector = ({
|
|
|
37
134
|
queryString?: string;
|
|
38
135
|
types?: Array<string>;
|
|
39
136
|
}) => {
|
|
40
|
-
const filterMap = {
|
|
41
|
-
[WorkStatus.DELETED]: { deleted: { $exists: true } },
|
|
42
|
-
[WorkStatus.NEW]: {
|
|
43
|
-
started: { $exists: false },
|
|
44
|
-
deleted: { $exists: false },
|
|
45
|
-
},
|
|
46
|
-
[WorkStatus.ALLOCATED]: {
|
|
47
|
-
started: { $exists: true },
|
|
48
|
-
finished: { $exists: false },
|
|
49
|
-
deleted: { $exists: false },
|
|
50
|
-
},
|
|
51
|
-
[WorkStatus.SUCCESS]: {
|
|
52
|
-
finished: { $exists: true },
|
|
53
|
-
success: true,
|
|
54
|
-
deleted: { $exists: false },
|
|
55
|
-
},
|
|
56
|
-
[WorkStatus.FAILED]: {
|
|
57
|
-
finished: { $exists: true },
|
|
58
|
-
success: false,
|
|
59
|
-
deleted: { $exists: false },
|
|
60
|
-
},
|
|
61
|
-
};
|
|
62
137
|
const statusQuery = {
|
|
63
|
-
$or: Object.entries(
|
|
138
|
+
$or: Object.entries(WORK_STATUS_FILTER_MAP).reduce(
|
|
64
139
|
(acc, [key, filter]) => (status?.includes(key as WorkStatus) ? [...acc, filter] : acc),
|
|
65
140
|
[],
|
|
66
141
|
),
|
|
@@ -91,6 +166,24 @@ export const buildQuerySelector = ({
|
|
|
91
166
|
return { ...query, ...rest };
|
|
92
167
|
};
|
|
93
168
|
|
|
169
|
+
const convertFilterMapToPipelineBranches = () => {
|
|
170
|
+
return Object.entries(WORK_STATUS_FILTER_MAP).map(([status, conditions]) => {
|
|
171
|
+
const caseConditions = Object.entries(conditions).map(([field, condition]) => {
|
|
172
|
+
if (typeof condition === 'object' && '$exists' in condition) {
|
|
173
|
+
return condition.$exists
|
|
174
|
+
? { $ne: [{ $type: `$${field}` }, 'missing'] }
|
|
175
|
+
: { $eq: [{ $type: `$${field}` }, 'missing'] };
|
|
176
|
+
}
|
|
177
|
+
return { $eq: [`$${field}`, condition] };
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
return {
|
|
181
|
+
case: caseConditions.length > 1 ? { $and: caseConditions } : caseConditions[0],
|
|
182
|
+
then: status,
|
|
183
|
+
};
|
|
184
|
+
});
|
|
185
|
+
};
|
|
186
|
+
|
|
94
187
|
const defaultSort: Array<{ key: string; value: SortDirection }> = [
|
|
95
188
|
{ key: 'started', value: SortDirection.DESC },
|
|
96
189
|
{ key: 'priority', value: SortDirection.DESC },
|
|
@@ -98,18 +191,48 @@ const defaultSort: Array<{ key: string; value: SortDirection }> = [
|
|
|
98
191
|
{ key: 'created', value: SortDirection.ASC },
|
|
99
192
|
];
|
|
100
193
|
|
|
194
|
+
const normalizeWorkQueueAggregateResult = (data = []): WorkerReport[] => {
|
|
195
|
+
const statusToFieldMap = {
|
|
196
|
+
NEW: 'newCount',
|
|
197
|
+
ALLOCATED: 'startCount',
|
|
198
|
+
FAILED: 'errorCount',
|
|
199
|
+
SUCCESS: 'successCount',
|
|
200
|
+
DELETED: 'deleteCount',
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
return data.map((item) => {
|
|
204
|
+
const workStatistics: WorkerReport = {
|
|
205
|
+
type: item.type,
|
|
206
|
+
...(Object.values(statusToFieldMap).reduce((acc, key) => ({ ...acc, [key]: 0 }), {}) as any),
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
item.statuses.forEach(({ status, count }) => {
|
|
210
|
+
if (statusToFieldMap[status]) {
|
|
211
|
+
workStatistics[statusToFieldMap[status]] = count;
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
const { errorCount, successCount, deleteCount } = workStatistics;
|
|
215
|
+
|
|
216
|
+
workStatistics.startCount += errorCount + successCount + deleteCount;
|
|
217
|
+
workStatistics.newCount += workStatistics.startCount;
|
|
218
|
+
|
|
219
|
+
return workStatistics;
|
|
220
|
+
}) as unknown as WorkerReport[];
|
|
221
|
+
};
|
|
222
|
+
|
|
101
223
|
export const configureWorkerModule = async ({
|
|
102
224
|
db,
|
|
225
|
+
migrationRepository,
|
|
103
226
|
options,
|
|
104
227
|
}: ModuleInput<WorkerSettingsOptions>): Promise<WorkerModule> => {
|
|
228
|
+
addMigrations(migrationRepository);
|
|
229
|
+
|
|
105
230
|
registerEvents(Object.values(WorkerEventTypes));
|
|
106
231
|
|
|
107
232
|
const WorkQueue = await WorkQueueCollection(db);
|
|
108
233
|
|
|
109
234
|
const removePrivateFields = buildObfuscatedFieldsFilter(options?.blacklistedVariables);
|
|
110
235
|
|
|
111
|
-
const mutations = generateDbMutations<Work>(WorkQueue, WorkQueueSchema) as ModuleMutations<Work>;
|
|
112
|
-
|
|
113
236
|
const allocateWork: WorkerModule['allocateWork'] = async ({ types, worker = UNCHAINED_WORKER_ID }) => {
|
|
114
237
|
// Find a work item that is scheduled for now and is not started.
|
|
115
238
|
// Also:
|
|
@@ -153,18 +276,21 @@ export const configureWorkerModule = async ({
|
|
|
153
276
|
|
|
154
277
|
if (!workBeforeUpdate) return null;
|
|
155
278
|
|
|
156
|
-
await
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
279
|
+
const work = await WorkQueue.findOneAndUpdate(
|
|
280
|
+
generateDbFilterById(workId),
|
|
281
|
+
{
|
|
282
|
+
$set: {
|
|
283
|
+
updated: new Date(),
|
|
284
|
+
finished,
|
|
285
|
+
success,
|
|
286
|
+
error,
|
|
287
|
+
result,
|
|
288
|
+
...(!workBeforeUpdate.started ? { started } : {}),
|
|
289
|
+
worker,
|
|
290
|
+
},
|
|
164
291
|
},
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
const work = await WorkQueue.findOne(generateDbFilterById(workId), {});
|
|
292
|
+
{ returnDocument: 'after' },
|
|
293
|
+
);
|
|
168
294
|
|
|
169
295
|
const duration = new Date(work.finished).getTime() - new Date(work.started).getTime();
|
|
170
296
|
if (work.success) {
|
|
@@ -332,14 +458,15 @@ export const configureWorkerModule = async ({
|
|
|
332
458
|
}
|
|
333
459
|
|
|
334
460
|
const created = new Date();
|
|
335
|
-
const workId = await
|
|
461
|
+
const { insertedId: workId } = await WorkQueue.insertOne({
|
|
462
|
+
_id: generateDbObjectId(),
|
|
463
|
+
created,
|
|
336
464
|
type,
|
|
337
465
|
input,
|
|
338
466
|
priority,
|
|
339
467
|
scheduled: scheduled || created,
|
|
340
468
|
originalWorkId,
|
|
341
469
|
retries,
|
|
342
|
-
created,
|
|
343
470
|
worker,
|
|
344
471
|
});
|
|
345
472
|
|
|
@@ -354,13 +481,17 @@ export const configureWorkerModule = async ({
|
|
|
354
481
|
},
|
|
355
482
|
|
|
356
483
|
rescheduleWork: async (currentWork, scheduled) => {
|
|
357
|
-
await
|
|
358
|
-
|
|
359
|
-
|
|
484
|
+
const work = await WorkQueue.findOneAndUpdate(
|
|
485
|
+
generateDbFilterById(currentWork._id),
|
|
486
|
+
{
|
|
487
|
+
$set: {
|
|
488
|
+
updated: new Date(),
|
|
489
|
+
scheduled,
|
|
490
|
+
},
|
|
360
491
|
},
|
|
361
|
-
|
|
492
|
+
{ returnDocument: 'after' },
|
|
493
|
+
);
|
|
362
494
|
|
|
363
|
-
const work = await WorkQueue.findOne(generateDbFilterById(currentWork._id), {});
|
|
364
495
|
emit(WorkerEventTypes.RESCHEDULED, {
|
|
365
496
|
work: removePrivateFields(work),
|
|
366
497
|
oldScheduled: currentWork.scheduled,
|
|
@@ -371,12 +502,13 @@ export const configureWorkerModule = async ({
|
|
|
371
502
|
|
|
372
503
|
allocateWork,
|
|
373
504
|
|
|
374
|
-
ensureNoWork: async ({ type, priority = 0 }) => {
|
|
505
|
+
ensureNoWork: async ({ type, priority = 0, scheduleId }) => {
|
|
375
506
|
const query = buildQuerySelector({
|
|
376
507
|
type,
|
|
377
508
|
status: [WorkStatus.NEW],
|
|
378
509
|
priority,
|
|
379
510
|
autoscheduled: true,
|
|
511
|
+
scheduleId,
|
|
380
512
|
});
|
|
381
513
|
|
|
382
514
|
await WorkQueue.updateMany(query, {
|
|
@@ -386,16 +518,25 @@ export const configureWorkerModule = async ({
|
|
|
386
518
|
});
|
|
387
519
|
},
|
|
388
520
|
|
|
389
|
-
ensureOneWork: async (
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
521
|
+
ensureOneWork: async ({
|
|
522
|
+
type,
|
|
523
|
+
input,
|
|
524
|
+
priority = 0,
|
|
525
|
+
scheduled,
|
|
526
|
+
timeout,
|
|
527
|
+
originalWorkId,
|
|
528
|
+
retries = 20,
|
|
529
|
+
scheduleId,
|
|
530
|
+
}) => {
|
|
531
|
+
const workId = `${scheduleId}:${scheduled.getTime()}`;
|
|
532
|
+
|
|
393
533
|
const created = new Date();
|
|
394
534
|
const query = buildQuerySelector({
|
|
395
535
|
type,
|
|
396
536
|
status: [WorkStatus.NEW],
|
|
397
537
|
priority,
|
|
398
538
|
autoscheduled: true,
|
|
539
|
+
scheduleId,
|
|
399
540
|
});
|
|
400
541
|
try {
|
|
401
542
|
const result = await WorkQueue.findOneAndUpdate(
|
|
@@ -414,6 +555,7 @@ export const configureWorkerModule = async ({
|
|
|
414
555
|
},
|
|
415
556
|
$setOnInsert: {
|
|
416
557
|
_id: workId,
|
|
558
|
+
scheduleId,
|
|
417
559
|
scheduled,
|
|
418
560
|
type,
|
|
419
561
|
created,
|
|
@@ -428,6 +570,7 @@ export const configureWorkerModule = async ({
|
|
|
428
570
|
upsert: true,
|
|
429
571
|
},
|
|
430
572
|
);
|
|
573
|
+
|
|
431
574
|
if (!result.lastErrorObject.updatedExisting) {
|
|
432
575
|
logger.info(`${type} auto-scheduled @ ${new Date(scheduled).toISOString()}`, {
|
|
433
576
|
workId,
|
|
@@ -435,7 +578,7 @@ export const configureWorkerModule = async ({
|
|
|
435
578
|
emit(WorkerEventTypes.ADDED, removePrivateFields(result.value));
|
|
436
579
|
}
|
|
437
580
|
return result.value;
|
|
438
|
-
} catch
|
|
581
|
+
} catch {
|
|
439
582
|
/* TODO:
|
|
440
583
|
If the findOneAndUpdate call failed because of _id conflict with a DELETED work,
|
|
441
584
|
we should permanently remove the conflicting deleted work
|
|
@@ -458,9 +601,16 @@ export const configureWorkerModule = async ({
|
|
|
458
601
|
);
|
|
459
602
|
if (!workBeforeRemoval) return null;
|
|
460
603
|
|
|
461
|
-
await
|
|
604
|
+
const work = await WorkQueue.findOneAndUpdate(
|
|
605
|
+
generateDbFilterById(workId),
|
|
606
|
+
{
|
|
607
|
+
$set: {
|
|
608
|
+
deleted: new Date(),
|
|
609
|
+
},
|
|
610
|
+
},
|
|
611
|
+
{ returnDocument: 'after' },
|
|
612
|
+
);
|
|
462
613
|
|
|
463
|
-
const work = await WorkQueue.findOne(generateDbFilterById(workId), {});
|
|
464
614
|
emit(WorkerEventTypes.DELETED, removePrivateFields(work));
|
|
465
615
|
|
|
466
616
|
return work;
|
|
@@ -493,5 +643,84 @@ export const configureWorkerModule = async ({
|
|
|
493
643
|
),
|
|
494
644
|
);
|
|
495
645
|
},
|
|
646
|
+
|
|
647
|
+
getReport: async ({ types, from, to } = { types: null, from: null, to: null }) => {
|
|
648
|
+
const pipeline = [];
|
|
649
|
+
const matchConditions = [];
|
|
650
|
+
// build date filter based on provided values it can be a range if both to and from is supplied
|
|
651
|
+
// a upper or lowe limit if either from or to is provided
|
|
652
|
+
// or all if none is provided
|
|
653
|
+
if (from || to) {
|
|
654
|
+
const dateConditions = [];
|
|
655
|
+
if (from) {
|
|
656
|
+
const fromDate = new Date(from);
|
|
657
|
+
dateConditions.push({
|
|
658
|
+
$or: [{ created: { $gte: fromDate } }, { updated: { $gte: fromDate } }],
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
if (to) {
|
|
662
|
+
const toDate = new Date(to);
|
|
663
|
+
dateConditions.push({
|
|
664
|
+
$or: [{ created: { $lte: toDate } }, { updated: { $lte: toDate } }],
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
if (dateConditions.length > 0) {
|
|
668
|
+
matchConditions.push({ $and: dateConditions });
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
// build types filter if type is provided or ignore types if it is not provided
|
|
672
|
+
if (types && Array.isArray(types) && types.length) {
|
|
673
|
+
matchConditions.push({ type: { $in: types } });
|
|
674
|
+
}
|
|
675
|
+
if (matchConditions.length > 0) {
|
|
676
|
+
pipeline.push({
|
|
677
|
+
$match: {
|
|
678
|
+
$and: matchConditions,
|
|
679
|
+
},
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
pipeline.push(
|
|
683
|
+
{
|
|
684
|
+
$addFields: {
|
|
685
|
+
status: {
|
|
686
|
+
$switch: {
|
|
687
|
+
branches: convertFilterMapToPipelineBranches(),
|
|
688
|
+
default: 'UNKNOWN',
|
|
689
|
+
},
|
|
690
|
+
},
|
|
691
|
+
},
|
|
692
|
+
},
|
|
693
|
+
{
|
|
694
|
+
$group: {
|
|
695
|
+
_id: {
|
|
696
|
+
type: '$type',
|
|
697
|
+
status: '$status',
|
|
698
|
+
},
|
|
699
|
+
count: { $sum: 1 },
|
|
700
|
+
},
|
|
701
|
+
},
|
|
702
|
+
{
|
|
703
|
+
$group: {
|
|
704
|
+
_id: '$_id.type',
|
|
705
|
+
statuses: {
|
|
706
|
+
$push: {
|
|
707
|
+
status: '$_id.status',
|
|
708
|
+
count: '$count',
|
|
709
|
+
},
|
|
710
|
+
},
|
|
711
|
+
},
|
|
712
|
+
},
|
|
713
|
+
|
|
714
|
+
{
|
|
715
|
+
$project: {
|
|
716
|
+
_id: 0,
|
|
717
|
+
type: '$_id',
|
|
718
|
+
statuses: 1,
|
|
719
|
+
},
|
|
720
|
+
},
|
|
721
|
+
);
|
|
722
|
+
|
|
723
|
+
return normalizeWorkQueueAggregateResult(await WorkQueue.aggregate(pipeline).toArray()) as any;
|
|
724
|
+
},
|
|
496
725
|
};
|
|
497
726
|
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { MigrationRepository } from '@unchainedshop/mongodb';
|
|
2
|
+
import { WorkQueueCollection } from '../../db/WorkQueueCollection.js';
|
|
3
|
+
|
|
4
|
+
export default function addMigrations(repository: MigrationRepository) {
|
|
5
|
+
repository?.register({
|
|
6
|
+
id: 20240905161400,
|
|
7
|
+
name: 'Remove all autoscheduled work items so they get scheduled freshly',
|
|
8
|
+
up: async () => {
|
|
9
|
+
const WorkQueue = await WorkQueueCollection(repository.db);
|
|
10
|
+
|
|
11
|
+
await WorkQueue.deleteMany({
|
|
12
|
+
autoscheduled: true,
|
|
13
|
+
started: null,
|
|
14
|
+
deleted: null,
|
|
15
|
+
});
|
|
16
|
+
},
|
|
17
|
+
});
|
|
18
|
+
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import { IScheduler, Work, WorkData } from '@unchainedshop/types/worker.js';
|
|
2
1
|
import { log } from '@unchainedshop/logger';
|
|
3
2
|
import { subscribe } from '@unchainedshop/events';
|
|
4
3
|
import { WorkerEventTypes } from '../director/WorkerEventTypes.js';
|
|
4
|
+
import { UnchainedCore } from '@unchainedshop/core';
|
|
5
|
+
import { WorkData } from '../worker-index.js';
|
|
6
|
+
import { Work } from '../types.js';
|
|
5
7
|
|
|
6
8
|
export interface FailedReschedulerParams {
|
|
7
9
|
retryInput?: (
|
|
@@ -10,6 +12,20 @@ export interface FailedReschedulerParams {
|
|
|
10
12
|
) => Promise<Record<string, any> | null>;
|
|
11
13
|
}
|
|
12
14
|
|
|
15
|
+
export type IScheduler<P> = {
|
|
16
|
+
key: string;
|
|
17
|
+
label: string;
|
|
18
|
+
version: string;
|
|
19
|
+
|
|
20
|
+
actions: (
|
|
21
|
+
params: P,
|
|
22
|
+
unchainedAPI: UnchainedCore,
|
|
23
|
+
) => {
|
|
24
|
+
start: () => void;
|
|
25
|
+
stop: () => void;
|
|
26
|
+
};
|
|
27
|
+
};
|
|
28
|
+
|
|
13
29
|
export const FailedRescheduler: IScheduler<FailedReschedulerParams> = {
|
|
14
30
|
key: 'shop.unchained.scheduler.failed',
|
|
15
31
|
label: 'Reschedule failed works',
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { TimestampFields } from '@unchainedshop/mongodb';
|
|
2
|
+
|
|
3
|
+
export type Work = {
|
|
4
|
+
_id?: string;
|
|
5
|
+
priority: number;
|
|
6
|
+
retries: number;
|
|
7
|
+
scheduled: Date;
|
|
8
|
+
type: string;
|
|
9
|
+
input: Record<string, any>;
|
|
10
|
+
error?: any;
|
|
11
|
+
finished?: Date;
|
|
12
|
+
originalWorkId?: string;
|
|
13
|
+
result?: any;
|
|
14
|
+
started?: Date;
|
|
15
|
+
success?: boolean;
|
|
16
|
+
timeout?: number;
|
|
17
|
+
worker?: string;
|
|
18
|
+
autoscheduled?: boolean;
|
|
19
|
+
scheduleId?: string;
|
|
20
|
+
} & TimestampFields;
|
package/src/worker-index.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
export
|
|
1
|
+
export * from './types.js';
|
|
2
2
|
|
|
3
|
-
export
|
|
4
|
-
export
|
|
3
|
+
export * from './module/configureWorkerModule.js';
|
|
4
|
+
export * from './director/WorkerDirector.js';
|
|
5
|
+
export * from './director/WorkerAdapter.js';
|
|
5
6
|
export { WorkStatus } from './director/WorkStatus.js';
|
|
6
7
|
export { WorkerEventTypes } from './director/WorkerEventTypes.js';
|
|
7
8
|
|
|
8
|
-
export
|
|
9
|
-
|
|
10
|
-
export
|
|
11
|
-
export
|
|
12
|
-
export { IntervalWorker } from './workers/IntervalWorker.js';
|
|
9
|
+
export * from './schedulers/FailedRescheduler.js';
|
|
10
|
+
export * from './workers/BaseWorker.js';
|
|
11
|
+
export * from './workers/EventListenerWorker.js';
|
|
12
|
+
export * from './workers/IntervalWorker.js';
|
|
@@ -1,7 +1,34 @@
|
|
|
1
|
-
import { IWorker, WorkData } from '@unchainedshop/types/worker.js';
|
|
2
1
|
import later from '@breejs/later';
|
|
3
2
|
import { log } from '@unchainedshop/logger';
|
|
4
3
|
import { WorkerDirector } from '../director/WorkerDirector.js';
|
|
4
|
+
import { UnchainedCore } from '@unchainedshop/core';
|
|
5
|
+
import { Work } from '../types.js';
|
|
6
|
+
|
|
7
|
+
export type WorkData = Pick<
|
|
8
|
+
Partial<Work>,
|
|
9
|
+
'input' | 'originalWorkId' | 'priority' | 'retries' | 'timeout' | 'scheduled' | 'worker' | 'scheduleId'
|
|
10
|
+
> & { type: string };
|
|
11
|
+
|
|
12
|
+
export type IWorker<P extends { workerId?: string }> = {
|
|
13
|
+
key: string;
|
|
14
|
+
label: string;
|
|
15
|
+
version: string;
|
|
16
|
+
type: string;
|
|
17
|
+
external: boolean;
|
|
18
|
+
|
|
19
|
+
getFloorDate: (date?: Date) => Date;
|
|
20
|
+
|
|
21
|
+
actions: (
|
|
22
|
+
params: P,
|
|
23
|
+
unchainedAPI: UnchainedCore,
|
|
24
|
+
) => {
|
|
25
|
+
autorescheduleTypes: (options: { referenceDate: Date }) => Promise<Array<Work | null>>;
|
|
26
|
+
process: (options: { maxWorkItemCount?: number; referenceDate?: Date }) => Promise<void>;
|
|
27
|
+
reset: () => Promise<void>;
|
|
28
|
+
start: () => void;
|
|
29
|
+
stop: () => void;
|
|
30
|
+
};
|
|
31
|
+
};
|
|
5
32
|
|
|
6
33
|
interface WorkerParams {
|
|
7
34
|
workerId?: string;
|
|
@@ -47,6 +74,7 @@ export const BaseWorker: IWorker<WorkerParams> = {
|
|
|
47
74
|
await unchainedAPI.modules.worker.ensureNoWork({
|
|
48
75
|
type: workConfig.type,
|
|
49
76
|
priority: workConfig.priority || 0,
|
|
77
|
+
scheduleId,
|
|
50
78
|
});
|
|
51
79
|
return null;
|
|
52
80
|
}
|
|
@@ -57,6 +85,7 @@ export const BaseWorker: IWorker<WorkerParams> = {
|
|
|
57
85
|
nextDate.setMilliseconds(0);
|
|
58
86
|
const workData: WorkData = {
|
|
59
87
|
type: workConfig.type,
|
|
88
|
+
scheduleId,
|
|
60
89
|
scheduled: nextDate,
|
|
61
90
|
timeout: workConfig.timeout,
|
|
62
91
|
priority: workConfig.priority || 0,
|
|
@@ -68,8 +97,7 @@ export const BaseWorker: IWorker<WorkerParams> = {
|
|
|
68
97
|
// when it explicitly returns a falsish input instead of a dictionary
|
|
69
98
|
if (!workData.input) return null;
|
|
70
99
|
}
|
|
71
|
-
|
|
72
|
-
return unchainedAPI.modules.worker.ensureOneWork(workData, workId);
|
|
100
|
+
return unchainedAPI.modules.worker.ensureOneWork(workData);
|
|
73
101
|
}),
|
|
74
102
|
);
|
|
75
103
|
},
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { IWorker } from '@unchainedshop/types/worker.js';
|
|
2
1
|
import { subscribe } from '@unchainedshop/events';
|
|
3
2
|
import { WorkerEventTypes } from '../director/WorkerEventTypes.js';
|
|
4
|
-
import { BaseWorker } from './BaseWorker.js';
|
|
3
|
+
import { BaseWorker, IWorker } from './BaseWorker.js';
|
|
5
4
|
|
|
6
5
|
function debounce<T extends (...args: any) => any>(func: T, wait) {
|
|
7
6
|
let timeout;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { IWorker, WorkerSchedule } from '@unchainedshop/types/worker.js';
|
|
2
1
|
import later from '@breejs/later';
|
|
3
|
-
import { BaseWorker } from './BaseWorker.js';
|
|
2
|
+
import { BaseWorker, IWorker } from './BaseWorker.js';
|
|
3
|
+
import { WorkerSchedule } from '../worker-index.js';
|
|
4
4
|
|
|
5
5
|
const { NODE_ENV } = process.env;
|
|
6
6
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"WorkQueueSchema.d.ts","sourceRoot":"","sources":["../../src/db/WorkQueueSchema.ts"],"names":[],"mappings":"AACA,OAAO,YAAY,MAAM,cAAc,CAAC;AAExC,eAAO,MAAM,eAAe,cAmB3B,CAAC"}
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import { Schemas } from '@unchainedshop/utils';
|
|
2
|
-
import SimpleSchema from 'simpl-schema';
|
|
3
|
-
export const WorkQueueSchema = new SimpleSchema({
|
|
4
|
-
finished: Date,
|
|
5
|
-
originalWorkId: { type: String },
|
|
6
|
-
priority: { type: Number, required: true },
|
|
7
|
-
retries: { type: Number, required: true },
|
|
8
|
-
scheduled: { type: Date, required: true },
|
|
9
|
-
started: Date,
|
|
10
|
-
success: Boolean,
|
|
11
|
-
timeout: Number,
|
|
12
|
-
type: { type: String, required: true },
|
|
13
|
-
worker: String,
|
|
14
|
-
// Worker data
|
|
15
|
-
input: { type: Object, blackbox: true },
|
|
16
|
-
result: { type: Object, blackbox: true },
|
|
17
|
-
error: { type: Object, blackbox: true },
|
|
18
|
-
...Schemas.timestampFields,
|
|
19
|
-
}, { requiredByDefault: false });
|
|
20
|
-
//# sourceMappingURL=WorkQueueSchema.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"WorkQueueSchema.js","sourceRoot":"","sources":["../../src/db/WorkQueueSchema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,YAAY,MAAM,cAAc,CAAC;AAExC,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,YAAY,CAC7C;IACE,QAAQ,EAAE,IAAI;IACd,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;IAChC,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC1C,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IACzC,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE;IACzC,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,OAAO;IAChB,OAAO,EAAE,MAAM;IACf,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IACtC,MAAM,EAAE,MAAM;IACd,cAAc;IACd,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IACvC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IACxC,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;IACvC,GAAG,OAAO,CAAC,eAAe;CAC3B,EACD,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAC7B,CAAC"}
|