@unchainedshop/core-worker 3.0.0-alpha6 → 3.0.0-rc.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/db/WorkQueueCollection.d.ts +26 -1
- package/lib/db/WorkQueueCollection.d.ts.map +1 -1
- package/lib/db/WorkQueueCollection.js +8 -0
- package/lib/db/WorkQueueCollection.js.map +1 -1
- package/lib/module/configureWorkerModule.d.ts +66 -54
- package/lib/module/configureWorkerModule.d.ts.map +1 -1
- package/lib/module/configureWorkerModule.js +54 -88
- package/lib/module/configureWorkerModule.js.map +1 -1
- package/lib/worker-index.d.ts +1 -9
- package/lib/worker-index.d.ts.map +1 -1
- package/lib/worker-index.js +1 -9
- package/lib/worker-index.js.map +1 -1
- package/package.json +13 -13
- package/src/db/WorkQueueCollection.ts +28 -1
- package/src/module/configureWorkerModule.ts +145 -171
- package/src/worker-index.ts +1 -10
- package/tsconfig.json +5 -7
- package/lib/director/WorkStatus.d.ts +0 -8
- package/lib/director/WorkStatus.d.ts.map +0 -1
- package/lib/director/WorkStatus.js +0 -9
- package/lib/director/WorkStatus.js.map +0 -1
- package/lib/director/WorkerAdapter.d.ts +0 -15
- package/lib/director/WorkerAdapter.d.ts.map +0 -1
- package/lib/director/WorkerAdapter.js +0 -11
- package/lib/director/WorkerAdapter.js.map +0 -1
- package/lib/director/WorkerDirector.d.ts +0 -27
- package/lib/director/WorkerDirector.d.ts.map +0 -1
- package/lib/director/WorkerDirector.js +0 -72
- package/lib/director/WorkerDirector.js.map +0 -1
- package/lib/director/WorkerEventTypes.d.ts +0 -8
- package/lib/director/WorkerEventTypes.d.ts.map +0 -1
- package/lib/director/WorkerEventTypes.js +0 -9
- package/lib/director/WorkerEventTypes.js.map +0 -1
- package/lib/schedulers/FailedRescheduler.d.ts +0 -16
- package/lib/schedulers/FailedRescheduler.d.ts.map +0 -1
- package/lib/schedulers/FailedRescheduler.js +0 -45
- package/lib/schedulers/FailedRescheduler.js.map +0 -1
- package/lib/types.d.ts +0 -20
- package/lib/types.d.ts.map +0 -1
- package/lib/types.js +0 -2
- package/lib/types.js.map +0 -1
- package/lib/workers/BaseWorker.d.ts +0 -34
- package/lib/workers/BaseWorker.d.ts.map +0 -1
- package/lib/workers/BaseWorker.js +0 -81
- package/lib/workers/BaseWorker.js.map +0 -1
- package/lib/workers/EventListenerWorker.d.ts +0 -6
- package/lib/workers/EventListenerWorker.d.ts.map +0 -1
- package/lib/workers/EventListenerWorker.js +0 -46
- package/lib/workers/EventListenerWorker.js.map +0 -1
- package/lib/workers/IntervalWorker.d.ts +0 -10
- package/lib/workers/IntervalWorker.d.ts.map +0 -1
- package/lib/workers/IntervalWorker.js +0 -38
- package/lib/workers/IntervalWorker.js.map +0 -1
- package/src/director/WorkStatus.ts +0 -7
- package/src/director/WorkerAdapter.ts +0 -28
- package/src/director/WorkerDirector.ts +0 -120
- package/src/director/WorkerEventTypes.ts +0 -7
- package/src/schedulers/FailedRescheduler.ts +0 -83
- package/src/types.ts +0 -20
- package/src/workers/BaseWorker.ts +0 -125
- package/src/workers/EventListenerWorker.ts +0 -61
- package/src/workers/IntervalWorker.ts +0 -56
|
@@ -1,8 +1,35 @@
|
|
|
1
|
+
import { TimestampFields } from '@unchainedshop/mongodb';
|
|
1
2
|
import { mongodb, buildDbIndexes } from '@unchainedshop/mongodb';
|
|
2
|
-
import { Work } from '../types.js';
|
|
3
3
|
|
|
4
4
|
const ONE_DAY_IN_SECONDS = 86400;
|
|
5
5
|
|
|
6
|
+
export enum WorkStatus {
|
|
7
|
+
NEW = 'NEW',
|
|
8
|
+
ALLOCATED = 'ALLOCATED',
|
|
9
|
+
SUCCESS = 'SUCCESS',
|
|
10
|
+
FAILED = 'FAILED',
|
|
11
|
+
DELETED = 'DELETED',
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export type Work = {
|
|
15
|
+
_id?: string;
|
|
16
|
+
priority: number;
|
|
17
|
+
retries: number;
|
|
18
|
+
scheduled: Date;
|
|
19
|
+
type: string;
|
|
20
|
+
input: Record<string, any>;
|
|
21
|
+
error?: any;
|
|
22
|
+
finished?: Date;
|
|
23
|
+
originalWorkId?: string;
|
|
24
|
+
result?: any;
|
|
25
|
+
started?: Date;
|
|
26
|
+
success?: boolean;
|
|
27
|
+
timeout?: number;
|
|
28
|
+
worker?: string;
|
|
29
|
+
autoscheduled?: boolean;
|
|
30
|
+
scheduleId?: string;
|
|
31
|
+
} & TimestampFields;
|
|
32
|
+
|
|
6
33
|
export const WorkQueueCollection = async (db: mongodb.Db) => {
|
|
7
34
|
const WorkQueue = db.collection<Work>('work_queue');
|
|
8
35
|
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
import
|
|
2
|
-
import type { UnchainedCore } from '@unchainedshop/core';
|
|
3
|
-
|
|
4
|
-
import os from 'os';
|
|
1
|
+
import os from 'node:os';
|
|
5
2
|
import { createLogger } from '@unchainedshop/logger';
|
|
6
3
|
import {
|
|
7
4
|
generateDbFilterById,
|
|
@@ -11,18 +8,40 @@ import {
|
|
|
11
8
|
ModuleInput,
|
|
12
9
|
} from '@unchainedshop/mongodb';
|
|
13
10
|
import { emit, registerEvents } from '@unchainedshop/events';
|
|
14
|
-
import {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
11
|
+
import {
|
|
12
|
+
buildObfuscatedFieldsFilter,
|
|
13
|
+
DateFilterInput,
|
|
14
|
+
SortDirection,
|
|
15
|
+
SortOption,
|
|
16
|
+
} from '@unchainedshop/utils';
|
|
17
|
+
import { Work, WorkQueueCollection, WorkStatus } from '../db/WorkQueueCollection.js';
|
|
20
18
|
import addMigrations from './migrations/addMigrations.js';
|
|
21
19
|
|
|
22
20
|
const { UNCHAINED_WORKER_ID = os.hostname() } = process.env;
|
|
23
21
|
|
|
22
|
+
export const DIRECTOR_MARKED_FAILED_ERROR = 'DIRECTOR_MARKED_FAILED';
|
|
23
|
+
|
|
24
24
|
const logger = createLogger('unchained:core-worker');
|
|
25
25
|
|
|
26
|
+
export type WorkData = Pick<
|
|
27
|
+
Partial<Work>,
|
|
28
|
+
'input' | 'originalWorkId' | 'priority' | 'retries' | 'timeout' | 'scheduled' | 'worker' | 'scheduleId'
|
|
29
|
+
> & { type: string };
|
|
30
|
+
|
|
31
|
+
export interface WorkResult<Result = unknown> {
|
|
32
|
+
success: boolean;
|
|
33
|
+
result?: Result;
|
|
34
|
+
error?: any;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export enum WorkerEventTypes {
|
|
38
|
+
ADDED = 'WORK_ADDED',
|
|
39
|
+
ALLOCATED = 'WORK_ALLOCATED',
|
|
40
|
+
FINISHED = 'WORK_FINISHED',
|
|
41
|
+
DELETED = 'WORK_DELETED',
|
|
42
|
+
RESCHEDULED = 'WORK_RESCHEDULED',
|
|
43
|
+
}
|
|
44
|
+
|
|
26
45
|
export interface WorkerSettingsOptions {
|
|
27
46
|
blacklistedVariables?: string[];
|
|
28
47
|
}
|
|
@@ -44,57 +63,6 @@ export type WorkQueueQuery = {
|
|
|
44
63
|
scheduled?: { end?: Date; start?: Date };
|
|
45
64
|
};
|
|
46
65
|
|
|
47
|
-
export type WorkerModule = {
|
|
48
|
-
activeWorkTypes: () => Promise<Array<string>>;
|
|
49
|
-
findWork: (query: { workId?: string; originalWorkId?: string }) => Promise<Work>;
|
|
50
|
-
findWorkQueue: (
|
|
51
|
-
query: WorkQueueQuery & {
|
|
52
|
-
sort?: Array<SortOption>;
|
|
53
|
-
limit?: number;
|
|
54
|
-
skip?: number;
|
|
55
|
-
},
|
|
56
|
-
) => Promise<Array<Work>>;
|
|
57
|
-
count: (query: WorkQueueQuery) => Promise<number>;
|
|
58
|
-
workExists: (query: { workId?: string; originalWorkId?: string }) => Promise<boolean>;
|
|
59
|
-
|
|
60
|
-
// Transformations
|
|
61
|
-
status: (work: Work) => WorkStatus;
|
|
62
|
-
|
|
63
|
-
type: (work: Work) => string;
|
|
64
|
-
|
|
65
|
-
// Mutations
|
|
66
|
-
addWork: (data: WorkData) => Promise<Work>;
|
|
67
|
-
|
|
68
|
-
allocateWork: (doc: { types: Array<string>; worker: string }) => Promise<Work>;
|
|
69
|
-
|
|
70
|
-
processNextWork: (unchainedAPI: UnchainedCore, workerId?: string) => Promise<Work>;
|
|
71
|
-
|
|
72
|
-
rescheduleWork: (work: Work, scheduled: Date, unchainedAPI: UnchainedCore) => Promise<Work>;
|
|
73
|
-
|
|
74
|
-
ensureOneWork: (work: WorkData) => Promise<Work>;
|
|
75
|
-
|
|
76
|
-
ensureNoWork: (work: { priority: number; type: string; scheduleId: string }) => Promise<void>;
|
|
77
|
-
|
|
78
|
-
finishWork: (
|
|
79
|
-
_id: string,
|
|
80
|
-
data: WorkResult<any> & {
|
|
81
|
-
finished?: Date;
|
|
82
|
-
started?: Date;
|
|
83
|
-
worker?: string;
|
|
84
|
-
},
|
|
85
|
-
) => Promise<Work | null>;
|
|
86
|
-
|
|
87
|
-
deleteWork: (_id: string) => Promise<Work | null>;
|
|
88
|
-
|
|
89
|
-
markOldWorkAsFailed: (params: {
|
|
90
|
-
types: Array<string>;
|
|
91
|
-
worker: string;
|
|
92
|
-
referenceDate: Date;
|
|
93
|
-
}) => Promise<Array<Work>>;
|
|
94
|
-
|
|
95
|
-
getReport: (params?: { types?: string[]; from?: Date; to?: Date }) => Promise<WorkerReport[]>;
|
|
96
|
-
};
|
|
97
|
-
|
|
98
66
|
const WORK_STATUS_FILTER_MAP = {
|
|
99
67
|
[WorkStatus.DELETED]: { deleted: { $exists: true } },
|
|
100
68
|
[WorkStatus.NEW]: {
|
|
@@ -224,7 +192,7 @@ export const configureWorkerModule = async ({
|
|
|
224
192
|
db,
|
|
225
193
|
migrationRepository,
|
|
226
194
|
options,
|
|
227
|
-
}: ModuleInput<WorkerSettingsOptions>)
|
|
195
|
+
}: ModuleInput<WorkerSettingsOptions>) => {
|
|
228
196
|
addMigrations(migrationRepository);
|
|
229
197
|
|
|
230
198
|
registerEvents(Object.values(WorkerEventTypes));
|
|
@@ -233,7 +201,13 @@ export const configureWorkerModule = async ({
|
|
|
233
201
|
|
|
234
202
|
const removePrivateFields = buildObfuscatedFieldsFilter(options?.blacklistedVariables);
|
|
235
203
|
|
|
236
|
-
const allocateWork
|
|
204
|
+
const allocateWork = async ({
|
|
205
|
+
types,
|
|
206
|
+
worker = UNCHAINED_WORKER_ID,
|
|
207
|
+
}: {
|
|
208
|
+
types: Array<string>;
|
|
209
|
+
worker: string;
|
|
210
|
+
}): Promise<Work> => {
|
|
237
211
|
// Find a work item that is scheduled for now and is not started.
|
|
238
212
|
// Also:
|
|
239
213
|
// - Restrict by types and worker if provided
|
|
@@ -259,8 +233,8 @@ export const configureWorkerModule = async ({
|
|
|
259
233
|
return result;
|
|
260
234
|
};
|
|
261
235
|
|
|
262
|
-
const finishWork
|
|
263
|
-
workId,
|
|
236
|
+
const finishWork = async (
|
|
237
|
+
workId: string,
|
|
264
238
|
{
|
|
265
239
|
error,
|
|
266
240
|
finished = new Date(),
|
|
@@ -268,8 +242,12 @@ export const configureWorkerModule = async ({
|
|
|
268
242
|
started = new Date(),
|
|
269
243
|
success,
|
|
270
244
|
worker = UNCHAINED_WORKER_ID,
|
|
245
|
+
}: WorkResult<any> & {
|
|
246
|
+
finished?: Date;
|
|
247
|
+
started?: Date;
|
|
248
|
+
worker?: string;
|
|
271
249
|
},
|
|
272
|
-
) => {
|
|
250
|
+
): Promise<Work> => {
|
|
273
251
|
const workBeforeUpdate = await WorkQueue.findOne(
|
|
274
252
|
buildQuerySelector({ workId, status: [WorkStatus.ALLOCATED] }),
|
|
275
253
|
);
|
|
@@ -310,88 +288,35 @@ export const configureWorkerModule = async ({
|
|
|
310
288
|
return work;
|
|
311
289
|
};
|
|
312
290
|
|
|
313
|
-
const processNextWork: WorkerModule['processNextWork'] = async (unchainedAPI, workerId) => {
|
|
314
|
-
const adapters = WorkerDirector.getAdapters();
|
|
315
|
-
|
|
316
|
-
const alreadyAllocatedWork = await WorkQueue.aggregate(
|
|
317
|
-
[
|
|
318
|
-
{
|
|
319
|
-
$match: {
|
|
320
|
-
started: {
|
|
321
|
-
$exists: true,
|
|
322
|
-
},
|
|
323
|
-
finished: {
|
|
324
|
-
$exists: false,
|
|
325
|
-
},
|
|
326
|
-
deleted: {
|
|
327
|
-
$exists: false,
|
|
328
|
-
},
|
|
329
|
-
},
|
|
330
|
-
},
|
|
331
|
-
{
|
|
332
|
-
$group: {
|
|
333
|
-
_id: '$type',
|
|
334
|
-
count: {
|
|
335
|
-
$sum: 1,
|
|
336
|
-
},
|
|
337
|
-
},
|
|
338
|
-
},
|
|
339
|
-
],
|
|
340
|
-
{
|
|
341
|
-
allowDiskUse: false,
|
|
342
|
-
},
|
|
343
|
-
).toArray();
|
|
344
|
-
|
|
345
|
-
const allocationMap = Object.fromEntries(alreadyAllocatedWork.map((w) => [w._id, w.count]));
|
|
346
|
-
|
|
347
|
-
const types = adapters
|
|
348
|
-
.filter((adapter) => {
|
|
349
|
-
// Filter out the external
|
|
350
|
-
if (adapter.external) return false;
|
|
351
|
-
if (
|
|
352
|
-
adapter.maxParallelAllocations &&
|
|
353
|
-
adapter.maxParallelAllocations <= allocationMap[adapter.type]
|
|
354
|
-
)
|
|
355
|
-
return false;
|
|
356
|
-
return true;
|
|
357
|
-
})
|
|
358
|
-
.map((adapter) => adapter.type);
|
|
359
|
-
|
|
360
|
-
const worker = workerId ?? UNCHAINED_WORKER_ID;
|
|
361
|
-
const work = await allocateWork({
|
|
362
|
-
types,
|
|
363
|
-
worker,
|
|
364
|
-
});
|
|
365
|
-
|
|
366
|
-
if (work) {
|
|
367
|
-
const output = await WorkerDirector.doWork(work, unchainedAPI);
|
|
368
|
-
|
|
369
|
-
return finishWork(work._id, {
|
|
370
|
-
...output,
|
|
371
|
-
finished: work.finished || new Date(),
|
|
372
|
-
started: work.started,
|
|
373
|
-
worker,
|
|
374
|
-
});
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
return null;
|
|
378
|
-
};
|
|
379
|
-
|
|
380
291
|
return {
|
|
381
292
|
// Queries
|
|
382
|
-
|
|
293
|
+
|
|
294
|
+
workerId: UNCHAINED_WORKER_ID,
|
|
295
|
+
|
|
296
|
+
activeWorkTypes: async (): Promise<Array<string>> => {
|
|
383
297
|
const typeList = await WorkQueue.aggregate([{ $group: { _id: '$type' } }]).toArray();
|
|
384
|
-
return typeList
|
|
385
|
-
.map((t) => t._id as string)
|
|
386
|
-
.filter((type) => {
|
|
387
|
-
return WorkerDirector.getActivePluginTypes().includes(type);
|
|
388
|
-
});
|
|
298
|
+
return typeList.map((t) => t._id as string);
|
|
389
299
|
},
|
|
390
300
|
|
|
391
|
-
findWork: async ({
|
|
301
|
+
findWork: async ({
|
|
302
|
+
workId,
|
|
303
|
+
originalWorkId,
|
|
304
|
+
}: {
|
|
305
|
+
workId?: string;
|
|
306
|
+
originalWorkId?: string;
|
|
307
|
+
}): Promise<Work> =>
|
|
392
308
|
WorkQueue.findOne(workId ? generateDbFilterById(workId) : { originalWorkId }, {}),
|
|
393
309
|
|
|
394
|
-
findWorkQueue: async ({
|
|
310
|
+
findWorkQueue: async ({
|
|
311
|
+
limit,
|
|
312
|
+
skip,
|
|
313
|
+
sort,
|
|
314
|
+
...selectorOptions
|
|
315
|
+
}: WorkQueueQuery & {
|
|
316
|
+
sort?: Array<SortOption>;
|
|
317
|
+
limit?: number;
|
|
318
|
+
skip?: number;
|
|
319
|
+
}): Promise<Array<Work>> => {
|
|
395
320
|
const selector = buildQuerySelector(selectorOptions);
|
|
396
321
|
return WorkQueue.find(selector, {
|
|
397
322
|
skip,
|
|
@@ -400,11 +325,51 @@ export const configureWorkerModule = async ({
|
|
|
400
325
|
}).toArray();
|
|
401
326
|
},
|
|
402
327
|
|
|
403
|
-
count: async (query) => {
|
|
328
|
+
count: async (query: WorkQueueQuery) => {
|
|
404
329
|
return WorkQueue.countDocuments(buildQuerySelector(query));
|
|
405
330
|
},
|
|
406
331
|
|
|
407
|
-
|
|
332
|
+
allocationMap: async (): Promise<Record<string, number>> => {
|
|
333
|
+
const alreadyAllocatedWork = await WorkQueue.aggregate(
|
|
334
|
+
[
|
|
335
|
+
{
|
|
336
|
+
$match: {
|
|
337
|
+
started: {
|
|
338
|
+
$exists: true,
|
|
339
|
+
},
|
|
340
|
+
finished: {
|
|
341
|
+
$exists: false,
|
|
342
|
+
},
|
|
343
|
+
deleted: {
|
|
344
|
+
$exists: false,
|
|
345
|
+
},
|
|
346
|
+
},
|
|
347
|
+
},
|
|
348
|
+
{
|
|
349
|
+
$group: {
|
|
350
|
+
_id: '$type',
|
|
351
|
+
count: {
|
|
352
|
+
$sum: 1,
|
|
353
|
+
},
|
|
354
|
+
},
|
|
355
|
+
},
|
|
356
|
+
],
|
|
357
|
+
{
|
|
358
|
+
allowDiskUse: false,
|
|
359
|
+
},
|
|
360
|
+
).toArray();
|
|
361
|
+
|
|
362
|
+
const allocationMap = Object.fromEntries(alreadyAllocatedWork.map((w) => [w._id, w.count]));
|
|
363
|
+
return allocationMap;
|
|
364
|
+
},
|
|
365
|
+
|
|
366
|
+
workExists: async ({
|
|
367
|
+
workId,
|
|
368
|
+
originalWorkId,
|
|
369
|
+
}: {
|
|
370
|
+
workId?: string;
|
|
371
|
+
originalWorkId?: string;
|
|
372
|
+
}): Promise<boolean> => {
|
|
408
373
|
const queueCount = await WorkQueue.countDocuments(
|
|
409
374
|
workId ? generateDbFilterById(workId) : { originalWorkId },
|
|
410
375
|
{ limit: 1 },
|
|
@@ -412,16 +377,7 @@ export const configureWorkerModule = async ({
|
|
|
412
377
|
return !!queueCount;
|
|
413
378
|
},
|
|
414
379
|
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
type: (work) => {
|
|
418
|
-
if (WorkerDirector.getActivePluginTypes().includes(work.type)) {
|
|
419
|
-
return work.type;
|
|
420
|
-
}
|
|
421
|
-
return 'UNKNOWN';
|
|
422
|
-
},
|
|
423
|
-
|
|
424
|
-
status: (work) => {
|
|
380
|
+
status: (work: Work): WorkStatus => {
|
|
425
381
|
if (work.deleted) {
|
|
426
382
|
return WorkStatus.DELETED;
|
|
427
383
|
}
|
|
@@ -452,11 +408,7 @@ export const configureWorkerModule = async ({
|
|
|
452
408
|
originalWorkId,
|
|
453
409
|
worker = null,
|
|
454
410
|
retries = 20,
|
|
455
|
-
}) => {
|
|
456
|
-
if (!WorkerDirector.getAdapterByType(type)) {
|
|
457
|
-
throw new Error(`No plugin registered for type ${type}`);
|
|
458
|
-
}
|
|
459
|
-
|
|
411
|
+
}: WorkData): Promise<Work> => {
|
|
460
412
|
const created = new Date();
|
|
461
413
|
const { insertedId: workId } = await WorkQueue.insertOne({
|
|
462
414
|
_id: generateDbObjectId(),
|
|
@@ -480,7 +432,7 @@ export const configureWorkerModule = async ({
|
|
|
480
432
|
return work;
|
|
481
433
|
},
|
|
482
434
|
|
|
483
|
-
rescheduleWork: async (currentWork, scheduled) => {
|
|
435
|
+
rescheduleWork: async (currentWork: Work, scheduled: Date): Promise<Work> => {
|
|
484
436
|
const work = await WorkQueue.findOneAndUpdate(
|
|
485
437
|
generateDbFilterById(currentWork._id),
|
|
486
438
|
{
|
|
@@ -502,7 +454,15 @@ export const configureWorkerModule = async ({
|
|
|
502
454
|
|
|
503
455
|
allocateWork,
|
|
504
456
|
|
|
505
|
-
ensureNoWork: async ({
|
|
457
|
+
ensureNoWork: async ({
|
|
458
|
+
type,
|
|
459
|
+
priority = 0,
|
|
460
|
+
scheduleId,
|
|
461
|
+
}: {
|
|
462
|
+
priority: number;
|
|
463
|
+
type: string;
|
|
464
|
+
scheduleId: string;
|
|
465
|
+
}): Promise<void> => {
|
|
506
466
|
const query = buildQuerySelector({
|
|
507
467
|
type,
|
|
508
468
|
status: [WorkStatus.NEW],
|
|
@@ -527,7 +487,7 @@ export const configureWorkerModule = async ({
|
|
|
527
487
|
originalWorkId,
|
|
528
488
|
retries = 20,
|
|
529
489
|
scheduleId,
|
|
530
|
-
}) => {
|
|
490
|
+
}: WorkData): Promise<Work> => {
|
|
531
491
|
const workId = `${scheduleId}:${scheduled.getTime()}`;
|
|
532
492
|
|
|
533
493
|
const created = new Date();
|
|
@@ -588,11 +548,9 @@ export const configureWorkerModule = async ({
|
|
|
588
548
|
}
|
|
589
549
|
},
|
|
590
550
|
|
|
591
|
-
processNextWork,
|
|
592
|
-
|
|
593
551
|
finishWork,
|
|
594
552
|
|
|
595
|
-
deleteWork: async (workId) => {
|
|
553
|
+
deleteWork: async (workId: string): Promise<Work> => {
|
|
596
554
|
const workBeforeRemoval = await WorkQueue.findOne(
|
|
597
555
|
buildQuerySelector({
|
|
598
556
|
workId,
|
|
@@ -616,7 +574,15 @@ export const configureWorkerModule = async ({
|
|
|
616
574
|
return work;
|
|
617
575
|
},
|
|
618
576
|
|
|
619
|
-
markOldWorkAsFailed: async ({
|
|
577
|
+
markOldWorkAsFailed: async ({
|
|
578
|
+
types,
|
|
579
|
+
worker = UNCHAINED_WORKER_ID,
|
|
580
|
+
referenceDate,
|
|
581
|
+
}: {
|
|
582
|
+
types: Array<string>;
|
|
583
|
+
worker: string;
|
|
584
|
+
referenceDate: Date;
|
|
585
|
+
}): Promise<Array<Work>> => {
|
|
620
586
|
const workQueue = await WorkQueue.find(
|
|
621
587
|
buildQuerySelector({
|
|
622
588
|
status: [WorkStatus.ALLOCATED],
|
|
@@ -644,22 +610,28 @@ export const configureWorkerModule = async ({
|
|
|
644
610
|
);
|
|
645
611
|
},
|
|
646
612
|
|
|
647
|
-
getReport: async ({
|
|
613
|
+
getReport: async ({
|
|
614
|
+
types,
|
|
615
|
+
dateRange,
|
|
616
|
+
}: {
|
|
617
|
+
types?: Array<string>;
|
|
618
|
+
dateRange?: DateFilterInput;
|
|
619
|
+
}): Promise<WorkerReport[]> => {
|
|
648
620
|
const pipeline = [];
|
|
649
621
|
const matchConditions = [];
|
|
650
622
|
// build date filter based on provided values it can be a range if both to and from is supplied
|
|
651
623
|
// a upper or lowe limit if either from or to is provided
|
|
652
624
|
// or all if none is provided
|
|
653
|
-
if (
|
|
625
|
+
if (dateRange?.start || dateRange?.end) {
|
|
654
626
|
const dateConditions = [];
|
|
655
|
-
if (
|
|
656
|
-
const fromDate = new Date(
|
|
627
|
+
if (dateRange?.start) {
|
|
628
|
+
const fromDate = new Date(dateRange?.start);
|
|
657
629
|
dateConditions.push({
|
|
658
630
|
$or: [{ created: { $gte: fromDate } }, { updated: { $gte: fromDate } }],
|
|
659
631
|
});
|
|
660
632
|
}
|
|
661
|
-
if (
|
|
662
|
-
const toDate = new Date(
|
|
633
|
+
if (dateRange?.end) {
|
|
634
|
+
const toDate = new Date(dateRange?.end);
|
|
663
635
|
dateConditions.push({
|
|
664
636
|
$or: [{ created: { $lte: toDate } }, { updated: { $lte: toDate } }],
|
|
665
637
|
});
|
|
@@ -724,3 +696,5 @@ export const configureWorkerModule = async ({
|
|
|
724
696
|
},
|
|
725
697
|
};
|
|
726
698
|
};
|
|
699
|
+
|
|
700
|
+
export type WorkerModule = Awaited<ReturnType<typeof configureWorkerModule>>;
|
package/src/worker-index.ts
CHANGED
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './db/WorkQueueCollection.js';
|
|
2
2
|
|
|
3
3
|
export * from './module/configureWorkerModule.js';
|
|
4
|
-
export * from './director/WorkerDirector.js';
|
|
5
|
-
export * from './director/WorkerAdapter.js';
|
|
6
|
-
export { WorkStatus } from './director/WorkStatus.js';
|
|
7
|
-
export { WorkerEventTypes } from './director/WorkerEventTypes.js';
|
|
8
|
-
|
|
9
|
-
export * from './schedulers/FailedRescheduler.js';
|
|
10
|
-
export * from './workers/BaseWorker.js';
|
|
11
|
-
export * from './workers/EventListenerWorker.js';
|
|
12
|
-
export * from './workers/IntervalWorker.js';
|
package/tsconfig.json
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"extends": "../shared/base.tsconfig.json",
|
|
3
3
|
"compilerOptions": {
|
|
4
|
-
"
|
|
5
|
-
"
|
|
4
|
+
"declarationDir": "./lib",
|
|
5
|
+
"rootDir": "./src",
|
|
6
|
+
"outDir": "./lib"
|
|
6
7
|
},
|
|
7
|
-
"exclude": ["**/*.test.ts", "**/*.test.js", "tests"]
|
|
8
|
-
|
|
9
|
-
"./src"
|
|
10
|
-
]
|
|
11
|
-
}
|
|
8
|
+
"exclude": ["**/*.test.ts", "**/*.test.js", "tests", "lib"]
|
|
9
|
+
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"WorkStatus.d.ts","sourceRoot":"","sources":["../../src/director/WorkStatus.ts"],"names":[],"mappings":"AAAA,oBAAY,UAAU;IACpB,GAAG,QAAQ;IACX,SAAS,cAAc;IACvB,OAAO,YAAY;IACnB,MAAM,WAAW;IACjB,OAAO,YAAY;CACpB"}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
export var WorkStatus;
|
|
2
|
-
(function (WorkStatus) {
|
|
3
|
-
WorkStatus["NEW"] = "NEW";
|
|
4
|
-
WorkStatus["ALLOCATED"] = "ALLOCATED";
|
|
5
|
-
WorkStatus["SUCCESS"] = "SUCCESS";
|
|
6
|
-
WorkStatus["FAILED"] = "FAILED";
|
|
7
|
-
WorkStatus["DELETED"] = "DELETED";
|
|
8
|
-
})(WorkStatus || (WorkStatus = {}));
|
|
9
|
-
//# sourceMappingURL=WorkStatus.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"WorkStatus.js","sourceRoot":"","sources":["../../src/director/WorkStatus.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,UAMX;AAND,WAAY,UAAU;IACpB,yBAAW,CAAA;IACX,qCAAuB,CAAA;IACvB,iCAAmB,CAAA;IACnB,+BAAiB,CAAA;IACjB,iCAAmB,CAAA;AACrB,CAAC,EANW,UAAU,KAAV,UAAU,QAMrB"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import type { IBaseAdapter } from '@unchainedshop/utils';
|
|
2
|
-
import type { UnchainedCore } from '@unchainedshop/core';
|
|
3
|
-
export interface WorkResult<Result> {
|
|
4
|
-
success: boolean;
|
|
5
|
-
result?: Result;
|
|
6
|
-
error?: any;
|
|
7
|
-
}
|
|
8
|
-
export type IWorkerAdapter<Input, Output> = IBaseAdapter & {
|
|
9
|
-
type: string;
|
|
10
|
-
external: boolean;
|
|
11
|
-
maxParallelAllocations?: number;
|
|
12
|
-
doWork: (input: Input, unchainedAPI: UnchainedCore, workId: string) => Promise<WorkResult<Output>>;
|
|
13
|
-
};
|
|
14
|
-
export declare const WorkerAdapter: Omit<IWorkerAdapter<any, void>, 'key' | 'label' | 'type' | 'version'>;
|
|
15
|
-
//# sourceMappingURL=WorkerAdapter.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"WorkerAdapter.d.ts","sourceRoot":"","sources":["../../src/director/WorkerAdapter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,MAAM,WAAW,UAAU,CAAC,MAAM;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,GAAG,CAAC;CACb;AAED,MAAM,MAAM,cAAc,CAAC,KAAK,EAAE,MAAM,IAAI,YAAY,GAAG;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,OAAO,CAAC;IAClB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;CACpG,CAAC;AAEF,eAAO,MAAM,aAAa,EAAE,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAU/F,CAAC"}
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { log, LogLevel } from '@unchainedshop/logger';
|
|
2
|
-
export const WorkerAdapter = {
|
|
3
|
-
external: false,
|
|
4
|
-
async doWork() {
|
|
5
|
-
return { success: false, result: null };
|
|
6
|
-
},
|
|
7
|
-
log(message, { level = LogLevel.Debug, ...options } = {}) {
|
|
8
|
-
return log(message, { level, ...options });
|
|
9
|
-
},
|
|
10
|
-
};
|
|
11
|
-
//# sourceMappingURL=WorkerAdapter.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"WorkerAdapter.js","sourceRoot":"","sources":["../../src/director/WorkerAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAiBtD,MAAM,CAAC,MAAM,aAAa,GAA0E;IAClG,QAAQ,EAAE,KAAK;IAEf,KAAK,CAAC,MAAM;QACV,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC1C,CAAC;IAED,GAAG,CAAC,OAAe,EAAE,EAAE,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,GAAG,OAAO,EAAE,GAAG,EAAE;QAC9D,OAAO,GAAG,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC7C,CAAC;CACF,CAAC"}
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import type { WorkData, IWorkerAdapter, WorkResult } from '../worker-index.js';
|
|
2
|
-
import type { UnchainedCore } from '@unchainedshop/core';
|
|
3
|
-
import { IBaseDirector } from '@unchainedshop/utils';
|
|
4
|
-
import { Work } from '../types.js';
|
|
5
|
-
export declare const DIRECTOR_MARKED_FAILED_ERROR = "DIRECTOR_MARKED_FAILED";
|
|
6
|
-
export interface WorkerSchedule {
|
|
7
|
-
schedules: Array<Record<string, any>>;
|
|
8
|
-
exceptions: Array<Record<string, any>>;
|
|
9
|
-
}
|
|
10
|
-
export type WorkScheduleConfiguration = Pick<WorkData, 'timeout' | 'retries' | 'priority' | 'worker' | 'scheduleId'> & {
|
|
11
|
-
type: string;
|
|
12
|
-
input?: (workData: Omit<WorkData, 'input'>) => Promise<Record<string, any> | null>;
|
|
13
|
-
schedule: WorkerSchedule;
|
|
14
|
-
scheduleId?: string;
|
|
15
|
-
};
|
|
16
|
-
export type IWorkerDirector = IBaseDirector<IWorkerAdapter<any, any>> & {
|
|
17
|
-
getActivePluginTypes: (options?: {
|
|
18
|
-
external?: boolean;
|
|
19
|
-
}) => Array<string>;
|
|
20
|
-
getAdapterByType: (type: string) => IWorkerAdapter<any, any>;
|
|
21
|
-
disableAutoscheduling: (scheduleId: string) => void;
|
|
22
|
-
configureAutoscheduling: (workScheduleConfiguration: WorkScheduleConfiguration) => void;
|
|
23
|
-
getAutoSchedules: () => Array<[string, WorkScheduleConfiguration]>;
|
|
24
|
-
doWork: (work: Work, unchainedAPI: UnchainedCore) => Promise<WorkResult<any>>;
|
|
25
|
-
};
|
|
26
|
-
export declare const WorkerDirector: IWorkerDirector;
|
|
27
|
-
//# sourceMappingURL=WorkerDirector.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"WorkerDirector.d.ts","sourceRoot":"","sources":["../../src/director/WorkerDirector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD,OAAO,EAAgB,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,eAAO,MAAM,4BAA4B,2BAA2B,CAAC;AAErE,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IACtC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;CACxC;AAED,MAAM,MAAM,yBAAyB,GAAG,IAAI,CAC1C,QAAQ,EACR,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,QAAQ,GAAG,YAAY,CAC7D,GAAG;IACF,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACnF,QAAQ,EAAE,cAAc,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,aAAa,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG;IACtE,oBAAoB,EAAE,CAAC,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1E,gBAAgB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC7D,qBAAqB,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,IAAI,CAAC;IACpD,uBAAuB,EAAE,CAAC,yBAAyB,EAAE,yBAAyB,KAAK,IAAI,CAAC;IACxF,gBAAgB,EAAE,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC,CAAC;IACnE,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,aAAa,KAAK,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;CAC/E,CAAC;AAQF,eAAO,MAAM,cAAc,EAAE,eAgF5B,CAAC"}
|