@tstdl/base 0.93.77 → 0.93.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/authentication/client/http-client.middleware.js +2 -2
- package/authentication/models/authentication-credentials.model.d.ts +2 -2
- package/authentication/models/authentication-credentials.model.js +5 -3
- package/authentication/models/authentication-session.model.d.ts +2 -2
- package/authentication/models/authentication-session.model.js +5 -3
- package/authentication/models/index.d.ts +4 -0
- package/authentication/models/index.js +4 -0
- package/authentication/models/service-account.model.d.ts +7 -0
- package/authentication/models/service-account.model.js +31 -0
- package/authentication/models/subject.model.d.ts +16 -0
- package/authentication/models/subject.model.js +59 -0
- package/authentication/models/system-account.model.d.ts +5 -0
- package/authentication/models/system-account.model.js +25 -0
- package/authentication/models/user.model.d.ts +15 -0
- package/authentication/models/user.model.js +47 -0
- package/authentication/server/drizzle/0001_condemned_pretty_boy.sql +70 -0
- package/authentication/server/drizzle/meta/0001_snapshot.json +651 -0
- package/authentication/server/drizzle/meta/_journal.json +7 -0
- package/authentication/server/index.d.ts +1 -0
- package/authentication/server/index.js +1 -0
- package/authentication/server/schemas.d.ts +16 -1
- package/authentication/server/schemas.js +7 -1
- package/authentication/server/subject.service.d.ts +6 -0
- package/authentication/server/subject.service.js +44 -0
- package/circuit-breaker/circuit-breaker.d.ts +32 -0
- package/circuit-breaker/circuit-breaker.js +9 -0
- package/circuit-breaker/index.d.ts +2 -0
- package/circuit-breaker/index.js +2 -0
- package/circuit-breaker/postgres/circuit-breaker.d.ts +7 -0
- package/circuit-breaker/postgres/circuit-breaker.js +78 -0
- package/circuit-breaker/postgres/drizzle/0000_hard_shocker.sql +9 -0
- package/circuit-breaker/postgres/drizzle/meta/0000_snapshot.json +82 -0
- package/circuit-breaker/postgres/drizzle/meta/_journal.json +13 -0
- package/circuit-breaker/postgres/drizzle.config.d.ts +2 -0
- package/circuit-breaker/postgres/drizzle.config.js +11 -0
- package/circuit-breaker/postgres/index.d.ts +5 -0
- package/circuit-breaker/postgres/index.js +5 -0
- package/circuit-breaker/postgres/model.d.ts +9 -0
- package/circuit-breaker/postgres/model.js +40 -0
- package/circuit-breaker/postgres/module.d.ts +6 -0
- package/circuit-breaker/postgres/module.js +25 -0
- package/circuit-breaker/postgres/provider.d.ts +6 -0
- package/circuit-breaker/postgres/provider.js +21 -0
- package/circuit-breaker/postgres/schemas.d.ts +8 -0
- package/circuit-breaker/postgres/schemas.js +6 -0
- package/circuit-breaker/provider.d.ts +4 -0
- package/circuit-breaker/provider.js +2 -0
- package/circuit-breaker/tests/circuit-breaker.test.js +113 -0
- package/document-management/models/document.model.d.ts +0 -1
- package/document-management/models/document.model.js +0 -5
- package/document-management/server/api/document-management.api.js +1 -2
- package/document-management/server/drizzle/0002_round_warbird.sql +1 -0
- package/document-management/server/drizzle/meta/0002_snapshot.json +2722 -0
- package/document-management/server/drizzle/meta/_journal.json +7 -0
- package/document-management/server/services/document-collection.service.js +3 -3
- package/document-management/server/services/document-management-ancillary.service.d.ts +1 -1
- package/document-management/server/services/document-management.service.js +1 -1
- package/document-management/server/services/document-workflow.service.js +5 -5
- package/document-management/server/services/document.service.d.ts +0 -2
- package/document-management/server/services/document.service.js +1 -2
- package/document-management/service-models/enriched/enriched-document.view.d.ts +1 -1
- package/examples/document-management/main.d.ts +1 -1
- package/examples/document-management/main.js +1 -1
- package/logger/transports/console.d.ts +1 -1
- package/logger/transports/console.js +4 -1
- package/message-bus/message-bus-base.js +1 -1
- package/package.json +6 -3
- package/queue/enqueue-batch.d.ts +11 -11
- package/queue/enqueue-batch.js +2 -3
- package/queue/index.d.ts +1 -0
- package/queue/index.js +1 -0
- package/queue/postgres/drizzle/0003_tricky_venom.sql +30 -0
- package/queue/postgres/drizzle/meta/0003_snapshot.json +288 -0
- package/queue/postgres/drizzle/meta/_journal.json +7 -0
- package/queue/postgres/drizzle.config.js +2 -2
- package/queue/postgres/index.d.ts +1 -1
- package/queue/postgres/index.js +1 -1
- package/queue/postgres/module.d.ts +1 -1
- package/queue/postgres/module.js +1 -1
- package/queue/postgres/queue.d.ts +52 -23
- package/queue/postgres/queue.js +582 -64
- package/queue/postgres/queue.provider.d.ts +1 -1
- package/queue/postgres/schemas.d.ts +13 -2
- package/queue/postgres/schemas.js +4 -2
- package/queue/postgres/task.model.d.ts +24 -0
- package/queue/postgres/task.model.js +115 -0
- package/queue/provider.d.ts +1 -1
- package/queue/queue.d.ts +158 -37
- package/queue/queue.js +97 -19
- package/queue/task-context.d.ts +38 -0
- package/queue/task-context.js +102 -0
- package/queue/tests/queue.test.d.ts +1 -0
- package/queue/tests/queue.test.js +623 -0
- package/test4.d.ts +1 -1
- package/test4.js +1 -1
- package/utils/format-error.d.ts +17 -20
- package/utils/format-error.js +105 -47
- package/queue/postgres/job.model.d.ts +0 -12
- package/queue/postgres/job.model.js +0 -53
- package/test6.js +0 -33
- /package/{test6.d.ts → circuit-breaker/tests/circuit-breaker.test.d.ts} +0 -0
|
@@ -3,5 +3,5 @@ import type { ObjectLiteral } from '../../types/index.js';
|
|
|
3
3
|
import { PostgresQueue } from './queue.js';
|
|
4
4
|
export declare class PostgresQueueProvider extends QueueProvider {
|
|
5
5
|
#private;
|
|
6
|
-
get<
|
|
6
|
+
get<Data extends ObjectLiteral, State extends ObjectLiteral, Result extends ObjectLiteral>(name: string, config?: QueueConfig): PostgresQueue<Data, State, Result>;
|
|
7
7
|
}
|
|
@@ -1,3 +1,14 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { PostgresTask } from './task.model.js';
|
|
2
2
|
export declare const queueSchema: import("../../orm/server/index.js").DatabaseSchema<"queue">;
|
|
3
|
-
export declare const
|
|
3
|
+
export declare const taskState: import("../../orm/enums.js").PgEnumFromEnumeration<{
|
|
4
|
+
readonly Pending: "pending";
|
|
5
|
+
readonly Running: "running";
|
|
6
|
+
readonly Completed: "completed";
|
|
7
|
+
readonly Cancelled: "cancelled";
|
|
8
|
+
readonly Waiting: "waiting";
|
|
9
|
+
readonly Dead: "dead";
|
|
10
|
+
}>;
|
|
11
|
+
export declare const task: import("../../orm/server/types.js").PgTableFromType<{
|
|
12
|
+
new (): PostgresTask<any, any, any>;
|
|
13
|
+
readonly entityName: "Task";
|
|
14
|
+
}, "queue">;
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { databaseSchema } from '../../orm/server/index.js';
|
|
2
|
-
import {
|
|
2
|
+
import { TaskState } from '../queue.js';
|
|
3
|
+
import { PostgresTask } from './task.model.js';
|
|
3
4
|
export const queueSchema = databaseSchema('queue');
|
|
4
|
-
export const
|
|
5
|
+
export const taskState = queueSchema.getEnum(TaskState);
|
|
6
|
+
export const task = queueSchema.getTable((PostgresTask));
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { BaseEntity, type Json, type Timestamp } from '../../orm/index.js';
|
|
2
|
+
import type { ObjectLiteral } from '../../types/types.js';
|
|
3
|
+
import { type Task, TaskState } from '../queue.js';
|
|
4
|
+
export declare class PostgresTask<Data extends ObjectLiteral = ObjectLiteral, State extends ObjectLiteral = ObjectLiteral, Result extends ObjectLiteral = ObjectLiteral> extends BaseEntity implements Task<Data, State, Result> {
|
|
5
|
+
static readonly entityName = "Task";
|
|
6
|
+
queue: string;
|
|
7
|
+
status: TaskState;
|
|
8
|
+
tag: string | null;
|
|
9
|
+
priority: number;
|
|
10
|
+
parentId: string | null;
|
|
11
|
+
lease: string | null;
|
|
12
|
+
enqueueTimestamp: Timestamp;
|
|
13
|
+
scheduleTimestamp: Timestamp;
|
|
14
|
+
startTimestamp: Timestamp | null;
|
|
15
|
+
expirationTimestamp: Timestamp | null;
|
|
16
|
+
lockExpirationTimestamp: Timestamp | null;
|
|
17
|
+
completeTimestamp: Timestamp | null;
|
|
18
|
+
tries: number;
|
|
19
|
+
progress: number;
|
|
20
|
+
data: Json<Data>;
|
|
21
|
+
state: Json<State> | null;
|
|
22
|
+
result: Json<Result> | null;
|
|
23
|
+
error: Json<ObjectLiteral> | null;
|
|
24
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
|
+
};
|
|
10
|
+
import { BaseEntity, Index, JsonProperty, Reference, Table, TimestampProperty, Unique, UuidProperty } from '../../orm/index.js';
|
|
11
|
+
import { Enumeration, Integer, NumberProperty, StringProperty } from '../../schema/index.js';
|
|
12
|
+
import { TaskState } from '../queue.js';
|
|
13
|
+
let PostgresTask = class PostgresTask extends BaseEntity {
|
|
14
|
+
static entityName = 'Task';
|
|
15
|
+
queue;
|
|
16
|
+
status;
|
|
17
|
+
tag;
|
|
18
|
+
priority;
|
|
19
|
+
parentId;
|
|
20
|
+
lease;
|
|
21
|
+
enqueueTimestamp;
|
|
22
|
+
scheduleTimestamp;
|
|
23
|
+
startTimestamp;
|
|
24
|
+
expirationTimestamp;
|
|
25
|
+
lockExpirationTimestamp;
|
|
26
|
+
completeTimestamp;
|
|
27
|
+
tries;
|
|
28
|
+
progress;
|
|
29
|
+
data;
|
|
30
|
+
state;
|
|
31
|
+
result;
|
|
32
|
+
error;
|
|
33
|
+
};
|
|
34
|
+
__decorate([
|
|
35
|
+
StringProperty(),
|
|
36
|
+
__metadata("design:type", String)
|
|
37
|
+
], PostgresTask.prototype, "queue", void 0);
|
|
38
|
+
__decorate([
|
|
39
|
+
Enumeration(TaskState),
|
|
40
|
+
__metadata("design:type", String)
|
|
41
|
+
], PostgresTask.prototype, "status", void 0);
|
|
42
|
+
__decorate([
|
|
43
|
+
StringProperty({ nullable: true }),
|
|
44
|
+
__metadata("design:type", Object)
|
|
45
|
+
], PostgresTask.prototype, "tag", void 0);
|
|
46
|
+
__decorate([
|
|
47
|
+
Integer(),
|
|
48
|
+
__metadata("design:type", Number)
|
|
49
|
+
], PostgresTask.prototype, "priority", void 0);
|
|
50
|
+
__decorate([
|
|
51
|
+
Reference(() => PostgresTask),
|
|
52
|
+
UuidProperty({ nullable: true }),
|
|
53
|
+
__metadata("design:type", Object)
|
|
54
|
+
], PostgresTask.prototype, "parentId", void 0);
|
|
55
|
+
__decorate([
|
|
56
|
+
UuidProperty({ nullable: true }),
|
|
57
|
+
__metadata("design:type", Object)
|
|
58
|
+
], PostgresTask.prototype, "lease", void 0);
|
|
59
|
+
__decorate([
|
|
60
|
+
TimestampProperty(),
|
|
61
|
+
__metadata("design:type", Number)
|
|
62
|
+
], PostgresTask.prototype, "enqueueTimestamp", void 0);
|
|
63
|
+
__decorate([
|
|
64
|
+
TimestampProperty(),
|
|
65
|
+
__metadata("design:type", Number)
|
|
66
|
+
], PostgresTask.prototype, "scheduleTimestamp", void 0);
|
|
67
|
+
__decorate([
|
|
68
|
+
TimestampProperty({ nullable: true }),
|
|
69
|
+
__metadata("design:type", Object)
|
|
70
|
+
], PostgresTask.prototype, "startTimestamp", void 0);
|
|
71
|
+
__decorate([
|
|
72
|
+
TimestampProperty({ nullable: true }),
|
|
73
|
+
__metadata("design:type", Object)
|
|
74
|
+
], PostgresTask.prototype, "expirationTimestamp", void 0);
|
|
75
|
+
__decorate([
|
|
76
|
+
TimestampProperty({ nullable: true }),
|
|
77
|
+
__metadata("design:type", Object)
|
|
78
|
+
], PostgresTask.prototype, "lockExpirationTimestamp", void 0);
|
|
79
|
+
__decorate([
|
|
80
|
+
TimestampProperty({ nullable: true }),
|
|
81
|
+
__metadata("design:type", Object)
|
|
82
|
+
], PostgresTask.prototype, "completeTimestamp", void 0);
|
|
83
|
+
__decorate([
|
|
84
|
+
Integer(),
|
|
85
|
+
__metadata("design:type", Number)
|
|
86
|
+
], PostgresTask.prototype, "tries", void 0);
|
|
87
|
+
__decorate([
|
|
88
|
+
NumberProperty(),
|
|
89
|
+
__metadata("design:type", Number)
|
|
90
|
+
], PostgresTask.prototype, "progress", void 0);
|
|
91
|
+
__decorate([
|
|
92
|
+
JsonProperty({ nullable: true }),
|
|
93
|
+
__metadata("design:type", Object)
|
|
94
|
+
], PostgresTask.prototype, "data", void 0);
|
|
95
|
+
__decorate([
|
|
96
|
+
JsonProperty({ nullable: true }),
|
|
97
|
+
__metadata("design:type", Object)
|
|
98
|
+
], PostgresTask.prototype, "state", void 0);
|
|
99
|
+
__decorate([
|
|
100
|
+
JsonProperty({ nullable: true }),
|
|
101
|
+
__metadata("design:type", Object)
|
|
102
|
+
], PostgresTask.prototype, "result", void 0);
|
|
103
|
+
__decorate([
|
|
104
|
+
JsonProperty({ nullable: true }),
|
|
105
|
+
__metadata("design:type", Object)
|
|
106
|
+
], PostgresTask.prototype, "error", void 0);
|
|
107
|
+
PostgresTask = __decorate([
|
|
108
|
+
Table('task', { schema: 'queue' }),
|
|
109
|
+
Unique(['queue', 'tag']),
|
|
110
|
+
Index(['queue', 'status', 'priority', 'scheduleTimestamp']),
|
|
111
|
+
Index(['queue', 'status', 'lockExpirationTimestamp']),
|
|
112
|
+
Index(['queue', 'completeTimestamp']),
|
|
113
|
+
Index(['parentId', 'status'])
|
|
114
|
+
], PostgresTask);
|
|
115
|
+
export { PostgresTask };
|
package/queue/provider.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ObjectLiteral } from '../types/index.js';
|
|
2
2
|
import type { Queue, QueueConfig } from './queue.js';
|
|
3
3
|
export declare abstract class QueueProvider {
|
|
4
|
-
abstract get<
|
|
4
|
+
abstract get<Data extends ObjectLiteral, State extends ObjectLiteral, Result extends ObjectLiteral>(key: string, config?: QueueConfig): Queue<Data, State, Result>;
|
|
5
5
|
}
|
package/queue/queue.d.ts
CHANGED
|
@@ -2,90 +2,211 @@ import type { CancellationSignal } from '../cancellation/token.js';
|
|
|
2
2
|
import { type EnumType } from '../enumeration/enumeration.js';
|
|
3
3
|
import type { Resolvable, resolveArgumentType } from '../injector/interfaces.js';
|
|
4
4
|
import type { Logger } from '../logger/logger.js';
|
|
5
|
+
import type { Transaction } from '../orm/server/transaction.js';
|
|
6
|
+
import type { UndefinableJson } from '../types/types.js';
|
|
5
7
|
import { QueueEnqueueBatch } from './enqueue-batch.js';
|
|
6
|
-
|
|
7
|
-
export
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
import { BatchTaskContext, type TaskContext } from './task-context.js';
|
|
9
|
+
export interface ProcessWorker<Data, State, Result> {
|
|
10
|
+
/**
|
|
11
|
+
* A worker function that processes a single task.
|
|
12
|
+
* @param context The task context providing data, logger, and orchestration helpers.
|
|
13
|
+
*/
|
|
14
|
+
(context: TaskContext<Data, State, Result>): void | Result | Promise<void | Result>;
|
|
15
|
+
}
|
|
16
|
+
export interface ProcessBatchWorker<Data, State, Result> {
|
|
17
|
+
/**
|
|
18
|
+
* A worker function that processes a batch of tasks.
|
|
19
|
+
* @param context The batch context providing tasks and helpers.
|
|
20
|
+
*/
|
|
21
|
+
(context: BatchTaskContext<Data, State, Result>): void | Result[] | Promise<void | Result[]>;
|
|
22
|
+
}
|
|
23
|
+
export type TaskTag = string | null;
|
|
24
|
+
export declare const TaskState: {
|
|
25
|
+
/**
|
|
26
|
+
* The task is waiting to be processed.
|
|
27
|
+
*/
|
|
28
|
+
readonly Pending: "pending";
|
|
29
|
+
/**
|
|
30
|
+
* The task is currently being processed.
|
|
31
|
+
*/
|
|
32
|
+
readonly Running: "running";
|
|
33
|
+
/**
|
|
34
|
+
* The task has been completed successfully.
|
|
35
|
+
*/
|
|
36
|
+
readonly Completed: "completed";
|
|
37
|
+
/**
|
|
38
|
+
* The task has been cancelled and will not be processed.
|
|
39
|
+
*/
|
|
40
|
+
readonly Cancelled: "cancelled";
|
|
41
|
+
/**
|
|
42
|
+
* The task is scheduled to be processed in the future when all children have completed.
|
|
43
|
+
*/
|
|
44
|
+
readonly Waiting: "waiting";
|
|
45
|
+
/**
|
|
46
|
+
* The task has failed and will not be retried.
|
|
47
|
+
*/
|
|
48
|
+
readonly Dead: "dead";
|
|
49
|
+
};
|
|
50
|
+
export type TaskState = EnumType<typeof TaskState>;
|
|
51
|
+
export type Task<Data = unknown, State = unknown, Result = unknown> = {
|
|
10
52
|
id: string;
|
|
53
|
+
queue: string;
|
|
54
|
+
status: TaskState;
|
|
55
|
+
lease: string | null;
|
|
11
56
|
/**
|
|
12
57
|
* The lower the number, the higher the priority.
|
|
13
58
|
* @default 1000
|
|
14
59
|
*/
|
|
15
60
|
priority: number;
|
|
16
|
-
tag:
|
|
17
|
-
data:
|
|
18
|
-
|
|
19
|
-
lastDequeueTimestamp: number | null;
|
|
61
|
+
tag: TaskTag;
|
|
62
|
+
data: Data;
|
|
63
|
+
parentId: string | null;
|
|
20
64
|
tries: number;
|
|
65
|
+
enqueueTimestamp: number;
|
|
66
|
+
scheduleTimestamp: number;
|
|
67
|
+
/**
|
|
68
|
+
* Timestamp when the task most recently switched to Running state.
|
|
69
|
+
* Used for Hard Execution Timeouts.
|
|
70
|
+
*/
|
|
71
|
+
startTimestamp: number | null;
|
|
72
|
+
/**
|
|
73
|
+
* Timestamp after which the task is considered expired if it hasn't started Running.
|
|
74
|
+
* If null, the task never expires in the queue.
|
|
75
|
+
*/
|
|
76
|
+
expirationTimestamp: number | null;
|
|
77
|
+
lockExpirationTimestamp: number | null;
|
|
78
|
+
completeTimestamp: number | null;
|
|
79
|
+
/** A number between 0 and 1 indicating the progress of the task. */
|
|
80
|
+
progress: number;
|
|
81
|
+
/** A snapshot of the current state of the task. */
|
|
82
|
+
state: State | null;
|
|
83
|
+
result: Result | null;
|
|
84
|
+
error: UndefinableJson | null;
|
|
21
85
|
};
|
|
22
|
-
export declare const
|
|
86
|
+
export declare const defaultTaskPriority = 1000;
|
|
23
87
|
export declare const UniqueTagStrategy: {
|
|
24
88
|
readonly KeepOld: 0;
|
|
25
89
|
readonly TakeNew: 1;
|
|
26
90
|
};
|
|
27
91
|
export type UniqueTagStrategy = EnumType<typeof UniqueTagStrategy>;
|
|
28
92
|
export type EnqueueOptions = {
|
|
29
|
-
tag?:
|
|
93
|
+
tag?: TaskTag;
|
|
30
94
|
priority?: number;
|
|
95
|
+
parentId?: string;
|
|
96
|
+
scheduleTimestamp?: number;
|
|
97
|
+
expirationTimestamp?: number;
|
|
98
|
+
transaction?: Transaction;
|
|
31
99
|
};
|
|
32
|
-
export type EnqueueOneOptions = {
|
|
33
|
-
tag?: JobTag;
|
|
100
|
+
export type EnqueueOneOptions = EnqueueOptions & {
|
|
34
101
|
uniqueTag?: UniqueTagStrategy;
|
|
35
|
-
priority?: number;
|
|
36
102
|
};
|
|
37
103
|
export type EnqueueManyItem<T> = EnqueueOptions & {
|
|
38
104
|
data: T;
|
|
39
105
|
};
|
|
40
106
|
export type EnqueueManyOptions = {
|
|
41
107
|
uniqueTag?: UniqueTagStrategy;
|
|
42
|
-
|
|
108
|
+
returnTasks?: boolean;
|
|
109
|
+
transaction?: Transaction;
|
|
43
110
|
};
|
|
44
111
|
export type QueueConfig = {
|
|
112
|
+
/**
|
|
113
|
+
* Duration in ms for the worker lease. The worker must heartbeat within this window.
|
|
114
|
+
*/
|
|
45
115
|
processTimeout?: number;
|
|
116
|
+
/**
|
|
117
|
+
* Maximum duration a task is allowed to run (in ms) before being forcefully killed.
|
|
118
|
+
* This protects against infinite loops even if the worker keeps heartbeating.
|
|
119
|
+
* **WARN:** CancellationSignal must still be observed by the worker!
|
|
120
|
+
* Default: 15 minutes
|
|
121
|
+
*/
|
|
122
|
+
executionTimeout?: number;
|
|
123
|
+
/** Maximum number of attempts to process a task before marking it as failed. Default 3 */
|
|
46
124
|
maxTries?: number;
|
|
125
|
+
/** Milliseconds to keep completed tasks */
|
|
126
|
+
retentionPeriod?: number;
|
|
127
|
+
/** Maximum number of running tasks across all workers (`null` = unlimited) */
|
|
128
|
+
globalConcurrency?: number | null;
|
|
129
|
+
/** Number of consecutive failures before tripping the circuit breaker */
|
|
130
|
+
circuitBreakerThreshold?: number;
|
|
131
|
+
/** Milliseconds to wait before trying to close the circuit breaker */
|
|
132
|
+
circuitBreakerResetTimeout?: number;
|
|
133
|
+
/** Minimum delay for retries (the floor). Default: 5 seconds */
|
|
134
|
+
retryDelayMinimum?: number;
|
|
135
|
+
/** Maximum delay for retries (the ceiling). Default: 5 minutes */
|
|
136
|
+
retryDelayMaximum?: number;
|
|
137
|
+
/** Base for exponential backoff (delay = min * (base ^ tries)). Default: 2 */
|
|
138
|
+
retryDelayGrowth?: number;
|
|
47
139
|
};
|
|
48
140
|
export type QueueArgument = string | (QueueConfig & {
|
|
49
141
|
name: string;
|
|
50
142
|
});
|
|
51
143
|
export declare const defaultQueueConfig: Required<QueueConfig>;
|
|
52
|
-
export declare abstract class Queue<
|
|
144
|
+
export declare abstract class Queue<Data, State = unknown, Result = unknown> implements Resolvable<QueueArgument> {
|
|
53
145
|
readonly [resolveArgumentType]: QueueArgument;
|
|
54
146
|
abstract readonly processTimeout: number;
|
|
55
147
|
abstract readonly maxTries: number;
|
|
56
|
-
batch(): QueueEnqueueBatch<
|
|
57
|
-
abstract enqueue(data:
|
|
58
|
-
abstract enqueueMany(items: EnqueueManyItem<
|
|
59
|
-
|
|
148
|
+
batch(): QueueEnqueueBatch<Data, State, Result>;
|
|
149
|
+
abstract enqueue(data: Data, options?: EnqueueOneOptions): Promise<Task<Data, State, Result>>;
|
|
150
|
+
abstract enqueueMany(items: EnqueueManyItem<Data>[], options?: EnqueueManyOptions & {
|
|
151
|
+
returnTasks?: false;
|
|
60
152
|
}): Promise<void>;
|
|
61
|
-
abstract enqueueMany(items: EnqueueManyItem<
|
|
62
|
-
|
|
63
|
-
}): Promise<
|
|
64
|
-
abstract enqueueMany(items: EnqueueManyItem<
|
|
153
|
+
abstract enqueueMany(items: EnqueueManyItem<Data>[], options: EnqueueManyOptions & {
|
|
154
|
+
returnTasks: true;
|
|
155
|
+
}): Promise<Task<Data, State, Result>[]>;
|
|
156
|
+
abstract enqueueMany(items: EnqueueManyItem<Data>[], options?: EnqueueManyOptions): Promise<Task<Data, State, Result>[] | undefined>;
|
|
65
157
|
abstract has(id: string): Promise<boolean>;
|
|
66
|
-
abstract
|
|
67
|
-
abstract
|
|
68
|
-
abstract
|
|
69
|
-
abstract
|
|
158
|
+
abstract get(id: string): Promise<Task<Data, State, Result> | undefined>;
|
|
159
|
+
abstract getByTag(tag: TaskTag): Promise<Task<Data, State, Result>[]>;
|
|
160
|
+
abstract getByTags(tags: TaskTag[]): Promise<Task<Data, State, Result>[]>;
|
|
161
|
+
abstract countByTag(tag: TaskTag): Promise<number>;
|
|
162
|
+
abstract getTree(rootId: string | string[]): Promise<Task<unknown, unknown, unknown>[]>;
|
|
70
163
|
abstract cancel(id: string): Promise<void>;
|
|
71
164
|
abstract cancelMany(ids: string[]): Promise<void>;
|
|
72
|
-
abstract cancelByTag(tag:
|
|
73
|
-
abstract
|
|
74
|
-
|
|
75
|
-
abstract
|
|
76
|
-
abstract
|
|
77
|
-
abstract
|
|
78
|
-
|
|
79
|
-
|
|
165
|
+
abstract cancelByTag(tag: TaskTag): Promise<void>;
|
|
166
|
+
abstract cancelManyByTag(tags: TaskTag[]): Promise<void>;
|
|
167
|
+
/** Clears all tasks from the queue. Use with caution! */
|
|
168
|
+
abstract clear(): Promise<void>;
|
|
169
|
+
abstract dequeue(): Promise<Task<Data, State, Result> | undefined>;
|
|
170
|
+
abstract dequeueMany(count: number): Promise<Task<Data, State, Result>[]>;
|
|
171
|
+
/**
|
|
172
|
+
* Reschedules a task to run at a specific time.
|
|
173
|
+
* NOTE: If the task is currently running, its retry count is decremented (refunded) so this attempt doesn't count against maxTries.
|
|
174
|
+
*/
|
|
175
|
+
abstract reschedule(id: string, timestamp: number, transaction?: Transaction): Promise<void>;
|
|
176
|
+
abstract rescheduleMany(ids: string[], timestamp: number, transaction?: Transaction): Promise<void>;
|
|
177
|
+
abstract rescheduleByTag(tag: TaskTag, timestamp: number, transaction?: Transaction): Promise<void>;
|
|
178
|
+
abstract rescheduleManyByTag(tags: TaskTag[], timestamp: number, transaction?: Transaction): Promise<void>;
|
|
179
|
+
/**
|
|
180
|
+
* Updates task progress, state and lock.
|
|
181
|
+
* Returns the updated task if successful, `undefined` if task is lost/cancelled/timed out.
|
|
182
|
+
*/
|
|
183
|
+
abstract touch(task: Task, options?: {
|
|
184
|
+
progress?: number;
|
|
185
|
+
state?: State;
|
|
186
|
+
transaction?: Transaction;
|
|
187
|
+
}): Promise<Task<Data, State, Result> | undefined>;
|
|
188
|
+
/**
|
|
189
|
+
* Updates multiple tasks' progress, state and lock.
|
|
190
|
+
* Returns the IDs of the successfully updated tasks.
|
|
191
|
+
*/
|
|
192
|
+
abstract touchMany(tasks: Task[], progresses?: number[], states?: State[], transaction?: Transaction): Promise<string[]>;
|
|
193
|
+
abstract acknowledge<R>(task: Task<any, any, R>, result?: R, transaction?: Transaction): Promise<void>;
|
|
194
|
+
abstract acknowledgeMany<R>(tasks: Task<any, any, R>[], results?: R[], transaction?: Transaction): Promise<void>;
|
|
195
|
+
abstract fail(task: Task, error: any, fatal?: boolean, transaction?: Transaction): Promise<void>;
|
|
196
|
+
abstract failMany(tasks: Task[], error: any, transaction?: Transaction): Promise<void>;
|
|
197
|
+
abstract prune(): Promise<void>;
|
|
198
|
+
abstract restart(id: string, transaction?: Transaction): Promise<void>;
|
|
199
|
+
abstract getConsumer(cancellationSignal: CancellationSignal): AsyncIterableIterator<Task<Data, State, Result>>;
|
|
200
|
+
abstract getBatchConsumer(size: number, cancellationSignal: CancellationSignal): AsyncIterableIterator<Task<Data, State, Result>[]>;
|
|
80
201
|
process({ concurrency, cancellationSignal }: {
|
|
81
202
|
concurrency?: number;
|
|
82
203
|
cancellationSignal: CancellationSignal;
|
|
83
|
-
}, handler: ProcessWorker<
|
|
204
|
+
}, handler: ProcessWorker<Data, State, Result>, logger: Logger): void;
|
|
84
205
|
processBatch({ batchSize, concurrency, cancellationSignal }: {
|
|
85
206
|
batchSize?: number;
|
|
86
207
|
concurrency?: number;
|
|
87
208
|
cancellationSignal: CancellationSignal;
|
|
88
|
-
}, handler: ProcessBatchWorker<
|
|
209
|
+
}, handler: ProcessBatchWorker<Data, State, Result>, logger: Logger): void;
|
|
89
210
|
private processWorker;
|
|
90
211
|
private processBatchWorker;
|
|
91
212
|
}
|
package/queue/queue.js
CHANGED
|
@@ -1,14 +1,52 @@
|
|
|
1
1
|
import { defineEnum } from '../enumeration/enumeration.js';
|
|
2
|
-
import {
|
|
2
|
+
import { createArray } from '../utils/array/array.js';
|
|
3
|
+
import { cancelableTimeout } from '../utils/timing.js';
|
|
4
|
+
import { isDefined } from '../utils/type-guards.js';
|
|
5
|
+
import { millisecondsPerDay, millisecondsPerMinute, millisecondsPerSecond } from '../utils/units.js';
|
|
3
6
|
import { QueueEnqueueBatch } from './enqueue-batch.js';
|
|
4
|
-
|
|
7
|
+
import { BatchTaskContext } from './task-context.js';
|
|
8
|
+
export const TaskState = defineEnum('TaskState', {
|
|
9
|
+
/**
|
|
10
|
+
* The task is waiting to be processed.
|
|
11
|
+
*/
|
|
12
|
+
Pending: 'pending',
|
|
13
|
+
/**
|
|
14
|
+
* The task is currently being processed.
|
|
15
|
+
*/
|
|
16
|
+
Running: 'running',
|
|
17
|
+
/**
|
|
18
|
+
* The task has been completed successfully.
|
|
19
|
+
*/
|
|
20
|
+
Completed: 'completed',
|
|
21
|
+
/**
|
|
22
|
+
* The task has been cancelled and will not be processed.
|
|
23
|
+
*/
|
|
24
|
+
Cancelled: 'cancelled',
|
|
25
|
+
/**
|
|
26
|
+
* The task is scheduled to be processed in the future when all children have completed.
|
|
27
|
+
*/
|
|
28
|
+
Waiting: 'waiting',
|
|
29
|
+
/**
|
|
30
|
+
* The task has failed and will not be retried.
|
|
31
|
+
*/
|
|
32
|
+
Dead: 'dead',
|
|
33
|
+
});
|
|
34
|
+
export const defaultTaskPriority = 1000;
|
|
5
35
|
export const UniqueTagStrategy = defineEnum('UniqueTagStrategy', {
|
|
6
36
|
KeepOld: 0,
|
|
7
37
|
TakeNew: 1,
|
|
8
38
|
});
|
|
9
39
|
export const defaultQueueConfig = {
|
|
10
|
-
processTimeout: millisecondsPerMinute,
|
|
40
|
+
processTimeout: millisecondsPerMinute * 5,
|
|
41
|
+
executionTimeout: millisecondsPerMinute * 60,
|
|
11
42
|
maxTries: 3,
|
|
43
|
+
retentionPeriod: 30 * millisecondsPerDay,
|
|
44
|
+
globalConcurrency: null,
|
|
45
|
+
circuitBreakerThreshold: 5,
|
|
46
|
+
circuitBreakerResetTimeout: 30 * millisecondsPerSecond,
|
|
47
|
+
retryDelayMinimum: 5 * millisecondsPerSecond,
|
|
48
|
+
retryDelayMaximum: 5 * millisecondsPerMinute,
|
|
49
|
+
retryDelayGrowth: 2,
|
|
12
50
|
};
|
|
13
51
|
export class Queue {
|
|
14
52
|
batch() {
|
|
@@ -25,28 +63,68 @@ export class Queue {
|
|
|
25
63
|
}
|
|
26
64
|
}
|
|
27
65
|
async processWorker(cancellationSignal, handler, logger) {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
}
|
|
35
|
-
catch (error) {
|
|
36
|
-
logger?.error(error);
|
|
66
|
+
await this.processBatchWorker(1, cancellationSignal, async (batchContext) => {
|
|
67
|
+
const task = batchContext.tasks[0];
|
|
68
|
+
const context = batchContext.for(task);
|
|
69
|
+
const result = await handler(context);
|
|
70
|
+
if (isDefined(result)) {
|
|
71
|
+
return [result];
|
|
37
72
|
}
|
|
38
|
-
|
|
73
|
+
return undefined;
|
|
74
|
+
}, logger);
|
|
39
75
|
}
|
|
40
76
|
async processBatchWorker(size, cancellationSignal, handler, logger) {
|
|
41
|
-
for await (const
|
|
42
|
-
|
|
77
|
+
for await (const tasks of this.getBatchConsumer(size, cancellationSignal)) {
|
|
78
|
+
const batchToken = cancellationSignal.createChild();
|
|
79
|
+
const context = new BatchTaskContext(this, tasks, batchToken, logger);
|
|
80
|
+
let activeTaskIds = new Set(tasks.map((t) => t.id));
|
|
81
|
+
context.logger.verbose(`Processing batch of ${tasks.length}`);
|
|
82
|
+
void (async () => {
|
|
83
|
+
while (batchToken.isUnset) {
|
|
84
|
+
await cancelableTimeout(Math.min(this.processTimeout / 2, 5000), batchToken);
|
|
85
|
+
if (batchToken.isSet) {
|
|
86
|
+
break;
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
const tasksToTouch = tasks.filter((t) => activeTaskIds.has(t.id));
|
|
90
|
+
if (tasksToTouch.length > 0) {
|
|
91
|
+
const touchedIds = await this.touchMany(tasksToTouch);
|
|
92
|
+
if (touchedIds.length != tasksToTouch.length) {
|
|
93
|
+
const lostCount = tasksToTouch.length - touchedIds.length;
|
|
94
|
+
context.logger.warn(`Batch integrity compromised: ${lostCount} tasks lost lease. Aborting batch.`);
|
|
95
|
+
activeTaskIds = new Set(touchedIds);
|
|
96
|
+
batchToken.set();
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
activeTaskIds = new Set(touchedIds);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (activeTaskIds.size == 0 && batchToken.isUnset) {
|
|
103
|
+
context.logger.warn(`All tasks in batch lost lease. Stopping worker.`);
|
|
104
|
+
batchToken.set();
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
context.logger.error(error);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
})();
|
|
43
112
|
try {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
113
|
+
if (batchToken.isSet) {
|
|
114
|
+
throw new Error('Tasks cancelled before start');
|
|
115
|
+
}
|
|
116
|
+
const results = await handler(context);
|
|
117
|
+
if (isDefined(results)) {
|
|
118
|
+
context.logger.verbose(`Acknowledging batch`);
|
|
119
|
+
await this.acknowledgeMany(tasks, results);
|
|
120
|
+
}
|
|
47
121
|
}
|
|
48
122
|
catch (error) {
|
|
49
|
-
logger
|
|
123
|
+
context.logger.error(error);
|
|
124
|
+
await this.failMany(tasks, createArray(tasks.length, () => error));
|
|
125
|
+
}
|
|
126
|
+
finally {
|
|
127
|
+
batchToken.set();
|
|
50
128
|
}
|
|
51
129
|
}
|
|
52
130
|
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { CancellationSignal, CancellationToken } from '../cancellation/index.js';
|
|
2
|
+
import type { Logger } from '../logger/index.js';
|
|
3
|
+
import type { Transaction } from '../orm/server/index.js';
|
|
4
|
+
import { type EnqueueManyItem, type EnqueueOptions, Queue, type Task } from './queue.js';
|
|
5
|
+
export declare class TaskContext<Data, State = unknown, Result = unknown> {
|
|
6
|
+
#private;
|
|
7
|
+
constructor(queue: Queue<Data, State, Result>, task: Task<Data, State, Result>, signal: CancellationToken, logger: Logger);
|
|
8
|
+
get id(): string;
|
|
9
|
+
get data(): Data;
|
|
10
|
+
get attempt(): number;
|
|
11
|
+
get triesLeft(): number;
|
|
12
|
+
get signal(): CancellationSignal;
|
|
13
|
+
get logger(): Logger;
|
|
14
|
+
acknowledge(result?: Result, transaction?: Transaction): Promise<void>;
|
|
15
|
+
checkpoint(options: {
|
|
16
|
+
progress?: number;
|
|
17
|
+
state?: State;
|
|
18
|
+
}): Promise<void>;
|
|
19
|
+
spawn(data: Data, options?: Omit<EnqueueOptions, 'parentId'>): Promise<Task<Data, State, Result>>;
|
|
20
|
+
spawn<D, S, R>(queue: Queue<D, S, R>, data: D, options?: Omit<EnqueueOptions, 'parentId'>): Promise<Task<D, S, R>>;
|
|
21
|
+
spawnMany(items: EnqueueManyItem<Data>[]): Promise<Task<Data, State, Result>[]>;
|
|
22
|
+
spawnMany<D = Data, S = any, R = any>(queue: Queue<D, S, R>, items: EnqueueManyItem<D>[]): Promise<Task<D, S, R>[]>;
|
|
23
|
+
/** Stop execution and reschedule the task for later without incrementing tries if possible */
|
|
24
|
+
reschedule(timestamp: number): Promise<void>;
|
|
25
|
+
reschedule(options: {
|
|
26
|
+
delay: number;
|
|
27
|
+
}): Promise<void>;
|
|
28
|
+
fail(error: any, fatal?: boolean, transaction?: Transaction): Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
export declare class BatchTaskContext<Data, State, Result> {
|
|
31
|
+
#private;
|
|
32
|
+
constructor(queue: Queue<Data, State, Result>, tasks: Task<Data, State, Result>[], signal: CancellationToken, logger: Logger);
|
|
33
|
+
get tasks(): Task<Data, State, Result>[];
|
|
34
|
+
get signal(): CancellationSignal;
|
|
35
|
+
get logger(): Logger;
|
|
36
|
+
for(task: Task<Data, State, Result>): TaskContext<Data, State, Result>;
|
|
37
|
+
checkpointAll(progresses?: number[], states?: State[]): Promise<void>;
|
|
38
|
+
}
|