@tstdl/base 0.93.76 → 0.93.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/ai/prompts/instructions-formatter.d.ts +68 -5
  2. package/ai/prompts/instructions-formatter.js +11 -3
  3. package/authentication/client/http-client.middleware.js +2 -2
  4. package/authentication/models/authentication-credentials.model.d.ts +2 -2
  5. package/authentication/models/authentication-credentials.model.js +5 -3
  6. package/authentication/models/authentication-session.model.d.ts +2 -2
  7. package/authentication/models/authentication-session.model.js +5 -3
  8. package/authentication/models/index.d.ts +4 -0
  9. package/authentication/models/index.js +4 -0
  10. package/authentication/models/service-account.model.d.ts +7 -0
  11. package/authentication/models/service-account.model.js +31 -0
  12. package/authentication/models/subject.model.d.ts +16 -0
  13. package/authentication/models/subject.model.js +59 -0
  14. package/authentication/models/system-account.model.d.ts +5 -0
  15. package/authentication/models/system-account.model.js +25 -0
  16. package/authentication/models/user.model.d.ts +15 -0
  17. package/authentication/models/user.model.js +47 -0
  18. package/authentication/server/drizzle/0001_condemned_pretty_boy.sql +70 -0
  19. package/authentication/server/drizzle/meta/0001_snapshot.json +651 -0
  20. package/authentication/server/drizzle/meta/_journal.json +7 -0
  21. package/authentication/server/index.d.ts +1 -0
  22. package/authentication/server/index.js +1 -0
  23. package/authentication/server/schemas.d.ts +16 -1
  24. package/authentication/server/schemas.js +7 -1
  25. package/authentication/server/subject.service.d.ts +6 -0
  26. package/authentication/server/subject.service.js +44 -0
  27. package/circuit-breaker/circuit-breaker.d.ts +32 -0
  28. package/circuit-breaker/circuit-breaker.js +9 -0
  29. package/circuit-breaker/index.d.ts +2 -0
  30. package/circuit-breaker/index.js +2 -0
  31. package/circuit-breaker/postgres/circuit-breaker.d.ts +7 -0
  32. package/circuit-breaker/postgres/circuit-breaker.js +78 -0
  33. package/circuit-breaker/postgres/drizzle/0000_hard_shocker.sql +9 -0
  34. package/circuit-breaker/postgres/drizzle/meta/0000_snapshot.json +82 -0
  35. package/circuit-breaker/postgres/drizzle/meta/_journal.json +13 -0
  36. package/circuit-breaker/postgres/drizzle.config.d.ts +2 -0
  37. package/circuit-breaker/postgres/drizzle.config.js +11 -0
  38. package/circuit-breaker/postgres/index.d.ts +5 -0
  39. package/circuit-breaker/postgres/index.js +5 -0
  40. package/circuit-breaker/postgres/model.d.ts +9 -0
  41. package/circuit-breaker/postgres/model.js +40 -0
  42. package/circuit-breaker/postgres/module.d.ts +6 -0
  43. package/circuit-breaker/postgres/module.js +25 -0
  44. package/circuit-breaker/postgres/provider.d.ts +6 -0
  45. package/circuit-breaker/postgres/provider.js +21 -0
  46. package/circuit-breaker/postgres/schemas.d.ts +8 -0
  47. package/circuit-breaker/postgres/schemas.js +6 -0
  48. package/circuit-breaker/provider.d.ts +4 -0
  49. package/circuit-breaker/provider.js +2 -0
  50. package/circuit-breaker/tests/circuit-breaker.test.js +113 -0
  51. package/document-management/models/document.model.d.ts +0 -1
  52. package/document-management/models/document.model.js +0 -5
  53. package/document-management/server/api/document-management.api.js +1 -2
  54. package/document-management/server/drizzle/0002_round_warbird.sql +1 -0
  55. package/document-management/server/drizzle/meta/0002_snapshot.json +2722 -0
  56. package/document-management/server/drizzle/meta/_journal.json +7 -0
  57. package/document-management/server/services/document-collection.service.js +3 -3
  58. package/document-management/server/services/document-management-ancillary.service.d.ts +1 -1
  59. package/document-management/server/services/document-management.service.js +1 -1
  60. package/document-management/server/services/document-workflow.service.js +5 -5
  61. package/document-management/server/services/document.service.d.ts +0 -2
  62. package/document-management/server/services/document.service.js +1 -2
  63. package/document-management/service-models/enriched/enriched-document.view.d.ts +1 -1
  64. package/examples/document-management/main.d.ts +1 -1
  65. package/examples/document-management/main.js +1 -1
  66. package/logger/transports/console.d.ts +1 -1
  67. package/logger/transports/console.js +4 -1
  68. package/message-bus/message-bus-base.js +1 -1
  69. package/package.json +8 -5
  70. package/queue/enqueue-batch.d.ts +11 -11
  71. package/queue/enqueue-batch.js +2 -3
  72. package/queue/index.d.ts +1 -0
  73. package/queue/index.js +1 -0
  74. package/queue/postgres/drizzle/0003_tricky_venom.sql +30 -0
  75. package/queue/postgres/drizzle/meta/0003_snapshot.json +288 -0
  76. package/queue/postgres/drizzle/meta/_journal.json +7 -0
  77. package/queue/postgres/drizzle.config.js +2 -2
  78. package/queue/postgres/index.d.ts +1 -1
  79. package/queue/postgres/index.js +1 -1
  80. package/queue/postgres/module.d.ts +1 -1
  81. package/queue/postgres/module.js +1 -1
  82. package/queue/postgres/queue.d.ts +52 -23
  83. package/queue/postgres/queue.js +582 -64
  84. package/queue/postgres/queue.provider.d.ts +1 -1
  85. package/queue/postgres/schemas.d.ts +13 -2
  86. package/queue/postgres/schemas.js +4 -2
  87. package/queue/postgres/task.model.d.ts +24 -0
  88. package/queue/postgres/task.model.js +115 -0
  89. package/queue/provider.d.ts +1 -1
  90. package/queue/queue.d.ts +158 -37
  91. package/queue/queue.js +97 -19
  92. package/queue/task-context.d.ts +38 -0
  93. package/queue/task-context.js +102 -0
  94. package/queue/tests/queue.test.d.ts +1 -0
  95. package/queue/tests/queue.test.js +623 -0
  96. package/test4.d.ts +1 -1
  97. package/test4.js +1 -1
  98. package/utils/format-error.d.ts +17 -20
  99. package/utils/format-error.js +105 -47
  100. package/queue/postgres/job.model.d.ts +0 -12
  101. package/queue/postgres/job.model.js +0 -53
  102. package/test6.js +0 -33
  103. /package/{test6.d.ts → circuit-breaker/tests/circuit-breaker.test.d.ts} +0 -0
@@ -3,5 +3,5 @@ import type { ObjectLiteral } from '../../types/index.js';
3
3
  import { PostgresQueue } from './queue.js';
4
4
  export declare class PostgresQueueProvider extends QueueProvider {
5
5
  #private;
6
- get<T extends ObjectLiteral>(name: string, config?: QueueConfig): PostgresQueue<T>;
6
+ get<Data extends ObjectLiteral, State extends ObjectLiteral, Result extends ObjectLiteral>(name: string, config?: QueueConfig): PostgresQueue<Data, State, Result>;
7
7
  }
@@ -1,3 +1,14 @@
1
- import { PostgresJob } from './job.model.js';
1
+ import { PostgresTask } from './task.model.js';
2
2
  export declare const queueSchema: import("../../orm/server/index.js").DatabaseSchema<"queue">;
3
- export declare const job: import("../../orm/server/types.js").PgTableFromType<typeof PostgresJob, "queue">;
3
+ export declare const taskState: import("../../orm/enums.js").PgEnumFromEnumeration<{
4
+ readonly Pending: "pending";
5
+ readonly Running: "running";
6
+ readonly Completed: "completed";
7
+ readonly Cancelled: "cancelled";
8
+ readonly Waiting: "waiting";
9
+ readonly Dead: "dead";
10
+ }>;
11
+ export declare const task: import("../../orm/server/types.js").PgTableFromType<{
12
+ new (): PostgresTask<any, any, any>;
13
+ readonly entityName: "Task";
14
+ }, "queue">;
@@ -1,4 +1,6 @@
1
1
  import { databaseSchema } from '../../orm/server/index.js';
2
- import { PostgresJob } from './job.model.js';
2
+ import { TaskState } from '../queue.js';
3
+ import { PostgresTask } from './task.model.js';
3
4
  export const queueSchema = databaseSchema('queue');
4
- export const job = queueSchema.getTable(PostgresJob);
5
+ export const taskState = queueSchema.getEnum(TaskState);
6
+ export const task = queueSchema.getTable((PostgresTask));
@@ -0,0 +1,24 @@
1
+ import { BaseEntity, type Json, type Timestamp } from '../../orm/index.js';
2
+ import type { ObjectLiteral } from '../../types/types.js';
3
+ import { type Task, TaskState } from '../queue.js';
4
+ export declare class PostgresTask<Data extends ObjectLiteral = ObjectLiteral, State extends ObjectLiteral = ObjectLiteral, Result extends ObjectLiteral = ObjectLiteral> extends BaseEntity implements Task<Data, State, Result> {
5
+ static readonly entityName = "Task";
6
+ queue: string;
7
+ status: TaskState;
8
+ tag: string | null;
9
+ priority: number;
10
+ parentId: string | null;
11
+ lease: string | null;
12
+ enqueueTimestamp: Timestamp;
13
+ scheduleTimestamp: Timestamp;
14
+ startTimestamp: Timestamp | null;
15
+ expirationTimestamp: Timestamp | null;
16
+ lockExpirationTimestamp: Timestamp | null;
17
+ completeTimestamp: Timestamp | null;
18
+ tries: number;
19
+ progress: number;
20
+ data: Json<Data>;
21
+ state: Json<State> | null;
22
+ result: Json<Result> | null;
23
+ error: Json<ObjectLiteral> | null;
24
+ }
@@ -0,0 +1,115 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ var __metadata = (this && this.__metadata) || function (k, v) {
8
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
9
+ };
10
+ import { BaseEntity, Index, JsonProperty, Reference, Table, TimestampProperty, Unique, UuidProperty } from '../../orm/index.js';
11
+ import { Enumeration, Integer, NumberProperty, StringProperty } from '../../schema/index.js';
12
+ import { TaskState } from '../queue.js';
13
+ let PostgresTask = class PostgresTask extends BaseEntity {
14
+ static entityName = 'Task';
15
+ queue;
16
+ status;
17
+ tag;
18
+ priority;
19
+ parentId;
20
+ lease;
21
+ enqueueTimestamp;
22
+ scheduleTimestamp;
23
+ startTimestamp;
24
+ expirationTimestamp;
25
+ lockExpirationTimestamp;
26
+ completeTimestamp;
27
+ tries;
28
+ progress;
29
+ data;
30
+ state;
31
+ result;
32
+ error;
33
+ };
34
+ __decorate([
35
+ StringProperty(),
36
+ __metadata("design:type", String)
37
+ ], PostgresTask.prototype, "queue", void 0);
38
+ __decorate([
39
+ Enumeration(TaskState),
40
+ __metadata("design:type", String)
41
+ ], PostgresTask.prototype, "status", void 0);
42
+ __decorate([
43
+ StringProperty({ nullable: true }),
44
+ __metadata("design:type", Object)
45
+ ], PostgresTask.prototype, "tag", void 0);
46
+ __decorate([
47
+ Integer(),
48
+ __metadata("design:type", Number)
49
+ ], PostgresTask.prototype, "priority", void 0);
50
+ __decorate([
51
+ Reference(() => PostgresTask),
52
+ UuidProperty({ nullable: true }),
53
+ __metadata("design:type", Object)
54
+ ], PostgresTask.prototype, "parentId", void 0);
55
+ __decorate([
56
+ UuidProperty({ nullable: true }),
57
+ __metadata("design:type", Object)
58
+ ], PostgresTask.prototype, "lease", void 0);
59
+ __decorate([
60
+ TimestampProperty(),
61
+ __metadata("design:type", Number)
62
+ ], PostgresTask.prototype, "enqueueTimestamp", void 0);
63
+ __decorate([
64
+ TimestampProperty(),
65
+ __metadata("design:type", Number)
66
+ ], PostgresTask.prototype, "scheduleTimestamp", void 0);
67
+ __decorate([
68
+ TimestampProperty({ nullable: true }),
69
+ __metadata("design:type", Object)
70
+ ], PostgresTask.prototype, "startTimestamp", void 0);
71
+ __decorate([
72
+ TimestampProperty({ nullable: true }),
73
+ __metadata("design:type", Object)
74
+ ], PostgresTask.prototype, "expirationTimestamp", void 0);
75
+ __decorate([
76
+ TimestampProperty({ nullable: true }),
77
+ __metadata("design:type", Object)
78
+ ], PostgresTask.prototype, "lockExpirationTimestamp", void 0);
79
+ __decorate([
80
+ TimestampProperty({ nullable: true }),
81
+ __metadata("design:type", Object)
82
+ ], PostgresTask.prototype, "completeTimestamp", void 0);
83
+ __decorate([
84
+ Integer(),
85
+ __metadata("design:type", Number)
86
+ ], PostgresTask.prototype, "tries", void 0);
87
+ __decorate([
88
+ NumberProperty(),
89
+ __metadata("design:type", Number)
90
+ ], PostgresTask.prototype, "progress", void 0);
91
+ __decorate([
92
+ JsonProperty({ nullable: true }),
93
+ __metadata("design:type", Object)
94
+ ], PostgresTask.prototype, "data", void 0);
95
+ __decorate([
96
+ JsonProperty({ nullable: true }),
97
+ __metadata("design:type", Object)
98
+ ], PostgresTask.prototype, "state", void 0);
99
+ __decorate([
100
+ JsonProperty({ nullable: true }),
101
+ __metadata("design:type", Object)
102
+ ], PostgresTask.prototype, "result", void 0);
103
+ __decorate([
104
+ JsonProperty({ nullable: true }),
105
+ __metadata("design:type", Object)
106
+ ], PostgresTask.prototype, "error", void 0);
107
+ PostgresTask = __decorate([
108
+ Table('task', { schema: 'queue' }),
109
+ Unique(['queue', 'tag']),
110
+ Index(['queue', 'status', 'priority', 'scheduleTimestamp']),
111
+ Index(['queue', 'status', 'lockExpirationTimestamp']),
112
+ Index(['queue', 'completeTimestamp']),
113
+ Index(['parentId', 'status'])
114
+ ], PostgresTask);
115
+ export { PostgresTask };
@@ -1,5 +1,5 @@
1
1
  import type { ObjectLiteral } from '../types/index.js';
2
2
  import type { Queue, QueueConfig } from './queue.js';
3
3
  export declare abstract class QueueProvider {
4
- abstract get<T extends ObjectLiteral>(key: string, config?: QueueConfig): Queue<T>;
4
+ abstract get<Data extends ObjectLiteral, State extends ObjectLiteral, Result extends ObjectLiteral>(key: string, config?: QueueConfig): Queue<Data, State, Result>;
5
5
  }
package/queue/queue.d.ts CHANGED
@@ -2,90 +2,211 @@ import type { CancellationSignal } from '../cancellation/token.js';
2
2
  import { type EnumType } from '../enumeration/enumeration.js';
3
3
  import type { Resolvable, resolveArgumentType } from '../injector/interfaces.js';
4
4
  import type { Logger } from '../logger/logger.js';
5
+ import type { Transaction } from '../orm/server/transaction.js';
6
+ import type { UndefinableJson } from '../types/types.js';
5
7
  import { QueueEnqueueBatch } from './enqueue-batch.js';
6
- export type ProcessWorker<T> = (job: Job<T>) => void | Promise<void>;
7
- export type ProcessBatchWorker<T> = (jobs: Job<T>[]) => void | Promise<void>;
8
- export type JobTag = string | null;
9
- export type Job<T> = {
8
+ import { BatchTaskContext, type TaskContext } from './task-context.js';
9
+ export interface ProcessWorker<Data, State, Result> {
10
+ /**
11
+ * A worker function that processes a single task.
12
+ * @param context The task context providing data, logger, and orchestration helpers.
13
+ */
14
+ (context: TaskContext<Data, State, Result>): void | Result | Promise<void | Result>;
15
+ }
16
+ export interface ProcessBatchWorker<Data, State, Result> {
17
+ /**
18
+ * A worker function that processes a batch of tasks.
19
+ * @param context The batch context providing tasks and helpers.
20
+ */
21
+ (context: BatchTaskContext<Data, State, Result>): void | Result[] | Promise<void | Result[]>;
22
+ }
23
+ export type TaskTag = string | null;
24
+ export declare const TaskState: {
25
+ /**
26
+ * The task is waiting to be processed.
27
+ */
28
+ readonly Pending: "pending";
29
+ /**
30
+ * The task is currently being processed.
31
+ */
32
+ readonly Running: "running";
33
+ /**
34
+ * The task has been completed successfully.
35
+ */
36
+ readonly Completed: "completed";
37
+ /**
38
+ * The task has been cancelled and will not be processed.
39
+ */
40
+ readonly Cancelled: "cancelled";
41
+ /**
42
+ * The task is scheduled to be processed in the future when all children have completed.
43
+ */
44
+ readonly Waiting: "waiting";
45
+ /**
46
+ * The task has failed and will not be retried.
47
+ */
48
+ readonly Dead: "dead";
49
+ };
50
+ export type TaskState = EnumType<typeof TaskState>;
51
+ export type Task<Data = unknown, State = unknown, Result = unknown> = {
10
52
  id: string;
53
+ queue: string;
54
+ status: TaskState;
55
+ lease: string | null;
11
56
  /**
12
57
  * The lower the number, the higher the priority.
13
58
  * @default 1000
14
59
  */
15
60
  priority: number;
16
- tag: JobTag;
17
- data: T;
18
- enqueueTimestamp: number;
19
- lastDequeueTimestamp: number | null;
61
+ tag: TaskTag;
62
+ data: Data;
63
+ parentId: string | null;
20
64
  tries: number;
65
+ enqueueTimestamp: number;
66
+ scheduleTimestamp: number;
67
+ /**
68
+ * Timestamp when the task most recently switched to Running state.
69
+ * Used for Hard Execution Timeouts.
70
+ */
71
+ startTimestamp: number | null;
72
+ /**
73
+ * Timestamp after which the task is considered expired if it hasn't started Running.
74
+ * If null, the task never expires in the queue.
75
+ */
76
+ expirationTimestamp: number | null;
77
+ lockExpirationTimestamp: number | null;
78
+ completeTimestamp: number | null;
79
+ /** A number between 0 and 1 indicating the progress of the task. */
80
+ progress: number;
81
+ /** A snapshot of the current state of the task. */
82
+ state: State | null;
83
+ result: Result | null;
84
+ error: UndefinableJson | null;
21
85
  };
22
- export declare const defaultJobPriority = 1000;
86
+ export declare const defaultTaskPriority = 1000;
23
87
  export declare const UniqueTagStrategy: {
24
88
  readonly KeepOld: 0;
25
89
  readonly TakeNew: 1;
26
90
  };
27
91
  export type UniqueTagStrategy = EnumType<typeof UniqueTagStrategy>;
28
92
  export type EnqueueOptions = {
29
- tag?: JobTag;
93
+ tag?: TaskTag;
30
94
  priority?: number;
95
+ parentId?: string;
96
+ scheduleTimestamp?: number;
97
+ expirationTimestamp?: number;
98
+ transaction?: Transaction;
31
99
  };
32
- export type EnqueueOneOptions = {
33
- tag?: JobTag;
100
+ export type EnqueueOneOptions = EnqueueOptions & {
34
101
  uniqueTag?: UniqueTagStrategy;
35
- priority?: number;
36
102
  };
37
103
  export type EnqueueManyItem<T> = EnqueueOptions & {
38
104
  data: T;
39
105
  };
40
106
  export type EnqueueManyOptions = {
41
107
  uniqueTag?: UniqueTagStrategy;
42
- returnJobs?: boolean;
108
+ returnTasks?: boolean;
109
+ transaction?: Transaction;
43
110
  };
44
111
  export type QueueConfig = {
112
+ /**
113
+ * Duration in ms for the worker lease. The worker must heartbeat within this window.
114
+ */
45
115
  processTimeout?: number;
116
+ /**
117
+ * Maximum duration a task is allowed to run (in ms) before being forcefully killed.
118
+ * This protects against infinite loops even if the worker keeps heartbeating.
119
+ * **WARN:** CancellationSignal must still be observed by the worker!
120
+ * Default: 15 minutes
121
+ */
122
+ executionTimeout?: number;
123
+ /** Maximum number of attempts to process a task before marking it as failed. Default 3 */
46
124
  maxTries?: number;
125
+ /** Milliseconds to keep completed tasks */
126
+ retentionPeriod?: number;
127
+ /** Maximum number of running tasks across all workers (`null` = unlimited) */
128
+ globalConcurrency?: number | null;
129
+ /** Number of consecutive failures before tripping the circuit breaker */
130
+ circuitBreakerThreshold?: number;
131
+ /** Milliseconds to wait before trying to close the circuit breaker */
132
+ circuitBreakerResetTimeout?: number;
133
+ /** Minimum delay for retries (the floor). Default: 5 seconds */
134
+ retryDelayMinimum?: number;
135
+ /** Maximum delay for retries (the ceiling). Default: 5 minutes */
136
+ retryDelayMaximum?: number;
137
+ /** Base for exponential backoff (delay = min * (base ^ tries)). Default: 2 */
138
+ retryDelayGrowth?: number;
47
139
  };
48
140
  export type QueueArgument = string | (QueueConfig & {
49
141
  name: string;
50
142
  });
51
143
  export declare const defaultQueueConfig: Required<QueueConfig>;
52
- export declare abstract class Queue<T> implements Resolvable<QueueArgument> {
144
+ export declare abstract class Queue<Data, State = unknown, Result = unknown> implements Resolvable<QueueArgument> {
53
145
  readonly [resolveArgumentType]: QueueArgument;
54
146
  abstract readonly processTimeout: number;
55
147
  abstract readonly maxTries: number;
56
- batch(): QueueEnqueueBatch<T>;
57
- abstract enqueue(data: T, options?: EnqueueOneOptions): Promise<Job<T>>;
58
- abstract enqueueMany(items: EnqueueManyItem<T>[], options?: EnqueueManyOptions & {
59
- returnJobs?: false;
148
+ batch(): QueueEnqueueBatch<Data, State, Result>;
149
+ abstract enqueue(data: Data, options?: EnqueueOneOptions): Promise<Task<Data, State, Result>>;
150
+ abstract enqueueMany(items: EnqueueManyItem<Data>[], options?: EnqueueManyOptions & {
151
+ returnTasks?: false;
60
152
  }): Promise<void>;
61
- abstract enqueueMany(items: EnqueueManyItem<T>[], options: EnqueueManyOptions & {
62
- returnJobs: true;
63
- }): Promise<Job<T>[]>;
64
- abstract enqueueMany(items: EnqueueManyItem<T>[], options?: EnqueueManyOptions): Promise<Job<T>[] | undefined>;
153
+ abstract enqueueMany(items: EnqueueManyItem<Data>[], options: EnqueueManyOptions & {
154
+ returnTasks: true;
155
+ }): Promise<Task<Data, State, Result>[]>;
156
+ abstract enqueueMany(items: EnqueueManyItem<Data>[], options?: EnqueueManyOptions): Promise<Task<Data, State, Result>[] | undefined>;
65
157
  abstract has(id: string): Promise<boolean>;
66
- abstract countByTag(tag: JobTag): Promise<number>;
67
- abstract get(id: string): Promise<Job<T> | undefined>;
68
- abstract getByTag(tag: JobTag): Promise<Job<T>[]>;
69
- abstract getByTags(tags: JobTag[]): Promise<Job<T>[]>;
158
+ abstract get(id: string): Promise<Task<Data, State, Result> | undefined>;
159
+ abstract getByTag(tag: TaskTag): Promise<Task<Data, State, Result>[]>;
160
+ abstract getByTags(tags: TaskTag[]): Promise<Task<Data, State, Result>[]>;
161
+ abstract countByTag(tag: TaskTag): Promise<number>;
162
+ abstract getTree(rootId: string | string[]): Promise<Task<unknown, unknown, unknown>[]>;
70
163
  abstract cancel(id: string): Promise<void>;
71
164
  abstract cancelMany(ids: string[]): Promise<void>;
72
- abstract cancelByTag(tag: JobTag): Promise<void>;
73
- abstract cancelByTags(tags: JobTag[]): Promise<void>;
74
- abstract dequeue(): Promise<Job<T> | undefined>;
75
- abstract dequeueMany(count: number): Promise<Job<T>[]>;
76
- abstract acknowledge(job: Job<T>): Promise<void>;
77
- abstract acknowledgeMany(jobs: Job<T>[]): Promise<void>;
78
- abstract getConsumer(cancellationSignal: CancellationSignal): AsyncIterableIterator<Job<T>>;
79
- abstract getBatchConsumer(size: number, cancellationSignal: CancellationSignal): AsyncIterableIterator<Job<T>[]>;
165
+ abstract cancelByTag(tag: TaskTag): Promise<void>;
166
+ abstract cancelManyByTag(tags: TaskTag[]): Promise<void>;
167
+ /** Clears all tasks from the queue. Use with caution! */
168
+ abstract clear(): Promise<void>;
169
+ abstract dequeue(): Promise<Task<Data, State, Result> | undefined>;
170
+ abstract dequeueMany(count: number): Promise<Task<Data, State, Result>[]>;
171
+ /**
172
+ * Reschedules a task to run at a specific time.
173
+ * NOTE: If the task is currently running, its retry count is decremented (refunded) so this attempt doesn't count against maxTries.
174
+ */
175
+ abstract reschedule(id: string, timestamp: number, transaction?: Transaction): Promise<void>;
176
+ abstract rescheduleMany(ids: string[], timestamp: number, transaction?: Transaction): Promise<void>;
177
+ abstract rescheduleByTag(tag: TaskTag, timestamp: number, transaction?: Transaction): Promise<void>;
178
+ abstract rescheduleManyByTag(tags: TaskTag[], timestamp: number, transaction?: Transaction): Promise<void>;
179
+ /**
180
+ * Updates task progress, state and lock.
181
+ * Returns the updated task if successful, `undefined` if task is lost/cancelled/timed out.
182
+ */
183
+ abstract touch(task: Task, options?: {
184
+ progress?: number;
185
+ state?: State;
186
+ transaction?: Transaction;
187
+ }): Promise<Task<Data, State, Result> | undefined>;
188
+ /**
189
+ * Updates multiple tasks' progress, state and lock.
190
+ * Returns the IDs of the successfully updated tasks.
191
+ */
192
+ abstract touchMany(tasks: Task[], progresses?: number[], states?: State[], transaction?: Transaction): Promise<string[]>;
193
+ abstract acknowledge<R>(task: Task<any, any, R>, result?: R, transaction?: Transaction): Promise<void>;
194
+ abstract acknowledgeMany<R>(tasks: Task<any, any, R>[], results?: R[], transaction?: Transaction): Promise<void>;
195
+ abstract fail(task: Task, error: any, fatal?: boolean, transaction?: Transaction): Promise<void>;
196
+ abstract failMany(tasks: Task[], error: any, transaction?: Transaction): Promise<void>;
197
+ abstract prune(): Promise<void>;
198
+ abstract restart(id: string, transaction?: Transaction): Promise<void>;
199
+ abstract getConsumer(cancellationSignal: CancellationSignal): AsyncIterableIterator<Task<Data, State, Result>>;
200
+ abstract getBatchConsumer(size: number, cancellationSignal: CancellationSignal): AsyncIterableIterator<Task<Data, State, Result>[]>;
80
201
  process({ concurrency, cancellationSignal }: {
81
202
  concurrency?: number;
82
203
  cancellationSignal: CancellationSignal;
83
- }, handler: ProcessWorker<T>, logger: Logger): void;
204
+ }, handler: ProcessWorker<Data, State, Result>, logger: Logger): void;
84
205
  processBatch({ batchSize, concurrency, cancellationSignal }: {
85
206
  batchSize?: number;
86
207
  concurrency?: number;
87
208
  cancellationSignal: CancellationSignal;
88
- }, handler: ProcessBatchWorker<T>, logger: Logger): void;
209
+ }, handler: ProcessBatchWorker<Data, State, Result>, logger: Logger): void;
89
210
  private processWorker;
90
211
  private processBatchWorker;
91
212
  }
package/queue/queue.js CHANGED
@@ -1,14 +1,52 @@
1
1
  import { defineEnum } from '../enumeration/enumeration.js';
2
- import { millisecondsPerMinute } from '../utils/units.js';
2
+ import { createArray } from '../utils/array/array.js';
3
+ import { cancelableTimeout } from '../utils/timing.js';
4
+ import { isDefined } from '../utils/type-guards.js';
5
+ import { millisecondsPerDay, millisecondsPerMinute, millisecondsPerSecond } from '../utils/units.js';
3
6
  import { QueueEnqueueBatch } from './enqueue-batch.js';
4
- export const defaultJobPriority = 1000;
7
+ import { BatchTaskContext } from './task-context.js';
8
+ export const TaskState = defineEnum('TaskState', {
9
+ /**
10
+ * The task is waiting to be processed.
11
+ */
12
+ Pending: 'pending',
13
+ /**
14
+ * The task is currently being processed.
15
+ */
16
+ Running: 'running',
17
+ /**
18
+ * The task has been completed successfully.
19
+ */
20
+ Completed: 'completed',
21
+ /**
22
+ * The task has been cancelled and will not be processed.
23
+ */
24
+ Cancelled: 'cancelled',
25
+ /**
26
+ * The task is scheduled to be processed in the future when all children have completed.
27
+ */
28
+ Waiting: 'waiting',
29
+ /**
30
+ * The task has failed and will not be retried.
31
+ */
32
+ Dead: 'dead',
33
+ });
34
+ export const defaultTaskPriority = 1000;
5
35
  export const UniqueTagStrategy = defineEnum('UniqueTagStrategy', {
6
36
  KeepOld: 0,
7
37
  TakeNew: 1,
8
38
  });
9
39
  export const defaultQueueConfig = {
10
- processTimeout: millisecondsPerMinute,
40
+ processTimeout: millisecondsPerMinute * 5,
41
+ executionTimeout: millisecondsPerMinute * 60,
11
42
  maxTries: 3,
43
+ retentionPeriod: 30 * millisecondsPerDay,
44
+ globalConcurrency: null,
45
+ circuitBreakerThreshold: 5,
46
+ circuitBreakerResetTimeout: 30 * millisecondsPerSecond,
47
+ retryDelayMinimum: 5 * millisecondsPerSecond,
48
+ retryDelayMaximum: 5 * millisecondsPerMinute,
49
+ retryDelayGrowth: 2,
12
50
  };
13
51
  export class Queue {
14
52
  batch() {
@@ -25,28 +63,68 @@ export class Queue {
25
63
  }
26
64
  }
27
65
  async processWorker(cancellationSignal, handler, logger) {
28
- for await (const job of this.getConsumer(cancellationSignal)) {
29
- logger?.verbose(`Processing job ${job.id}`);
30
- try {
31
- await handler(job);
32
- logger?.verbose(`Acknowledge job ${job.id}`);
33
- await this.acknowledge(job);
34
- }
35
- catch (error) {
36
- logger?.error(error);
66
+ await this.processBatchWorker(1, cancellationSignal, async (batchContext) => {
67
+ const task = batchContext.tasks[0];
68
+ const context = batchContext.for(task);
69
+ const result = await handler(context);
70
+ if (isDefined(result)) {
71
+ return [result];
37
72
  }
38
- }
73
+ return undefined;
74
+ }, logger);
39
75
  }
40
76
  async processBatchWorker(size, cancellationSignal, handler, logger) {
41
- for await (const jobs of this.getBatchConsumer(size, cancellationSignal)) {
42
- logger?.verbose(`Processing ${jobs.length} jobs`);
77
+ for await (const tasks of this.getBatchConsumer(size, cancellationSignal)) {
78
+ const batchToken = cancellationSignal.createChild();
79
+ const context = new BatchTaskContext(this, tasks, batchToken, logger);
80
+ let activeTaskIds = new Set(tasks.map((t) => t.id));
81
+ context.logger.verbose(`Processing batch of ${tasks.length}`);
82
+ void (async () => {
83
+ while (batchToken.isUnset) {
84
+ await cancelableTimeout(Math.min(this.processTimeout / 2, 5000), batchToken);
85
+ if (batchToken.isSet) {
86
+ break;
87
+ }
88
+ try {
89
+ const tasksToTouch = tasks.filter((t) => activeTaskIds.has(t.id));
90
+ if (tasksToTouch.length > 0) {
91
+ const touchedIds = await this.touchMany(tasksToTouch);
92
+ if (touchedIds.length != tasksToTouch.length) {
93
+ const lostCount = tasksToTouch.length - touchedIds.length;
94
+ context.logger.warn(`Batch integrity compromised: ${lostCount} tasks lost lease. Aborting batch.`);
95
+ activeTaskIds = new Set(touchedIds);
96
+ batchToken.set();
97
+ }
98
+ else {
99
+ activeTaskIds = new Set(touchedIds);
100
+ }
101
+ }
102
+ if (activeTaskIds.size == 0 && batchToken.isUnset) {
103
+ context.logger.warn(`All tasks in batch lost lease. Stopping worker.`);
104
+ batchToken.set();
105
+ }
106
+ }
107
+ catch (error) {
108
+ context.logger.error(error);
109
+ }
110
+ }
111
+ })();
43
112
  try {
44
- await handler(jobs);
45
- logger?.verbose(`Acknowledge ${jobs.length} jobs`);
46
- await this.acknowledgeMany(jobs);
113
+ if (batchToken.isSet) {
114
+ throw new Error('Tasks cancelled before start');
115
+ }
116
+ const results = await handler(context);
117
+ if (isDefined(results)) {
118
+ context.logger.verbose(`Acknowledging batch`);
119
+ await this.acknowledgeMany(tasks, results);
120
+ }
47
121
  }
48
122
  catch (error) {
49
- logger?.error(error);
123
+ context.logger.error(error);
124
+ await this.failMany(tasks, createArray(tasks.length, () => error));
125
+ }
126
+ finally {
127
+ batchToken.set();
50
128
  }
51
129
  }
52
130
  }
@@ -0,0 +1,38 @@
1
+ import type { CancellationSignal, CancellationToken } from '../cancellation/index.js';
2
+ import type { Logger } from '../logger/index.js';
3
+ import type { Transaction } from '../orm/server/index.js';
4
+ import { type EnqueueManyItem, type EnqueueOptions, Queue, type Task } from './queue.js';
5
+ export declare class TaskContext<Data, State = unknown, Result = unknown> {
6
+ #private;
7
+ constructor(queue: Queue<Data, State, Result>, task: Task<Data, State, Result>, signal: CancellationToken, logger: Logger);
8
+ get id(): string;
9
+ get data(): Data;
10
+ get attempt(): number;
11
+ get triesLeft(): number;
12
+ get signal(): CancellationSignal;
13
+ get logger(): Logger;
14
+ acknowledge(result?: Result, transaction?: Transaction): Promise<void>;
15
+ checkpoint(options: {
16
+ progress?: number;
17
+ state?: State;
18
+ }): Promise<void>;
19
+ spawn(data: Data, options?: Omit<EnqueueOptions, 'parentId'>): Promise<Task<Data, State, Result>>;
20
+ spawn<D, S, R>(queue: Queue<D, S, R>, data: D, options?: Omit<EnqueueOptions, 'parentId'>): Promise<Task<D, S, R>>;
21
+ spawnMany(items: EnqueueManyItem<Data>[]): Promise<Task<Data, State, Result>[]>;
22
+ spawnMany<D = Data, S = any, R = any>(queue: Queue<D, S, R>, items: EnqueueManyItem<D>[]): Promise<Task<D, S, R>[]>;
23
+ /** Stop execution and reschedule the task for later without incrementing tries if possible */
24
+ reschedule(timestamp: number): Promise<void>;
25
+ reschedule(options: {
26
+ delay: number;
27
+ }): Promise<void>;
28
+ fail(error: any, fatal?: boolean, transaction?: Transaction): Promise<void>;
29
+ }
30
+ export declare class BatchTaskContext<Data, State, Result> {
31
+ #private;
32
+ constructor(queue: Queue<Data, State, Result>, tasks: Task<Data, State, Result>[], signal: CancellationToken, logger: Logger);
33
+ get tasks(): Task<Data, State, Result>[];
34
+ get signal(): CancellationSignal;
35
+ get logger(): Logger;
36
+ for(task: Task<Data, State, Result>): TaskContext<Data, State, Result>;
37
+ checkpointAll(progresses?: number[], states?: State[]): Promise<void>;
38
+ }