@balena/pinejs 16.2.0-build-joshbwlng-tasks-046f828587ae6d1889a3ae3298b3e44e96c74f90-1 → 17.0.0-build-wip-large-file-uploads-d6522dad962bc0bff6ee7c596df8f43f596b6aaa-1
Sign up to get free protection for your applications and to get access to all the features.
- package/.husky/pre-commit +0 -2
- package/.pinejs-cache.json +1 -1
- package/.versionbot/CHANGELOG.yml +33 -7
- package/CHANGELOG.md +12 -2
- package/VERSION +1 -1
- package/out/config-loader/env.d.ts +0 -4
- package/out/config-loader/env.js +1 -5
- package/out/config-loader/env.js.map +1 -1
- package/out/database-layer/db.d.ts +0 -3
- package/out/database-layer/db.js +0 -14
- package/out/database-layer/db.js.map +1 -1
- package/out/migrator/utils.js +2 -2
- package/out/migrator/utils.js.map +1 -1
- package/out/sbvr-api/sbvr-utils.d.ts +0 -1
- package/out/sbvr-api/sbvr-utils.js +1 -6
- package/out/sbvr-api/sbvr-utils.js.map +1 -1
- package/out/server-glue/module.d.ts +0 -1
- package/out/server-glue/module.js +1 -2
- package/out/server-glue/module.js.map +1 -1
- package/out/webresource-handler/handlers/NoopHandler.d.ts +3 -0
- package/out/webresource-handler/handlers/NoopHandler.js +6 -0
- package/out/webresource-handler/handlers/NoopHandler.js.map +1 -1
- package/out/webresource-handler/handlers/S3Handler.d.ts +7 -0
- package/out/webresource-handler/handlers/S3Handler.js +68 -2
- package/out/webresource-handler/handlers/S3Handler.js.map +1 -1
- package/out/webresource-handler/index.d.ts +5 -0
- package/out/webresource-handler/index.js +10 -5
- package/out/webresource-handler/index.js.map +1 -1
- package/out/webresource-handler/multipartUpload.d.ts +40 -0
- package/out/webresource-handler/multipartUpload.js +125 -0
- package/out/webresource-handler/multipartUpload.js.map +1 -0
- package/package.json +7 -10
- package/src/config-loader/env.ts +1 -6
- package/src/database-layer/db.ts +0 -24
- package/src/migrator/utils.ts +1 -1
- package/src/sbvr-api/sbvr-utils.ts +1 -5
- package/src/server-glue/module.ts +0 -1
- package/src/webresource-handler/handlers/NoopHandler.ts +21 -0
- package/src/webresource-handler/handlers/S3Handler.ts +130 -4
- package/src/webresource-handler/index.ts +24 -1
- package/src/webresource-handler/multipartUpload.ts +214 -0
- package/out/tasks/common.d.ts +0 -4
- package/out/tasks/common.js +0 -13
- package/out/tasks/common.js.map +0 -1
- package/out/tasks/index.d.ts +0 -10
- package/out/tasks/index.js +0 -139
- package/out/tasks/index.js.map +0 -1
- package/out/tasks/model.sbvr +0 -60
- package/out/tasks/types.d.ts +0 -38
- package/out/tasks/types.js +0 -10
- package/out/tasks/types.js.map +0 -1
- package/out/tasks/worker.d.ts +0 -16
- package/out/tasks/worker.js +0 -191
- package/out/tasks/worker.js.map +0 -1
- package/src/tasks/common.ts +0 -14
- package/src/tasks/index.ts +0 -158
- package/src/tasks/model.sbvr +0 -60
- package/src/tasks/types.ts +0 -58
- package/src/tasks/worker.ts +0 -246
package/src/tasks/model.sbvr
DELETED
@@ -1,60 +0,0 @@
|
|
1
|
-
Vocabulary: tasks
|
2
|
-
|
3
|
-
Term: id
|
4
|
-
Concept Type: Big Serial (Type)
|
5
|
-
Term: actor
|
6
|
-
Concept Type: Integer (Type)
|
7
|
-
Term: attempt count
|
8
|
-
Concept Type: Integer (Type)
|
9
|
-
Term: attempt limit
|
10
|
-
Concept Type: Integer (Type)
|
11
|
-
Term: cron expression
|
12
|
-
Concept Type: Short Text (Type)
|
13
|
-
Term: error message
|
14
|
-
Concept Type: Short Text (Type)
|
15
|
-
Term: handler
|
16
|
-
Concept Type: Short Text (Type)
|
17
|
-
Term: key
|
18
|
-
Concept Type: Short Text (Type)
|
19
|
-
Term: parameter set
|
20
|
-
Concept Type: JSON (Type)
|
21
|
-
Term: priority
|
22
|
-
Concept Type: Integer (Type)
|
23
|
-
Term: status
|
24
|
-
Concept Type: Short Text (Type)
|
25
|
-
Term: time
|
26
|
-
Concept Type: Date Time (Type)
|
27
|
-
|
28
|
-
Term: task
|
29
|
-
Fact type: task has id
|
30
|
-
Necessity: each task has exactly one id
|
31
|
-
Fact type: task has key
|
32
|
-
Necessity: each task has at most one key
|
33
|
-
Fact type: task is created by actor
|
34
|
-
Necessity: each task is created by exactly one actor
|
35
|
-
Fact type: task is executed by handler
|
36
|
-
Necessity: each task is executed by exactly one handler
|
37
|
-
Fact type: task is executed with parameter set
|
38
|
-
Necessity: each task is executed with at most one parameter set
|
39
|
-
Fact type: task has priority
|
40
|
-
Necessity: each task has exactly one priority
|
41
|
-
Necessity: each task has a priority that is greater than or equal to 0
|
42
|
-
Fact type: task is scheduled with cron expression
|
43
|
-
Necessity: each task is scheduled with at most one cron expression
|
44
|
-
Fact type: task is scheduled to execute on time
|
45
|
-
Necessity: each task is scheduled to execute on at most one time
|
46
|
-
Fact type: task has status
|
47
|
-
Necessity: each task has exactly one status
|
48
|
-
Definition: "pending" or "cancelled" or "success" or "failed"
|
49
|
-
Fact type: task started on time
|
50
|
-
Necessity: each task started on at most one time
|
51
|
-
Fact type: task ended on time
|
52
|
-
Necessity: each task ended on at most one time
|
53
|
-
Fact type: task has error message
|
54
|
-
Necessity: each task has at most one error message
|
55
|
-
Fact type: task has attempt count
|
56
|
-
Necessity: each task has exactly one attempt count
|
57
|
-
Fact type: task has attempt limit
|
58
|
-
Necessity: each task has exactly one attempt limit
|
59
|
-
Necessity: each task has an attempt limit that is greater than or equal to 1
|
60
|
-
|
package/src/tasks/types.ts
DELETED
@@ -1,58 +0,0 @@
|
|
1
|
-
import type { ValidateFunction } from 'ajv';
|
2
|
-
import type { AnyObject } from 'pinejs-client-core';
|
3
|
-
import type * as Db from '../database-layer/db';
|
4
|
-
import type { PinejsClient } from '../sbvr-api/sbvr-utils';
|
5
|
-
|
6
|
-
export const taskStatuses = [
|
7
|
-
'pending',
|
8
|
-
'cancelled',
|
9
|
-
'success',
|
10
|
-
'failed',
|
11
|
-
] as const;
|
12
|
-
export type TaskStatus = (typeof taskStatuses)[number];
|
13
|
-
export interface Task {
|
14
|
-
id: number;
|
15
|
-
created_at: Date;
|
16
|
-
modified_at: Date;
|
17
|
-
is_created_by__actor: number;
|
18
|
-
is_executed_by__handler: string;
|
19
|
-
is_executed_with__parameter_set: object | null;
|
20
|
-
is_scheduled_with__cron_expression: string | null;
|
21
|
-
is_scheduled_to_execute_on__time: Date | null;
|
22
|
-
priority: number;
|
23
|
-
status: TaskStatus;
|
24
|
-
started_on__time: Date | null;
|
25
|
-
ended_on__time: Date | null;
|
26
|
-
error_message: string | null;
|
27
|
-
attempt_count: number;
|
28
|
-
attempt_limit: number;
|
29
|
-
}
|
30
|
-
|
31
|
-
export type PartialTask = Pick<
|
32
|
-
Task,
|
33
|
-
| 'id'
|
34
|
-
| 'is_created_by__actor'
|
35
|
-
| 'is_executed_by__handler'
|
36
|
-
| 'is_executed_with__parameter_set'
|
37
|
-
| 'is_scheduled_with__cron_expression'
|
38
|
-
| 'priority'
|
39
|
-
| 'attempt_count'
|
40
|
-
| 'attempt_limit'
|
41
|
-
>;
|
42
|
-
|
43
|
-
export interface TaskArgs {
|
44
|
-
api: PinejsClient;
|
45
|
-
params: AnyObject;
|
46
|
-
tx: Db.Tx;
|
47
|
-
}
|
48
|
-
|
49
|
-
export type TaskResponse = Promise<{
|
50
|
-
status: TaskStatus;
|
51
|
-
error?: string;
|
52
|
-
}>;
|
53
|
-
|
54
|
-
export interface TaskHandler {
|
55
|
-
name: string;
|
56
|
-
fn: (options: TaskArgs) => TaskResponse;
|
57
|
-
validate?: ValidateFunction;
|
58
|
-
}
|
package/src/tasks/worker.ts
DELETED
@@ -1,246 +0,0 @@
|
|
1
|
-
import type { AnyObject } from 'pinejs-client-core';
|
2
|
-
import { tasks as tasksEnv } from '../config-loader/env';
|
3
|
-
import type * as Db from '../database-layer/db';
|
4
|
-
import * as permissions from '../sbvr-api/permissions';
|
5
|
-
import { PinejsClient } from '../sbvr-api/sbvr-utils';
|
6
|
-
import { sbvrUtils } from '../server-glue/module';
|
7
|
-
import { ajv, channel } from './common';
|
8
|
-
import type { PartialTask, TaskHandler, TaskStatus } from './types';
|
9
|
-
|
10
|
-
// Map of column names with SBVR names used in SELECT queries
|
11
|
-
const selectColumns = Object.entries({
|
12
|
-
id: 'id',
|
13
|
-
'is executed by-handler': 'is_executed_by__handler',
|
14
|
-
'is executed with-parameter set': 'is_executed_with__parameter_set',
|
15
|
-
'is scheduled with-cron expression': 'is_scheduled_with__cron_expression',
|
16
|
-
'attempt count': 'attempt_count',
|
17
|
-
'attempt limit': 'attempt_limit',
|
18
|
-
priority: 'priority',
|
19
|
-
'is created by-actor': 'is_created_by__actor',
|
20
|
-
})
|
21
|
-
.map(([key, value]) => `t."${key}" AS "${value}"`)
|
22
|
-
.join(', ');
|
23
|
-
|
24
|
-
// The worker is responsible for executing tasks in the queue. It listens for
|
25
|
-
// notifications and polls the database for tasks to execute. It will execute
|
26
|
-
// tasks in parallel up to a certain concurrency limit.
|
27
|
-
export class Worker {
|
28
|
-
public handlers: Record<string, TaskHandler> = {};
|
29
|
-
private readonly concurrency: number;
|
30
|
-
private readonly interval: number;
|
31
|
-
private client: PinejsClient;
|
32
|
-
private executing = 0;
|
33
|
-
|
34
|
-
constructor(client: PinejsClient) {
|
35
|
-
this.client = client;
|
36
|
-
this.concurrency = tasksEnv.queueConcurrency;
|
37
|
-
this.interval = tasksEnv.queueIntervalMS;
|
38
|
-
}
|
39
|
-
|
40
|
-
// Check if instance can execute more tasks
|
41
|
-
private canExecute(): boolean {
|
42
|
-
return (
|
43
|
-
this.executing < this.concurrency && Object.keys(this.handlers).length > 0
|
44
|
-
);
|
45
|
-
}
|
46
|
-
|
47
|
-
// Execute a task
|
48
|
-
private async execute(tx: Db.Tx, task: PartialTask): Promise<void> {
|
49
|
-
this.executing++;
|
50
|
-
try {
|
51
|
-
// Get specified handler
|
52
|
-
const handler = this.handlers[task.is_executed_by__handler];
|
53
|
-
const startedOnTime = new Date();
|
54
|
-
if (handler == null) {
|
55
|
-
await this.finalize(
|
56
|
-
tx,
|
57
|
-
task,
|
58
|
-
startedOnTime,
|
59
|
-
'failed',
|
60
|
-
'Matching task handler not found',
|
61
|
-
);
|
62
|
-
return;
|
63
|
-
}
|
64
|
-
|
65
|
-
// Validate parameters before execution so we can fail early if
|
66
|
-
// the parameter set is invalid. This can happen if the handler
|
67
|
-
// definition changes after a task is added to the queue.
|
68
|
-
if (
|
69
|
-
handler.validate != null &&
|
70
|
-
!handler.validate(task.is_executed_with__parameter_set)
|
71
|
-
) {
|
72
|
-
await this.finalize(
|
73
|
-
tx,
|
74
|
-
task,
|
75
|
-
startedOnTime,
|
76
|
-
'failed',
|
77
|
-
`Invalid parameter set: ${ajv.errorsText(handler.validate.errors)}`,
|
78
|
-
);
|
79
|
-
return;
|
80
|
-
}
|
81
|
-
|
82
|
-
// Execute handler
|
83
|
-
const result = await handler.fn({
|
84
|
-
api: new PinejsClient({
|
85
|
-
passthrough: {
|
86
|
-
tx,
|
87
|
-
},
|
88
|
-
}),
|
89
|
-
params: task.is_executed_with__parameter_set ?? {},
|
90
|
-
tx,
|
91
|
-
});
|
92
|
-
|
93
|
-
// Update task with results
|
94
|
-
await this.finalize(tx, task, startedOnTime, result.status, result.error);
|
95
|
-
} catch (err) {
|
96
|
-
// This shouldn't happen, but if it does we want to log and kill the process
|
97
|
-
console.error('Task execution failed:', err);
|
98
|
-
process.exit(1);
|
99
|
-
} finally {
|
100
|
-
this.executing--;
|
101
|
-
}
|
102
|
-
}
|
103
|
-
|
104
|
-
// Update task and schedule next attempt if needed
|
105
|
-
private async finalize(
|
106
|
-
tx: Db.Tx,
|
107
|
-
task: PartialTask,
|
108
|
-
startedOnTime: Date,
|
109
|
-
status: TaskStatus,
|
110
|
-
errorMessage?: string,
|
111
|
-
): Promise<void> {
|
112
|
-
const attemptCount = task.attempt_count + 1;
|
113
|
-
const body: AnyObject = {
|
114
|
-
started_on__time: startedOnTime,
|
115
|
-
ended_on__time: new Date(),
|
116
|
-
status,
|
117
|
-
attempt_count: attemptCount,
|
118
|
-
...(errorMessage != null && { error_message: errorMessage }),
|
119
|
-
};
|
120
|
-
|
121
|
-
// Re-enqueue if the task failed but has retries left, remember that
|
122
|
-
// attemptCount includes the initial attempt while attempt_limit does not
|
123
|
-
if (status === 'failed' && attemptCount < task.attempt_limit) {
|
124
|
-
body.status = 'pending';
|
125
|
-
|
126
|
-
// Schedule next attempt using exponential backoff
|
127
|
-
body.is_scheduled_to_execute_on__time =
|
128
|
-
this.getNextAttemptTime(attemptCount);
|
129
|
-
}
|
130
|
-
|
131
|
-
// Patch current task
|
132
|
-
await this.client.patch({
|
133
|
-
resource: 'task',
|
134
|
-
passthrough: {
|
135
|
-
tx,
|
136
|
-
req: permissions.root,
|
137
|
-
},
|
138
|
-
id: task.id,
|
139
|
-
body,
|
140
|
-
});
|
141
|
-
|
142
|
-
// Create new task with same configuration if previous
|
143
|
-
// iteration completed and has a cron expression
|
144
|
-
if (
|
145
|
-
['failed', 'success'].includes(body.status) &&
|
146
|
-
task.is_scheduled_with__cron_expression != null
|
147
|
-
) {
|
148
|
-
await this.client.post({
|
149
|
-
resource: 'task',
|
150
|
-
passthrough: {
|
151
|
-
tx,
|
152
|
-
req: permissions.root,
|
153
|
-
},
|
154
|
-
body: {
|
155
|
-
attempt_limit: task.attempt_limit,
|
156
|
-
is_created_by__actor: task.is_created_by__actor,
|
157
|
-
is_executed_by__handler: task.is_executed_by__handler,
|
158
|
-
is_executed_with__parameter_set: task.is_executed_with__parameter_set,
|
159
|
-
is_scheduled_with__cron_expression:
|
160
|
-
task.is_scheduled_with__cron_expression,
|
161
|
-
priority: task.priority,
|
162
|
-
},
|
163
|
-
});
|
164
|
-
}
|
165
|
-
}
|
166
|
-
|
167
|
-
// Calculate next attempt time using exponential backoff
|
168
|
-
private getNextAttemptTime(attempt: number): Date | null {
|
169
|
-
const delay = Math.ceil(Math.exp(Math.min(10, attempt))) * 1000;
|
170
|
-
return new Date(Date.now() + delay);
|
171
|
-
}
|
172
|
-
|
173
|
-
// Poll for tasks to execute
|
174
|
-
private poll(): void {
|
175
|
-
let executed = false;
|
176
|
-
const handlerNames = Object.keys(this.handlers);
|
177
|
-
const binds = handlerNames.map((_, index) => `$${index + 1}`).join(', ');
|
178
|
-
sbvrUtils.db
|
179
|
-
.transaction(async (tx) => {
|
180
|
-
if (!this.canExecute()) {
|
181
|
-
return;
|
182
|
-
}
|
183
|
-
|
184
|
-
const result = await tx.executeSql(
|
185
|
-
`SELECT ${selectColumns}
|
186
|
-
FROM task AS t
|
187
|
-
WHERE
|
188
|
-
t."is executed by-handler" IN (${binds}) AND
|
189
|
-
t."status" = 'pending' AND
|
190
|
-
t."attempt count" <= t."attempt limit" AND
|
191
|
-
(
|
192
|
-
t."is scheduled to execute on-time" IS NULL OR
|
193
|
-
t."is scheduled to execute on-time" <= CURRENT_TIMESTAMP + INTERVAL '${Math.ceil(this.interval / 1000)} second'
|
194
|
-
)
|
195
|
-
ORDER BY
|
196
|
-
t."is scheduled to execute on-time" ASC,
|
197
|
-
t."priority" DESC,
|
198
|
-
t."id" ASC
|
199
|
-
LIMIT ${Math.max(this.concurrency - this.executing, 0)}
|
200
|
-
FOR UPDATE SKIP LOCKED`,
|
201
|
-
handlerNames,
|
202
|
-
);
|
203
|
-
if (result.rows.length === 0) {
|
204
|
-
return;
|
205
|
-
}
|
206
|
-
|
207
|
-
// Tasks found, execute them in parallel
|
208
|
-
await Promise.all(
|
209
|
-
result.rows.map(async (row) => {
|
210
|
-
await this.execute(tx, row as PartialTask);
|
211
|
-
}),
|
212
|
-
);
|
213
|
-
executed = true;
|
214
|
-
})
|
215
|
-
.catch((err) => {
|
216
|
-
console.error('Failed polling for tasks:', err);
|
217
|
-
})
|
218
|
-
.finally(() => {
|
219
|
-
setTimeout(() => this.poll(), executed ? 0 : this.interval);
|
220
|
-
});
|
221
|
-
}
|
222
|
-
|
223
|
-
// Start listening and polling for tasks
|
224
|
-
public start(): void {
|
225
|
-
sbvrUtils.db.on?.(
|
226
|
-
'notification',
|
227
|
-
async (msg) => {
|
228
|
-
if (this.canExecute()) {
|
229
|
-
await sbvrUtils.db.transaction(async (tx) => {
|
230
|
-
const result = await tx.executeSql(
|
231
|
-
`SELECT ${selectColumns} FROM task AS t WHERE id = $1 FOR UPDATE SKIP LOCKED`,
|
232
|
-
[msg.payload],
|
233
|
-
);
|
234
|
-
if (result.rows.length > 0) {
|
235
|
-
await this.execute(tx, result.rows[0] as PartialTask);
|
236
|
-
}
|
237
|
-
});
|
238
|
-
}
|
239
|
-
},
|
240
|
-
{
|
241
|
-
channel,
|
242
|
-
},
|
243
|
-
);
|
244
|
-
this.poll();
|
245
|
-
}
|
246
|
-
}
|